From 2a79b3dd2f72be1a53ab0f1a83886bdb159cb3fd Mon Sep 17 00:00:00 2001 From: Jason Farrar Date: Thu, 6 Aug 2026 11:33:44 +0100 Subject: [PATCH 1/3] chore: resolve all ruff lint warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - C408: dict() → {} in import_config.py - C403: set([...]) → {...} in statements.py - RUF015: [x][0] → next(...) in statement_functions.py - RUF022: Sort __all__ in __init__.py and modules/__init__.py - RUF023: Sort __slots__ in Statement, StatementBatch, TestHarness - RUF034: Remove useless if-else ({} if x else {}) in statements.py - RUF012: Add ClassVar annotations in housekeeping.py and test_cli.py - SIM102: Combine nested if in statement_functions.py and test_docs.py - SIM103: Inline return condition in statements.py - PLW1510: Add check=False to subprocess.run in testing.py - PYI034: Use Self return type for __enter__ in testing.py - PIE810: Merge startswith calls in generate_docs.py - PERF102: .items() → .values() in import_config.py and scripts - DTZ005/DTZ001/DTZ007: Suppress intentional local-time uses with noqa - BLE001/S110/S112: Suppress intentional broad exception catches with noqa - Formatting fixes via ruff format Signed-off-by: Jason Farrar --- AGENTS.md | 6 +- docs/guides/exports.md | 7 +- scripts/generate_docs.py | 24 +-- scripts/generate_supported_banks.py | 8 +- scripts/generate_test_metadata.py | 6 +- src/bank_statement_parser/__init__.py | 115 +++++++------- src/bank_statement_parser/cli.py | 6 +- src/bank_statement_parser/data/__init__.py | 2 +- .../data/build_datamart.py | 6 +- .../data/create_project_db.py | 2 +- .../data/create_project_db_views.py | 2 +- .../data/housekeeping.py | 7 +- .../data/mock_project_data.py | 8 +- src/bank_statement_parser/modules/__init__.py | 27 ++-- src/bank_statement_parser/modules/data.py | 104 ++++++------- src/bank_statement_parser/modules/database.py | 10 +- src/bank_statement_parser/modules/debug.py | 2 +- src/bank_statement_parser/modules/errors.py | 8 +- .../modules/export_spec.py | 4 +- src/bank_statement_parser/modules/forex.py | 6 +- .../modules/import_config.py | 26 ++-- src/bank_statement_parser/modules/parquet.py | 2 +- src/bank_statement_parser/modules/paths.py | 6 +- .../modules/reports_db.py | 6 +- .../modules/statement_functions.py | 33 ++--- .../modules/statements.py | 140 ++++++++---------- src/bank_statement_parser/testing.py | 9 +- tests/conftest.py | 6 +- tests/test_cli.py | 7 +- tests/test_docs.py | 16 +- tests/test_forex.py | 7 +- tests/test_statements.py | 4 +- 32 files changed, 286 insertions(+), 336 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e08489e..1a2124f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,10 +100,11 @@ Private constants use a leading underscore: `_ALLOWED_TABLES`. Public shared con ### Classes — use `__slots__` on every class. Regular classes declare a tuple; dataclasses use `slots=True`. Subclasses declare only their own additional slots. ```python -@dataclass(frozen=True, slots=True) # immutable value objects +@dataclass(frozen=True, slots=True) # immutable value objects class AccountType: account_type: str + @dataclass(frozen=False, slots=True) # mutable state holders class Account: account: str @@ -114,7 +115,7 @@ class Account: ```python class Housekeeping: - FK_RELATIONSHIPS = [...] # class variable, not in __slots__ + FK_RELATIONSHIPS = [...] # class variable, not in __slots__ _ALLOWED_TABLES: frozenset[str] = frozenset([...]) ``` @@ -135,6 +136,7 @@ Use `frozenset[str]` for any identifier or name whitelist used in membership tes def __enter__(self) -> "TestHarness": return self.setup() + def __exit__(self, *args: object) -> None: self.teardown() ``` diff --git a/docs/guides/exports.md b/docs/guides/exports.md index 69c175c..332ad48 100644 --- a/docs/guides/exports.md +++ b/docs/guides/exports.md @@ -238,7 +238,7 @@ import bank_statement_parser as bsp bsp.db.export_csv() # Export multi star-schema tables to Excel -bsp.db.export_excel(type='multi') +bsp.db.export_excel(type="multi") # Export JSON bsp.db.export_json() @@ -248,8 +248,9 @@ bsp.db.export_reporting_data() # Export to a custom directory from pathlib import Path -bsp.db.export_csv(folder=Path('~/exports')) -bsp.db.export_excel(path=Path('~/exports/report.xlsx')) + +bsp.db.export_csv(folder=Path("~/exports")) +bsp.db.export_excel(path=Path("~/exports/report.xlsx")) ``` ## Report Classes diff --git a/scripts/generate_docs.py b/scripts/generate_docs.py index a237311..a21aef7 100644 --- a/scripts/generate_docs.py +++ b/scripts/generate_docs.py @@ -993,9 +993,8 @@ def w(text: str = "") -> None: acct_lines: list[str] = [] entry_count = 0 for line in accounts_toml.splitlines(): - if line.startswith("[") and not line.startswith("[HSBC_UK_CRD_RCC"): - if entry_count >= 1: - break + if line.startswith("[") and not line.startswith("[HSBC_UK_CRD_RCC") and entry_count >= 1: + break if line.startswith("[HSBC_UK_CRD_RCC"): entry_count = 1 if entry_count >= 1: @@ -1391,14 +1390,7 @@ def w(text: str = "") -> None: in_code = False for line in qs_lines: stripped = line.strip() - if ( - stripped.startswith("import ") - or stripped.startswith("from ") - or stripped.startswith("batch") - or stripped.startswith("flat") - or stripped.startswith("bsp.") - or stripped.startswith("#") - ): + if stripped.startswith(("import ", "from ", "batch", "flat", "bsp.", "#")): in_code = True if in_code: if stripped and not re.match(r"^-+$", stripped) and not re.match(r"^\w.*\w$", stripped.rstrip(":")): @@ -2014,12 +2006,12 @@ def _extract_banks(config_dir: Path) -> list[tuple[str, list[str]]]: try: with open(companies_file, "rb") as f: companies_data = tomllib.load(f) - except Exception: + except Exception: # noqa: BLE001, S112 continue # Find the company name (first [[section]] in companies.toml) bank_name = None - for section_name, section_data in companies_data.items(): + for section_data in companies_data.values(): if isinstance(section_data, dict) and "name" in section_data: bank_name = section_data["name"] break @@ -2032,11 +2024,11 @@ def _extract_banks(config_dir: Path) -> list[tuple[str, list[str]]]: try: with open(accounts_file, "rb") as f: accounts_data = tomllib.load(f) - except Exception: + except Exception: # noqa: BLE001 accounts = [] else: accounts = [] - for section_name, section_data in accounts_data.items(): + for section_data in accounts_data.values(): if isinstance(section_data, dict) and "account" in section_data: account_name = section_data["account"] if account_name and account_name not in accounts: @@ -2155,7 +2147,7 @@ def main() -> None: table = _generate_banks_table(config_dir) _update_index_md_table(_INDEX_MD, table) print(f"Updated banks table in {_INDEX_MD.relative_to(_REPO_ROOT)}") - except Exception as e: + except Exception as e: # noqa: BLE001 print(f"Warning: Failed to update banks table: {e}") diff --git a/scripts/generate_supported_banks.py b/scripts/generate_supported_banks.py index c233e8c..a293d1a 100644 --- a/scripts/generate_supported_banks.py +++ b/scripts/generate_supported_banks.py @@ -66,12 +66,12 @@ def extract_banks(config_dir: Path) -> list[tuple[str, list[str]]]: try: with open(companies_file, "rb") as f: companies_data = tomllib.load(f) - except Exception: + except Exception: # noqa: BLE001, S112 continue # Find the company name (first [[section]] in companies.toml) bank_name = None - for section_name, section_data in companies_data.items(): + for section_data in companies_data.values(): if isinstance(section_data, dict) and "name" in section_data: bank_name = section_data["name"] break @@ -84,11 +84,11 @@ def extract_banks(config_dir: Path) -> list[tuple[str, list[str]]]: try: with open(accounts_file, "rb") as f: accounts_data = tomllib.load(f) - except Exception: + except Exception: # noqa: BLE001 accounts = [] else: accounts = [] - for section_name, section_data in accounts_data.items(): + for section_data in accounts_data.values(): if isinstance(section_data, dict) and "account" in section_data: account_name = section_data["account"] if account_name and account_name not in accounts: diff --git a/scripts/generate_test_metadata.py b/scripts/generate_test_metadata.py index 34f2133..fd90625 100644 --- a/scripts/generate_test_metadata.py +++ b/scripts/generate_test_metadata.py @@ -76,7 +76,7 @@ def _get_transaction_count(project_path: Path, id_statement: str) -> int: ) count = cursor.fetchone()[0] return count - except Exception as e: + except Exception as e: # noqa: BLE001 print(f" ⚠️ Could not query transaction count from database: {e}") return 0 @@ -148,7 +148,7 @@ def _generate_metadata_for_good_pdfs() -> int: print(f"✓ {pdf_path.name} → {metadata_path.name}") successful += 1 - except Exception as e: + except Exception as e: # noqa: BLE001 print(f"✗ {pdf_path.name}: {type(e).__name__}: {e}") # Cleanup @@ -211,7 +211,7 @@ def _generate_metadata_for_bad_pdfs() -> int: print(f"✓ {pdf_path.name} → {metadata_path.name}") successful += 1 - except Exception as e: + except Exception as e: # noqa: BLE001 print(f"✗ {pdf_path.name}: {type(e).__name__}: {e}") # Cleanup diff --git a/src/bank_statement_parser/__init__.py b/src/bank_statement_parser/__init__.py index e7f1b3e..b71e605 100644 --- a/src/bank_statement_parser/__init__.py +++ b/src/bank_statement_parser/__init__.py @@ -97,26 +97,14 @@ import bank_statement_parser.modules.reports_db as db # --------------------------------------------------------------------------- -# Statement processing +# Data structures # --------------------------------------------------------------------------- -from bank_statement_parser.modules.statements import ( - Statement, - StatementBatch, - copy_statements_to_project, - delete_temp_files, - process_pdf_statement, -) +from bank_statement_parser.modules.data import Failure, ParquetFiles, PdfResult, Review, StatementInfo, Success # --------------------------------------------------------------------------- # Low-level persistence helpers — consumed by openstan's StanBatch # --------------------------------------------------------------------------- from bank_statement_parser.modules.database import update_db -from bank_statement_parser.modules.parquet import update_parquet - -# --------------------------------------------------------------------------- -# Data structures -# --------------------------------------------------------------------------- -from bank_statement_parser.modules.data import Failure, ParquetFiles, PdfResult, Review, StatementInfo, Success # --------------------------------------------------------------------------- # Debug / diagnostics @@ -137,8 +125,20 @@ # Config helpers # --------------------------------------------------------------------------- from bank_statement_parser.modules.import_config import copy_default_import_config +from bank_statement_parser.modules.parquet import update_parquet from bank_statement_parser.modules.paths import ProjectPaths, copy_project_folders, validate_or_initialise_project +# --------------------------------------------------------------------------- +# Statement processing +# --------------------------------------------------------------------------- +from bank_statement_parser.modules.statements import ( + Statement, + StatementBatch, + copy_statements_to_project, + delete_temp_files, + process_pdf_statement, +) + # --------------------------------------------------------------------------- # PDF anonymisation utility # --------------------------------------------------------------------------- @@ -148,24 +148,23 @@ # --------------------------------------------------------------------------- # Low-level PDF helpers # --------------------------------------------------------------------------- -from bank_statement_parser.modules.pdf_functions import ( - get_table_from_region, - page_crop, - page_text, - pdf_open, - region_search, -) - # --------------------------------------------------------------------------- # Database / data-mart utilities # --------------------------------------------------------------------------- from bank_statement_parser.data import Housekeeping, build_datamart, create_db +from bank_statement_parser.modules.data import ForexApiConfig # --------------------------------------------------------------------------- # Forex / currency conversion # --------------------------------------------------------------------------- from bank_statement_parser.modules.forex import get_exchange_rates -from bank_statement_parser.modules.data import ForexApiConfig +from bank_statement_parser.modules.pdf_functions import ( + get_table_from_region, + page_crop, + page_text, + pdf_open, + region_search, +) # --------------------------------------------------------------------------- # Testing harness @@ -173,55 +172,43 @@ from bank_statement_parser.testing import TestHarness __all__ = [ - # Meta - "__app_name__", - "__version__", - # Namespaced report backend - "db", - # Statement processing - "Statement", - "StatementBatch", - "process_pdf_statement", - "copy_statements_to_project", - "delete_temp_files", - # Low-level persistence helpers - "update_parquet", - "update_db", - # Data structures - "PdfResult", - "Success", - "Review", "Failure", - "StatementInfo", + "ForexApiConfig", + "Housekeeping", "ParquetFiles", - # Debug / diagnostics - "debug_pdf_statement", - "debug_statements", - # Errors - "StatementError", - "ProjectDatabaseMissing", + "PdfResult", "ProjectConfigMissing", - # Config helpers + "ProjectDatabaseMissing", + "ProjectPaths", + "Review", + "Statement", + "StatementBatch", + "StatementError", + "StatementInfo", + "Success", + "TestGateFailure", + "TestHarness", + "__app_name__", + "__version__", + "build_datamart", "copy_default_import_config", "copy_project_folders", - "validate_or_initialise_project", - "ProjectPaths", - # Low-level PDF helpers - "pdf_open", + "copy_statements_to_project", + "create_db", + "db", + "debug_pdf_statement", + "debug_statements", + "delete_temp_files", + "get_exchange_rates", + "get_table_from_region", "page_crop", "page_text", + "pdf_open", + "process_pdf_statement", "region_search", - "get_table_from_region", - # Data-mart / database - "build_datamart", - "create_db", - "Housekeeping", - # Forex / currency conversion - "get_exchange_rates", - "ForexApiConfig", - # Testing harness - "TestHarness", - "TestGateFailure", + "update_db", + "update_parquet", + "validate_or_initialise_project", ] # Conditionally add anonymise_pdf if uk-bank-statement-anonymiser is installed diff --git a/src/bank_statement_parser/cli.py b/src/bank_statement_parser/cli.py index a5a3e83..7e983d6 100644 --- a/src/bank_statement_parser/cli.py +++ b/src/bank_statement_parser/cli.py @@ -100,7 +100,7 @@ def _cmd_anonymise(args: argparse.Namespace) -> int: except ImportError as exc: print(f"Error: {exc}", file=sys.stderr) return 1 - except (ValueError, FileNotFoundError, IOError, OSError) as exc: + except (ValueError, FileNotFoundError, OSError) as exc: print(f"Error: {type(exc).__name__}: {exc}", file=sys.stderr) return 1 @@ -173,9 +173,9 @@ def _cmd_process(args: argparse.Namespace) -> int: # -- summary ------------------------------------------------------------- paths = ProjectPaths.resolve(project_path) - print("") + print() print(f"Done — processed {batch.pdf_count} PDF(s) ({batch.errors} error(s)) in {batch.duration_secs:.1f}s.") - print("") + print() print(f"Database: {paths.project_db}") print(f"Exports: {paths.exports}") diff --git a/src/bank_statement_parser/data/__init__.py b/src/bank_statement_parser/data/__init__.py index b21a879..9c15781 100644 --- a/src/bank_statement_parser/data/__init__.py +++ b/src/bank_statement_parser/data/__init__.py @@ -29,7 +29,7 @@ from bank_statement_parser.data.housekeeping import Housekeeping __all__ = [ + "Housekeeping", "build_datamart", "create_db", - "Housekeeping", ] diff --git a/src/bank_statement_parser/data/build_datamart.py b/src/bank_statement_parser/data/build_datamart.py index 2f8443d..a9b5759 100644 --- a/src/bank_statement_parser/data/build_datamart.py +++ b/src/bank_statement_parser/data/build_datamart.py @@ -269,7 +269,7 @@ def _drop_mart_objects(conn: sqlite3.Connection) -> None: if name not in _ALLOWED_MART_NAMES: # defence-in-depth: name is a literal above raise ValueError(f"Unexpected mart object name {name!r}; refusing to DROP.") kw = "VIEW" if row[0] == "view" else "TABLE" - conn.execute(f"DROP {kw} IF EXISTS {name}") # noqa: S608 + conn.execute(f"DROP {kw} IF EXISTS {name}") def _ensure_mart_structure(conn: sqlite3.Connection) -> None: @@ -680,7 +680,7 @@ def _build_fact_balance(conn: sqlite3.Connection, verbose: bool) -> float: for tbl in ("_fb_agg", "_fb_bk", "_fb_grid"): if tbl not in _ALLOWED_TEMP: raise ValueError(f"Unexpected temp table name {tbl!r}; refusing to DROP.") - conn.execute(f"DROP TABLE IF EXISTS {tbl}") # noqa: S608 + conn.execute(f"DROP TABLE IF EXISTS {tbl}") elapsed = time.monotonic() - t0 n = conn.execute("SELECT COUNT(*) FROM FactBalance").fetchone()[0] @@ -743,7 +743,7 @@ def build_datamart(db_path: Path, verbose: bool = True) -> dict: # --------------------------------------------------------------------------- if __name__ == "__main__": - from bank_statement_parser.modules.paths import ProjectPaths # noqa: PLC0415 + from bank_statement_parser.modules.paths import ProjectPaths parser = argparse.ArgumentParser(description="Build (or rebuild) the data mart tables from raw source data.") parser.add_argument( diff --git a/src/bank_statement_parser/data/create_project_db.py b/src/bank_statement_parser/data/create_project_db.py index b8b1708..eb593a9 100644 --- a/src/bank_statement_parser/data/create_project_db.py +++ b/src/bank_statement_parser/data/create_project_db.py @@ -230,6 +230,6 @@ def create_indexes(db_path: Path): if __name__ == "__main__": - from bank_statement_parser.modules.paths import ProjectPaths # noqa: PLC0415 + from bank_statement_parser.modules.paths import ProjectPaths main(db_path=ProjectPaths.resolve().project_db, with_fk=True) diff --git a/src/bank_statement_parser/data/create_project_db_views.py b/src/bank_statement_parser/data/create_project_db_views.py index af513d5..2bff93a 100644 --- a/src/bank_statement_parser/data/create_project_db_views.py +++ b/src/bank_statement_parser/data/create_project_db_views.py @@ -290,6 +290,6 @@ def create_views(db_path: Path): if __name__ == "__main__": - from bank_statement_parser.modules.paths import ProjectPaths # noqa: PLC0415 + from bank_statement_parser.modules.paths import ProjectPaths create_views(ProjectPaths.resolve().project_db) diff --git a/src/bank_statement_parser/data/housekeeping.py b/src/bank_statement_parser/data/housekeeping.py index 3f28ce2..71190b6 100644 --- a/src/bank_statement_parser/data/housekeeping.py +++ b/src/bank_statement_parser/data/housekeeping.py @@ -18,6 +18,7 @@ import argparse import sqlite3 from pathlib import Path +from typing import ClassVar class Housekeeping: @@ -31,7 +32,7 @@ class Housekeeping: db_path: Path to the project's ``database/project.db`` SQLite file. """ - FK_RELATIONSHIPS = [ + FK_RELATIONSHIPS: ClassVar[list[tuple[str, str, str, str]]] = [ ("checks_and_balances", "ID_BATCHLINE", "batch_lines", "ID_BATCHLINE"), ("checks_and_balances", "ID_BATCH", "batch_heads", "ID_BATCH"), ("statement_heads", "ID_BATCHLINE", "batch_lines", "ID_BATCHLINE"), @@ -39,7 +40,7 @@ class Housekeeping: ("batch_lines", "ID_BATCH", "batch_heads", "ID_BATCH"), ] - DELETE_ORDER = [ + DELETE_ORDER: ClassVar[list[str]] = [ "checks_and_balances", "statement_lines", "batch_lines", @@ -184,7 +185,7 @@ def cleanup(self, delete: bool = False) -> dict[str, dict]: if __name__ == "__main__": - from bank_statement_parser.modules.paths import ProjectPaths # noqa: PLC0415 + from bank_statement_parser.modules.paths import ProjectPaths parser = argparse.ArgumentParser(description="Database integrity housekeeping") parser.add_argument( diff --git a/src/bank_statement_parser/data/mock_project_data.py b/src/bank_statement_parser/data/mock_project_data.py index b1c1cfe..b280447 100644 --- a/src/bank_statement_parser/data/mock_project_data.py +++ b/src/bank_statement_parser/data/mock_project_data.py @@ -74,7 +74,7 @@ def generate_mock_data(db_path: Path, num_batches: int = 10, statements_per_batc batch_ids = [str(uuid.uuid4()) for _ in range(num_batches)] session_ids = [str(uuid.uuid4()) for _ in range(num_batches)] - start_date = datetime(2024, 1, 1) + start_date = datetime(2024, 1, 1) # noqa: DTZ001 batch_dates = [(start_date + timedelta(days=i * 30)).strftime("%Y-%m-%d %H:%M:%S") for i in range(num_batches)] batch_heads_data = [] @@ -102,7 +102,7 @@ def generate_mock_data(db_path: Path, num_batches: int = 10, statements_per_batc statement_dates = [] for batch_idx in range(num_batches): - base_date = datetime(2024, 1, 1) + timedelta(days=batch_idx * 30) + base_date = datetime(2024, 1, 1) + timedelta(days=batch_idx * 30) # noqa: DTZ001 for stmt_idx in range(statements_per_batch): stmt_date = (base_date + timedelta(days=random.randint(10, 25))).strftime("%Y-%m-%d") statement_dates.append(stmt_date) @@ -182,7 +182,7 @@ def generate_mock_data(db_path: Path, num_batches: int = 10, statements_per_batc for trn_idx in range(num_transactions): transaction_id = str(uuid.uuid4()) page_number = 1 - transaction_date = (datetime.strptime(statement_dates[stmt_idx], "%Y-%m-%d") - timedelta(days=random.randint(1, 28))).strftime( + transaction_date = (datetime.strptime(statement_dates[stmt_idx], "%Y-%m-%d") - timedelta(days=random.randint(1, 28))).strftime( # noqa: DTZ007 "%Y-%m-%d" ) transaction_number = trn_idx + 1 @@ -287,7 +287,7 @@ def generate_mock_data(db_path: Path, num_batches: int = 10, statements_per_batc if __name__ == "__main__": - from bank_statement_parser.modules.paths import ProjectPaths # noqa: PLC0415 + from bank_statement_parser.modules.paths import ProjectPaths generate_mock_data( db_path=ProjectPaths.resolve().project_db, diff --git a/src/bank_statement_parser/modules/__init__.py b/src/bank_statement_parser/modules/__init__.py index da83427..69132bd 100644 --- a/src/bank_statement_parser/modules/__init__.py +++ b/src/bank_statement_parser/modules/__init__.py @@ -54,28 +54,23 @@ ) __all__ = [ - # Statement processing - "Statement", - "StatementBatch", - "process_pdf_statement", + "Failure", + "ParquetFiles", "PdfResult", - "Success", "Review", - "Failure", + "Statement", + "StatementBatch", + "StatementError", "StatementInfo", - "ParquetFiles", - "delete_temp_files", - # Config helpers + "Success", "copy_default_import_config", "copy_project_folders", - # Errors - "StatementError", - # Low-level PDF helpers - "pdf_open", + "db", + "delete_temp_files", + "get_table_from_region", "page_crop", "page_text", + "pdf_open", + "process_pdf_statement", "region_search", - "get_table_from_region", - # Namespaced report backend - "db", ] diff --git a/src/bank_statement_parser/modules/data.py b/src/bank_statement_parser/modules/data.py index b516046..d230f52 100644 --- a/src/bank_statement_parser/modules/data.py +++ b/src/bank_statement_parser/modules/data.py @@ -302,36 +302,36 @@ class StdRefs: # [ACTIVE] — Key used to select this rule; matched against the statement type string # of the PDF being processed (e.g. "HSBC UK Current Account"). - field: Optional[str] + field: str | None # [ACTIVE] — Name of the raw extracted column to promote. Set to None (or omit) # when a literal default value should be used instead of a column value. - concat_fields: Optional[list] + concat_fields: list | None # [ACTIVE] — Name of the raw extracted columns to concatenate and promote. Set to None (or omit) # in order to revert to a single field and it's fallback - format: Optional[str] + format: str | None # [ACTIVE] — strptime format string applied when StandardFields.type == "date" # (e.g. "%-d %B %Y"). Ignored for numeric and string types. - default: Optional[str] + default: str | None # [ACTIVE] — Literal string value used as the output when ``field`` is None/absent. # Useful for injecting constant metadata (e.g. transaction_type = "CC"). - multiplier: Optional[float] = 1 + multiplier: float | None = 1 # [ACTIVE] — Scalar applied to the value after casting when # StandardFields.type == "numeric". Use -1 to invert sign (e.g. to convert a # credit amount stored as positive into a negative figure). - exclude_positive_values: Optional[bool] = False + exclude_positive_values: bool | None = False # [ACTIVE] — When True, any positive numeric value is replaced with 0 after # casting. Used to isolate debit-side figures from a combined amount column. - exclude_negative_values: Optional[bool] = False + exclude_negative_values: bool | None = False # [ACTIVE] — When True, any negative numeric value is replaced with 0 after # casting. Used to isolate credit-side figures from a combined amount column. - terminator: Optional[str] = None + terminator: str | None = None # [ACTIVE] — Regex pattern; when present the string value is truncated at the # first match position before being written to the standard column. Useful for # stripping trailing boilerplate appended by merge_fields @@ -437,12 +437,12 @@ class NumericModifier: Example TOML: ``numeric_modifier = {suffix = "D", multiplier = -1.0}`` """ - prefix: Optional[str] + prefix: str | None # [ACTIVE] — If the raw value starts with this string the prefix is stripped and # the multiplier applied. Use for formats like "(123.45)" where "(" signals a # negative value. - suffix: Optional[str] + suffix: str | None # [ACTIVE] — If the raw value ends with this string the suffix is stripped and # the multiplier applied. Use for formats like "123.45 CR" or "123.45D". @@ -499,7 +499,7 @@ class FieldOffset: # offset value when type == "numeric". Overrides the account-level currency. # When type == "currency" the account-level currency is used and this is ignored. - numeric_modifier: Optional[NumericModifier] = None + numeric_modifier: NumericModifier | None = None # [ACTIVE] — Sign/multiplier modifier for the offset value. Overrides the # parent Field.numeric_modifier. @@ -524,7 +524,7 @@ class Field: # Used as the field identifier throughout the pipeline and in the output Parquet # files. - cell: Optional[Cell] + cell: Cell | None # [ACTIVE] — Row/column address for summary or detail table extraction. # Mutually exclusive with ``column``; set to None for transaction tables. @@ -549,12 +549,12 @@ class Field: # amount fields; reserve "numeric" for non-monetary numerics (e.g. # APR, sort code). - strip_characters_start: Optional[str] = None + strip_characters_start: str | None = None # [ACTIVE] — Characters to strip from the start of the raw string before pattern # matching (passed to Polars str.strip_chars_start()). Useful for leading # currency symbols not covered by the account currency spec. - strip_characters_end: Optional[str] = None + strip_characters_end: str | None = None # [ACTIVE] — Characters to strip from the end of the raw string before pattern # matching (passed to Polars str.strip_chars_end()). @@ -565,22 +565,22 @@ class Field: # (which always uses the account-level currency). Omit for non-monetary numeric # fields (e.g. APR, sort code) where no currency stripping is required. - numeric_modifier: Optional[NumericModifier] = None + numeric_modifier: NumericModifier | None = None # [ACTIVE] — Sign/multiplier transformation applied after numeric casting. # See NumericModifier. Omit for straightforward positive numeric values. - string_pattern: Optional[str] = None + string_pattern: str | None = None # [ACTIVE] — Regex pattern the extracted string must match. Extraction is # marked as failed (success = False) if the value does not match. Used to # validate field contents (e.g. date format) and to skip blank or irrelevant # rows. - string_max_length: Optional[int] = None + string_max_length: int | None = None # [ACTIVE] — Maximum character length for string values; longer strings are # truncated via str.head(). Useful for capping free-text description fields. # Defaults to 999 if not set. - date_format: Optional[str] = None + date_format: str | None = None # [STUB] — Intended strptime format for date parsing at the Field level. # Declared but never read by the pipeline; date format parsing is handled via # StdRefs.format in get_standard_fields() instead. @@ -592,7 +592,7 @@ class Field: # column is still extracted normally; the offset column value replaces it in the # output. See FieldOffset. - regex_groups: Optional[int] = None + regex_groups: int | None = None # [ACTIVE] — When set, extracts the specified capture group (1-indexed) from the # string_pattern regex match instead of the entire match (group 0). Useful for # splitting a single PDF column into multiple fields via regex capture groups. @@ -655,37 +655,37 @@ class Location: ``{page_number = 1, top_left = [50, 120], bottom_right = [560, 400], vertical_lines = [50, 150, 150, 320]}`` """ - page_number: Optional[int] = None + page_number: int | None = None # [ACTIVE] — 1-based page number. When set the location is used only on that # page. When None the location is cloned for every page (spawn_locations()). - top_left: Optional[list[int]] = None + top_left: list[int] | None = None # [ACTIVE] — [x, y] coordinates of the top-left corner of the crop rectangle. # Must be set together with bottom_right. When both are None the full page is # used. - bottom_right: Optional[list[int]] = None + bottom_right: list[int] | None = None # [ACTIVE] — [x, y] coordinates of the bottom-right corner of the crop # rectangle. Must be set together with top_left. - vertical_lines: Optional[list[int]] = None + vertical_lines: list[int] | None = None # [ACTIVE] — Explicit x-coordinates of vertical column dividers supplied to # pdfplumber as explicit_vertical_lines. Pairs of identical values create a # zero-width gap that forces a column boundary (e.g. [100, 100, 200, 200]). # When set, pdfplumber's automatic column detection is disabled for this region. - dynamic_last_vertical_line: Optional[DynamicLineSpec] = None + dynamic_last_vertical_line: DynamicLineSpec | None = None # [ACTIVE] — When set, the final value in vertical_lines is replaced at runtime # with an x-coordinate derived from a PDF image's bounding box. See # DynamicLineSpec. Used where the rightmost column boundary floats with a logo. - allow_text_failover: Optional[bool] = False + allow_text_failover: bool | None = False # [ACTIVE] — When True and the extracted table has the wrong number of columns, # the extraction is retried without vertical_lines, falling back to pdfplumber's # text-based column detection. Useful as a safety net for pages where the # explicit dividers produce a malformed table. - try_shift_down: Optional[int] = None + try_shift_down: int | None = None # [ACTIVE] — Number of PDF points to shift the crop rectangle downward (applied # to both top_left[1] and bottom_right[1]) when the initial extraction returns an # empty region. Handles statements where the table top boundary varies slightly @@ -760,19 +760,19 @@ class TransactionBookend: # [ACTIVE] — Minimum number of end_fields that must have extracted # successfully for a row to be flagged as transaction_end = True. - extra_validation_start: Optional[FieldValidation] + extra_validation_start: FieldValidation | None # [ACTIVE] — When set, any row where the named field's value does NOT match the # pattern is excluded from being a start-bookend candidate for this bookend. # Rows excluded here may still be captured by another bookend in the list. # Useful for bookends that should only trigger on a specific row shape # (e.g. an interest charge line identified by its details text). - extra_validation_end: Optional[FieldValidation] + extra_validation_end: FieldValidation | None # [STUB] — Symmetric counterpart to extra_validation_start for end rows. # Declared but not yet implemented in the pipeline; no code currently reads # this field. Reserved for future use. - sticky_fields: Optional[list[str]] + sticky_fields: list[str] | None # [STUB] — Intended to forward-fill named fields from the start row of a # transaction down to its end row, scoped within a single transaction (as # opposed to fill_forward_fields which fills across transactions). Declared @@ -822,17 +822,17 @@ class TransactionSpec: # boundaries. Evaluated in order; a row matched by an earlier bookend is not # re-matched by a later one. At least one bookend is required. - fill_forward_fields: Optional[list[str]] + fill_forward_fields: list[str] | None # [ACTIVE] — Field names whose null values should be forward-filled across rows # within the same page after pivot. Use for sparse columns where a value # (e.g. a date or payment type) appears only on the first row of a multi-row # block and needs propagating to the end row. - merge_fields: Optional[MergeFields] + merge_fields: MergeFields | None # [ACTIVE] — When set, collapses multi-row text fields within each transaction # into a single joined string. See MergeFields. - exclude_rows: Optional[list[FieldValidation]] + exclude_rows: list[FieldValidation] | None # [ACTIVE] — Rows where any rule's field value matches its pattern are removed # from the results before bookend detection runs. Use to suppress known # non-transaction rows (e.g. a closing balance summary line) that would @@ -876,12 +876,12 @@ class StatementTable: # [STUB] — Human-readable table label (e.g. "Transactions", "Account Summary"). # Loaded from TOML for documentation purposes but not consumed by the pipeline. - header_text: Optional[str] + header_text: str | None # [ACTIVE] — When set, the first table row whose text matches this string is # stripped before extraction. Use when pdfplumber includes the column header # row in the extracted data. - remove_header: Optional[bool] + remove_header: bool | None # [ACTIVE] — When True the first table row is unconditionally stripped. Use # when the header row is always present but its text varies (making header_text # impractical). @@ -895,43 +895,43 @@ class StatementTable: # each field must have a column; for summary/detail tables each field must have # a cell. - table_columns: Optional[int] + table_columns: int | None # [ACTIVE] — Expected minimum number of columns in the extracted table. Passed # to pdfplumber as min_words_horizontal and used to validate column count after # extraction. Also triggers allow_text_failover retry logic. - table_rows: Optional[int] + table_rows: int | None # [ACTIVE] — Expected minimum number of rows in the extracted table. Passed to # pdfplumber as min_words_vertical. - row_spacing: Optional[int] + row_spacing: int | None # [ACTIVE] — pdfplumber snap_y_tolerance in PDF points. Rows whose top edges # fall within this distance of each other are merged into the same table row. # Increase if the statement uses tight line spacing that splits a single visual # row across multiple pdfplumber rows. - tests: Optional[list[Test]] + tests: list[Test] | None # [STUB] — Declarative post-extraction assertions. Declared and accepted in # TOML but no pipeline code evaluates them. Reserved for a future config # validation pass. - delete_success_false: Optional[bool] + delete_success_false: bool | None # [STUB] — Intended to drop rows where any field extraction returned # success = False. Declared and set in TOML (typically True) but no pipeline # code currently reads or acts on this flag. - delete_cast_success_false: Optional[bool] + delete_cast_success_false: bool | None # [STUB] — Intended to drop rows where numeric casting failed. Declared and # set in TOML (typically True) but no pipeline code currently reads or acts on # this flag. - delete_rows_with_missing_vital_fields: Optional[bool] + delete_rows_with_missing_vital_fields: bool | None # [STUB] — Intended to drop rows where any vital field is missing after # extraction. Declared and set in TOML (typically True) but no pipeline code # currently reads or acts on this flag. Note: vital-field hard-failure logic # exists in validate() but is separate from this flag. - transaction_spec: Optional[TransactionSpec] + transaction_spec: TransactionSpec | None # [ACTIVE] — When set, the table is processed as a transaction table using the # bookend-based multi-row extraction path. Must be None for summary/detail # tables. @@ -954,20 +954,20 @@ class Config: # Balances"). Written into the "config" column of the long-format results # DataFrame for traceability. - statement_table_key: Optional[str] + statement_table_key: str | None # [ACTIVE] — Key into statement_tables.toml that identifies the StatementTable # to use. Resolved to statement_table at load time. Set to None for inline # single-field configs. - statement_table: Optional[StatementTable] + statement_table: StatementTable | None # [ACTIVE] — Resolved at load time from statement_table_key. The StatementTable # object used during extraction. Not set directly in TOML. - locations: Optional[list[Location]] + locations: list[Location] | None # [ACTIVE] — Used only for inline single-field configs (where statement_table is # None). Defines where on the page to find the field value. - field: Optional[Field] + field: Field | None # [ACTIVE] — Used only for inline single-field configs. Defines the extraction # spec for the single value to read from the location. @@ -981,7 +981,7 @@ class ConfigGroup: Defined in ``statement_types.toml``. """ - configs: Optional[list[Config]] + configs: list[Config] | None # [ACTIVE] — Ordered list of Config steps. Executed in sequence during # extraction; results are stacked into the section's results DataFrame. @@ -1038,12 +1038,12 @@ class Company: # [ACTIVE] — Human-readable company name (e.g. "HSBC UK"). Used to populate # the STD_COMPANY standard field. - config: Optional[Config] + config: Config | None # [ACTIVE] — Extraction config used during the company-identification pass. # Extracts a discriminating field (e.g. a bank-specific header string) to # confirm the PDF belongs to this company before attempting account matching. - accounts: Optional[dict] + accounts: dict | None # [STUB] — Declared but never accessed by the pipeline after load. Intended # as a lookup from account key to Account object but currently unused. @@ -1081,7 +1081,7 @@ class Account: # [ACTIVE] — Key into companies.toml identifying the issuing bank. Used to # build ID_ACCOUNT and to look up the Company object at load time. - company: Optional[Company] + company: Company | None # [ACTIVE] — Resolved at load time from company_key. Provides the company name # and company-level identification config. @@ -1089,7 +1089,7 @@ class Account: # [ACTIVE] — Key into account_types.toml (e.g. "CRD", "CUR", "SAV"). Used to # look up the AccountType object at load time. - account_type: Optional[AccountType] + account_type: AccountType | None # [STUB] — Resolved at load time from account_type_key. The AccountType object # is populated but never subsequently read by any pipeline consumer. @@ -1098,7 +1098,7 @@ class Account: # this account's statements. Used to look up the StatementType object at load # time. - statement_type: Optional[StatementType] + statement_type: StatementType | None # [ACTIVE] — Resolved at load time from statement_type_key. Provides the header # and lines ConfigGroups used during extraction. diff --git a/src/bank_statement_parser/modules/database.py b/src/bank_statement_parser/modules/database.py index 25fd3cf..610e505 100644 --- a/src/bank_statement_parser/modules/database.py +++ b/src/bank_statement_parser/modules/database.py @@ -29,7 +29,7 @@ import polars as pl -from bank_statement_parser.data.build_datamart import build_datamart, _ensure_mart_structure +from bank_statement_parser.data.build_datamart import _ensure_mart_structure, build_datamart from bank_statement_parser.modules.data import PdfResult, Success from bank_statement_parser.modules.errors import ProjectDatabaseMissing from bank_statement_parser.modules.paths import ProjectPaths @@ -362,9 +362,9 @@ def _migrate_db(conn: sqlite3.Connection) -> None: _validate_migration_identifier(table, _ALLOWED_MIGRATION_TABLES, "table") _validate_migration_identifier(column, _ALLOWED_MIGRATION_COLUMNS, "column") _validate_migration_identifier(col_type, _ALLOWED_MIGRATION_TYPES, "column type") - existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})").fetchall()} # noqa: S608 + existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})").fetchall()} if column not in existing: - conn.execute(f'ALTER TABLE {table} ADD COLUMN "{column}" {col_type} DEFAULT {default}') # noqa: S608 + conn.execute(f'ALTER TABLE {table} ADD COLUMN "{column}" {col_type} DEFAULT {default}') print(f"[migrate] added column {column} to {table}") existing_tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()} @@ -531,7 +531,5 @@ def _insert_df(df: pl.DataFrame, table_name: str) -> None: if pdf_count > (errors + reviews): # if all pdf statements have failed/are under review no point in re-building the datamart try: build_datamart(db_path=db_path) - except Exception as e: + except Exception as e: # noqa: BLE001 print(f"[update_db] ** Datamart Rebuild Failed **: {type(e).__name__}: {e}") - - return db_secs diff --git a/src/bank_statement_parser/modules/debug.py b/src/bank_statement_parser/modules/debug.py index 4a6d697..2429793 100644 --- a/src/bank_statement_parser/modules/debug.py +++ b/src/bank_statement_parser/modules/debug.py @@ -86,7 +86,7 @@ def debug_pdf_statement( return debug_json_path - except Exception as e: + except Exception as e: # noqa: BLE001 print(f"[debug] unexpected error processing {pdf.name}: {e}") return None diff --git a/src/bank_statement_parser/modules/errors.py b/src/bank_statement_parser/modules/errors.py index 2ec0fdd..f4deab6 100644 --- a/src/bank_statement_parser/modules/errors.py +++ b/src/bank_statement_parser/modules/errors.py @@ -21,14 +21,10 @@ class StatementError(Exception): """Root exception for statement processing errors.""" - pass - class ConfigError(StatementError): """Configuration error during setup.""" - pass - class ConfigFileError(ConfigError): """Configuration file not found.""" @@ -48,8 +44,6 @@ def __init__(self, config_path: Path, missing_files: list[str]) -> None: class ProjectError(StatementError): """Project error during setup or operation.""" - pass - class ProjectFolderNotFound(ProjectError): """Project folder not found at specified path.""" @@ -88,7 +82,7 @@ class TestGateFailure(StatementError): output: Captured stdout/stderr from the pytest run. """ - __slots__ = ("failed", "errors", "output") + __slots__ = ("errors", "failed", "output") def __init__(self, failed: int, errors: int, output: str) -> None: self.failed = failed diff --git a/src/bank_statement_parser/modules/export_spec.py b/src/bank_statement_parser/modules/export_spec.py index e63932a..274f1c1 100644 --- a/src/bank_statement_parser/modules/export_spec.py +++ b/src/bank_statement_parser/modules/export_spec.py @@ -90,7 +90,7 @@ def _ts() -> str: Returns: Datetime formatted as ``"yyyymmddHHMMSS"``, e.g. ``"20250331143022"``. """ - return datetime.now().strftime("%Y%m%d%H%M%S") + return datetime.now().strftime("%Y%m%d%H%M%S") # noqa: DTZ005 # --------------------------------------------------------------------------- @@ -306,7 +306,7 @@ def _build_frame( where_clauses.append("da.id_account = ?") else: # For other tables that already carry id_account directly - base_query = f"SELECT * FROM {spec.source_table}" # noqa: S608 + base_query = f"SELECT * FROM {spec.source_table}" where_clauses.append("id_account = ?") if date_from is not None: diff --git a/src/bank_statement_parser/modules/forex.py b/src/bank_statement_parser/modules/forex.py index e50904e..43cb4ae 100644 --- a/src/bank_statement_parser/modules/forex.py +++ b/src/bank_statement_parser/modules/forex.py @@ -94,9 +94,9 @@ def _load_forex_config(project_path: Path | None) -> ForexApiConfig: Returns: A :class:`~bank_statement_parser.modules.data.ForexApiConfig` instance. """ - import tomllib # noqa: PLC0415 — stdlib, Python ≥ 3.11 + import tomllib - import dacite # noqa: PLC0415 + import dacite paths = ProjectPaths.resolve(project_path) config_file = paths.forex_config @@ -119,7 +119,7 @@ def _provider_frankfurter( currencies: list[str], date_from: str, date_to: str, - api_key: str, # noqa: ARG001 — Frankfurter does not use an API key + api_key: str, ) -> list[tuple[str, str, float]]: """Fetch USD-based rates from Frankfurter for a date range. diff --git a/src/bank_statement_parser/modules/import_config.py b/src/bank_statement_parser/modules/import_config.py index 8d5d61e..d449e4e 100644 --- a/src/bank_statement_parser/modules/import_config.py +++ b/src/bank_statement_parser/modules/import_config.py @@ -31,12 +31,12 @@ from copy import deepcopy from datetime import datetime from pathlib import Path +from tomllib import load from typing import Any, TypedDict import polars as pl from dacite import from_dict from pdfplumber.pdf import PDF -from tomllib import load from bank_statement_parser.modules.data import ( Account, @@ -142,7 +142,7 @@ class ImportConfigManager: >>> accounts = config.get_accounts_for_company("my_company") """ - __slots__ = ("_project_path", "_config_dict", "_accounts_df", "_statement_types_df", "_companies_df") + __slots__ = ("_accounts_df", "_companies_df", "_config_dict", "_project_path", "_statement_types_df") def __init__(self, project_path: Path | None = None) -> None: """ @@ -257,12 +257,12 @@ def _load_config(self) -> None: self._require_config_dir() config_dict: dict[str, _ConfigEntry] = { - "companies": {"dataclass": Company, "config": dict()}, - "account_types": {"dataclass": AccountType, "config": dict()}, - "accounts": {"dataclass": Account, "config": dict()}, - "statement_types": {"dataclass": StatementType, "config": dict()}, - "statement_tables": {"dataclass": StatementTable, "config": dict()}, - "standard_fields": {"dataclass": StandardFields, "config": dict()}, + "companies": {"dataclass": Company, "config": {}}, + "account_types": {"dataclass": AccountType, "config": {}}, + "accounts": {"dataclass": Account, "config": {}}, + "statement_types": {"dataclass": StatementType, "config": {}}, + "statement_tables": {"dataclass": StatementTable, "config": {}}, + "standard_fields": {"dataclass": StandardFields, "config": {}}, } for key in config_dict: @@ -314,7 +314,7 @@ def _link_statement_tables(self, config_dict: dict[str, _ConfigEntry]) -> None: Args: config_dict: The fully loaded (but not yet linked) config dictionary. """ - for key, statement_type in config_dict["statement_types"]["config"].items(): + for statement_type in config_dict["statement_types"]["config"].values(): for config_group in [statement_type.header.configs, statement_type.lines.configs]: if config_group: for idx, cfg in enumerate(config_group): @@ -411,7 +411,7 @@ def identify_from_pdf(self, pdf: PDF, file_path: str, logs: pl.DataFrame) -> tup if len(result) > 0: logs.vstack( pl.DataFrame( - [[file_path, "config", "identify_from_pdf", time.time() - start, 1, datetime.now(), ""]], + [[file_path, "config", "identify_from_pdf", time.time() - start, 1, datetime.now(), ""]], # noqa: DTZ005 schema=logs.schema, orient="row", ), @@ -443,7 +443,7 @@ def get_config_from_account(self, account_key: str, logs: pl.DataFrame, file_pat raise StatementError(f"Unable to identify the account from the statement provided: {file_path}") logs.vstack( pl.DataFrame( - [[file_path, "config", "get_config_from_account", time.time() - start, 1, datetime.now(), ""]], + [[file_path, "config", "get_config_from_account", time.time() - start, 1, datetime.now(), ""]], # noqa: DTZ005 schema=logs.schema, orient="row", ), @@ -483,7 +483,7 @@ def get_config_from_company(self, company_key: str, pdf: PDF, logs: pl.DataFrame if len(result) > 0: logs.vstack( pl.DataFrame( - [[file_path, "config", "get_config_from_company", time.time() - start, 1, datetime.now(), ""]], + [[file_path, "config", "get_config_from_company", time.time() - start, 1, datetime.now(), ""]], # noqa: DTZ005 schema=logs.schema, orient="row", ), @@ -522,7 +522,7 @@ def get_config_from_statement(self, pdf: PDF, file_path: str, logs: pl.DataFrame account = self.get_config_from_company(key, pdf, logs, file_path) logs.vstack( pl.DataFrame( - [[file_path, "config", "get_config_from_statement", time.time() - start, 1, datetime.now(), ""]], + [[file_path, "config", "get_config_from_statement", time.time() - start, 1, datetime.now(), ""]], # noqa: DTZ005 schema=logs.schema, orient="row", ), diff --git a/src/bank_statement_parser/modules/parquet.py b/src/bank_statement_parser/modules/parquet.py index 6b9eba2..64c12a1 100644 --- a/src/bank_statement_parser/modules/parquet.py +++ b/src/bank_statement_parser/modules/parquet.py @@ -44,7 +44,7 @@ class Parquet: - __slots__ = ("file", "schema", "records", "key", "db_records") + __slots__ = ("db_records", "file", "key", "records", "schema") def __init__(self, file: Path, schema: pl.DataFrame, records: pl.DataFrame | None, key: str | None) -> None: self.file = file diff --git a/src/bank_statement_parser/modules/paths.py b/src/bank_statement_parser/modules/paths.py index ece0f45..6d06e92 100644 --- a/src/bank_statement_parser/modules/paths.py +++ b/src/bank_statement_parser/modules/paths.py @@ -353,7 +353,7 @@ def statement_lines_temp_stem(self, idx: int, batch_id: str) -> str: # ------------------------------------------------------------------ @classmethod - def resolve(cls, project_path: Path | None = None) -> "ProjectPaths": + def resolve(cls, project_path: Path | None = None) -> ProjectPaths: """Return a :class:`ProjectPaths` for *project_path*. When *project_path* is ``None``, the default project folder bundled @@ -581,7 +581,7 @@ def _create_project_db(paths: ProjectPaths) -> None: paths: A :class:`ProjectPaths` instance for the target project. """ paths.ensure_subdir_for_write(paths.database) - from bank_statement_parser.data.create_project_db import main as create_db # noqa: PLC0415 + from bank_statement_parser.data.create_project_db import main as create_db create_db(db_path=paths.project_db, with_fk=True) print(f"[scaffold] created missing database: {paths.project_db}") @@ -641,6 +641,6 @@ def _scaffold_new_project(paths: ProjectPaths) -> None: # 6. Create the SQLite database with the full schema. # Import here to avoid a circular dependency at module level # (database.py → paths.py; paths.py must not import database.py at top). - from bank_statement_parser.data.create_project_db import main as create_db # noqa: PLC0415 + from bank_statement_parser.data.create_project_db import main as create_db create_db(db_path=paths.project_db, with_fk=True) diff --git a/src/bank_statement_parser/modules/reports_db.py b/src/bank_statement_parser/modules/reports_db.py index 991bf68..39e5f73 100644 --- a/src/bank_statement_parser/modules/reports_db.py +++ b/src/bank_statement_parser/modules/reports_db.py @@ -105,7 +105,7 @@ def _read_data(db_path: Path, table_name: str) -> pl.LazyFrame: ValueError: If *table_name* is not in the allowed whitelist. """ _validate_read_target(table_name) - query = f"SELECT * FROM {table_name}" # noqa: S608 + query = f"SELECT * FROM {table_name}" with sqlite3.connect(db_path) as conn: return pl.read_database(query, connection=conn, infer_schema_length=None).lazy() @@ -134,7 +134,7 @@ def _read_data_filtered(db_path: Path, table_name: str, batch_table: str, batch_ if batch_id is None: return _read_data(db_path, table_name) _validate_read_target(batch_table) - query = f"SELECT * FROM {batch_table} WHERE batch_id = ?" # noqa: S608 + query = f"SELECT * FROM {batch_table} WHERE batch_id = ?" with sqlite3.connect(db_path) as conn: return pl.read_database(query, connection=conn, execute_options={"parameters": [batch_id]}, infer_schema_length=None).lazy() @@ -194,7 +194,7 @@ def _ts() -> str: Returns: Datetime formatted as ``"yyyymmddHHMMSS"``, e.g. ``"20250331143022"``. """ - return datetime.now().strftime("%Y%m%d%H%M%S") + return datetime.now().strftime("%Y%m%d%H%M%S") # noqa: DTZ005 def _collect_report_frames( diff --git a/src/bank_statement_parser/modules/statement_functions.py b/src/bank_statement_parser/modules/statement_functions.py index 2f5c4de..a4ffb03 100644 --- a/src/bank_statement_parser/modules/statement_functions.py +++ b/src/bank_statement_parser/modules/statement_functions.py @@ -740,18 +740,16 @@ def get_results( ) results.vstack(result, in_place=True) - if statement_table := config.statement_table: - # process transactions if there's a transaction spec - if spec := statement_table.transaction_spec: - results = results.pipe( - process_transactions, - transaction_spec=spec, - logs=logs, - file_path=file_path, - debug_collector=debug_collector, - debug_dataframes=debug_dataframes, - ) - return results + if (statement_table := config.statement_table) and (spec := statement_table.transaction_spec): + results = results.pipe( + process_transactions, + transaction_spec=spec, + logs=logs, + file_path=file_path, + debug_collector=debug_collector, + debug_dataframes=debug_dataframes, + ) + return results if scope == "all": return results @@ -797,10 +795,7 @@ def get_standard_fields( for std_field, std_config in config_standard_fields.items(): if std_config.section == section: - try: - ref = [ref for ref in std_config.std_refs if ref.statement_type == statement_type][0] - except IndexError: - ref = None + ref = next((ref for ref in std_config.std_refs if ref.statement_type == statement_type), None) if ref: if ref.concat_fields: data = data.with_columns(pl.concat_str([f"{field}" for field in ref.concat_fields]).alias(std_field)) @@ -924,9 +919,7 @@ def get_standard_fields( "snapshot_stored": True, } ) - except Exception: # noqa: BLE001 - pass # Silently fail if snapshot cannot be created - - # # add a GUID to each record + except Exception: # noqa: BLE001, S110 + pass # # add a GUID to each record # data = data.with_columns(STD_GUID=pl.lit(f"{uuid4()}")) return data diff --git a/src/bank_statement_parser/modules/statements.py b/src/bank_statement_parser/modules/statements.py index 8048364..33ed3a1 100644 --- a/src/bank_statement_parser/modules/statements.py +++ b/src/bank_statement_parser/modules/statements.py @@ -169,7 +169,7 @@ def _write_debug_excel( debug_dataframes: Dict mapping section names to lists of dataframes. """ try: - from xlsxwriter import Workbook # noqa: PLC0415 + from xlsxwriter import Workbook out_file = debug_dir / "debug_dataframes.xlsx" wb = Workbook(str(out_file)) @@ -222,8 +222,8 @@ def _write_debug_json(stmt: "Statement", include_lines: bool = False) -> Path | Path to the debug.json file that was written, or ``None`` if an error prevented writing (the error is printed to stdout). """ - import json # noqa: PLC0415 - from datetime import datetime # noqa: PLC0415 + import json + from datetime import datetime try: paths = ProjectPaths.resolve(stmt.project_path) @@ -256,7 +256,7 @@ def _write_debug_json(stmt: "Statement", include_lines: bool = False) -> Path | "id_batch": stmt.ID_BATCH or "", "success": stmt.success, "error_message": stmt.error_message, - "debug_timestamp": datetime.now().isoformat(timespec="seconds"), + "debug_timestamp": datetime.now().isoformat(timespec="seconds"), # noqa: DTZ005 }, "events": stmt._debug_collector or [], "checks_and_balances": cab_data, @@ -310,41 +310,38 @@ class Statement: """ __slots__ = ( - "company_key", - "file", - "file_absolute", - "file_renamed", - "account_key", - "ID_BATCH", "ID_ACCOUNT", - "checks_and_balances", - "pdf", + "ID_BATCH", "ID_STATEMENT", - "config", - "company", + "_debug_collector", + "_debug_dataframes", "account", - "statement_type", + "account_key", + "checks_and_balances", + "company", + "company_key", + "config", "config_header", "config_lines", + "debug", + "error_detail", + "error_message", + "file", + "file_absolute", + "file_renamed", "header_results", "lines_results", - "success", - "error_message", + "logs", + "pdf", "project_path", "skip_project_validation", - "logs", - "error_detail", - # Lightweight summary scalars — populated on success; None otherwise. - # Avoids a second .collect() call in process_pdf_statement(). - "std_statement_date", + "statement_type", + "std_closing_balance", + "std_opening_balance", "std_payments_in", "std_payments_out", - "std_opening_balance", - "std_closing_balance", - # Debug support — populated when debug=True; None otherwise. - "debug", - "_debug_collector", - "_debug_dataframes", + "std_statement_date", + "success", ) def __init__( @@ -413,7 +410,7 @@ def __init__( self.skip_project_validation = skip_project_validation self.debug: bool = debug self._debug_collector: list | None = [] if debug else None - self._debug_dataframes: dict[str, list[pl.DataFrame]] = {} if debug else {} + self._debug_dataframes: dict[str, list[pl.DataFrame]] = {} # Safe defaults — ensure every slot is initialised before the processing # try block so that is_successfull() and cleanup() never hit an @@ -619,27 +616,17 @@ def is_successfull(self): self.checks_and_balances.filter(pl.col("ZERO_TRANSACTION_STATEMENT")).height > 0 ): # some statments are just a header so there's nothing really to fail return True - if self.header_results.collect().height == 0: - return False - elif self.lines_results.collect().height == 0: - return False - elif self.checks_and_balances.height == 0: - return False - elif self.checks_and_balances.filter(~pl.col("BAL_PAYMENTS_IN")).height > 0: - return False - elif self.checks_and_balances.filter(~pl.col("BAL_PAYMENTS_OUT")).height > 0: - return False - elif self.checks_and_balances.filter(~pl.col("BAL_MOVEMENT")).height > 0: - return False - elif self.checks_and_balances.filter(~pl.col("BAL_CLOSING")).height > 0: - return False - # Check that no transaction lines have null dates (datamart integrity) - elif self.checks_and_balances.filter(pl.col("TRANSACTION_LINES_WITH_NULL_DATE") > 0).height > 0: - return False - # Check that no transaction lines have null descriptions - elif self.checks_and_balances.filter(pl.col("TRANSACTION_LINES_WITH_NULL_DESC") > 0).height > 0: - return False - return True + return not ( + self.header_results.collect().height == 0 + or self.lines_results.collect().height == 0 + or self.checks_and_balances.height == 0 + or self.checks_and_balances.filter(~pl.col("BAL_PAYMENTS_IN")).height > 0 + or self.checks_and_balances.filter(~pl.col("BAL_PAYMENTS_OUT")).height > 0 + or self.checks_and_balances.filter(~pl.col("BAL_MOVEMENT")).height > 0 + or self.checks_and_balances.filter(~pl.col("BAL_CLOSING")).height > 0 + or self.checks_and_balances.filter(pl.col("TRANSACTION_LINES_WITH_NULL_DATE") > 0).height > 0 + or self.checks_and_balances.filter(pl.col("TRANSACTION_LINES_WITH_NULL_DESC") > 0).height > 0 + ) def get_results(self, section: str) -> pl.LazyFrame: """ @@ -917,7 +904,7 @@ def process_pdf_statement( batch_line["STD_FILENAME"] = pdf.name batch_line["STD_ACCOUNT"] = "" batch_line["STD_DURATION_SECS"] = 0.00 - batch_line["STD_UPDATETIME"] = datetime.now() + batch_line["STD_UPDATETIME"] = datetime.now() # noqa: DTZ005 batch_line["STD_SUCCESS"] = False batch_line["STD_ERROR_MESSAGE"] = "" batch_line["ERROR_CAB"] = False @@ -980,7 +967,7 @@ def process_pdf_statement( statement_heads_path = paths.statement_heads_temp(idx, batch_id) pq_statement_heads.cleanup() pq_statement_heads = None - except (IOError, OSError, ValueError, pl.exceptions.PolarsError) as e: + except (OSError, ValueError, pl.exceptions.PolarsError) as e: _handle_parquet_write_error("StatementHeads", batch_line, [error_message], pdf, e) error_data = True @@ -995,7 +982,7 @@ def process_pdf_statement( statement_lines_path = paths.statement_lines_temp(idx, batch_id) pq_statement_lines.cleanup() pq_statement_lines = None - except (IOError, OSError, ValueError, pl.exceptions.PolarsError) as e: + except (OSError, ValueError, pl.exceptions.PolarsError) as e: _handle_parquet_write_error("StatementLines", batch_line, [error_message], pdf, e) error_data = True @@ -1038,15 +1025,13 @@ def process_pdf_statement( cab_path = paths.cab_temp(idx, batch_id) pq_cab.cleanup() pq_cab = None - except (IOError, OSError, ValueError, pl.exceptions.PolarsError) as e: + except (OSError, ValueError, pl.exceptions.PolarsError) as e: _handle_parquet_write_error("ChecksAndBalances", batch_line, [error_message], pdf, e) error_data = True stmt.cleanup() stmt = None - except Exception as e: - # Last-resort guard — intentionally broad to catch any unexpected failures outside - # the statement constructor (e.g. path resolution errors, import issues). + except Exception as e: # noqa: BLE001 — last-resort guard, intentionally broad # All recoverable statement-level errors are caught by inner try/except blocks above. error_other = True batch_line["ERROR_CONFIG"] = True @@ -1058,7 +1043,7 @@ def process_pdf_statement( # Record processing time and timestamp line_end = time() batch_line["STD_DURATION_SECS"] = line_end - line_start - batch_line["STD_UPDATETIME"] = datetime.now() + batch_line["STD_UPDATETIME"] = datetime.now() # noqa: DTZ005 # Save batch line data — always written regardless of success/failure pq_batch_lines = pq.BatchLines(file=paths.batch_lines_temp(idx, batch_id), batch_lines=[batch_line]) @@ -1263,31 +1248,31 @@ class StatementBatch: """ __slots__ = ( - "process_time", - "path", "ID_BATCH", "ID_SESSION", "ID_USER", "__type", - "company_key", "account_key", - "print_log", - "pdfs", - "pdf_count", - "log", - "errors", - "reviews", + "batch_lines", + "company_key", + "db_secs", "duration_secs", - "process_secs", + "errors", + "log", "parquet_secs", - "db_secs", - "batch_lines", - "timer_start", - "statements", - "turbo", + "path", + "pdf_count", + "pdfs", + "print_log", + "process_secs", + "process_time", + "processed_pdfs", "project_path", + "reviews", "skip_project_validation", - "processed_pdfs", + "statements", + "timer_start", + "turbo", ) def __init__( @@ -1326,7 +1311,7 @@ def __init__( if not skip_project_validation: validate_or_initialise_project(ProjectPaths.resolve(project_path).root) print("processing...") - self.process_time: datetime = datetime.now() + self.process_time: datetime = datetime.now() # noqa: DTZ005 self.timer_start = time() self.ID_BATCH: str = str(uuid4()) self.ID_SESSION: str = str(uuid4()) @@ -1339,7 +1324,7 @@ def __init__( self.skip_project_validation = skip_project_validation self.pdfs = pdfs # Build path string from unique parent directories of all PDFs - self.path: str = ", ".join(map(str, set([p.parent for p in self.pdfs]))) + self.path: str = ", ".join(map(str, {p.parent for p in self.pdfs})) self.pdf_count: int = len(self.pdfs) self.log: list = [] self.errors: int = 0 @@ -1677,7 +1662,7 @@ def export( return resolved_project_path = project_path if project_path is not None else self.project_path - import bank_statement_parser.modules.reports_db as _rd # noqa: PLC0415 + import bank_statement_parser.modules.reports_db as _rd if filetype == "excel": _rd.export_excel( @@ -1768,4 +1753,3 @@ def debug(self, project_path: Path | None = None) -> int: def __del__(self): """Destructor to ensure temporary files are cleaned up.""" # self.delete_temp_files() - pass diff --git a/src/bank_statement_parser/testing.py b/src/bank_statement_parser/testing.py index b0e827a..c13de70 100644 --- a/src/bank_statement_parser/testing.py +++ b/src/bank_statement_parser/testing.py @@ -53,6 +53,7 @@ import subprocess import tempfile from pathlib import Path +from typing import Self from bank_statement_parser.modules.errors import TestGateFailure from bank_statement_parser.modules.paths import ProjectPaths, validate_or_initialise_project @@ -106,6 +107,7 @@ def _clone_test_data() -> Path | None: capture_output=True, text=True, timeout=60, + check=False, ) # Now pull the latest changes result = subprocess.run( @@ -113,6 +115,7 @@ def _clone_test_data() -> Path | None: capture_output=True, text=True, timeout=60, + check=False, ) if result.returncode != 0: # Pull failed — return existing cache if available @@ -124,6 +127,7 @@ def _clone_test_data() -> Path | None: capture_output=True, text=True, timeout=60, + check=False, ) if result.returncode != 0: return None @@ -240,7 +244,7 @@ class TestHarness: compare_databases(h.db_path, my_db_path) """ - __slots__ = ("_project_path", "_owned", "_test_results", "_batch", "_ready", "_skip_bsp_tests") + __slots__ = ("_batch", "_owned", "_project_path", "_ready", "_skip_bsp_tests", "_test_results") def __init__(self, skip_bsp_tests: bool = False) -> None: self._project_path: Path | None = None @@ -332,7 +336,7 @@ def teardown(self) -> None: # Context manager # ------------------------------------------------------------------ - def __enter__(self) -> "TestHarness": + def __enter__(self) -> "Self": """Set up the harness and return it for use as a context manager. Returns: @@ -424,6 +428,7 @@ def _run_bsp_tests(self) -> None: ["python", "-m", "pytest", str(tests_path), "-q", "--tb=short"], capture_output=True, text=True, + check=False, ) output = result.stdout + result.stderr diff --git a/tests/conftest.py b/tests/conftest.py index 6f45045..721967f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -126,7 +126,7 @@ def good_project() -> Generator[ProjectContext, None, None]: # type: ignore[mis pytest.skip("No good PDFs with metadata sidecars found") # Count banks for session-end summary - global _good_bank_counts # noqa: PLW0603 + global _good_bank_counts _good_bank_counts = {} for pdf in pdfs_with_metadata: match = re.match(r"anonymised_([A-Za-z]+)", pdf.name) @@ -141,7 +141,7 @@ def good_project() -> Generator[ProjectContext, None, None]: # type: ignore[mis batch.update_data() batch.delete_temp_files() - global _good_pdf_count # noqa: PLW0603 + global _good_pdf_count _good_pdf_count = len(pdfs_with_metadata) yield ProjectContext(project_path=project_path, batch=batch, pdfs=pdfs_with_metadata) @@ -189,7 +189,7 @@ def bad_project() -> Generator[ProjectContext, None, None]: # type: ignore[misc project_path=project_path, ) - global _bad_pdf_count # noqa: PLW0603 + global _bad_pdf_count _bad_pdf_count = len(pdfs_with_metadata) yield ProjectContext(project_path=project_path, batch=batch, pdfs=pdfs_with_metadata) diff --git a/tests/test_cli.py b/tests/test_cli.py index af72787..e80282a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -33,6 +33,7 @@ import argparse import ast from pathlib import Path +from typing import ClassVar from unittest.mock import patch import pytest @@ -90,7 +91,7 @@ def _capture_parse_args(self: argparse.ArgumentParser, *args: object, **kwargs: class TestSubcommands: """Verify that the expected subcommands are registered.""" - EXPECTED_SUBCOMMANDS = {"anonymise", "forex", "process"} + EXPECTED_SUBCOMMANDS: ClassVar[set[str]] = {"anonymise", "forex", "process"} def test_expected_subcommands_exist(self) -> None: """All expected subcommands must be present in the parser.""" @@ -124,7 +125,7 @@ def _get_positional_actions(subcommand: str) -> list[argparse.Action]: class TestProcessOptions: """Validate the ``process`` subcommand options.""" - EXPECTED_FLAGS = { + EXPECTED_FLAGS: ClassVar[set[str]] = { "--project", "--pdfs", "--pattern", @@ -189,7 +190,7 @@ def test_pattern_default(self) -> None: class TestAnonymiseOptions: """Validate the ``anonymise`` subcommand options.""" - EXPECTED_FLAGS = {"--output", "--always-anonymise", "--never-anonymise", "--debug"} + EXPECTED_FLAGS: ClassVar[set[str]] = {"--output", "--always-anonymise", "--never-anonymise", "--debug"} def test_all_expected_flags_exist(self) -> None: """Every expected --flag must be registered on the anonymise subparser.""" diff --git a/tests/test_docs.py b/tests/test_docs.py index e5234e4..50521d8 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -43,8 +43,7 @@ # Ensure scripts/ is importable sys.path.insert(0, str(_SCRIPTS)) -import generate_docs # noqa: E402 - +import generate_docs # =================================================================== # Freshness tests — generated output matches committed files @@ -90,13 +89,12 @@ def _parse_all_symbols() -> list[str]: for node in ast.walk(tree): if isinstance(node, ast.Assign): for target in node.targets: - if isinstance(target, ast.Name) and target.id == "__all__": - if isinstance(node.value, ast.List): - return [ - elt.value # type: ignore[union-attr] - for elt in node.value.elts - if isinstance(elt, ast.Constant) and isinstance(elt.value, str) - ] + if isinstance(target, ast.Name) and target.id == "__all__" and isinstance(node.value, ast.List): + return [ + elt.value # type: ignore[union-attr] + for elt in node.value.elts + if isinstance(elt, ast.Constant) and isinstance(elt.value, str) + ] return [] def test_every_all_symbol_in_api_docs(self) -> None: diff --git a/tests/test_forex.py b/tests/test_forex.py index 437718c..e62c537 100644 --- a/tests/test_forex.py +++ b/tests/test_forex.py @@ -33,16 +33,15 @@ import pytest +from bank_statement_parser.modules.data import ForexApiConfig +from bank_statement_parser.modules.errors import ProjectDatabaseMissing from bank_statement_parser.modules.forex import ( _forward_fill, _load_forex_config, - _provider_frankfurter, _provider_exchangerate_api, + _provider_frankfurter, get_exchange_rates, ) -from bank_statement_parser.modules.data import ForexApiConfig -from bank_statement_parser.modules.errors import ProjectDatabaseMissing - # --------------------------------------------------------------------------- # Helpers diff --git a/tests/test_statements.py b/tests/test_statements.py index 0b8fc47..1b3a3fa 100644 --- a/tests/test_statements.py +++ b/tests/test_statements.py @@ -263,7 +263,7 @@ def test_csv_multi_row_counts(self, good_project): paths = ProjectPaths.resolve(good_project.project_path) with sqlite3.connect(str(paths.project_db)) as conn: for stem, table in _MULTI_STEM_TO_DB_TABLE.items(): - db_rows = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 + db_rows = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] df = pl.read_csv(paths.csv / "multi" / f"{stem}.csv", infer_schema_length=0) assert df.height == db_rows, f"{stem}.csv: CSV rows={df.height} != DB rows={db_rows}" @@ -339,7 +339,7 @@ def test_json_multi_row_counts(self, good_project): paths = ProjectPaths.resolve(good_project.project_path) with sqlite3.connect(str(paths.project_db)) as conn: for stem, table in _MULTI_STEM_TO_DB_TABLE.items(): - db_rows = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 + db_rows = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] df = pl.read_json(paths.json / "multi" / f"{stem}.json", infer_schema_length=None) assert df.height == db_rows, f"{stem}.json: JSON rows={df.height} != DB rows={db_rows}" From fa96eb88ad806e8bae4baa3d0faf52509811d6ec Mon Sep 17 00:00:00 2001 From: Jason Farrar Date: Thu, 6 Aug 2026 11:42:58 +0100 Subject: [PATCH 2/3] fix: restore return db_secs in update_db and regenerate stale docs - Restore 'return db_secs' accidentally removed by ruff --fix in database.py - Fix generate_docs.py exports guide to use double quotes (ruff format style) - Add missing blank line in exports code example to match formatter output - Regenerate all docs to match current generator output Signed-off-by: Jason Farrar --- docs/guides/new-bank-config.md | 246 +++++++++--------- docs/reference/python-api.md | 218 +++++++--------- scripts/generate_docs.py | 7 +- src/bank_statement_parser/modules/database.py | 2 + 4 files changed, 227 insertions(+), 246 deletions(-) diff --git a/docs/guides/new-bank-config.md b/docs/guides/new-bank-config.md index 7e730c4..75e53d8 100644 --- a/docs/guides/new-bank-config.md +++ b/docs/guides/new-bank-config.md @@ -139,8 +139,8 @@ Configuration for a financial institution (bank/provider). | Field | Type | Status | Description | | --- | --- | --- | --- | | `company` | `str` | ACTIVE | Human-readable company name (e.g. "HSBC UK"). Used to populate the STD_COMPANY standard field. | -| `config` | `Config` | ACTIVE | Extraction config used during the company-identification pass. Extracts a discriminating field (e.g. a bank-specific header string) to confirm the PDF belongs to this company before attempting account matching. | -| `accounts` | `dict` | STUB | Declared but never accessed by the pipeline after load. Intended as a lookup from account key to Account object but currently unused. | +| `config` | `Config | None` | ACTIVE | Extraction config used during the company-identification pass. Extracts a discriminating field (e.g. a bank-specific header string) to confirm the PDF belongs to this company before attempting account matching. | +| `accounts` | `dict | None` | STUB | Declared but never accessed by the pipeline after load. Intended as a lookup from account key to Account object but currently unused. | #### `Config` @@ -149,10 +149,10 @@ A single extraction step: one table (or one standalone field) from one location. | Field | Type | Status | Description | | --- | --- | --- | --- | | `config` | `str` | ACTIVE | Human-readable label for this extraction step (e.g. "Statement Balances"). Written into the "config" column of the long-format results DataFrame for traceability. | -| `statement_table_key` | `str` | ACTIVE | Key into statement_tables.toml that identifies the StatementTable to use. Resolved to statement_table at load time. Set to None for inline single-field configs. | -| `statement_table` | `StatementTable` | ACTIVE | Resolved at load time from statement_table_key. The StatementTable object used during extraction. Not set directly in TOML. | -| `locations` | `list[Location` | ACTIVE | Used only for inline single-field configs (where statement_table is None). Defines where on the page to find the field value. | -| `field` | `Field` | ACTIVE | Used only for inline single-field configs. Defines the extraction spec for the single value to read from the location. | +| `statement_table_key` | `str | None` | ACTIVE | Key into statement_tables.toml that identifies the StatementTable to use. Resolved to statement_table at load time. Set to None for inline single-field configs. | +| `statement_table` | `StatementTable | None` | ACTIVE | Resolved at load time from statement_table_key. The StatementTable object used during extraction. Not set directly in TOML. | +| `locations` | `list[Location] | None` | ACTIVE | Used only for inline single-field configs (where statement_table is None). Defines where on the page to find the field value. | +| `field` | `Field | None` | ACTIVE | Used only for inline single-field configs. Defines the extraction spec for the single value to read from the location. | #### `Location` @@ -160,13 +160,13 @@ Describes a rectangular region on a PDF page from which a table or text is extra | Field | Type | Status | Description | | --- | --- | --- | --- | -| `page_number` | `int` | ACTIVE | 1-based page number. When set the location is used only on that page. When None the location is cloned for every page (spawn_locations()). | -| `top_left` | `list[int` | ACTIVE | [x, y] coordinates of the top-left corner of the crop rectangle. Must be set together with bottom_right. When both are None the full page is used. | -| `bottom_right` | `list[int` | ACTIVE | [x, y] coordinates of the bottom-right corner of the crop rectangle. Must be set together with top_left. | -| `vertical_lines` | `list[int` | ACTIVE | Explicit x-coordinates of vertical column dividers supplied to pdfplumber as explicit_vertical_lines. Pairs of identical values create a zero-width gap that forces a column boundary (e.g. [100, 100, 200, 200]). When set, pdfplumber's automatic column detection is disabled for this region. | -| `dynamic_last_vertical_line` | `DynamicLineSpec` | ACTIVE | When set, the final value in vertical_lines is replaced at runtime with an x-coordinate derived from a PDF image's bounding box. See DynamicLineSpec. Used where the rightmost column boundary floats with a logo. | -| `allow_text_failover` | `bool` | ACTIVE | When True and the extracted table has the wrong number of columns, the extraction is retried without vertical_lines, falling back to pdfplumber's text-based column detection. Useful as a safety net for pages where the explicit dividers produce a malformed table. | -| `try_shift_down` | `int` | ACTIVE | Number of PDF points to shift the crop rectangle downward (applied to both top_left[1] and bottom_right[1]) when the initial extraction returns an empty region. Handles statements where the table top boundary varies slightly between pages. | +| `page_number` | `int | None` | ACTIVE | 1-based page number. When set the location is used only on that page. When None the location is cloned for every page (spawn_locations()). | +| `top_left` | `list[int] | None` | ACTIVE | [x, y] coordinates of the top-left corner of the crop rectangle. Must be set together with bottom_right. When both are None the full page is used. | +| `bottom_right` | `list[int] | None` | ACTIVE | [x, y] coordinates of the bottom-right corner of the crop rectangle. Must be set together with top_left. | +| `vertical_lines` | `list[int] | None` | ACTIVE | Explicit x-coordinates of vertical column dividers supplied to pdfplumber as explicit_vertical_lines. Pairs of identical values create a zero-width gap that forces a column boundary (e.g. [100, 100, 200, 200]). When set, pdfplumber's automatic column detection is disabled for this region. | +| `dynamic_last_vertical_line` | `DynamicLineSpec | None` | ACTIVE | When set, the final value in vertical_lines is replaced at runtime with an x-coordinate derived from a PDF image's bounding box. See DynamicLineSpec. Used where the rightmost column boundary floats with a logo. | +| `allow_text_failover` | `bool | None` | ACTIVE | When True and the extracted table has the wrong number of columns, the extraction is retried without vertical_lines, falling back to pdfplumber's text-based column detection. Useful as a safety net for pages where the explicit dividers produce a malformed table. | +| `try_shift_down` | `int | None` | ACTIVE | Number of PDF points to shift the crop rectangle downward (applied to both top_left[1] and bottom_right[1]) when the initial extraction returns an empty region. Handles statements where the table top boundary varies slightly between pages. | #### `Field` @@ -175,19 +175,19 @@ Extraction specification for a single column or cell within a PDF table. | Field | Type | Status | Description | | --- | --- | --- | --- | | `field` | `str` | ACTIVE | Output column name for this field (e.g. "date", "£_paid_out"). Used as the field identifier throughout the pipeline and in the output Parquet files. | -| `cell` | `Cell` | ACTIVE | Row/column address for summary or detail table extraction. Mutually exclusive with ``column``; set to None for transaction tables. | +| `cell` | `Cell | None` | ACTIVE | Row/column address for summary or detail table extraction. Mutually exclusive with ``column``; set to None for transaction tables. | | `column` | `int | None` | ACTIVE | Zero-based column index for transaction table extraction. Mutually exclusive with ``cell``; set to None for summary/detail tables. | | `vital` | `bool` | ACTIVE | When True, extraction failure for this field causes the row to be flagged as a hard failure and excluded from output. When False, failure is recorded but the row is retained. | | `type` | `str` | ACTIVE | Data type: "string", "numeric", or "currency". * "string" — raw text extraction; pattern matching and trimming applied. * "numeric" — numeric extraction with optional explicit currency stripping via ``currency_override``. * "currency" — identical to "numeric" but inherits the CurrencySpec from the account's ``Account.currency`` rather than requiring an explicit ``currency_override`` on every field. Use this for all monetary amount fields; reserve "numeric" for non-monetary numerics (e.g. APR, sort code). | -| `strip_characters_start` | `str` | ACTIVE | Characters to strip from the start of the raw string before pattern matching (passed to Polars str.strip_chars_start()). Useful for leading currency symbols not covered by the account currency spec. | -| `strip_characters_end` | `str` | ACTIVE | Characters to strip from the end of the raw string before pattern matching (passed to Polars str.strip_chars_end()). | +| `strip_characters_start` | `str | None` | ACTIVE | Characters to strip from the start of the raw string before pattern matching (passed to Polars str.strip_chars_start()). Useful for leading currency symbols not covered by the account currency spec. | +| `strip_characters_end` | `str | None` | ACTIVE | Characters to strip from the end of the raw string before pattern matching (passed to Polars str.strip_chars_end()). | | `currency_override` | `str | None` | ACTIVE | Explicit ISO 4217 currency key (e.g. "GBP") used when ``type == "numeric"`` and currency stripping is needed but should differ from the account-level ``Account.currency``. Ignored when ``type == "currency"`` (which always uses the account-level currency). Omit for non-monetary numeric fields (e.g. APR, sort code) where no currency stripping is required. | -| `numeric_modifier` | `NumericModifier` | ACTIVE | Sign/multiplier transformation applied after numeric casting. See NumericModifier. Omit for straightforward positive numeric values. | -| `string_pattern` | `str` | ACTIVE | Regex pattern the extracted string must match. Extraction is marked as failed (success = False) if the value does not match. Used to validate field contents (e.g. date format) and to skip blank or irrelevant rows. | -| `string_max_length` | `int` | ACTIVE | Maximum character length for string values; longer strings are truncated via str.head(). Useful for capping free-text description fields. Defaults to 999 if not set. | -| `date_format` | `str` | STUB | Intended strptime format for date parsing at the Field level. Declared but never read by the pipeline; date format parsing is handled via StdRefs.format in get_standard_fields() instead. | +| `numeric_modifier` | `NumericModifier | None` | ACTIVE | Sign/multiplier transformation applied after numeric casting. See NumericModifier. Omit for straightforward positive numeric values. | +| `string_pattern` | `str | None` | ACTIVE | Regex pattern the extracted string must match. Extraction is marked as failed (success = False) if the value does not match. Used to validate field contents (e.g. date format) and to skip blank or irrelevant rows. | +| `string_max_length` | `int | None` | ACTIVE | Maximum character length for string values; longer strings are truncated via str.head(). Useful for capping free-text description fields. Defaults to 999 if not set. | +| `date_format` | `str | None` | STUB | Intended strptime format for date parsing at the Field level. Declared but never read by the pipeline; date format parsing is handled via StdRefs.format in get_standard_fields() instead. | | `value_offset` | `'FieldOffset'` | ACTIVE | When set, reads the field's value from an adjacent column (Field.column + FieldOffset.cols_offset) using the type and currency rules defined in the FieldOffset rather than those on this Field. The primary field column is still extracted normally; the offset column value replaces it in the output. See FieldOffset. | -| `regex_groups` | `int` | ACTIVE | When set, extracts the specified capture group (1-indexed) from the string_pattern regex match instead of the entire match (group 0). Useful for splitting a single PDF column into multiple fields via regex capture groups. Example: string_pattern = '^([A-Z ]+)\s+([A-Z0-9]+)$' with regex_groups = 1 extracts group 1; regex_groups = 2 extracts group 2. When None, defaults to group 0 (entire match, backward compatible). Omit for standard extraction. | +| `regex_groups` | `int | None` | ACTIVE | When set, extracts the specified capture group (1-indexed) from the string_pattern regex match instead of the entire match (group 0). Useful for splitting a single PDF column into multiple fields via regex capture groups. Example: string_pattern = '^([A-Z ]+)\s+([A-Z0-9]+)$' with regex_groups = 1 extracts group 1; regex_groups = 2 extracts group 2. When None, defaults to group 0 (entire match, backward compatible). Omit for standard extraction. | ## Step 4: Define Statement Tables @@ -277,18 +277,18 @@ Full configuration for extracting one table from a PDF statement. | --- | --- | --- | --- | | `type` | `str` | STUB | Table type label: "transaction", "summary", or "detail". Loaded from TOML but not currently read by the pipeline; the extraction path is determined by whether transaction_spec is present rather than this field. | | `statement_table` | `str` | STUB | Human-readable table label (e.g. "Transactions", "Account Summary"). Loaded from TOML for documentation purposes but not consumed by the pipeline. | -| `header_text` | `str` | ACTIVE | When set, the first table row whose text matches this string is stripped before extraction. Use when pdfplumber includes the column header row in the extracted data. | -| `remove_header` | `bool` | ACTIVE | When True the first table row is unconditionally stripped. Use when the header row is always present but its text varies (making header_text impractical). | +| `header_text` | `str | None` | ACTIVE | When set, the first table row whose text matches this string is stripped before extraction. Use when pdfplumber includes the column header row in the extracted data. | +| `remove_header` | `bool | None` | ACTIVE | When True the first table row is unconditionally stripped. Use when the header row is always present but its text varies (making header_text impractical). | | `locations` | `list[Location]` | ACTIVE | One or more Location entries describing where on the page to find this table. Locations without a page_number are cloned for every page. | | `fields` | `list[Field]` | ACTIVE | Ordered list of field extraction specs. For transaction tables each field must have a column; for summary/detail tables each field must have a cell. | -| `table_columns` | `int` | ACTIVE | Expected minimum number of columns in the extracted table. Passed to pdfplumber as min_words_horizontal and used to validate column count after extraction. Also triggers allow_text_failover retry logic. | -| `table_rows` | `int` | ACTIVE | Expected minimum number of rows in the extracted table. Passed to pdfplumber as min_words_vertical. | -| `row_spacing` | `int` | ACTIVE | pdfplumber snap_y_tolerance in PDF points. Rows whose top edges fall within this distance of each other are merged into the same table row. Increase if the statement uses tight line spacing that splits a single visual row across multiple pdfplumber rows. | -| `tests` | `list[Test` | STUB | Declarative post-extraction assertions. Declared and accepted in TOML but no pipeline code evaluates them. Reserved for a future config validation pass. | -| `delete_success_false` | `bool` | STUB | Intended to drop rows where any field extraction returned success = False. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. | -| `delete_cast_success_false` | `bool` | STUB | Intended to drop rows where numeric casting failed. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. | -| `delete_rows_with_missing_vital_fields` | `bool` | STUB | Intended to drop rows where any vital field is missing after extraction. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. Note: vital-field hard-failure logic exists in validate() but is separate from this flag. | -| `transaction_spec` | `TransactionSpec` | ACTIVE | When set, the table is processed as a transaction table using the bookend-based multi-row extraction path. Must be None for summary/detail tables. | +| `table_columns` | `int | None` | ACTIVE | Expected minimum number of columns in the extracted table. Passed to pdfplumber as min_words_horizontal and used to validate column count after extraction. Also triggers allow_text_failover retry logic. | +| `table_rows` | `int | None` | ACTIVE | Expected minimum number of rows in the extracted table. Passed to pdfplumber as min_words_vertical. | +| `row_spacing` | `int | None` | ACTIVE | pdfplumber snap_y_tolerance in PDF points. Rows whose top edges fall within this distance of each other are merged into the same table row. Increase if the statement uses tight line spacing that splits a single visual row across multiple pdfplumber rows. | +| `tests` | `list[Test] | None` | STUB | Declarative post-extraction assertions. Declared and accepted in TOML but no pipeline code evaluates them. Reserved for a future config validation pass. | +| `delete_success_false` | `bool | None` | STUB | Intended to drop rows where any field extraction returned success = False. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. | +| `delete_cast_success_false` | `bool | None` | STUB | Intended to drop rows where numeric casting failed. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. | +| `delete_rows_with_missing_vital_fields` | `bool | None` | STUB | Intended to drop rows where any vital field is missing after extraction. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. Note: vital-field hard-failure logic exists in validate() but is separate from this flag. | +| `transaction_spec` | `TransactionSpec | None` | ACTIVE | When set, the table is processed as a transaction table using the bookend-based multi-row extraction path. Must be None for summary/detail tables. | #### `Location` @@ -296,13 +296,13 @@ Describes a rectangular region on a PDF page from which a table or text is extra | Field | Type | Status | Description | | --- | --- | --- | --- | -| `page_number` | `int` | ACTIVE | 1-based page number. When set the location is used only on that page. When None the location is cloned for every page (spawn_locations()). | -| `top_left` | `list[int` | ACTIVE | [x, y] coordinates of the top-left corner of the crop rectangle. Must be set together with bottom_right. When both are None the full page is used. | -| `bottom_right` | `list[int` | ACTIVE | [x, y] coordinates of the bottom-right corner of the crop rectangle. Must be set together with top_left. | -| `vertical_lines` | `list[int` | ACTIVE | Explicit x-coordinates of vertical column dividers supplied to pdfplumber as explicit_vertical_lines. Pairs of identical values create a zero-width gap that forces a column boundary (e.g. [100, 100, 200, 200]). When set, pdfplumber's automatic column detection is disabled for this region. | -| `dynamic_last_vertical_line` | `DynamicLineSpec` | ACTIVE | When set, the final value in vertical_lines is replaced at runtime with an x-coordinate derived from a PDF image's bounding box. See DynamicLineSpec. Used where the rightmost column boundary floats with a logo. | -| `allow_text_failover` | `bool` | ACTIVE | When True and the extracted table has the wrong number of columns, the extraction is retried without vertical_lines, falling back to pdfplumber's text-based column detection. Useful as a safety net for pages where the explicit dividers produce a malformed table. | -| `try_shift_down` | `int` | ACTIVE | Number of PDF points to shift the crop rectangle downward (applied to both top_left[1] and bottom_right[1]) when the initial extraction returns an empty region. Handles statements where the table top boundary varies slightly between pages. | +| `page_number` | `int | None` | ACTIVE | 1-based page number. When set the location is used only on that page. When None the location is cloned for every page (spawn_locations()). | +| `top_left` | `list[int] | None` | ACTIVE | [x, y] coordinates of the top-left corner of the crop rectangle. Must be set together with bottom_right. When both are None the full page is used. | +| `bottom_right` | `list[int] | None` | ACTIVE | [x, y] coordinates of the bottom-right corner of the crop rectangle. Must be set together with top_left. | +| `vertical_lines` | `list[int] | None` | ACTIVE | Explicit x-coordinates of vertical column dividers supplied to pdfplumber as explicit_vertical_lines. Pairs of identical values create a zero-width gap that forces a column boundary (e.g. [100, 100, 200, 200]). When set, pdfplumber's automatic column detection is disabled for this region. | +| `dynamic_last_vertical_line` | `DynamicLineSpec | None` | ACTIVE | When set, the final value in vertical_lines is replaced at runtime with an x-coordinate derived from a PDF image's bounding box. See DynamicLineSpec. Used where the rightmost column boundary floats with a logo. | +| `allow_text_failover` | `bool | None` | ACTIVE | When True and the extracted table has the wrong number of columns, the extraction is retried without vertical_lines, falling back to pdfplumber's text-based column detection. Useful as a safety net for pages where the explicit dividers produce a malformed table. | +| `try_shift_down` | `int | None` | ACTIVE | Number of PDF points to shift the crop rectangle downward (applied to both top_left[1] and bottom_right[1]) when the initial extraction returns an empty region. Handles statements where the table top boundary varies slightly between pages. | #### `DynamicLineSpec` @@ -320,19 +320,19 @@ Extraction specification for a single column or cell within a PDF table. | Field | Type | Status | Description | | --- | --- | --- | --- | | `field` | `str` | ACTIVE | Output column name for this field (e.g. "date", "£_paid_out"). Used as the field identifier throughout the pipeline and in the output Parquet files. | -| `cell` | `Cell` | ACTIVE | Row/column address for summary or detail table extraction. Mutually exclusive with ``column``; set to None for transaction tables. | +| `cell` | `Cell | None` | ACTIVE | Row/column address for summary or detail table extraction. Mutually exclusive with ``column``; set to None for transaction tables. | | `column` | `int | None` | ACTIVE | Zero-based column index for transaction table extraction. Mutually exclusive with ``cell``; set to None for summary/detail tables. | | `vital` | `bool` | ACTIVE | When True, extraction failure for this field causes the row to be flagged as a hard failure and excluded from output. When False, failure is recorded but the row is retained. | | `type` | `str` | ACTIVE | Data type: "string", "numeric", or "currency". * "string" — raw text extraction; pattern matching and trimming applied. * "numeric" — numeric extraction with optional explicit currency stripping via ``currency_override``. * "currency" — identical to "numeric" but inherits the CurrencySpec from the account's ``Account.currency`` rather than requiring an explicit ``currency_override`` on every field. Use this for all monetary amount fields; reserve "numeric" for non-monetary numerics (e.g. APR, sort code). | -| `strip_characters_start` | `str` | ACTIVE | Characters to strip from the start of the raw string before pattern matching (passed to Polars str.strip_chars_start()). Useful for leading currency symbols not covered by the account currency spec. | -| `strip_characters_end` | `str` | ACTIVE | Characters to strip from the end of the raw string before pattern matching (passed to Polars str.strip_chars_end()). | +| `strip_characters_start` | `str | None` | ACTIVE | Characters to strip from the start of the raw string before pattern matching (passed to Polars str.strip_chars_start()). Useful for leading currency symbols not covered by the account currency spec. | +| `strip_characters_end` | `str | None` | ACTIVE | Characters to strip from the end of the raw string before pattern matching (passed to Polars str.strip_chars_end()). | | `currency_override` | `str | None` | ACTIVE | Explicit ISO 4217 currency key (e.g. "GBP") used when ``type == "numeric"`` and currency stripping is needed but should differ from the account-level ``Account.currency``. Ignored when ``type == "currency"`` (which always uses the account-level currency). Omit for non-monetary numeric fields (e.g. APR, sort code) where no currency stripping is required. | -| `numeric_modifier` | `NumericModifier` | ACTIVE | Sign/multiplier transformation applied after numeric casting. See NumericModifier. Omit for straightforward positive numeric values. | -| `string_pattern` | `str` | ACTIVE | Regex pattern the extracted string must match. Extraction is marked as failed (success = False) if the value does not match. Used to validate field contents (e.g. date format) and to skip blank or irrelevant rows. | -| `string_max_length` | `int` | ACTIVE | Maximum character length for string values; longer strings are truncated via str.head(). Useful for capping free-text description fields. Defaults to 999 if not set. | -| `date_format` | `str` | STUB | Intended strptime format for date parsing at the Field level. Declared but never read by the pipeline; date format parsing is handled via StdRefs.format in get_standard_fields() instead. | +| `numeric_modifier` | `NumericModifier | None` | ACTIVE | Sign/multiplier transformation applied after numeric casting. See NumericModifier. Omit for straightforward positive numeric values. | +| `string_pattern` | `str | None` | ACTIVE | Regex pattern the extracted string must match. Extraction is marked as failed (success = False) if the value does not match. Used to validate field contents (e.g. date format) and to skip blank or irrelevant rows. | +| `string_max_length` | `int | None` | ACTIVE | Maximum character length for string values; longer strings are truncated via str.head(). Useful for capping free-text description fields. Defaults to 999 if not set. | +| `date_format` | `str | None` | STUB | Intended strptime format for date parsing at the Field level. Declared but never read by the pipeline; date format parsing is handled via StdRefs.format in get_standard_fields() instead. | | `value_offset` | `'FieldOffset'` | ACTIVE | When set, reads the field's value from an adjacent column (Field.column + FieldOffset.cols_offset) using the type and currency rules defined in the FieldOffset rather than those on this Field. The primary field column is still extracted normally; the offset column value replaces it in the output. See FieldOffset. | -| `regex_groups` | `int` | ACTIVE | When set, extracts the specified capture group (1-indexed) from the string_pattern regex match instead of the entire match (group 0). Useful for splitting a single PDF column into multiple fields via regex capture groups. Example: string_pattern = '^([A-Z ]+)\s+([A-Z0-9]+)$' with regex_groups = 1 extracts group 1; regex_groups = 2 extracts group 2. When None, defaults to group 0 (entire match, backward compatible). Omit for standard extraction. | +| `regex_groups` | `int | None` | ACTIVE | When set, extracts the specified capture group (1-indexed) from the string_pattern regex match instead of the entire match (group 0). Useful for splitting a single PDF column into multiple fields via regex capture groups. Example: string_pattern = '^([A-Z ]+)\s+([A-Z0-9]+)$' with regex_groups = 1 extracts group 1; regex_groups = 2 extracts group 2. When None, defaults to group 0 (entire match, backward compatible). Omit for standard extraction. | #### `Cell` @@ -349,8 +349,8 @@ Optional sign/multiplier transformation applied after numeric casting. | Field | Type | Status | Description | | --- | --- | --- | --- | -| `prefix` | `str` | ACTIVE | If the raw value starts with this string the prefix is stripped and the multiplier applied. Use for formats like "(123.45)" where "(" signals a negative value. | -| `suffix` | `str` | ACTIVE | If the raw value ends with this string the suffix is stripped and the multiplier applied. Use for formats like "123.45 CR" or "123.45D". | +| `prefix` | `str | None` | ACTIVE | If the raw value starts with this string the prefix is stripped and the multiplier applied. Use for formats like "(123.45)" where "(" signals a negative value. | +| `suffix` | `str | None` | ACTIVE | If the raw value ends with this string the suffix is stripped and the multiplier applied. Use for formats like "123.45 CR" or "123.45D". | | `multiplier` | `float` | ACTIVE | Scalar applied to the cast value when the prefix/suffix matches, or unconditionally if neither prefix nor suffix is set. Typically -1 to invert sign. | | `exclude_negative_values` | `bool` | ACTIVE | When True, any negative result after casting and multiplier application is replaced with 0. Useful for isolating one side of a combined debit/credit column. | | `exclude_positive_values` | `bool` | ACTIVE | When True, any positive result after casting and multiplier application is replaced with 0. Useful for isolating one side of a combined debit/credit column. | @@ -366,7 +366,7 @@ Reads a field's value from an adjacent column rather than the field's own column | `vital` | `bool` | ACTIVE | Passed to the extraction pipeline for the offset field; when True extraction failure is treated as a hard failure for that row. | | `type` | `str` | ACTIVE | Data type for the offset value: "string", "numeric", or "currency". Overrides the parent Field.type for this value read. | | `currency_override` | `str | None` | ACTIVE | Explicit currency key (e.g. "GBP") for numeric stripping of the offset value when type == "numeric". Overrides the account-level currency. When type == "currency" the account-level currency is used and this is ignored. | -| `numeric_modifier` | `NumericModifier` | ACTIVE | Sign/multiplier modifier for the offset value. Overrides the parent Field.numeric_modifier. | +| `numeric_modifier` | `NumericModifier | None` | ACTIVE | Sign/multiplier modifier for the offset value. Overrides the parent Field.numeric_modifier. | #### `CurrencySpec` @@ -388,9 +388,9 @@ Full specification for extracting transactions from a transaction-type table. | Field | Type | Status | Description | | --- | --- | --- | --- | | `transaction_bookends` | `list[TransactionBookend]` | ACTIVE | One or more bookend definitions that identify transaction boundaries. Evaluated in order; a row matched by an earlier bookend is not re-matched by a later one. At least one bookend is required. | -| `fill_forward_fields` | `list[str` | ACTIVE | Field names whose null values should be forward-filled across rows within the same page after pivot. Use for sparse columns where a value (e.g. a date or payment type) appears only on the first row of a multi-row block and needs propagating to the end row. | -| `merge_fields` | `MergeFields` | ACTIVE | When set, collapses multi-row text fields within each transaction into a single joined string. See MergeFields. | -| `exclude_rows` | `list[FieldValidation` | ACTIVE | Rows where any rule's field value matches its pattern are removed from the results before bookend detection runs. Use to suppress known non-transaction rows (e.g. a closing balance summary line) that would otherwise interfere with transaction counting or checks & balances. Each rule is a {field, pattern} pair; a row is excluded if any rule matches. | +| `fill_forward_fields` | `list[str] | None` | ACTIVE | Field names whose null values should be forward-filled across rows within the same page after pivot. Use for sparse columns where a value (e.g. a date or payment type) appears only on the first row of a multi-row block and needs propagating to the end row. | +| `merge_fields` | `MergeFields | None` | ACTIVE | When set, collapses multi-row text fields within each transaction into a single joined string. See MergeFields. | +| `exclude_rows` | `list[FieldValidation] | None` | ACTIVE | Rows where any rule's field value matches its pattern are removed from the results before bookend detection runs. Use to suppress known non-transaction rows (e.g. a closing balance summary line) that would otherwise interfere with transaction counting or checks & balances. Each rule is a {field, pattern} pair; a row is excluded if any rule matches. | #### `TransactionBookend` @@ -402,9 +402,9 @@ Defines how the start and end of a single transaction are detected within a tabl | `min_non_empty_start` | `int` | ACTIVE | Minimum number of start_fields that must have extracted successfully for a row to be flagged as transaction_start = True. | | `end_fields` | `list[str]` | ACTIVE | Field names checked to identify the last row of a transaction. A row qualifies as an end row when at least min_non_empty_end of these fields extracted successfully. | | `min_non_empty_end` | `int` | ACTIVE | Minimum number of end_fields that must have extracted successfully for a row to be flagged as transaction_end = True. | -| `extra_validation_start` | `FieldValidation` | ACTIVE | When set, any row where the named field's value does NOT match the pattern is excluded from being a start-bookend candidate for this bookend. Rows excluded here may still be captured by another bookend in the list. Useful for bookends that should only trigger on a specific row shape (e.g. an interest charge line identified by its details text). | -| `extra_validation_end` | `FieldValidation` | STUB | Symmetric counterpart to extra_validation_start for end rows. Declared but not yet implemented in the pipeline; no code currently reads this field. Reserved for future use. | -| `sticky_fields` | `list[str` | STUB | Intended to forward-fill named fields from the start row of a transaction down to its end row, scoped within a single transaction (as opposed to fill_forward_fields which fills across transactions). Declared but not implemented; no pipeline code reads this field. | +| `extra_validation_start` | `FieldValidation | None` | ACTIVE | When set, any row where the named field's value does NOT match the pattern is excluded from being a start-bookend candidate for this bookend. Rows excluded here may still be captured by another bookend in the list. Useful for bookends that should only trigger on a specific row shape (e.g. an interest charge line identified by its details text). | +| `extra_validation_end` | `FieldValidation | None` | STUB | Symmetric counterpart to extra_validation_start for end rows. Declared but not yet implemented in the pipeline; no code currently reads this field. Reserved for future use. | +| `sticky_fields` | `list[str] | None` | STUB | Intended to forward-fill named fields from the start row of a transaction down to its end row, scoped within a single transaction (as opposed to fill_forward_fields which fills across transactions). Declared but not implemented; no pipeline code reads this field. | #### `FieldValidation` @@ -531,7 +531,7 @@ An ordered list of Config extraction steps for one pipeline section. | Field | Type | Status | Description | | --- | --- | --- | --- | -| `configs` | `list[Config` | ACTIVE | Ordered list of Config steps. Executed in sequence during extraction; results are stacked into the section's results DataFrame. | +| `configs` | `list[Config] | None` | ACTIVE | Ordered list of Config steps. Executed in sequence during extraction; results are stacked into the section's results DataFrame. | #### `Config` @@ -540,10 +540,10 @@ A single extraction step: one table (or one standalone field) from one location. | Field | Type | Status | Description | | --- | --- | --- | --- | | `config` | `str` | ACTIVE | Human-readable label for this extraction step (e.g. "Statement Balances"). Written into the "config" column of the long-format results DataFrame for traceability. | -| `statement_table_key` | `str` | ACTIVE | Key into statement_tables.toml that identifies the StatementTable to use. Resolved to statement_table at load time. Set to None for inline single-field configs. | -| `statement_table` | `StatementTable` | ACTIVE | Resolved at load time from statement_table_key. The StatementTable object used during extraction. Not set directly in TOML. | -| `locations` | `list[Location` | ACTIVE | Used only for inline single-field configs (where statement_table is None). Defines where on the page to find the field value. | -| `field` | `Field` | ACTIVE | Used only for inline single-field configs. Defines the extraction spec for the single value to read from the location. | +| `statement_table_key` | `str | None` | ACTIVE | Key into statement_tables.toml that identifies the StatementTable to use. Resolved to statement_table at load time. Set to None for inline single-field configs. | +| `statement_table` | `StatementTable | None` | ACTIVE | Resolved at load time from statement_table_key. The StatementTable object used during extraction. Not set directly in TOML. | +| `locations` | `list[Location] | None` | ACTIVE | Used only for inline single-field configs (where statement_table is None). Defines where on the page to find the field value. | +| `field` | `Field | None` | ACTIVE | Used only for inline single-field configs. Defines the extraction spec for the single value to read from the location. | ## Step 6: Define Accounts @@ -585,11 +585,11 @@ Full runtime configuration for one bank account. | --- | --- | --- | --- | | `account` | `str` | ACTIVE | Human-readable account name (e.g. "Current Account"). Written to the STD_ACCOUNT standard field in the output. | | `company_key` | `str` | ACTIVE | Key into companies.toml identifying the issuing bank. Used to build ID_ACCOUNT and to look up the Company object at load time. | -| `company` | `Company` | ACTIVE | Resolved at load time from company_key. Provides the company name and company-level identification config. | +| `company` | `Company | None` | ACTIVE | Resolved at load time from company_key. Provides the company name and company-level identification config. | | `account_type_key` | `str` | ACTIVE | Key into account_types.toml (e.g. "CRD", "CUR", "SAV"). Used to look up the AccountType object at load time. | -| `account_type` | `AccountType` | STUB | Resolved at load time from account_type_key. The AccountType object is populated but never subsequently read by any pipeline consumer. | +| `account_type` | `AccountType | None` | STUB | Resolved at load time from account_type_key. The AccountType object is populated but never subsequently read by any pipeline consumer. | | `statement_type_key` | `str` | ACTIVE | Key into statement_types.toml identifying the extraction layout for this account's statements. Used to look up the StatementType object at load time. | -| `statement_type` | `StatementType` | ACTIVE | Resolved at load time from statement_type_key. Provides the header and lines ConfigGroups used during extraction. | +| `statement_type` | `StatementType | None` | ACTIVE | Resolved at load time from statement_type_key. Provides the header and lines ConfigGroups used during extraction. | | `exclude_last_n_pages` | `int` | ACTIVE | Number of trailing pages to skip when cloning per-page locations. Set to 1 (or more) when the final page(s) contain terms & conditions or other non-transaction content that would otherwise be passed to the extraction pipeline. | | `currency` | `str` | ACTIVE | ISO 4217 currency code for all monetary fields on this account (e.g. "GBP", "USD", "PHP"). Must be a key in ``currency_spec`` in ``currency.py``; validated at config load time. Used by the extraction pipeline to resolve the CurrencySpec for fields of type "currency". | | `config` | `Config` | ACTIVE | Account-level identification config. A lightweight extraction step run to confirm a PDF belongs to this account before the full extraction pass. Defined inline under ``[ACCOUNT_KEY.config]`` in accounts.toml. | @@ -655,14 +655,14 @@ Mapping rule that promotes a raw extracted field to a standard output column. | Field | Type | Status | Description | | --- | --- | --- | --- | | `statement_type` | `str` | ACTIVE | Key used to select this rule; matched against the statement type string of the PDF being processed (e.g. "HSBC UK Current Account"). | -| `field` | `str` | ACTIVE | Name of the raw extracted column to promote. Set to None (or omit) when a literal default value should be used instead of a column value. | -| `concat_fields` | `list` | ACTIVE | Name of the raw extracted columns to concatenate and promote. Set to None (or omit) in order to revert to a single field and it's fallback | -| `format` | `str` | ACTIVE | strptime format string applied when StandardFields.type == "date" (e.g. "%-d %B %Y"). Ignored for numeric and string types. | -| `default` | `str` | ACTIVE | Literal string value used as the output when ``field`` is None/absent. Useful for injecting constant metadata (e.g. transaction_type = "CC"). | -| `multiplier` | `float` | ACTIVE | Scalar applied to the value after casting when StandardFields.type == "numeric". Use -1 to invert sign (e.g. to convert a credit amount stored as positive into a negative figure). | -| `exclude_positive_values` | `bool` | ACTIVE | When True, any positive numeric value is replaced with 0 after casting. Used to isolate debit-side figures from a combined amount column. | -| `exclude_negative_values` | `bool` | ACTIVE | When True, any negative numeric value is replaced with 0 after casting. Used to isolate credit-side figures from a combined amount column. | -| `terminator` | `str` | ACTIVE | Regex pattern; when present the string value is truncated at the first match position before being written to the standard column. Useful for stripping trailing boilerplate appended by merge_fields (e.g. " \| BALANCE CARRIED FORWARD"). | +| `field` | `str | None` | ACTIVE | Name of the raw extracted column to promote. Set to None (or omit) when a literal default value should be used instead of a column value. | +| `concat_fields` | `list | None` | ACTIVE | Name of the raw extracted columns to concatenate and promote. Set to None (or omit) in order to revert to a single field and it's fallback | +| `format` | `str | None` | ACTIVE | strptime format string applied when StandardFields.type == "date" (e.g. "%-d %B %Y"). Ignored for numeric and string types. | +| `default` | `str | None` | ACTIVE | Literal string value used as the output when ``field`` is None/absent. Useful for injecting constant metadata (e.g. transaction_type = "CC"). | +| `multiplier` | `float | None` | ACTIVE | Scalar applied to the value after casting when StandardFields.type == "numeric". Use -1 to invert sign (e.g. to convert a credit amount stored as positive into a negative figure). | +| `exclude_positive_values` | `bool | None` | ACTIVE | When True, any positive numeric value is replaced with 0 after casting. Used to isolate debit-side figures from a combined amount column. | +| `exclude_negative_values` | `bool | None` | ACTIVE | When True, any negative numeric value is replaced with 0 after casting. Used to isolate credit-side figures from a combined amount column. | +| `terminator` | `str | None` | ACTIVE | Regex pattern; when present the string value is truncated at the first match position before being written to the standard column. Useful for stripping trailing boilerplate appended by merge_fields (e.g. " \| BALANCE CARRIED FORWARD"). | #### `StandardFields` @@ -703,8 +703,8 @@ Configuration for a financial institution (bank/provider). | Field | Type | Status | Description | | --- | --- | --- | --- | | `company` | `str` | ACTIVE | Human-readable company name (e.g. "HSBC UK"). Used to populate the STD_COMPANY standard field. | -| `config` | `Config` | ACTIVE | Extraction config used during the company-identification pass. Extracts a discriminating field (e.g. a bank-specific header string) to confirm the PDF belongs to this company before attempting account matching. | -| `accounts` | `dict` | STUB | Declared but never accessed by the pipeline after load. Intended as a lookup from account key to Account object but currently unused. | +| `config` | `Config | None` | ACTIVE | Extraction config used during the company-identification pass. Extracts a discriminating field (e.g. a bank-specific header string) to confirm the PDF belongs to this company before attempting account matching. | +| `accounts` | `dict | None` | STUB | Declared but never accessed by the pipeline after load. Intended as a lookup from account key to Account object but currently unused. | ### `Account` @@ -714,11 +714,11 @@ Full runtime configuration for one bank account. | --- | --- | --- | --- | | `account` | `str` | ACTIVE | Human-readable account name (e.g. "Current Account"). Written to the STD_ACCOUNT standard field in the output. | | `company_key` | `str` | ACTIVE | Key into companies.toml identifying the issuing bank. Used to build ID_ACCOUNT and to look up the Company object at load time. | -| `company` | `Company` | ACTIVE | Resolved at load time from company_key. Provides the company name and company-level identification config. | +| `company` | `Company | None` | ACTIVE | Resolved at load time from company_key. Provides the company name and company-level identification config. | | `account_type_key` | `str` | ACTIVE | Key into account_types.toml (e.g. "CRD", "CUR", "SAV"). Used to look up the AccountType object at load time. | -| `account_type` | `AccountType` | STUB | Resolved at load time from account_type_key. The AccountType object is populated but never subsequently read by any pipeline consumer. | +| `account_type` | `AccountType | None` | STUB | Resolved at load time from account_type_key. The AccountType object is populated but never subsequently read by any pipeline consumer. | | `statement_type_key` | `str` | ACTIVE | Key into statement_types.toml identifying the extraction layout for this account's statements. Used to look up the StatementType object at load time. | -| `statement_type` | `StatementType` | ACTIVE | Resolved at load time from statement_type_key. Provides the header and lines ConfigGroups used during extraction. | +| `statement_type` | `StatementType | None` | ACTIVE | Resolved at load time from statement_type_key. Provides the header and lines ConfigGroups used during extraction. | | `exclude_last_n_pages` | `int` | ACTIVE | Number of trailing pages to skip when cloning per-page locations. Set to 1 (or more) when the final page(s) contain terms & conditions or other non-transaction content that would otherwise be passed to the extraction pipeline. | | `currency` | `str` | ACTIVE | ISO 4217 currency code for all monetary fields on this account (e.g. "GBP", "USD", "PHP"). Must be a key in ``currency_spec`` in ``currency.py``; validated at config load time. Used by the extraction pipeline to resolve the CurrencySpec for fields of type "currency". | | `config` | `Config` | ACTIVE | Account-level identification config. A lightweight extraction step run to confirm a PDF belongs to this account before the full extraction pass. Defined inline under ``[ACCOUNT_KEY.config]`` in accounts.toml. | @@ -747,7 +747,7 @@ An ordered list of Config extraction steps for one pipeline section. | Field | Type | Status | Description | | --- | --- | --- | --- | -| `configs` | `list[Config` | ACTIVE | Ordered list of Config steps. Executed in sequence during extraction; results are stacked into the section's results DataFrame. | +| `configs` | `list[Config] | None` | ACTIVE | Ordered list of Config steps. Executed in sequence during extraction; results are stacked into the section's results DataFrame. | ### `Config` @@ -756,10 +756,10 @@ A single extraction step: one table (or one standalone field) from one location. | Field | Type | Status | Description | | --- | --- | --- | --- | | `config` | `str` | ACTIVE | Human-readable label for this extraction step (e.g. "Statement Balances"). Written into the "config" column of the long-format results DataFrame for traceability. | -| `statement_table_key` | `str` | ACTIVE | Key into statement_tables.toml that identifies the StatementTable to use. Resolved to statement_table at load time. Set to None for inline single-field configs. | -| `statement_table` | `StatementTable` | ACTIVE | Resolved at load time from statement_table_key. The StatementTable object used during extraction. Not set directly in TOML. | -| `locations` | `list[Location` | ACTIVE | Used only for inline single-field configs (where statement_table is None). Defines where on the page to find the field value. | -| `field` | `Field` | ACTIVE | Used only for inline single-field configs. Defines the extraction spec for the single value to read from the location. | +| `statement_table_key` | `str | None` | ACTIVE | Key into statement_tables.toml that identifies the StatementTable to use. Resolved to statement_table at load time. Set to None for inline single-field configs. | +| `statement_table` | `StatementTable | None` | ACTIVE | Resolved at load time from statement_table_key. The StatementTable object used during extraction. Not set directly in TOML. | +| `locations` | `list[Location] | None` | ACTIVE | Used only for inline single-field configs (where statement_table is None). Defines where on the page to find the field value. | +| `field` | `Field | None` | ACTIVE | Used only for inline single-field configs. Defines the extraction spec for the single value to read from the location. | ### `StatementTable` @@ -769,18 +769,18 @@ Full configuration for extracting one table from a PDF statement. | --- | --- | --- | --- | | `type` | `str` | STUB | Table type label: "transaction", "summary", or "detail". Loaded from TOML but not currently read by the pipeline; the extraction path is determined by whether transaction_spec is present rather than this field. | | `statement_table` | `str` | STUB | Human-readable table label (e.g. "Transactions", "Account Summary"). Loaded from TOML for documentation purposes but not consumed by the pipeline. | -| `header_text` | `str` | ACTIVE | When set, the first table row whose text matches this string is stripped before extraction. Use when pdfplumber includes the column header row in the extracted data. | -| `remove_header` | `bool` | ACTIVE | When True the first table row is unconditionally stripped. Use when the header row is always present but its text varies (making header_text impractical). | +| `header_text` | `str | None` | ACTIVE | When set, the first table row whose text matches this string is stripped before extraction. Use when pdfplumber includes the column header row in the extracted data. | +| `remove_header` | `bool | None` | ACTIVE | When True the first table row is unconditionally stripped. Use when the header row is always present but its text varies (making header_text impractical). | | `locations` | `list[Location]` | ACTIVE | One or more Location entries describing where on the page to find this table. Locations without a page_number are cloned for every page. | | `fields` | `list[Field]` | ACTIVE | Ordered list of field extraction specs. For transaction tables each field must have a column; for summary/detail tables each field must have a cell. | -| `table_columns` | `int` | ACTIVE | Expected minimum number of columns in the extracted table. Passed to pdfplumber as min_words_horizontal and used to validate column count after extraction. Also triggers allow_text_failover retry logic. | -| `table_rows` | `int` | ACTIVE | Expected minimum number of rows in the extracted table. Passed to pdfplumber as min_words_vertical. | -| `row_spacing` | `int` | ACTIVE | pdfplumber snap_y_tolerance in PDF points. Rows whose top edges fall within this distance of each other are merged into the same table row. Increase if the statement uses tight line spacing that splits a single visual row across multiple pdfplumber rows. | -| `tests` | `list[Test` | STUB | Declarative post-extraction assertions. Declared and accepted in TOML but no pipeline code evaluates them. Reserved for a future config validation pass. | -| `delete_success_false` | `bool` | STUB | Intended to drop rows where any field extraction returned success = False. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. | -| `delete_cast_success_false` | `bool` | STUB | Intended to drop rows where numeric casting failed. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. | -| `delete_rows_with_missing_vital_fields` | `bool` | STUB | Intended to drop rows where any vital field is missing after extraction. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. Note: vital-field hard-failure logic exists in validate() but is separate from this flag. | -| `transaction_spec` | `TransactionSpec` | ACTIVE | When set, the table is processed as a transaction table using the bookend-based multi-row extraction path. Must be None for summary/detail tables. | +| `table_columns` | `int | None` | ACTIVE | Expected minimum number of columns in the extracted table. Passed to pdfplumber as min_words_horizontal and used to validate column count after extraction. Also triggers allow_text_failover retry logic. | +| `table_rows` | `int | None` | ACTIVE | Expected minimum number of rows in the extracted table. Passed to pdfplumber as min_words_vertical. | +| `row_spacing` | `int | None` | ACTIVE | pdfplumber snap_y_tolerance in PDF points. Rows whose top edges fall within this distance of each other are merged into the same table row. Increase if the statement uses tight line spacing that splits a single visual row across multiple pdfplumber rows. | +| `tests` | `list[Test] | None` | STUB | Declarative post-extraction assertions. Declared and accepted in TOML but no pipeline code evaluates them. Reserved for a future config validation pass. | +| `delete_success_false` | `bool | None` | STUB | Intended to drop rows where any field extraction returned success = False. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. | +| `delete_cast_success_false` | `bool | None` | STUB | Intended to drop rows where numeric casting failed. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. | +| `delete_rows_with_missing_vital_fields` | `bool | None` | STUB | Intended to drop rows where any vital field is missing after extraction. Declared and set in TOML (typically True) but no pipeline code currently reads or acts on this flag. Note: vital-field hard-failure logic exists in validate() but is separate from this flag. | +| `transaction_spec` | `TransactionSpec | None` | ACTIVE | When set, the table is processed as a transaction table using the bookend-based multi-row extraction path. Must be None for summary/detail tables. | ### `Location` @@ -788,13 +788,13 @@ Describes a rectangular region on a PDF page from which a table or text is extra | Field | Type | Status | Description | | --- | --- | --- | --- | -| `page_number` | `int` | ACTIVE | 1-based page number. When set the location is used only on that page. When None the location is cloned for every page (spawn_locations()). | -| `top_left` | `list[int` | ACTIVE | [x, y] coordinates of the top-left corner of the crop rectangle. Must be set together with bottom_right. When both are None the full page is used. | -| `bottom_right` | `list[int` | ACTIVE | [x, y] coordinates of the bottom-right corner of the crop rectangle. Must be set together with top_left. | -| `vertical_lines` | `list[int` | ACTIVE | Explicit x-coordinates of vertical column dividers supplied to pdfplumber as explicit_vertical_lines. Pairs of identical values create a zero-width gap that forces a column boundary (e.g. [100, 100, 200, 200]). When set, pdfplumber's automatic column detection is disabled for this region. | -| `dynamic_last_vertical_line` | `DynamicLineSpec` | ACTIVE | When set, the final value in vertical_lines is replaced at runtime with an x-coordinate derived from a PDF image's bounding box. See DynamicLineSpec. Used where the rightmost column boundary floats with a logo. | -| `allow_text_failover` | `bool` | ACTIVE | When True and the extracted table has the wrong number of columns, the extraction is retried without vertical_lines, falling back to pdfplumber's text-based column detection. Useful as a safety net for pages where the explicit dividers produce a malformed table. | -| `try_shift_down` | `int` | ACTIVE | Number of PDF points to shift the crop rectangle downward (applied to both top_left[1] and bottom_right[1]) when the initial extraction returns an empty region. Handles statements where the table top boundary varies slightly between pages. | +| `page_number` | `int | None` | ACTIVE | 1-based page number. When set the location is used only on that page. When None the location is cloned for every page (spawn_locations()). | +| `top_left` | `list[int] | None` | ACTIVE | [x, y] coordinates of the top-left corner of the crop rectangle. Must be set together with bottom_right. When both are None the full page is used. | +| `bottom_right` | `list[int] | None` | ACTIVE | [x, y] coordinates of the bottom-right corner of the crop rectangle. Must be set together with top_left. | +| `vertical_lines` | `list[int] | None` | ACTIVE | Explicit x-coordinates of vertical column dividers supplied to pdfplumber as explicit_vertical_lines. Pairs of identical values create a zero-width gap that forces a column boundary (e.g. [100, 100, 200, 200]). When set, pdfplumber's automatic column detection is disabled for this region. | +| `dynamic_last_vertical_line` | `DynamicLineSpec | None` | ACTIVE | When set, the final value in vertical_lines is replaced at runtime with an x-coordinate derived from a PDF image's bounding box. See DynamicLineSpec. Used where the rightmost column boundary floats with a logo. | +| `allow_text_failover` | `bool | None` | ACTIVE | When True and the extracted table has the wrong number of columns, the extraction is retried without vertical_lines, falling back to pdfplumber's text-based column detection. Useful as a safety net for pages where the explicit dividers produce a malformed table. | +| `try_shift_down` | `int | None` | ACTIVE | Number of PDF points to shift the crop rectangle downward (applied to both top_left[1] and bottom_right[1]) when the initial extraction returns an empty region. Handles statements where the table top boundary varies slightly between pages. | ### `DynamicLineSpec` @@ -812,19 +812,19 @@ Extraction specification for a single column or cell within a PDF table. | Field | Type | Status | Description | | --- | --- | --- | --- | | `field` | `str` | ACTIVE | Output column name for this field (e.g. "date", "£_paid_out"). Used as the field identifier throughout the pipeline and in the output Parquet files. | -| `cell` | `Cell` | ACTIVE | Row/column address for summary or detail table extraction. Mutually exclusive with ``column``; set to None for transaction tables. | +| `cell` | `Cell | None` | ACTIVE | Row/column address for summary or detail table extraction. Mutually exclusive with ``column``; set to None for transaction tables. | | `column` | `int | None` | ACTIVE | Zero-based column index for transaction table extraction. Mutually exclusive with ``cell``; set to None for summary/detail tables. | | `vital` | `bool` | ACTIVE | When True, extraction failure for this field causes the row to be flagged as a hard failure and excluded from output. When False, failure is recorded but the row is retained. | | `type` | `str` | ACTIVE | Data type: "string", "numeric", or "currency". * "string" — raw text extraction; pattern matching and trimming applied. * "numeric" — numeric extraction with optional explicit currency stripping via ``currency_override``. * "currency" — identical to "numeric" but inherits the CurrencySpec from the account's ``Account.currency`` rather than requiring an explicit ``currency_override`` on every field. Use this for all monetary amount fields; reserve "numeric" for non-monetary numerics (e.g. APR, sort code). | -| `strip_characters_start` | `str` | ACTIVE | Characters to strip from the start of the raw string before pattern matching (passed to Polars str.strip_chars_start()). Useful for leading currency symbols not covered by the account currency spec. | -| `strip_characters_end` | `str` | ACTIVE | Characters to strip from the end of the raw string before pattern matching (passed to Polars str.strip_chars_end()). | +| `strip_characters_start` | `str | None` | ACTIVE | Characters to strip from the start of the raw string before pattern matching (passed to Polars str.strip_chars_start()). Useful for leading currency symbols not covered by the account currency spec. | +| `strip_characters_end` | `str | None` | ACTIVE | Characters to strip from the end of the raw string before pattern matching (passed to Polars str.strip_chars_end()). | | `currency_override` | `str | None` | ACTIVE | Explicit ISO 4217 currency key (e.g. "GBP") used when ``type == "numeric"`` and currency stripping is needed but should differ from the account-level ``Account.currency``. Ignored when ``type == "currency"`` (which always uses the account-level currency). Omit for non-monetary numeric fields (e.g. APR, sort code) where no currency stripping is required. | -| `numeric_modifier` | `NumericModifier` | ACTIVE | Sign/multiplier transformation applied after numeric casting. See NumericModifier. Omit for straightforward positive numeric values. | -| `string_pattern` | `str` | ACTIVE | Regex pattern the extracted string must match. Extraction is marked as failed (success = False) if the value does not match. Used to validate field contents (e.g. date format) and to skip blank or irrelevant rows. | -| `string_max_length` | `int` | ACTIVE | Maximum character length for string values; longer strings are truncated via str.head(). Useful for capping free-text description fields. Defaults to 999 if not set. | -| `date_format` | `str` | STUB | Intended strptime format for date parsing at the Field level. Declared but never read by the pipeline; date format parsing is handled via StdRefs.format in get_standard_fields() instead. | +| `numeric_modifier` | `NumericModifier | None` | ACTIVE | Sign/multiplier transformation applied after numeric casting. See NumericModifier. Omit for straightforward positive numeric values. | +| `string_pattern` | `str | None` | ACTIVE | Regex pattern the extracted string must match. Extraction is marked as failed (success = False) if the value does not match. Used to validate field contents (e.g. date format) and to skip blank or irrelevant rows. | +| `string_max_length` | `int | None` | ACTIVE | Maximum character length for string values; longer strings are truncated via str.head(). Useful for capping free-text description fields. Defaults to 999 if not set. | +| `date_format` | `str | None` | STUB | Intended strptime format for date parsing at the Field level. Declared but never read by the pipeline; date format parsing is handled via StdRefs.format in get_standard_fields() instead. | | `value_offset` | `'FieldOffset'` | ACTIVE | When set, reads the field's value from an adjacent column (Field.column + FieldOffset.cols_offset) using the type and currency rules defined in the FieldOffset rather than those on this Field. The primary field column is still extracted normally; the offset column value replaces it in the output. See FieldOffset. | -| `regex_groups` | `int` | ACTIVE | When set, extracts the specified capture group (1-indexed) from the string_pattern regex match instead of the entire match (group 0). Useful for splitting a single PDF column into multiple fields via regex capture groups. Example: string_pattern = '^([A-Z ]+)\s+([A-Z0-9]+)$' with regex_groups = 1 extracts group 1; regex_groups = 2 extracts group 2. When None, defaults to group 0 (entire match, backward compatible). Omit for standard extraction. | +| `regex_groups` | `int | None` | ACTIVE | When set, extracts the specified capture group (1-indexed) from the string_pattern regex match instead of the entire match (group 0). Useful for splitting a single PDF column into multiple fields via regex capture groups. Example: string_pattern = '^([A-Z ]+)\s+([A-Z0-9]+)$' with regex_groups = 1 extracts group 1; regex_groups = 2 extracts group 2. When None, defaults to group 0 (entire match, backward compatible). Omit for standard extraction. | ### `Cell` @@ -846,7 +846,7 @@ Reads a field's value from an adjacent column rather than the field's own column | `vital` | `bool` | ACTIVE | Passed to the extraction pipeline for the offset field; when True extraction failure is treated as a hard failure for that row. | | `type` | `str` | ACTIVE | Data type for the offset value: "string", "numeric", or "currency". Overrides the parent Field.type for this value read. | | `currency_override` | `str | None` | ACTIVE | Explicit currency key (e.g. "GBP") for numeric stripping of the offset value when type == "numeric". Overrides the account-level currency. When type == "currency" the account-level currency is used and this is ignored. | -| `numeric_modifier` | `NumericModifier` | ACTIVE | Sign/multiplier modifier for the offset value. Overrides the parent Field.numeric_modifier. | +| `numeric_modifier` | `NumericModifier | None` | ACTIVE | Sign/multiplier modifier for the offset value. Overrides the parent Field.numeric_modifier. | ### `NumericModifier` @@ -854,8 +854,8 @@ Optional sign/multiplier transformation applied after numeric casting. | Field | Type | Status | Description | | --- | --- | --- | --- | -| `prefix` | `str` | ACTIVE | If the raw value starts with this string the prefix is stripped and the multiplier applied. Use for formats like "(123.45)" where "(" signals a negative value. | -| `suffix` | `str` | ACTIVE | If the raw value ends with this string the suffix is stripped and the multiplier applied. Use for formats like "123.45 CR" or "123.45D". | +| `prefix` | `str | None` | ACTIVE | If the raw value starts with this string the prefix is stripped and the multiplier applied. Use for formats like "(123.45)" where "(" signals a negative value. | +| `suffix` | `str | None` | ACTIVE | If the raw value ends with this string the suffix is stripped and the multiplier applied. Use for formats like "123.45 CR" or "123.45D". | | `multiplier` | `float` | ACTIVE | Scalar applied to the cast value when the prefix/suffix matches, or unconditionally if neither prefix nor suffix is set. Typically -1 to invert sign. | | `exclude_negative_values` | `bool` | ACTIVE | When True, any negative result after casting and multiplier application is replaced with 0. Useful for isolating one side of a combined debit/credit column. | | `exclude_positive_values` | `bool` | ACTIVE | When True, any positive result after casting and multiplier application is replaced with 0. Useful for isolating one side of a combined debit/credit column. | @@ -880,9 +880,9 @@ Full specification for extracting transactions from a transaction-type table. | Field | Type | Status | Description | | --- | --- | --- | --- | | `transaction_bookends` | `list[TransactionBookend]` | ACTIVE | One or more bookend definitions that identify transaction boundaries. Evaluated in order; a row matched by an earlier bookend is not re-matched by a later one. At least one bookend is required. | -| `fill_forward_fields` | `list[str` | ACTIVE | Field names whose null values should be forward-filled across rows within the same page after pivot. Use for sparse columns where a value (e.g. a date or payment type) appears only on the first row of a multi-row block and needs propagating to the end row. | -| `merge_fields` | `MergeFields` | ACTIVE | When set, collapses multi-row text fields within each transaction into a single joined string. See MergeFields. | -| `exclude_rows` | `list[FieldValidation` | ACTIVE | Rows where any rule's field value matches its pattern are removed from the results before bookend detection runs. Use to suppress known non-transaction rows (e.g. a closing balance summary line) that would otherwise interfere with transaction counting or checks & balances. Each rule is a {field, pattern} pair; a row is excluded if any rule matches. | +| `fill_forward_fields` | `list[str] | None` | ACTIVE | Field names whose null values should be forward-filled across rows within the same page after pivot. Use for sparse columns where a value (e.g. a date or payment type) appears only on the first row of a multi-row block and needs propagating to the end row. | +| `merge_fields` | `MergeFields | None` | ACTIVE | When set, collapses multi-row text fields within each transaction into a single joined string. See MergeFields. | +| `exclude_rows` | `list[FieldValidation] | None` | ACTIVE | Rows where any rule's field value matches its pattern are removed from the results before bookend detection runs. Use to suppress known non-transaction rows (e.g. a closing balance summary line) that would otherwise interfere with transaction counting or checks & balances. Each rule is a {field, pattern} pair; a row is excluded if any rule matches. | ### `TransactionBookend` @@ -894,9 +894,9 @@ Defines how the start and end of a single transaction are detected within a tabl | `min_non_empty_start` | `int` | ACTIVE | Minimum number of start_fields that must have extracted successfully for a row to be flagged as transaction_start = True. | | `end_fields` | `list[str]` | ACTIVE | Field names checked to identify the last row of a transaction. A row qualifies as an end row when at least min_non_empty_end of these fields extracted successfully. | | `min_non_empty_end` | `int` | ACTIVE | Minimum number of end_fields that must have extracted successfully for a row to be flagged as transaction_end = True. | -| `extra_validation_start` | `FieldValidation` | ACTIVE | When set, any row where the named field's value does NOT match the pattern is excluded from being a start-bookend candidate for this bookend. Rows excluded here may still be captured by another bookend in the list. Useful for bookends that should only trigger on a specific row shape (e.g. an interest charge line identified by its details text). | -| `extra_validation_end` | `FieldValidation` | STUB | Symmetric counterpart to extra_validation_start for end rows. Declared but not yet implemented in the pipeline; no code currently reads this field. Reserved for future use. | -| `sticky_fields` | `list[str` | STUB | Intended to forward-fill named fields from the start row of a transaction down to its end row, scoped within a single transaction (as opposed to fill_forward_fields which fills across transactions). Declared but not implemented; no pipeline code reads this field. | +| `extra_validation_start` | `FieldValidation | None` | ACTIVE | When set, any row where the named field's value does NOT match the pattern is excluded from being a start-bookend candidate for this bookend. Rows excluded here may still be captured by another bookend in the list. Useful for bookends that should only trigger on a specific row shape (e.g. an interest charge line identified by its details text). | +| `extra_validation_end` | `FieldValidation | None` | STUB | Symmetric counterpart to extra_validation_start for end rows. Declared but not yet implemented in the pipeline; no code currently reads this field. Reserved for future use. | +| `sticky_fields` | `list[str] | None` | STUB | Intended to forward-fill named fields from the start row of a transaction down to its end row, scoped within a single transaction (as opposed to fill_forward_fields which fills across transactions). Declared but not implemented; no pipeline code reads this field. | ### `FieldValidation` @@ -934,14 +934,14 @@ Mapping rule that promotes a raw extracted field to a standard output column. | Field | Type | Status | Description | | --- | --- | --- | --- | | `statement_type` | `str` | ACTIVE | Key used to select this rule; matched against the statement type string of the PDF being processed (e.g. "HSBC UK Current Account"). | -| `field` | `str` | ACTIVE | Name of the raw extracted column to promote. Set to None (or omit) when a literal default value should be used instead of a column value. | -| `concat_fields` | `list` | ACTIVE | Name of the raw extracted columns to concatenate and promote. Set to None (or omit) in order to revert to a single field and it's fallback | -| `format` | `str` | ACTIVE | strptime format string applied when StandardFields.type == "date" (e.g. "%-d %B %Y"). Ignored for numeric and string types. | -| `default` | `str` | ACTIVE | Literal string value used as the output when ``field`` is None/absent. Useful for injecting constant metadata (e.g. transaction_type = "CC"). | -| `multiplier` | `float` | ACTIVE | Scalar applied to the value after casting when StandardFields.type == "numeric". Use -1 to invert sign (e.g. to convert a credit amount stored as positive into a negative figure). | -| `exclude_positive_values` | `bool` | ACTIVE | When True, any positive numeric value is replaced with 0 after casting. Used to isolate debit-side figures from a combined amount column. | -| `exclude_negative_values` | `bool` | ACTIVE | When True, any negative numeric value is replaced with 0 after casting. Used to isolate credit-side figures from a combined amount column. | -| `terminator` | `str` | ACTIVE | Regex pattern; when present the string value is truncated at the first match position before being written to the standard column. Useful for stripping trailing boilerplate appended by merge_fields (e.g. " \| BALANCE CARRIED FORWARD"). | +| `field` | `str | None` | ACTIVE | Name of the raw extracted column to promote. Set to None (or omit) when a literal default value should be used instead of a column value. | +| `concat_fields` | `list | None` | ACTIVE | Name of the raw extracted columns to concatenate and promote. Set to None (or omit) in order to revert to a single field and it's fallback | +| `format` | `str | None` | ACTIVE | strptime format string applied when StandardFields.type == "date" (e.g. "%-d %B %Y"). Ignored for numeric and string types. | +| `default` | `str | None` | ACTIVE | Literal string value used as the output when ``field`` is None/absent. Useful for injecting constant metadata (e.g. transaction_type = "CC"). | +| `multiplier` | `float | None` | ACTIVE | Scalar applied to the value after casting when StandardFields.type == "numeric". Use -1 to invert sign (e.g. to convert a credit amount stored as positive into a negative figure). | +| `exclude_positive_values` | `bool | None` | ACTIVE | When True, any positive numeric value is replaced with 0 after casting. Used to isolate debit-side figures from a combined amount column. | +| `exclude_negative_values` | `bool | None` | ACTIVE | When True, any negative numeric value is replaced with 0 after casting. Used to isolate credit-side figures from a combined amount column. | +| `terminator` | `str | None` | ACTIVE | Regex pattern; when present the string value is truncated at the first match position before being written to the standard column. Useful for stripping trailing boilerplate appended by merge_fields (e.g. " \| BALANCE CARRIED FORWARD"). | ### `Test` diff --git a/docs/reference/python-api.md b/docs/reference/python-api.md index c4baa33..5e53ddd 100644 --- a/docs/reference/python-api.md +++ b/docs/reference/python-api.md @@ -9,95 +9,79 @@ All public symbols are available from the top-level package: import bank_statement_parser as bsp ``` -## Meta +## Ungrouped -### `bsp.__app_name__` - -*constant* - -### `bsp.__version__` - -*constant* - -## Namespaced report backend - -### `bsp.db` - -*module* — `bank_statement_parser.modules.reports_db` - -SQLite-backed report classes and export helpers. - -## Statement processing +### `bsp.Failure` -### `bsp.Statement` +*class* — `bank_statement_parser.modules.data` -*class* — `bank_statement_parser.modules.statements` +Payload for a PDF result where no usable statement data was produced. -Represents a single bank statement PDF with data extraction and validation. +### `bsp.ForexApiConfig` -### `bsp.StatementBatch` +*class* — `bank_statement_parser.modules.data` -*class* — `bank_statement_parser.modules.statements` +Configuration for the forex exchange-rate fetching service. -Handles batch processing of multiple bank statement PDFs. +### `bsp.Housekeeping` -### `bsp.process_pdf_statement()` +*class* — `bank_statement_parser.data` -*function* — `bank_statement_parser.modules.statements` +Orphan-detection and cascaded-delete helper for the raw SQLite database. -Process a single bank statement PDF and save results to parquet files. +### `bsp.ParquetFiles` -### `bsp.copy_statements_to_project()` +*class* — `bank_statement_parser.modules.data` -*function* — `bank_statement_parser.modules.statements` +Paths to the statement-level temporary Parquet files written on the SUCCESS path. -Copy processed statement PDFs into the project ``statements/`` directory. +### `bsp.PdfResult` -### `bsp.delete_temp_files()` +*class* — `bank_statement_parser.modules.data` -*function* — `bank_statement_parser.modules.statements` +Top-level result returned by :func:`~bank_statement_parser.modules.statements.process_pdf_statement`. -Delete temporary parquet files created during batch processing. +### `bsp.ProjectConfigMissing` -## Low-level persistence helpers +*class* — `bank_statement_parser.modules.errors` -### `bsp.update_parquet()` +Project config folder missing or empty. -*function* — `bank_statement_parser.modules.parquet` +### `bsp.ProjectDatabaseMissing` -Update parquet files with processed results from all PDFs in a batch. +*class* — `bank_statement_parser.modules.errors` -### `bsp.update_db()` +Project database file not found. -*function* — `bank_statement_parser.modules.database` +### `bsp.ProjectPaths` -Insert processed batch results into the SQLite database. +*class* — `bank_statement_parser.modules.paths` -## Data structures +All file-system paths for a bank_statement_parser project, derived from a single root directory. -### `bsp.PdfResult` +### `bsp.Review` *class* — `bank_statement_parser.modules.data` -Top-level result returned by :func:`~bank_statement_parser.modules.statements.process_pdf_statement`. +Payload for a PDF where extraction succeeded but CAB validation failed. -### `bsp.Success` +### `bsp.Statement` -*class* — `bank_statement_parser.modules.data` +*class* — `bank_statement_parser.modules.statements` -Payload for a fully-validated PDF result. +Represents a single bank statement PDF with data extraction and validation. -### `bsp.Review` +### `bsp.StatementBatch` -*class* — `bank_statement_parser.modules.data` +*class* — `bank_statement_parser.modules.statements` -Payload for a PDF where extraction succeeded but CAB validation failed. +Handles batch processing of multiple bank statement PDFs. -### `bsp.Failure` +### `bsp.StatementError` -*class* — `bank_statement_parser.modules.data` +*class* — `bank_statement_parser.modules.errors` -Payload for a PDF result where no usable statement data was produced. +Root exception for statement processing errors. ### `bsp.StatementInfo` @@ -105,47 +89,37 @@ Payload for a PDF result where no usable statement data was produced. Statement-level metadata extracted from a successfully validated PDF. -### `bsp.ParquetFiles` +### `bsp.Success` *class* — `bank_statement_parser.modules.data` -Paths to the statement-level temporary Parquet files written on the SUCCESS path. - -## Debug / diagnostics - -### `bsp.debug_pdf_statement()` - -*function* — `bank_statement_parser.modules.debug` - -Re-process a single failing PDF and write a debug.json diagnostic file. - -### `bsp.debug_statements()` +Payload for a fully-validated PDF result. -*function* — `bank_statement_parser.modules.debug` +### `bsp.TestGateFailure` -Re-process all failing statements from a completed batch and write debug files. +*class* — `bank_statement_parser.modules.errors` -## Errors +Raised when bsp's own pytest suite fails during TestHarness.setup(). -### `bsp.StatementError` +### `bsp.TestHarness` -*class* — `bank_statement_parser.modules.errors` +*class* — `bank_statement_parser.testing` -Root exception for statement processing errors. +Programmatic test environment for integration testing by dependent projects. -### `bsp.ProjectDatabaseMissing` +### `bsp.__app_name__` -*class* — `bank_statement_parser.modules.errors` +*constant* -Project database file not found. +### `bsp.__version__` -### `bsp.ProjectConfigMissing` +*constant* -*class* — `bank_statement_parser.modules.errors` +### `bsp.build_datamart()` -Project config folder missing or empty. +*function* — `bank_statement_parser.data` -## Config helpers +Empty and rebuild all mart tables (DimDate, DimAccount, DimStatement, FactTransaction, FactBalance) from the raw source tables. ### `bsp.copy_default_import_config()` @@ -159,43 +133,47 @@ Copy all default import TOML configuration files to a destination directory. Copy the project folder structure (directories only) to a destination. -### `bsp.validate_or_initialise_project()` +### `bsp.copy_statements_to_project()` -*function* — `bank_statement_parser.modules.paths` +*function* — `bank_statement_parser.modules.statements` -Validate an existing project or initialise a new one at *project_path*. +Copy processed statement PDFs into the project ``statements/`` directory. -### `bsp.ProjectPaths` +### `bsp.create_db()` -*class* — `bank_statement_parser.modules.paths` +*function* — `bank_statement_parser.data` -All file-system paths for a bank_statement_parser project, derived from a single root directory. +Create (or recreate) the raw SQLite database with all tables and indexes. -## Low-level PDF helpers +### `bsp.db` -### `bsp.pdf_open()` +*module* — `bank_statement_parser.modules.reports_db` -*function* — `bank_statement_parser.modules.pdf_functions` +SQLite-backed report classes and export helpers. -Open a PDF file and return the PDF object with performance logging. +### `bsp.debug_pdf_statement()` -### `bsp.page_crop()` +*function* — `bank_statement_parser.modules.debug` -*function* — `bank_statement_parser.modules.pdf_functions` +Re-process a single failing PDF and write a debug.json diagnostic file. -Crop a PDF page to the specified bounding box coordinates, with smart defaults. +### `bsp.debug_statements()` -### `bsp.page_text()` +*function* — `bank_statement_parser.modules.debug` -*function* — `bank_statement_parser.modules.pdf_functions` +Re-process all failing statements from a completed batch and write debug files. -Extract all text content from a PDF page. +### `bsp.delete_temp_files()` -### `bsp.region_search()` +*function* — `bank_statement_parser.modules.statements` -*function* — `bank_statement_parser.modules.pdf_functions` +Delete temporary parquet files created during batch processing. -Search for a regex pattern within a PDF region and return the first match text. +### `bsp.get_exchange_rates()` + +*function* — `bank_statement_parser.modules.forex` + +Fetch daily USD-based exchange rates and persist them to ``exchange_rates``. ### `bsp.get_table_from_region()` @@ -203,53 +181,53 @@ Search for a regex pattern within a PDF region and return the first match text. Extract a structured table from a PDF region using configurable extraction settings. -## Data-mart / database +### `bsp.page_crop()` -### `bsp.build_datamart()` +*function* — `bank_statement_parser.modules.pdf_functions` -*function* — `bank_statement_parser.data` +Crop a PDF page to the specified bounding box coordinates, with smart defaults. -Empty and rebuild all mart tables (DimDate, DimAccount, DimStatement, FactTransaction, FactBalance) from the raw source tables. +### `bsp.page_text()` -### `bsp.create_db()` +*function* — `bank_statement_parser.modules.pdf_functions` -*function* — `bank_statement_parser.data` +Extract all text content from a PDF page. -Create (or recreate) the raw SQLite database with all tables and indexes. +### `bsp.pdf_open()` -### `bsp.Housekeeping` +*function* — `bank_statement_parser.modules.pdf_functions` -*class* — `bank_statement_parser.data` +Open a PDF file and return the PDF object with performance logging. -Orphan-detection and cascaded-delete helper for the raw SQLite database. +### `bsp.process_pdf_statement()` -## Forex / currency conversion +*function* — `bank_statement_parser.modules.statements` -### `bsp.get_exchange_rates()` +Process a single bank statement PDF and save results to parquet files. -*function* — `bank_statement_parser.modules.forex` +### `bsp.region_search()` -Fetch daily USD-based exchange rates and persist them to ``exchange_rates``. +*function* — `bank_statement_parser.modules.pdf_functions` -### `bsp.ForexApiConfig` +Search for a regex pattern within a PDF region and return the first match text. -*class* — `bank_statement_parser.modules.data` +### `bsp.update_db()` -Configuration for the forex exchange-rate fetching service. +*function* — `bank_statement_parser.modules.database` -## Testing harness +Insert processed batch results into the SQLite database. -### `bsp.TestHarness` +### `bsp.update_parquet()` -*class* — `bank_statement_parser.testing` +*function* — `bank_statement_parser.modules.parquet` -Programmatic test environment for integration testing by dependent projects. +Update parquet files with processed results from all PDFs in a batch. -### `bsp.TestGateFailure` +### `bsp.validate_or_initialise_project()` -*class* — `bank_statement_parser.modules.errors` +*function* — `bank_statement_parser.modules.paths` -Raised when bsp's own pytest suite fails during TestHarness.setup(). +Validate an existing project or initialise a new one at *project_path*. ## DB Report Backend diff --git a/scripts/generate_docs.py b/scripts/generate_docs.py index a21aef7..ad45bcd 100644 --- a/scripts/generate_docs.py +++ b/scripts/generate_docs.py @@ -1881,7 +1881,7 @@ def w(text: str = "") -> None: w("bsp.db.export_csv()") w() w("# Export multi star-schema tables to Excel") - w("bsp.db.export_excel(type='multi')") + w('bsp.db.export_excel(type="multi")') w() w("# Export JSON") w("bsp.db.export_json()") @@ -1891,8 +1891,9 @@ def w(text: str = "") -> None: w("") w("# Export to a custom directory") w("from pathlib import Path") - w("bsp.db.export_csv(folder=Path('~/exports'))") - w("bsp.db.export_excel(path=Path('~/exports/report.xlsx'))") + w("") + w('bsp.db.export_csv(folder=Path("~/exports"))') + w('bsp.db.export_excel(path=Path("~/exports/report.xlsx"))') w("```") w() diff --git a/src/bank_statement_parser/modules/database.py b/src/bank_statement_parser/modules/database.py index 610e505..1d7dbba 100644 --- a/src/bank_statement_parser/modules/database.py +++ b/src/bank_statement_parser/modules/database.py @@ -533,3 +533,5 @@ def _insert_df(df: pl.DataFrame, table_name: str) -> None: build_datamart(db_path=db_path) except Exception as e: # noqa: BLE001 print(f"[update_db] ** Datamart Rebuild Failed **: {type(e).__name__}: {e}") + + return db_secs From f3e4f15e90ff8affd07ce5250c24d44bb6ec6506 Mon Sep 17 00:00:00 2001 From: Jason Farrar Date: Thu, 6 Aug 2026 09:37:26 +0100 Subject: [PATCH 3/3] fix: restore missing indentation in anonymise_pdf function body The retain_descriptions parameter addition in d09e01c lost the 4-space indentation of the function body, placing the if/return statements at module level and causing a SyntaxError on import. Signed-off-by: Jason Farrar --- .../modules/anonymise.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/bank_statement_parser/modules/anonymise.py b/src/bank_statement_parser/modules/anonymise.py index 090a71c..813134a 100644 --- a/src/bank_statement_parser/modules/anonymise.py +++ b/src/bank_statement_parser/modules/anonymise.py @@ -107,14 +107,14 @@ def anonymise_pdf( If ``retain_descriptions`` is ``True`` but no ``always_anonymise_path`` was provided. """ -if retain_descriptions and always_anonymise_path is None: - raise ValueError("retain_descriptions=True requires always_anonymise_path") - -return _anonymise_pdf( - input_path, - output_path, - always_anonymise_path=always_anonymise_path, - never_anonymise_path=never_anonymise_path, - retain_descriptions=retain_descriptions, - debug=debug, -) + if retain_descriptions and always_anonymise_path is None: + raise ValueError("retain_descriptions=True requires always_anonymise_path") + + return _anonymise_pdf( + input_path, + output_path, + always_anonymise_path=always_anonymise_path, + never_anonymise_path=never_anonymise_path, + retain_descriptions=retain_descriptions, + debug=debug, + )