diff --git a/README.md b/README.md index 294782d..defa3a0 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,43 @@ trs.parse(ASNB_mt940_data()) print(pprint.pformat(trs.data, sort_dicts=False)) ``` +### Transaction grouping (opt-in) + +By default a new transaction is started only on the `:61:` statement tag. +Some banks delimit transactions differently — for example by repeating the +`:20:` transaction reference per block. Because changing the default grouping +would break existing users, this behaviour is **opt-in**: pass +`transaction_boundary` (an iterable of tag *slugs*) to start a new transaction +on those tags too. Omitting it preserves the historical behaviour. + +```python +import mt940 + +# Each `:20:` (transaction_reference_number) starts its own transaction: +transactions = mt940.parse( + data, transaction_boundary={'transaction_reference_number'} +) +``` + +The same option is accepted by `mt940.models.Transactions(transaction_boundary=...)`. + +### Banks with longer reference fields (opt-in) + +Some banks (e.g. GLS / Atruvia) put a customer reference longer than the SWIFT +16-character cap on the `:61:` line, followed by the `//` bank reference. +Relaxing the default would change how other banks (e.g. Rabobank) split +same-line data, so this is handled by an **opt-in** `StatementGLS` tag: + +```python +import mt940 + +gls = mt940.tags.StatementGLS() +transactions = mt940.parse(data, tags={gls.id: gls}) +``` + +(Longer *supplementary details* — issue #117, e.g. Wise — are handled by the +default parser and need no opt-in.) + ## Contributing Help is greatly appreciated. Please clone the **develop** branch and run `tox` diff --git a/mt940/models.py b/mt940/models.py index a6603cb..156c4ff 100644 --- a/mt940/models.py +++ b/mt940/models.py @@ -5,7 +5,13 @@ import re import typing import warnings -from collections.abc import Callable, Mapping, MutableMapping, Sequence +from collections.abc import ( + Callable, + Iterable, + Mapping, + MutableMapping, + Sequence, +) from typing import Any, ClassVar, overload import mt940 @@ -363,6 +369,7 @@ def __init__( self, processors: dict[str, list[Callable[..., Any]]] | None = None, tags: dict[int | str, mt940.tags.Tag] | None = None, + transaction_boundary: Iterable[str] | None = None, ) -> None: self.processors: dict[str, list[Callable[..., Any]]] = ( self.DEFAULT_PROCESSORS.copy() @@ -376,6 +383,19 @@ def __init__( if tags: self.tags.update(tags) + # Opt-in (issue #110): tag slugs that each open a new transaction. + # Banks differ in how they delimit transactions; by default only the + # `:61:` statement tag starts a transaction. Passing e.g. + # ``{'transaction_reference_number'}`` makes each `:20:` start its own + # transaction too. Empty (the default) preserves the legacy behaviour. + if isinstance(transaction_boundary, str): + # A bare string is almost certainly a single slug, not an iterable + # of single characters. + transaction_boundary = (transaction_boundary,) + self.transaction_boundary: frozenset[str] = frozenset( + transaction_boundary or () + ) + self.transactions: list[Transaction] = [] self.data: dict[str, Any] = {} @@ -479,7 +499,16 @@ def _process_match( result = processor(self, tag, tag_dict, result) if isinstance(tag, mt940.tags.Statement): + # Statement (:61:) handling always takes precedence so it cannot + # be bypassed by listing its slug in transaction_boundary. self._process_statement_tag(result) + elif tag.slug in self.transaction_boundary: + # Opt-in (issue #110): this tag opens a new transaction block. + self.transactions.append(Transaction(self, result)) + if issubclass(tag.scope, Transactions): + # Keep statement-level data (e.g. the :20: reference) global + # too, so later :61: tags in the same block can copy it. + self.data.update(result) elif issubclass(tag.scope, Transaction) and self.transactions: self._update_transaction(result) elif issubclass( # pragma: no branch diff --git a/mt940/parser.py b/mt940/parser.py index ffa645e..f4824b2 100644 --- a/mt940/parser.py +++ b/mt940/parser.py @@ -40,11 +40,19 @@ def parse( encoding: str | None = None, processors: dict[str, list[Any]] | None = None, tags: dict[Any, Any] | None = None, + transaction_boundary: Any = None, ) -> Transactions: """ Parses mt940 data and returns transactions object :param src: file handler to read, filename to read or raw data as string + :param encoding: optional encoding override for byte input + :param processors: optional extra pre/post processors + :param tags: optional extra/override tag parsers + :param transaction_boundary: optional iterable of tag *slugs* that each + start a new transaction (issue #110). By default only ``:61:`` starts a + transaction; pass e.g. ``{'transaction_reference_number'}`` to also + start one on every ``:20:``. Omitting it keeps the legacy behaviour. :return: Collection of transactions :rtype: Transactions """ @@ -83,7 +91,9 @@ def safe_is_file(filename: Any) -> bool: raise exception # pragma: no cover assert isinstance(data, str) - transactions = mt940.models.Transactions(processors, tags) + transactions = mt940.models.Transactions( + processors, tags, transaction_boundary=transaction_boundary + ) transactions.parse(data) return transactions diff --git a/mt940/processors.py b/mt940/processors.py index b4f0c45..f151f8c 100644 --- a/mt940/processors.py +++ b/mt940/processors.py @@ -278,12 +278,16 @@ def _process_segments( key32 = DETAIL_KEYS['32'] result[key32].append(value) elif key.startswith('2'): - # For segment keys beginning with '2', adjust specific - # segments by trimming trailing identifiers. - if key == '29' and value.endswith(' BIC'): - value = value.removesuffix(' BIC').rstrip() - elif key == '28D' and value.endswith(' IBAN'): - value = value.removesuffix(' IBAN').rstrip() + # Some banks append a bare ' BIC'/' IBAN' label with no value at + # the end of a detail segment (issue #109); strip the dangling + # label so it does not pollute the purpose. Segment keys are + # always two characters (see _parse_segments), so the historical + # '29'/'28D' key checks could never match the IBAN case -- the + # label is matched on the value instead. + for label in (' BIC', ' IBAN'): + if value.endswith(label): + value = value.removesuffix(label).rstrip() + break key20 = DETAIL_KEYS['20'] result[key20].append(value) elif key in {'60', '61', '62', '63', '64', '65'}: diff --git a/mt940/tags.py b/mt940/tags.py index b394f05..9a0addb 100644 --- a/mt940/tags.py +++ b/mt940/tags.py @@ -421,7 +421,10 @@ class Statement(Tag): (?P[A-Z][A-Z0-9 ]{3})? (?P((?!//)[^\n]){0,16}) (//(?P.{0,23}))? - (\n?(?P.{0,34}))? + # Supplementary details: the SWIFT spec caps this at 34x, but some banks + # (e.g. Wise, issue #117) send more, so the length limit is relaxed. This + # only ever turns a previous parse error into a successful parse. + (\n?(?P.*))? $""" def __call__( @@ -436,7 +439,7 @@ def __call__( entry_month = str(data.get('entry_month') or '') if entry_day.isdigit() and entry_month.isdigit(): - entry_date = data['entry_date'] = models.Date( + entry_date = models.Date( day=entry_day, month=entry_month, year=str(date.year) ) if date > entry_date and (date - entry_date).days >= 330: @@ -446,11 +449,18 @@ def __call__( else: year = 0 - data['guessed_entry_date'] = models.Date( - day=entry_date.day, - month=entry_date.month, - year=entry_date.year + year, - ) + # Correct the entry date's year when the entry date crosses a + # year boundary relative to the value date (issue #121). Both + # `entry_date` and `guessed_entry_date` expose the resolved value; + # `guessed_entry_date` is kept as a backwards-compatible alias. + if year: + entry_date = models.Date( + day=entry_date.day, + month=entry_date.month, + year=entry_date.year + year, + ) + data['entry_date'] = entry_date + data['guessed_entry_date'] = entry_date return data @@ -494,6 +504,41 @@ def __call__( return super().__call__(transactions, value) +class StatementGLS(Statement): + """Statement variant for GLS / Atruvia banks (issue #111). + + These banks send a customer reference longer than the SWIFT 16x cap, + followed by the ``//`` bank-reference delimiter (e.g. + ``...DR20,NTRFBIPI-dvT1FzfMqvzF5HaU4oetlH7SGRkonU//2022070616391534000``). + + This is an opt-in tag because relaxing the default customer-reference + length would change how banks that legitimately pack data after a 16x + reference (e.g. Rabobank) are parsed. Enable it explicitly:: + + import mt940 + + gls = mt940.tags.StatementGLS() + mt940.parse(data, tags={gls.id: gls}) + """ + + pattern = r"""^ + (?P\d{2}) # 6!n Value Date (YYMMDD) + (?P\d{2}) + (?P\d{2}) + (?P\d{2}|\s{2})? # [4!n] Entry Date (MMDD) + (?P\d{2}|\s{2})? + (?PR?[DC]) # 2a Debit/Credit Mark + (?P[A-Z])? + [\n ]? + (?P[\d,]{1,15}) # 15d Amount + (?P[A-Z][A-Z0-9 ]{3})? + # Customer reference of any length, up to the // bank reference. + (?P(?:(?!//)[^\n])*) + (//(?P.{0,23}))? + (\n?(?P.*))? + $""" + + class ClosingBalance(BalanceBase): id: str | int = 62 diff --git a/mt940_tests/test_entry_dates.py b/mt940_tests/test_entry_dates.py index 392f7db..5edaf0b 100644 --- a/mt940_tests/test_entry_dates.py +++ b/mt940_tests/test_entry_dates.py @@ -1,44 +1,46 @@ +import datetime + import mt940 +def _statement( + transactions: mt940.models.Transactions, **kwargs: object +) -> dict: + statement = mt940.tags.Statement() + data = dict(amount='123', status='D', **kwargs) + return statement(transactions, data) + + def test_entry_dates_wrapping_years(): transactions = mt940.models.Transactions() - statement = mt940.tags.Statement() - data = dict( - amount='123', - status='D', - year=2000, - ) - # Regular statement - statement( - transactions, - dict( - data.items(), - month=1, - day=1, - ), + # Regular statement without an entry date: only the value date is set. + regular = _statement(transactions, year=2000, month=1, day=1) + assert regular['date'] == datetime.date(2000, 1, 1) + assert 'entry_date' not in regular + + # Entry date in the same year as the value date: no correction applied. + same_year = _statement( + transactions, year=2000, month=6, day=15, entry_month=6, entry_day=10 ) + assert same_year['date'] == datetime.date(2000, 6, 15) + assert same_year['entry_date'] == datetime.date(2000, 6, 10) + assert same_year['guessed_entry_date'] == same_year['entry_date'] - # Statement which wraps to the future - statement( - transactions, - dict( - data.items(), - month=12, - day=31, - entry_day=1, - entry_month=1, - ), + # Entry date wraps into the next year (value date Dec, entry date Jan). + forward = _statement( + transactions, year=2000, month=12, day=31, entry_month=1, entry_day=1 ) - # Statement which wraps the past year - statement( - transactions, - dict( - data.items(), - month=1, - day=1, - entry_day=31, - entry_month=12, - ), + assert forward['date'] == datetime.date(2000, 12, 31) + assert forward['entry_date'] == datetime.date(2001, 1, 1) + assert forward['guessed_entry_date'] == forward['entry_date'] + + # Entry date wraps into the previous year (value date Jan, entry date + # Dec) -- issue #121. `entry_date` must be resolved, not left in the + # value date's year. + backward = _statement( + transactions, year=2000, month=1, day=1, entry_month=12, entry_day=31 ) + assert backward['date'] == datetime.date(2000, 1, 1) + assert backward['entry_date'] == datetime.date(1999, 12, 31) + assert backward['guessed_entry_date'] == backward['entry_date'] diff --git a/mt940_tests/test_issues/test_issue106_streams.py b/mt940_tests/test_issues/test_issue106_streams.py new file mode 100644 index 0000000..3874e86 --- /dev/null +++ b/mt940_tests/test_issues/test_issue106_streams.py @@ -0,0 +1,26 @@ +"""Issue #106: parsing must not depend on sys.stdout/sys.stderr being usable. + +Sandboxed / serverless environments (and some GUI apps) run with +``sys.stdout``/``sys.stderr`` set to ``None``. Parsing valid data must work +there without raising ``AttributeError: 'NoneType' object has no attribute +'encoding'``. +""" + +import mt940 + +VALID = """:20:STARTUMS +:25:NL12RABO0123456789 +:28C:0 +:60F:C220706EUR100,00 +:61:2207060706C10,00NTRFref//bank +:86:116?00Payment +:62F:C220706EUR110,00 +""" + + +def test_parses_without_stdout_and_stderr(monkeypatch): + monkeypatch.setattr('sys.stdout', None) + monkeypatch.setattr('sys.stderr', None) + transactions = mt940.parse(VALID) + assert len(transactions) == 1 + assert str(transactions[0].data['amount']) == '10.00 EUR' diff --git a/mt940_tests/test_issues/test_issue110_boundary.py b/mt940_tests/test_issues/test_issue110_boundary.py new file mode 100644 index 0000000..0d8040d --- /dev/null +++ b/mt940_tests/test_issues/test_issue110_boundary.py @@ -0,0 +1,89 @@ +"""Issue #110: opt-in transaction boundaries. + +By default only the ``:61:`` statement tag starts a new transaction, so a +statement that repeats the ``:20:`` transaction reference per block collapses +into a single transaction (legacy behaviour, kept for backwards +compatibility). The opt-in ``transaction_boundary`` option lets callers +nominate extra tag slugs that each open a new transaction. +""" + +import mt940 + +DATA = """:20:REF1 +:20:REF2 +:25:ACC +:60F:C200101EUR0,00 +:61:2001010101C5,00NTRFr//b +:86:116?00p +:62F:C200101EUR5,00 +""" + + +def test_default_grouping_unchanged(): + # Legacy behaviour: the second :20: overwrites the global reference and a + # single transaction is produced. + transactions = mt940.parse(DATA) + assert len(transactions) == 1 + assert transactions[0].data['transaction_reference'] == 'REF2' + + +def test_transaction_boundary_opt_in_via_parse(): + transactions = mt940.parse( + DATA, transaction_boundary={'transaction_reference_number'} + ) + refs = [t.data.get('transaction_reference') for t in transactions] + assert refs == ['REF1', 'REF2'] + + +def test_transaction_boundary_opt_in_via_transactions(): + transactions = mt940.models.Transactions( + transaction_boundary={'transaction_reference_number'} + ) + transactions.parse(DATA) + assert len(transactions) == 2 + + +def test_transaction_boundary_accepts_a_bare_string(): + # A single slug passed as a plain string must not be iterated per-char. + transactions = mt940.parse( + DATA, transaction_boundary='transaction_reference_number' + ) + refs = [t.data.get('transaction_reference') for t in transactions] + assert refs == ['REF1', 'REF2'] + + +def test_reference_propagates_to_later_statements_in_block(): + # A block with one :20: and several :61: tags: every transaction in the + # block must carry the block's reference (the global reference is kept in + # sync for the transactions_to_transaction post-processor). + data = """:20:REF1 +:61:2001010101C5,00NTRFa//b +:86:116?00p1 +:61:2001020102C6,00NTRFc//d +:86:116?00p2 +:20:REF2 +:61:2001030103C7,00NTRFe//f +:62F:C200101EUR18,00 +""" + transactions = mt940.parse( + data, transaction_boundary={'transaction_reference_number'} + ) + refs = [t.data.get('transaction_reference') for t in transactions] + assert refs == ['REF1', 'REF1', 'REF2'] + + +def test_transaction_scoped_boundary_tag(): + # A boundary tag whose scope is Transaction (not Transactions) opens a new + # transaction without touching the global statement data. + data = """:20:REF +:25:ACC +:60F:C200101EUR0,00 +:61:2001010101C5,00NTRFa//b +:86:116?00detail +:62F:C200101EUR5,00 +""" + transactions = mt940.parse( + data, transaction_boundary={'transaction_details'} + ) + # The :86: detail tag now opens its own transaction. + assert len(transactions) == 2 diff --git a/mt940_tests/test_issues/test_long_references.py b/mt940_tests/test_issues/test_long_references.py new file mode 100644 index 0000000..abfae77 --- /dev/null +++ b/mt940_tests/test_issues/test_long_references.py @@ -0,0 +1,82 @@ +"""Regression tests for banks that exceed the SWIFT field-length caps. + +- Issue #111 (GLS / Atruvia): customer_reference longer than 16 chars. +- Issue #117 (Wise): extra_details (supplementary details) over 34 chars. +""" + +import mt940 + + +def test_issue_111_long_customer_reference(): + # GLS / Atruvia sends a 35-char customer reference followed by the // + # bank-reference delimiter. This is handled by the opt-in StatementGLS tag + # (relaxing the default would change Rabobank-style same-line parsing). + gls = mt940.tags.StatementGLS() + data = """:20:STARTUMS +:25:GENODEF1XXX/1234567890 +:28C:0 +:60F:C220706EUR100,00 +:61:2207060706DR20,NTRFBIPI-dvT1FzfMqvzF5HaU4oetlH7SGRkonU//2022070616391534000 +:86:116?00Ueberweisung +:62F:C220706EUR80,00 +""" + transaction = mt940.parse(data, tags={gls.id: gls})[0] + assert ( + transaction.data['customer_reference'] + == 'BIPI-dvT1FzfMqvzF5HaU4oetlH7SGRkonU' + ) + assert transaction.data['bank_reference'] == '2022070616391534000' + assert str(transaction.data['amount']) == '-20 EUR' + + +def test_same_line_details_with_double_slash_preserved(): + # A // inside same-line details (e.g. a URL) must NOT be treated as the + # bank-reference delimiter by the default Statement tag (backwards-compat + # guard: the GLS support is opt-in for exactly this reason). + data = """:20:STARTUMS +:25:NL12RABO0123456789 +:28C:0 +:60F:C220706EUR100,00 +:61:2207060706C10,00N654NONREF HTTP://SHOP EXAMPLE +:86:116?00Payment +:62F:C220706EUR110,00 +""" + transaction = mt940.parse(data)[0] + assert transaction.data['customer_reference'] == 'NONREF ' + assert transaction.data['extra_details'] == 'HTTP://SHOP EXAMPLE' + assert transaction.data.get('bank_reference') is None + + +def test_issue_117_long_extra_details(): + # Wise sends supplementary details longer than the 34-char SWIFT cap. + long_details = 'Sent money to John Doe for invoice 12345 reference ABCDE' + assert len(long_details) > 34 + data = f""":20:STARTUMS +:25:WISE/1234567890 +:28C:0 +:60F:C220706EUR100,00 +:61:2207060706C50,00NTRFsomeref//bankref123 +{long_details} +:86:116?00Payment +:62F:C220706EUR150,00 +""" + transaction = mt940.parse(data)[0] + assert transaction.data['extra_details'] == long_details + assert str(transaction.data['amount']) == '50.00 EUR' + + +def test_rabobank_same_line_extra_details_unchanged(): + # Rabobank packs a 16-char (space-padded) customer reference plus extra + # text on the same line, with no // delimiter. The relaxation must keep + # splitting these (backwards compatibility). + data = """:20:STARTUMS +:25:NL12RABO0123456789 +:28C:0 +:60F:C220706EUR100,00 +:61:2207060706C10,00N654NONREF TOMTE TUMMETOT AMERSFOORT +:86:116?00Payment +:62F:C220706EUR110,00 +""" + transaction = mt940.parse(data)[0] + assert transaction.data['customer_reference'] == 'NONREF ' + assert transaction.data['extra_details'] == 'TOMTE TUMMETOT AMERSFOORT' diff --git a/mt940_tests/test_processors_issue109.py b/mt940_tests/test_processors_issue109.py index 4c158d4..47dc03e 100644 --- a/mt940_tests/test_processors_issue109.py +++ b/mt940_tests/test_processors_issue109.py @@ -1,43 +1,49 @@ -import collections - -from mt940 import processors - - -def test_process_segments_29_bic_removed(): - # Create a pseudo OrderedDict similar to what _parse_segments returns. - tmp = collections.OrderedDict() - # Segment with key "29" ending with " BIC" should have the trailing - # " BIC" removed. - tmp['29'] = 'DE69280123450012345670 BIC' - result = processors._process_segments(tmp) - key20 = processors.DETAIL_KEYS['20'] - # After removal, expect "DE69280123450012345670" to be appended. - assert key20 in result - assert result[key20] == ['DE69280123450012345670'] - - -def test_process_segments_28d_iban_removed(): - # Create a pseudo OrderedDict similar to what _parse_segments returns. - tmp = collections.OrderedDict() - # Segment with key "28D" ending with " IBAN" should have the trailing - # " IBAN" removed. - tmp['28D'] = 'DE69280123450012345670 IBAN' - result = processors._process_segments(tmp) - key20 = processors.DETAIL_KEYS['20'] - # After removal, expect "DE69280123450012345670" to be appended. - assert key20 in result - assert result[key20] == ['DE69280123450012345670'] - - -def test_process_segments_other_fields(): - # Test normal behavior for other segments. - tmp = collections.OrderedDict() - # Field "20" is present in DETAIL_KEYS so goes to its own key. - tmp['20'] = 'Purpose Text' - # Field "29" that does not end with " BIC" should remain unchanged. - tmp['29'] = 'Another Example' - result = processors._process_segments(tmp) - key20 = processors.DETAIL_KEYS['20'] - # Both values should be appended to the same key. - assert key20 in result - assert result[key20] == ['Purpose Text', 'Another Example'] +"""Issue #109: strip a dangling ' BIC'/' IBAN' label (a label with no value) +from the end of a transaction-detail segment. + +These tests drive the real segment pipeline (`_parse_segments` -> +`_process_segments`) instead of fabricating segment dictionaries, so the input +matches what banks actually send (segment keys are always two characters). +""" + +import mt940 + + +def _purpose(detail_str: str) -> list[str]: + processors = mt940.processors + segments = processors._parse_segments(detail_str) + return processors._process_segments(segments)[processors.DETAIL_KEYS['20']] + + +def test_dangling_bic_is_stripped(): + # ?29 ends with a bare ' BIC' label and no BIC value (issue #109). + purpose = _purpose('?20Purpose?29 DE69280123450012345670 BIC') + assert purpose == ['Purpose', ' DE69280123450012345670'] + + +def test_dangling_iban_is_stripped(): + # The same pattern with a bare ' IBAN' label must also be stripped. + purpose = _purpose('?20Purpose?29 DE69280123450012345670 IBAN') + assert purpose == ['Purpose', ' DE69280123450012345670'] + + +def test_value_not_ending_in_label_is_unchanged(): + purpose = _purpose('?20Legit purpose text?29trailing content') + assert purpose == ['Legit purpose text', 'trailing content'] + + +def test_issue_109_end_to_end(): + data = """:20:STARTUMS +:25:GENODEF1XXX/1234567890 +:28C:0 +:60F:C240123EUR100,00 +:61:2401231020DR30,00NDDTKREF+ +:86:105?00Basislastschrift?20EREF+DD-1?29 DE69280123450012345670 BIC +:62F:C240123EUR70,00 +""" + transaction = mt940.parse(data)[0] + # The dangling ' BIC' must not appear anywhere in the parsed details, + # while the actual reference value is preserved. + blob = ' '.join(str(v) for v in transaction.data.values()) + assert ' BIC' not in blob + assert 'DE69280123450012345670' in blob