Triage open/closed issues: parser fixes + opt-in transaction grouping

README.md

@@ -95,6 +95,43 @@ trs.parse(ASNB_mt940_data())
 print(pprint.pformat(trs.data, sort_dicts=False))
 ```
 
+### Transaction grouping (opt-in)
+
+By default a new transaction is started only on the `:61:` statement tag.
+Some banks delimit transactions differently — for example by repeating the
+`:20:` transaction reference per block. Because changing the default grouping
+would break existing users, this behaviour is **opt-in**: pass
+`transaction_boundary` (an iterable of tag *slugs*) to start a new transaction
+on those tags too. Omitting it preserves the historical behaviour.
+
+```python
+import mt940
+
+# Each `:20:` (transaction_reference_number) starts its own transaction:
+transactions = mt940.parse(
+    data, transaction_boundary={'transaction_reference_number'}
+)
+```
+
+The same option is accepted by `mt940.models.Transactions(transaction_boundary=...)`.
+
+### Banks with longer reference fields (opt-in)
+
+Some banks (e.g. GLS / Atruvia) put a customer reference longer than the SWIFT
+16-character cap on the `:61:` line, followed by the `//` bank reference.
+Relaxing the default would change how other banks (e.g. Rabobank) split
+same-line data, so this is handled by an **opt-in** `StatementGLS` tag:
+
+```python
+import mt940
+
+gls = mt940.tags.StatementGLS()
+transactions = mt940.parse(data, tags={gls.id: gls})
+```
+
+(Longer *supplementary details* — issue #117, e.g. Wise — are handled by the
+default parser and need no opt-in.)
+
 ## Contributing
 
 Help is greatly appreciated. Please clone the **develop** branch and run `tox`

mt940/models.py

@@ -5,7 +5,13 @@
 import re
 import typing
 import warnings
-from collections.abc import Callable, Mapping, MutableMapping, Sequence
+from collections.abc import (
+    Callable,
+    Iterable,
+    Mapping,
+    MutableMapping,
+    Sequence,
+)
 from typing import Any, ClassVar, overload
 
 import mt940
@@ -363,6 +369,7 @@ def __init__(
         self,
         processors: dict[str, list[Callable[..., Any]]] | None = None,
         tags: dict[int | str, mt940.tags.Tag] | None = None,
+        transaction_boundary: Iterable[str] | None = None,
     ) -> None:
         self.processors: dict[str, list[Callable[..., Any]]] = (
             self.DEFAULT_PROCESSORS.copy()
@@ -376,6 +383,19 @@ def __init__(
         if tags:
             self.tags.update(tags)
 
+        # Opt-in (issue #110): tag slugs that each open a new transaction.
+        # Banks differ in how they delimit transactions; by default only the
+        # `:61:` statement tag starts a transaction. Passing e.g.
+        # ``{'transaction_reference_number'}`` makes each `:20:` start its own
+        # transaction too. Empty (the default) preserves the legacy behaviour.
+        if isinstance(transaction_boundary, str):
+            # A bare string is almost certainly a single slug, not an iterable
+            # of single characters.
+            transaction_boundary = (transaction_boundary,)
+        self.transaction_boundary: frozenset[str] = frozenset(
+            transaction_boundary or ()
+        )
+
         self.transactions: list[Transaction] = []
         self.data: dict[str, Any] = {}
 
@@ -479,7 +499,16 @@ def _process_match(
             result = processor(self, tag, tag_dict, result)
 
         if isinstance(tag, mt940.tags.Statement):
+            # Statement (:61:) handling always takes precedence so it cannot
+            # be bypassed by listing its slug in transaction_boundary.
             self._process_statement_tag(result)
+        elif tag.slug in self.transaction_boundary:
+            # Opt-in (issue #110): this tag opens a new transaction block.
+            self.transactions.append(Transaction(self, result))
+            if issubclass(tag.scope, Transactions):
+                # Keep statement-level data (e.g. the :20: reference) global
+                # too, so later :61: tags in the same block can copy it.
+                self.data.update(result)
         elif issubclass(tag.scope, Transaction) and self.transactions:
             self._update_transaction(result)
         elif issubclass(  # pragma: no branch

mt940/parser.py

@@ -40,11 +40,19 @@ def parse(
     encoding: str | None = None,
     processors: dict[str, list[Any]] | None = None,
     tags: dict[Any, Any] | None = None,
+    transaction_boundary: Any = None,
 ) -> Transactions:
     """
     Parses mt940 data and returns transactions object
 
     :param src: file handler to read, filename to read or raw data as string
+    :param encoding: optional encoding override for byte input
+    :param processors: optional extra pre/post processors
+    :param tags: optional extra/override tag parsers
+    :param transaction_boundary: optional iterable of tag *slugs* that each
+        start a new transaction (issue #110). By default only ``:61:`` starts a
+        transaction; pass e.g. ``{'transaction_reference_number'}`` to also
+        start one on every ``:20:``. Omitting it keeps the legacy behaviour.
     :return: Collection of transactions
     :rtype: Transactions
     """
@@ -83,7 +91,9 @@ def safe_is_file(filename: Any) -> bool:
             raise exception  # pragma: no cover
 
     assert isinstance(data, str)
-    transactions = mt940.models.Transactions(processors, tags)
+    transactions = mt940.models.Transactions(
+        processors, tags, transaction_boundary=transaction_boundary
+    )
     transactions.parse(data)
 
     return transactions

mt940/processors.py

@@ -278,12 +278,16 @@ def _process_segments(
             key32 = DETAIL_KEYS['32']
             result[key32].append(value)
         elif key.startswith('2'):
-            # For segment keys beginning with '2', adjust specific
-            # segments by trimming trailing identifiers.
-            if key == '29' and value.endswith(' BIC'):
-                value = value.removesuffix(' BIC').rstrip()
-            elif key == '28D' and value.endswith(' IBAN'):
-                value = value.removesuffix(' IBAN').rstrip()
+            # Some banks append a bare ' BIC'/' IBAN' label with no value at
+            # the end of a detail segment (issue #109); strip the dangling
+            # label so it does not pollute the purpose. Segment keys are
+            # always two characters (see _parse_segments), so the historical
+            # '29'/'28D' key checks could never match the IBAN case -- the
+            # label is matched on the value instead.
+            for label in (' BIC', ' IBAN'):
+                if value.endswith(label):
+                    value = value.removesuffix(label).rstrip()
+                    break
             key20 = DETAIL_KEYS['20']
             result[key20].append(value)
         elif key in {'60', '61', '62', '63', '64', '65'}:

mt940/tags.py

@@ -421,7 +421,10 @@ class Statement(Tag):
     (?P<id>[A-Z][A-Z0-9 ]{3})?
     (?P<customer_reference>((?!//)[^\n]){0,16})
     (//(?P<bank_reference>.{0,23}))?
-    (\n?(?P<extra_details>.{0,34}))?
+    # Supplementary details: the SWIFT spec caps this at 34x, but some banks
+    # (e.g. Wise, issue #117) send more, so the length limit is relaxed. This
+    # only ever turns a previous parse error into a successful parse.
+    (\n?(?P<extra_details>.*))?
     $"""
 
     def __call__(
@@ -436,7 +439,7 @@ def __call__(
         entry_month = str(data.get('entry_month') or '')
 
         if entry_day.isdigit() and entry_month.isdigit():
-            entry_date = data['entry_date'] = models.Date(
+            entry_date = models.Date(
                 day=entry_day, month=entry_month, year=str(date.year)
             )
             if date > entry_date and (date - entry_date).days >= 330:
@@ -446,11 +449,18 @@ def __call__(
             else:
                 year = 0
 
-            data['guessed_entry_date'] = models.Date(
-                day=entry_date.day,
-                month=entry_date.month,
-                year=entry_date.year + year,
-            )
+            # Correct the entry date's year when the entry date crosses a
+            # year boundary relative to the value date (issue #121). Both
+            # `entry_date` and `guessed_entry_date` expose the resolved value;
+            # `guessed_entry_date` is kept as a backwards-compatible alias.
+            if year:
+                entry_date = models.Date(
+                    day=entry_date.day,
+                    month=entry_date.month,
+                    year=entry_date.year + year,
+                )
+            data['entry_date'] = entry_date
+            data['guessed_entry_date'] = entry_date
 
         return data
 
@@ -494,6 +504,41 @@ def __call__(
         return super().__call__(transactions, value)
 
 
+class StatementGLS(Statement):
+    """Statement variant for GLS / Atruvia banks (issue #111).
+
+    These banks send a customer reference longer than the SWIFT 16x cap,
+    followed by the ``//`` bank-reference delimiter (e.g.
+    ``...DR20,NTRFBIPI-dvT1FzfMqvzF5HaU4oetlH7SGRkonU//2022070616391534000``).
+
+    This is an opt-in tag because relaxing the default customer-reference
+    length would change how banks that legitimately pack data after a 16x
+    reference (e.g. Rabobank) are parsed. Enable it explicitly::
+
+        import mt940
+
+        gls = mt940.tags.StatementGLS()
+        mt940.parse(data, tags={gls.id: gls})
+    """
+
+    pattern = r"""^
+    (?P<year>\d{2})  # 6!n Value Date (YYMMDD)
+    (?P<month>\d{2})
+    (?P<day>\d{2})
+    (?P<entry_month>\d{2}|\s{2})?  # [4!n] Entry Date (MMDD)
+    (?P<entry_day>\d{2}|\s{2})?
+    (?P<status>R?[DC])  # 2a Debit/Credit Mark
+    (?P<funds_code>[A-Z])?
+    [\n ]?
+    (?P<amount>[\d,]{1,15})  # 15d Amount
+    (?P<id>[A-Z][A-Z0-9 ]{3})?
+    # Customer reference of any length, up to the // bank reference.
+    (?P<customer_reference>(?:(?!//)[^\n])*)
+    (//(?P<bank_reference>.{0,23}))?
+    (\n?(?P<extra_details>.*))?
+    $"""
+
+
 class ClosingBalance(BalanceBase):
     id: str | int = 62
 

mt940_tests/test_entry_dates.py

@@ -1,44 +1,46 @@
+import datetime
+
 import mt940
 
 
+def _statement(
+    transactions: mt940.models.Transactions, **kwargs: object
+) -> dict:
+    statement = mt940.tags.Statement()
+    data = dict(amount='123', status='D', **kwargs)
+    return statement(transactions, data)
+
+
 def test_entry_dates_wrapping_years():
     transactions = mt940.models.Transactions()
-    statement = mt940.tags.Statement()
-    data = dict(
-        amount='123',
-        status='D',
-        year=2000,
-    )
 
-    # Regular statement
-    statement(
-        transactions,
-        dict(
-            data.items(),
-            month=1,
-            day=1,
-        ),
+    # Regular statement without an entry date: only the value date is set.
+    regular = _statement(transactions, year=2000, month=1, day=1)
+    assert regular['date'] == datetime.date(2000, 1, 1)
+    assert 'entry_date' not in regular
+
+    # Entry date in the same year as the value date: no correction applied.
+    same_year = _statement(
+        transactions, year=2000, month=6, day=15, entry_month=6, entry_day=10
     )
+    assert same_year['date'] == datetime.date(2000, 6, 15)
+    assert same_year['entry_date'] == datetime.date(2000, 6, 10)
+    assert same_year['guessed_entry_date'] == same_year['entry_date']
 
-    # Statement which wraps to the future
-    statement(
-        transactions,
-        dict(
-            data.items(),
-            month=12,
-            day=31,
-            entry_day=1,
-            entry_month=1,
-        ),
+    # Entry date wraps into the next year (value date Dec, entry date Jan).
+    forward = _statement(
+        transactions, year=2000, month=12, day=31, entry_month=1, entry_day=1
     )
-    # Statement which wraps the past year
-    statement(
-        transactions,
-        dict(
-            data.items(),
-            month=1,
-            day=1,
-            entry_day=31,
-            entry_month=12,
-        ),
+    assert forward['date'] == datetime.date(2000, 12, 31)
+    assert forward['entry_date'] == datetime.date(2001, 1, 1)
+    assert forward['guessed_entry_date'] == forward['entry_date']
+
+    # Entry date wraps into the previous year (value date Jan, entry date
+    # Dec) -- issue #121. `entry_date` must be resolved, not left in the
+    # value date's year.
+    backward = _statement(
+        transactions, year=2000, month=1, day=1, entry_month=12, entry_day=31
     )
+    assert backward['date'] == datetime.date(2000, 1, 1)
+    assert backward['entry_date'] == datetime.date(1999, 12, 31)
+    assert backward['guessed_entry_date'] == backward['entry_date']

mt940_tests/test_issues/test_issue106_streams.py

@@ -0,0 +1,26 @@
+"""Issue #106: parsing must not depend on sys.stdout/sys.stderr being usable.
+
+Sandboxed / serverless environments (and some GUI apps) run with
+``sys.stdout``/``sys.stderr`` set to ``None``. Parsing valid data must work
+there without raising ``AttributeError: 'NoneType' object has no attribute
+'encoding'``.
+"""
+
+import mt940
+
+VALID = """:20:STARTUMS
+:25:NL12RABO0123456789
+:28C:0
+:60F:C220706EUR100,00
+:61:2207060706C10,00NTRFref//bank
+:86:116?00Payment
+:62F:C220706EUR110,00
+"""
+
+
+def test_parses_without_stdout_and_stderr(monkeypatch):
+    monkeypatch.setattr('sys.stdout', None)
+    monkeypatch.setattr('sys.stderr', None)
+    transactions = mt940.parse(VALID)
+    assert len(transactions) == 1
+    assert str(transactions[0].data['amount']) == '10.00 EUR'