fix: pad sub-microsecond digits when parsing nanosecond (#3614)

timestamp_to_nanos and timestamptz_to_nanos read the sub-microsecond
digits of the fraction (regex group 3 of ISO_TIMESTAMP_NANO /
ISO_TIMESTAMPTZ_NANO) as a bare integer. For a 7- or 8-digit fraction
that group is 1 or 2 digits and represents positions 7-8 of the
fraction, so it must be right-padded to 3 digits before being
interpreted as nanoseconds. Without padding, .1234567 was parsed as
+7 ns instead of +700 ns, silently corrupting the value.

Right-pad the group to 3 digits with ljust so all fraction widths
round-trip correctly, and add 7- and 8-digit cases to the tests, which
previously only covered 6- and 9-digit fractions.

Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com>

Author: anxkhn

pyiceberg/utils/datetime.py

@@ -121,7 +121,9 @@ def timestamp_to_nanos(timestamp_str: str) -> int:
     if match := ISO_TIMESTAMP_NANO.fullmatch(timestamp_str):
         # Python datetime does not have native nanoseconds support
         # Hence we need to extract nanoseconds timestamp manually
-        ns_str = match.group(3) or "0"
+        # group(3) holds the sub-microsecond digits (fraction positions 7-9), so
+        # right-pad to 3 digits before reading them as nanoseconds (e.g. "7" -> 700).
+        ns_str = (match.group(3) or "0").ljust(3, "0")
         ms_str = match.group(2) if match.group(2) else ""
         timestamp_str_without_ns_str = match.group(1) + ms_str
         return datetime_to_nanos(datetime.fromisoformat(timestamp_str_without_ns_str)) + int(ns_str)
@@ -136,7 +138,9 @@ def timestamptz_to_nanos(timestamptz_str: str) -> int:
     if match := ISO_TIMESTAMPTZ_NANO.fullmatch(timestamptz_str):
         # Python datetime does not have native nanoseconds support
         # Hence we need to extract nanoseconds timestamp manually
-        ns_str = match.group(3) or "0"
+        # group(3) holds the sub-microsecond digits (fraction positions 7-9), so
+        # right-pad to 3 digits before reading them as nanoseconds (e.g. "7" -> 700).
+        ns_str = (match.group(3) or "0").ljust(3, "0")
         ms_str = match.group(2) if match.group(2) else ""
         timestamptz_str_without_ns_str = match.group(1) + ms_str + match.group(4)
         return datetime_to_nanos(datetime.fromisoformat(timestamptz_str_without_ns_str)) + int(ns_str)

tests/utils/test_datetime.py

@@ -111,6 +111,8 @@ def test_datetime_to_nanos(datetime_: datetime, nanos: int) -> None:
     [
         ("1970-01-01T00:00:00", 0),
         ("2025-02-23T20:21:44.375612", 1740342104375612000),
+        ("2025-02-23T20:21:44.3756127", 1740342104375612700),
+        ("2025-02-23T20:21:44.37561278", 1740342104375612780),
         ("2025-02-23T20:21:44.375612001", 1740342104375612001),
     ],
 )
@@ -128,6 +130,8 @@ def test_timestamp_to_nanos_unexpected_zone_offset() -> None:
     [
         ("1970-01-01T00:00:00+00:00", 0),
         ("2025-02-23T16:21:44.375612-04:00", 1740342104375612000),
+        ("2025-02-23T16:21:44.3756127-04:00", 1740342104375612700),
+        ("2025-02-23T16:21:44.37561278-04:00", 1740342104375612780),
         ("2025-02-23T16:21:44.375612001-04:00", 1740342104375612001),
     ],
 )