diff --git a/parse_errors/context.py b/parse_errors/context.py
index beee9a5..404f8cc 100644
--- a/parse_errors/context.py
+++ b/parse_errors/context.py
@@ -3,13 +3,25 @@
 from __future__ import annotations
 
 import os
+import re
 import contextlib
 from pathlib import Path
 from typing import Iterator
 
-from .source_map import detect_format, build_source_map, closest_entry
+from .source_map import detect_format, build_source_map, closest_entry, Location
 from ._jsonpath import extract_jsonpath, jsonpath_to_pointer
 
+POSITIONAL_RE = re.compile(r"at line (\d+), column (\d+)")
+
+
+def extract_positional_reference(msg: str) -> Location | None:
+    if m := POSITIONAL_RE.search(msg):
+        return Location(
+            line=int(m.group(1)) - 1, column=int(m.group(2)) - 1, position=0
+        )
+    return None
+
+
 __all__ = ["ParseError", "ParseContext"]
 
 
@@ -50,19 +62,31 @@ def ParseContext(
         yield
     except Exception as exc:
         message = str(exc)
+        path = Path(filename)
+
         # This is focused on msgspec-style exceptions, which use JSONPath for
-        # some reason.  If there are other formats we know can be raised,
-        # adjust this.
+        # some reason.
         jsonpath = extract_jsonpath(message)
         if jsonpath is None:
-            raise
+            # These are raised by toml decoding
+            loc = extract_positional_reference(message)
+            if loc is None:
+                raise ParseError(
+                    f"{filename}: {exc!r}", filename=filename, line=1
+                ) from exc
+
+            raise ParseError(
+                f"{path}:{loc.line + 1}:{loc.column + 1}: {message}",
+                filename=path,
+                line=loc.line + 1,
+                column=loc.column + 1,
+            ) from exc
 
         try:
             pointer = jsonpath_to_pointer(jsonpath)
         except ValueError:  # pragma: no cover
             raise exc
 
-        path = Path(filename)
         fmt = format or detect_format(path)
         assert fmt is not None
 
diff --git a/tests/test_parse_context_json.py b/tests/test_parse_context_json.py
index 3147af0..9bd57b3 100644
--- a/tests/test_parse_context_json.py
+++ b/tests/test_parse_context_json.py
@@ -33,7 +33,7 @@ def test_json_no_error():
 
 
 def test_json_non_jsonpath_exception_passes_through():
-    with pytest.raises(ZeroDivisionError):
+    with pytest.raises(ParseError, match=r"config.json: ZeroDivisionError\('oops'\)"):
         with ParseContext("config.json", data=JSON_SOURCE):
             raise ZeroDivisionError("oops")
 
@@ -77,3 +77,11 @@ def test_json_original_exception_is_cause():
             msgspec.json.decode(JSON_SOURCE.encode(), type=Config)
 
     assert isinstance(exc_info.value.__cause__, msgspec.ValidationError)
+
+
+def test_incomplete_json():
+    with pytest.raises(
+        ParseError, match=r"config.json: DecodeError\('Input data was truncated'\)"
+    ):
+        with ParseContext("config.json", data=JSON_GOOD.decode()[:-2]):
+            msgspec.json.decode(JSON_GOOD[:-2], type=Config)
diff --git a/tests/test_parse_context_toml.py b/tests/test_parse_context_toml.py
index 128753b..f6aaad8 100644
--- a/tests/test_parse_context_toml.py
+++ b/tests/test_parse_context_toml.py
@@ -5,6 +5,7 @@
 
 import pytest
 import msgspec
+from msgspec.toml import decode as decode_toml
 
 from parse_errors import ParseContext, ParseError
 
@@ -24,7 +25,7 @@
 
 
 def test_passthrough_non_jsonspec():
-    with pytest.raises(ValueError, match="^foo$"):
+    with pytest.raises(ParseError, match=r"^config.toml: ValueError\('foo'\)$"):
         with ParseContext("config.toml", data=TOML_SOURCE):
             raise ValueError("foo")
 
@@ -80,3 +81,15 @@ def test_toml_fallback_to_parent():
                 "Expected `str`, got `int` - at `$.server.tls.cert`"
             )
     assert exc_info.value.line == 1
+
+
+INCOMPLETE_TOML = """\
+
+[foo
+"""
+
+
+def test_toml_decode_raises_decode_error():
+    with pytest.raises(ParseError, match="config.toml:2:5: Expected"):
+        with ParseContext("config.toml", data=INCOMPLETE_TOML):
+            decode_toml(INCOMPLETE_TOML, type=Config)