From 5dc8e6048dea958896b6e63b748fae9c776e1d01 Mon Sep 17 00:00:00 2001 From: hiltonhe Date: Tue, 8 Sep 2026 22:59:07 +0800 Subject: [PATCH 1/2] feat(dataset): expose manifest_size on Version Dataset::versions() already knows each manifest's on-disk size from ManifestLocation but drops it. Keep it on the Version struct (and expose it in the Python versions() dicts) so metadata growth per version is queryable: ds.versions()[-1]['manifest_size'] This is pure bookkeeping - no format change. The field is Option since manifest listing may not always provide a size (it is #[serde(default)] for backward-compatible deserialization). Motivation: on wide tables the full manifest is rewritten on every commit; being able to chart manifest_size across versions is the cheapest way to quantify metadata amplification. --- python/src/dataset.rs | 1 + rust/lance/src/dataset.rs | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/python/src/dataset.rs b/python/src/dataset.rs index 547ba91d16b..30f4657a7f4 100644 --- a/python/src/dataset.rs +++ b/python/src/dataset.rs @@ -2080,6 +2080,7 @@ impl Dataset { .unwrap(); let tup: Vec<(&String, &String)> = v.metadata.iter().collect(); dict.set_item("metadata", tup.into_py_dict(py)?).unwrap(); + dict.set_item("manifest_size", v.manifest_size).unwrap(); dict.into_py_any(py) }) .collect::>>()?; diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs index ef025388d9d..ed431aab2fd 100644 --- a/rust/lance/src/dataset.rs +++ b/rust/lance/src/dataset.rs @@ -278,6 +278,14 @@ pub struct Version { /// Key-value pairs of metadata. pub metadata: BTreeMap, + + /// Size of the manifest file for this version, in bytes, if known. + /// + /// On wide tables (many columns and/or fragments) the manifest is rewritten + /// in full on every commit; exposing its size per version makes metadata + /// growth observable via `list_versions`. + #[serde(default)] + pub manifest_size: Option, } /// A lightweight reference to an attached dataset version, which could be used to uniquely identify a version. @@ -295,6 +303,7 @@ impl From<&Manifest> for Version { version: m.version, timestamp: m.timestamp(), metadata: m.summary().into(), + manifest_size: None, } } } @@ -2611,8 +2620,13 @@ impl Dataset { .commit_handler .list_manifest_locations(&self.base, &self.object_store, false) .try_filter_map(|location| async move { - match read_manifest(&self.object_store, &location.path, location.size).await { - Ok(manifest) => Ok(Some(Version::from(&manifest))), + let manifest_size = location.size; + match read_manifest(&self.object_store, &location.path, manifest_size).await { + Ok(manifest) => { + let mut version = Version::from(&manifest); + version.manifest_size = manifest_size; + Ok(Some(version)) + } Err(e) => Err(e), } }) From 10d93dae4f3796e0852f94b5e208d10a5aa7762a Mon Sep 17 00:00:00 2001 From: hiltonhe Date: Fri, 18 Sep 2026 02:36:19 +0800 Subject: [PATCH 2/2] test(dataset): pin manifest_size in the public type and test_versions Addresses the two review requests on #9102: 1. Add `manifest_size: int | None` to the public `Version` TypedDict in `python/python/lance/dataset.py`. The binding already returns the key, but typed callers could not use it: Pyright reported "manifest_size" is not a defined key in "Version". The field is optional because `ManifestLocation::size` is `Option` and is documented as possibly unknown. Also documents the returned fields on `versions()`. 2. Extend `test_versions` to assert the returned `manifest_size` against the corresponding `_versions/*.manifest` file size, so both the Rust propagation and the binding key are pinned. Resolving the file needs to handle V1 (`{version}.manifest`), V2 (`{u64::MAX - version:020}.manifest`) and detached (`d{version}.manifest`) naming. --- python/python/lance/dataset.py | 10 ++++++++ python/python/tests/test_dataset.py | 40 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py index 6ae17bb1c00..b4b9c8623d6 100644 --- a/python/python/lance/dataset.py +++ b/python/python/lance/dataset.py @@ -3098,6 +3098,11 @@ def update( def versions(self) -> List[Version]: """ Return all versions in this dataset. + + Each entry is a :class:`Version` with ``version``, ``timestamp``, + ``metadata``, and ``manifest_size``. ``manifest_size`` is the size in + bytes of that version's manifest file, or ``None`` if the commit + handler could not report it. """ versions = self._ds.versions() for v in versions: @@ -5923,6 +5928,11 @@ class Version(TypedDict): version: int timestamp: int | datetime metadata: Dict[str, str] + #: Size of this version's manifest file, in bytes, or ``None`` when the + #: commit handler could not report the size. The manifest grows with the + #: number of columns and fragments, so this is useful for observing + #: metadata amplification in wide tables. + manifest_size: int | None class VersionRef(TypedDict): diff --git a/python/python/tests/test_dataset.py b/python/python/tests/test_dataset.py index 7891f7e4215..4bf903a8735 100644 --- a/python/python/tests/test_dataset.py +++ b/python/python/tests/test_dataset.py @@ -449,6 +449,36 @@ def test_schema_metadata(tmp_path: Path): assert ds.schema.field("b").metadata == {b"thisis": b"b"} +def _manifest_file_size(base_dir: Path, version: int) -> int: + """Return the on-disk size of the manifest file for ``version``. + + Manifest files may use V1 (``_versions/{version}.manifest``) or V2 + (``_versions/{u64::MAX - version:020}.manifest``) naming, so resolve by + parsing the version back out of the file name rather than guessing. + """ + candidates = [] + for path in (base_dir / "_versions").glob("*.manifest"): + try: + number = int(path.stem) + except ValueError: + # Detached versions are named ``d{version}.manifest``; they are not + # part of the attached version history, so skip them. + continue + # V2 stores the bitwise-inverted version so that the newest version + # sorts first lexicographically. + candidates.append( + (number if number < (1 << 63) else (1 << 64) - 1 - number, path) + ) + + matches = [ + path for candidate_version, path in candidates if candidate_version == version + ] + assert len(matches) == 1, ( + f"expected exactly one manifest file for version {version}" + ) + return matches[0].stat().st_size + + def test_versions(tmp_path: Path): table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}]) base_dir = tmp_path / "test" @@ -477,6 +507,16 @@ def test_versions(tmp_path: Path): assert isinstance(v1["metadata"], dict) assert isinstance(v2["metadata"], dict) + # manifest_size must be pinned to the on-disk manifest file for each version. + for version in (v1, v2): + manifest_size = version["manifest_size"] + assert manifest_size is not None + assert manifest_size > 0 + # The key is always present, but the value is optional by contract. + assert version.get("manifest_size") == _manifest_file_size( + base_dir, version["version"] + ) + def test_version_id(tmp_path: Path): table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}])