diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py index 6ae17bb1c00..b4b9c8623d6 100644 --- a/python/python/lance/dataset.py +++ b/python/python/lance/dataset.py @@ -3098,6 +3098,11 @@ def update( def versions(self) -> List[Version]: """ Return all versions in this dataset. + + Each entry is a :class:`Version` with ``version``, ``timestamp``, + ``metadata``, and ``manifest_size``. ``manifest_size`` is the size in + bytes of that version's manifest file, or ``None`` if the commit + handler could not report it. """ versions = self._ds.versions() for v in versions: @@ -5923,6 +5928,11 @@ class Version(TypedDict): version: int timestamp: int | datetime metadata: Dict[str, str] + #: Size of this version's manifest file, in bytes, or ``None`` when the + #: commit handler could not report the size. The manifest grows with the + #: number of columns and fragments, so this is useful for observing + #: metadata amplification in wide tables. + manifest_size: int | None class VersionRef(TypedDict): diff --git a/python/python/tests/test_dataset.py b/python/python/tests/test_dataset.py index 7891f7e4215..4bf903a8735 100644 --- a/python/python/tests/test_dataset.py +++ b/python/python/tests/test_dataset.py @@ -449,6 +449,36 @@ def test_schema_metadata(tmp_path: Path): assert ds.schema.field("b").metadata == {b"thisis": b"b"} +def _manifest_file_size(base_dir: Path, version: int) -> int: + """Return the on-disk size of the manifest file for ``version``. + + Manifest files may use V1 (``_versions/{version}.manifest``) or V2 + (``_versions/{u64::MAX - version:020}.manifest``) naming, so resolve by + parsing the version back out of the file name rather than guessing. + """ + candidates = [] + for path in (base_dir / "_versions").glob("*.manifest"): + try: + number = int(path.stem) + except ValueError: + # Detached versions are named ``d{version}.manifest``; they are not + # part of the attached version history, so skip them. + continue + # V2 stores the bitwise-inverted version so that the newest version + # sorts first lexicographically. + candidates.append( + (number if number < (1 << 63) else (1 << 64) - 1 - number, path) + ) + + matches = [ + path for candidate_version, path in candidates if candidate_version == version + ] + assert len(matches) == 1, ( + f"expected exactly one manifest file for version {version}" + ) + return matches[0].stat().st_size + + def test_versions(tmp_path: Path): table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}]) base_dir = tmp_path / "test" @@ -477,6 +507,16 @@ def test_versions(tmp_path: Path): assert isinstance(v1["metadata"], dict) assert isinstance(v2["metadata"], dict) + # manifest_size must be pinned to the on-disk manifest file for each version. + for version in (v1, v2): + manifest_size = version["manifest_size"] + assert manifest_size is not None + assert manifest_size > 0 + # The key is always present, but the value is optional by contract. + assert version.get("manifest_size") == _manifest_file_size( + base_dir, version["version"] + ) + def test_version_id(tmp_path: Path): table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}]) diff --git a/python/src/dataset.rs b/python/src/dataset.rs index 547ba91d16b..30f4657a7f4 100644 --- a/python/src/dataset.rs +++ b/python/src/dataset.rs @@ -2080,6 +2080,7 @@ impl Dataset { .unwrap(); let tup: Vec<(&String, &String)> = v.metadata.iter().collect(); dict.set_item("metadata", tup.into_py_dict(py)?).unwrap(); + dict.set_item("manifest_size", v.manifest_size).unwrap(); dict.into_py_any(py) }) .collect::>>()?; diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs index ef025388d9d..ed431aab2fd 100644 --- a/rust/lance/src/dataset.rs +++ b/rust/lance/src/dataset.rs @@ -278,6 +278,14 @@ pub struct Version { /// Key-value pairs of metadata. pub metadata: BTreeMap, + + /// Size of the manifest file for this version, in bytes, if known. + /// + /// On wide tables (many columns and/or fragments) the manifest is rewritten + /// in full on every commit; exposing its size per version makes metadata + /// growth observable via `list_versions`. + #[serde(default)] + pub manifest_size: Option, } /// A lightweight reference to an attached dataset version, which could be used to uniquely identify a version. @@ -295,6 +303,7 @@ impl From<&Manifest> for Version { version: m.version, timestamp: m.timestamp(), metadata: m.summary().into(), + manifest_size: None, } } } @@ -2611,8 +2620,13 @@ impl Dataset { .commit_handler .list_manifest_locations(&self.base, &self.object_store, false) .try_filter_map(|location| async move { - match read_manifest(&self.object_store, &location.path, location.size).await { - Ok(manifest) => Ok(Some(Version::from(&manifest))), + let manifest_size = location.size; + match read_manifest(&self.object_store, &location.path, manifest_size).await { + Ok(manifest) => { + let mut version = Version::from(&manifest); + version.manifest_size = manifest_size; + Ok(Some(version)) + } Err(e) => Err(e), } })