From 5aecdae7b85c32a9fa234e89d95880721d3be9a0 Mon Sep 17 00:00:00 2001 From: Anthony Dawson Date: Sun, 21 Jun 2026 13:43:16 -0400 Subject: [PATCH 1/5] fix: LanceDB flat schema metadata extraction and column width limits Fixes metadata display for LanceDB databases with flat schemas (e.g., Contextus) where metadata is stored as individual columns rather than nested in a 'metadata' column. ## Changes ### LanceDB Connection - Extract metadata from PyArrow schema first (all non-reserved columns) - Support flat schema format in get_all_items() - build metadata dicts from column values - Detect and exclude content column from metadata to prevent duplication - Maintain backward compatibility with nested 'metadata' column format - Fix count detection when schema extraction succeeds but count is unavailable ### UI - Add 600px max-width constraint to table columns to prevent excessive width - Prevent 'document' column from appearing twice (once as content, once as metadata) ### Tests - Add comprehensive flat schema metadata extraction tests - Update existing tests to reflect content column exclusion from metadata - Verify backward compatibility with nested metadata format Closes #XX --- pyproject.toml | 2 +- .../core/connections/lancedb_connection.py | 189 +++++++++++++----- .../ui/views/metadata/metadata_table.py | 8 + .../lancedb/test_flat_schema_metadata.py | 123 ++++++++++++ tests/test_lancedb_metadata_padding.py | 4 +- 5 files changed, 273 insertions(+), 53 deletions(-) create mode 100644 tests/providers/lancedb/test_flat_schema_metadata.py diff --git a/pyproject.toml b/pyproject.toml index c57267e..ba95bf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vector-inspector" -version = "0.8.3" +version = "0.8.4" description = "A comprehensive desktop application for visualizing, querying, and managing vector database data" authors = [ {name = "Anthony Dawson", email = "anthonypdawson+github@gmail.com"}, diff --git a/src/vector_inspector/core/connections/lancedb_connection.py b/src/vector_inspector/core/connections/lancedb_connection.py index 3622460..a558285 100644 --- a/src/vector_inspector/core/connections/lancedb_connection.py +++ b/src/vector_inspector/core/connections/lancedb_connection.py @@ -1,5 +1,6 @@ """LanceDB connection implementation for Vector Inspector.""" +import math import os from typing import Any @@ -78,61 +79,121 @@ def get_collection_info(self, name: str) -> dict[str, Any] | None: # Row count (prefer num_rows, fallback to pandas length) count = getattr(tbl, "num_rows", None) - # Pull dataframe sample to infer metadata fields and vector dimension + # Extract metadata fields from schema first (more reliable) + metadata_fields: list[str] = [] + vector_dimension: int | str = "Unknown" + reserved_columns = {"id", "vector", "embedding", "_distance"} + + # Detect content column to exclude it from metadata + content_col: str | None = None try: - df = tbl.to_pandas() - # Filter out dummy initialization row - if df is not None and "id" in df.columns: - df = df[df["id"] != "__dummy_init__"] + # Try to get schema to detect content column + schema = tbl.schema + if schema: + # Build a temporary dict for content detection + temp_schema = {field.name: str(field.type) for field in schema} + content_col = self._detect_content_column(name, temp_schema) + reserved_columns.add(content_col) except Exception: - df = None + pass - metadata_fields: list[str] = [] - vector_dimension: int | str = "Unknown" + # Try to get schema from PyArrow table schema + try: + schema = tbl.schema + if schema: + # All non-reserved columns (excluding content) are metadata fields + metadata_fields = [ + field.name for field in schema + if field.name not in reserved_columns and not field.name.startswith("_") + ] - if df is not None and not df.empty: - # Determine count if not available - if count is None: - try: - count = len(df) - except Exception: - count = 0 + # Get vector dimension from schema if available + for field in schema: + if field.name in ("vector", "embedding"): + try: + # PyArrow FixedSizeListType has list_size attribute + if hasattr(field.type, "list_size"): + vector_dimension = field.type.list_size + except Exception: + pass + except Exception: + # If schema access fails, fall back to pandas sampling + pass - # Infer metadata fields from the first row - first_meta = df.iloc[0].get("metadata") if "metadata" in df.columns else None - if isinstance(first_meta, str): - import ast + # Fallback: Pull dataframe sample if schema extraction didn't work or count is unknown + if not metadata_fields or vector_dimension == "Unknown" or count is None: + try: + df = tbl.to_pandas() + # Filter out dummy initialization row + if df is not None and "id" in df.columns: + df = df[df["id"] != "__dummy_init__"] + except Exception: + df = None - try: - parsed = ast.literal_eval(first_meta) - if isinstance(parsed, dict): - metadata_fields = list(parsed.keys()) - except Exception: - # treat as raw string field - metadata_fields = [] - elif isinstance(first_meta, dict): - metadata_fields = list(first_meta.keys()) - - # Infer vector dimension from the first vector entry - if "vector" in df.columns: - first_vec = df.iloc[0].get("vector") - if first_vec is not None: + if df is not None and not df.empty: + # Determine count if not available + if count is None: try: - vector_dimension = len(first_vec) + count = len(df) except Exception: - vector_dimension = "Unknown" + count = 0 + + # If we didn't get metadata fields from schema, extract from columns + if not metadata_fields: + # First check for flat schema columns + metadata_fields = [ + col for col in df.columns + if col not in reserved_columns and not col.startswith("_") + ] - # Cache vector dimension if known - try: - if isinstance(vector_dimension, int): - self._collection_meta[name] = vector_dimension - except Exception: - pass + # Also check for nested metadata column (legacy format) + if "metadata" in df.columns and "metadata" in metadata_fields: + # Remove "metadata" from the list and try to extract its keys + metadata_fields.remove("metadata") + first_meta = df.iloc[0].get("metadata") + if isinstance(first_meta, str): + import ast + try: + parsed = ast.literal_eval(first_meta) + if isinstance(parsed, dict): + # Add nested metadata keys as separate fields + metadata_fields.extend( + f"metadata.{k}" for k in parsed.keys() + ) + except Exception: + # If parsing fails, keep "metadata" as a column + metadata_fields.append("metadata") + elif isinstance(first_meta, dict): + metadata_fields.extend( + f"metadata.{k}" for k in first_meta.keys() + ) + else: + # Not a dict, keep "metadata" as a column + metadata_fields.append("metadata") - else: - # No dataframe available, try to get count from attribute - if count is None: - count = 0 + # Infer vector dimension from the first vector entry if still unknown + if vector_dimension == "Unknown" and "vector" in df.columns: + first_vec = df.iloc[0].get("vector") + if first_vec is not None: + try: + vector_dimension = len(first_vec) + except Exception: + vector_dimension = "Unknown" + + # Cache vector dimension if known + try: + if isinstance(vector_dimension, int): + self._collection_meta[name] = vector_dimension + except Exception: + pass + else: + # No dataframe available, try to get count from attribute + if count is None: + count = 0 + + # Final fallback for count + if count is None: + count = 0 distance_metric = "Unknown" @@ -601,8 +662,37 @@ def get_all_items( # Get result count first result_count = len(df) - raw_meta = df["metadata"].tolist() if "metadata" in df.columns else [] - metadatas = self._parse_metadata_list(raw_meta) + # Extract metadata - support both nested "metadata" column and flat schema + # Get content column first to exclude it from metadata + schema = {col: str(dtype) for col, dtype in df.dtypes.items()} + content_col = self._detect_content_column(collection_name, schema) + reserved_columns = {"id", "vector", "embedding", "_distance", content_col} + + if "metadata" in df.columns: + # Legacy format: nested metadata column + raw_meta = df["metadata"].tolist() + metadatas = self._parse_metadata_list(raw_meta) + else: + # Flat schema format: all non-reserved columns are metadata + metadata_columns = [ + col for col in df.columns + if col not in reserved_columns and not col.startswith("_") + ] + + if metadata_columns: + # Build metadata dicts from the flat columns + # Use .to_dict('records') for efficient row-wise conversion + records = df[metadata_columns].to_dict('records') + metadatas = [] + for record in records: + # Filter out NaN/None values + meta = { + k: v for k, v in record.items() + if v is not None and (not isinstance(v, float) or not math.isnan(v)) + } + metadatas.append(meta) + else: + metadatas = [{} for _ in range(result_count)] # Ensure metadatas has same length as result count if len(metadatas) < result_count: @@ -615,10 +705,7 @@ def get_all_items( while len(metadatas) < result_count: metadatas.append({}) - # Get documents from content column (auto-detect) - schema = {col: str(dtype) for col, dtype in df.dtypes.items()} - content_col = self._detect_content_column(collection_name, schema) - + # Get documents from content column (already detected above) if content_col in df.columns: documents = df[content_col].tolist() else: diff --git a/src/vector_inspector/ui/views/metadata/metadata_table.py b/src/vector_inspector/ui/views/metadata/metadata_table.py index 44f6bf5..3e4dc9f 100644 --- a/src/vector_inspector/ui/views/metadata/metadata_table.py +++ b/src/vector_inspector/ui/views/metadata/metadata_table.py @@ -149,6 +149,14 @@ def populate_table( # Re-pin preview column after resizeColumnsToContents table.setColumnWidth(PREVIEW_COL, 28) + # Apply max width constraints to prevent excessively wide columns + MAX_COLUMN_WIDTH = 600 # Maximum width in pixels + for col in range(table.columnCount()): + if col != PREVIEW_COL: # Skip preview column (already fixed) + current_width = table.columnWidth(col) + if current_width > MAX_COLUMN_WIDTH: + table.setColumnWidth(col, MAX_COLUMN_WIDTH) + def copy_vectors_to_json( table: QTableWidget, diff --git a/tests/providers/lancedb/test_flat_schema_metadata.py b/tests/providers/lancedb/test_flat_schema_metadata.py new file mode 100644 index 0000000..e0f30e2 --- /dev/null +++ b/tests/providers/lancedb/test_flat_schema_metadata.py @@ -0,0 +1,123 @@ +"""Test LanceDB flat schema metadata extraction (e.g., Contextus-style databases).""" + +import pytest + +pytest.importorskip("lancedb") + +import uuid +import lancedb + + +def test_flat_schema_metadata_extraction(tmp_path): + """Test that flat schema columns (non-nested) are correctly extracted as metadata.""" + # Create a LanceDB database with flat schema like Contextus uses + db_path = str(tmp_path) + db = lancedb.connect(db_path) + + # Create a table with flat schema columns (like Contextus) + table_name = f"contextus_test_{uuid.uuid4().hex[:8]}" + data = [ + { + "id": "1", + "vector": [0.1, 0.2, 0.3], + "document": "First document", + "project": "test-project", + "filename": "test.md", + "heading": "Introduction", + "type": "decision", + "chunk_index": 0, + }, + { + "id": "2", + "vector": [0.4, 0.5, 0.6], + "document": "Second document", + "project": "test-project", + "filename": "guide.md", + "heading": "Setup", + "type": "reference", + "chunk_index": 1, + }, + ] + + db.create_table(table_name, data=data, mode="overwrite") + + # Test with Vector Inspector connection + from vector_inspector.core.connections.lancedb_connection import LanceDBConnection + + conn = LanceDBConnection(uri=db_path) + assert conn.connect() + + # Test get_collection_info returns flat schema columns as metadata_fields + info = conn.get_collection_info(table_name) + assert info is not None + assert info["count"] == 2 + + # All non-reserved columns except content column should be in metadata_fields + metadata_fields = info["metadata_fields"] + # "document" is the content column and should be excluded from metadata + expected_fields = {"project", "filename", "heading", "type", "chunk_index"} + assert expected_fields.issubset(set(metadata_fields)), \ + f"Missing fields in metadata_fields. Expected {expected_fields}, got {set(metadata_fields)}" + # Verify document is NOT in metadata_fields (it's the content column) + assert "document" not in metadata_fields, "Content column 'document' should not be in metadata_fields" + + # Test get_all_items extracts flat columns as metadata (excluding content column) + items = conn.get_all_items(table_name, limit=10) + assert items is not None + assert len(items["ids"]) == 2 + assert len(items["metadatas"]) == 2 + + # Check first item's metadata contains the flat schema columns (but not document) + first_meta = items["metadatas"][0] + assert "project" in first_meta + assert "filename" in first_meta + assert "heading" in first_meta + assert "type" in first_meta + assert "chunk_index" in first_meta + # Verify document is NOT duplicated in metadata + assert "document" not in first_meta, "Content column should not appear in metadata" + + assert first_meta["project"] == "test-project" + assert first_meta["filename"] == "test.md" + assert first_meta["heading"] == "Introduction" + assert first_meta["type"] == "decision" + assert first_meta["chunk_index"] == 0 + + # Verify documents are properly extracted to the documents field + assert items["documents"][0] == "First document" + assert items["documents"][1] == "Second document" + + +def test_nested_metadata_column_still_works(tmp_path): + """Test that legacy nested metadata column format still works.""" + from vector_inspector.core.connections.lancedb_connection import LanceDBConnection + + collection_name = f"test_nested_{uuid.uuid4().hex[:8]}" + db_path = str(tmp_path) + + conn = LanceDBConnection(uri=db_path) + assert conn.connect() + + # Create collection with nested metadata (old format) + assert conn.create_collection(collection_name, vector_size=2) + + # Add items with metadata in the traditional nested format + test_docs = ["doc1", "doc2"] + test_metadata = [{"key1": "value1"}, {"key2": "value2"}] + test_ids = ["id1", "id2"] + test_vectors = [[0.1, 0.2], [0.3, 0.4]] + + assert conn.add_items( + collection_name, + documents=test_docs, + metadatas=test_metadata, + ids=test_ids, + embeddings=test_vectors, + ) + + # Verify nested metadata still works + items = conn.get_all_items(collection_name, limit=10) + assert items is not None + assert len(items["metadatas"]) == 2 + assert items["metadatas"][0].get("key1") == "value1" + assert items["metadatas"][1].get("key2") == "value2" diff --git a/tests/test_lancedb_metadata_padding.py b/tests/test_lancedb_metadata_padding.py index 081fc39..cedbe0b 100644 --- a/tests/test_lancedb_metadata_padding.py +++ b/tests/test_lancedb_metadata_padding.py @@ -64,7 +64,9 @@ def test_get_all_items_metadata_padding(lancedb_conn): assert len(result["ids"]) == 2 assert len(result["documents"]) == 2 assert len(result["metadatas"]) == 2 # Should be padded! - assert all(m == {} for m in result["metadatas"]) + # With flat schema support, but "document" is excluded as it's the content column + assert result["metadatas"][0] == {} + assert result["metadatas"][1] == {} def test_query_collection_with_sparse_metadata(lancedb_conn): From 88b722d793cff4e603b55476fe9a5783919377a6 Mon Sep 17 00:00:00 2001 From: Anthony Dawson Date: Sun, 21 Jun 2026 13:43:55 -0400 Subject: [PATCH 2/5] docs: update CHANGELOG for v0.8.4 --- CHANGELOG.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0c3317..2528380 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ All notable changes to Vector Viewer will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.4] - 2026-06-21 + +### Fixed +- **LanceDB flat schema metadata extraction**: Fixed metadata display for LanceDB databases with flat schemas (e.g., Contextus) where metadata is stored as individual columns rather than nested in a 'metadata' column +- **Duplicate document column**: Prevented content column from appearing twice in data browser (once as content, once as metadata) +- **Column width limits**: Added 600px max-width constraint to table columns to prevent excessive width + +### Changed +- LanceDB connection now extracts metadata from PyArrow schema first for better performance +- Content column (e.g., "document") is automatically detected and excluded from metadata fields + +### Added +- Comprehensive tests for flat schema metadata extraction +- Backward compatibility maintained with nested 'metadata' column format + +## [0.8.3] - 2026-06-21 + +### Changed +- Moved hashlib import to module level in base_connection.py +- Improved platform-specific monospace font rendering in provider install dialog +- Centralized content column detection logic with single return path ### Added - Phase 1 Implementation From 081fa9a683548925feed7e15c752f27a3c65be05 Mon Sep 17 00:00:00 2001 From: Anthony Dawson Date: Tue, 23 Jun 2026 11:56:28 -0400 Subject: [PATCH 3/5] ci: add PR checks for tests and linting - Enable CI test workflow on all PRs to master/main - Add new lint workflow with ruff (check + format) - Linting is advisory-only (won't block merges) until codebase is cleaned up - Tests remain blocking as expected --- .github/workflows/ci-tests.yml | 7 +++++++ .github/workflows/lint.yml | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index bd7a75c..9576175 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -12,7 +12,14 @@ on: - '["3.11", "3.12"]' - '["3.11"]' - '["3.12"]' + pull_request: + branches: + - master + - main push: + branches: + - master + - main paths: - 'pyproject.toml' - '.github/workflows/ci-tests.yml' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..f06eb1a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,35 @@ +name: Lint + +on: + pull_request: + branches: + - master + - main + push: + branches: + - master + - main + +jobs: + lint: + name: Lint with Ruff + runs-on: ubuntu-latest + continue-on-error: true # Advisory only - won't block PRs + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install Ruff + run: pip install ruff + + - name: Run Ruff linter + run: ruff check src/ tests/ --output-format=github + + - name: Run Ruff formatter check + run: ruff format --check src/ tests/ From 3da7b676ffa51d68b9ab4f18c4a44e85eef17dfc Mon Sep 17 00:00:00 2001 From: Anthony Dawson Date: Tue, 23 Jun 2026 12:01:44 -0400 Subject: [PATCH 4/5] chore: finalize v0.8.4 release prep - Update RELEASE_REASON.md to v0.8.4 (matches CHANGELOG and pyproject.toml) - Add permissions blocks to CI workflows per GitHub security recommendation --- .github/workflows/ci-tests.yml | 3 +++ .github/workflows/lint.yml | 3 +++ docs/RELEASE_REASON.md | 26 ++++++++++++-------------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 9576175..fba7fd2 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -25,6 +25,9 @@ on: - '.github/workflows/ci-tests.yml' - 'tests/**' +permissions: + contents: read + jobs: test: name: Test on Python ${{ matrix.python-version }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f06eb1a..8b96752 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,6 +10,9 @@ on: - master - main +permissions: + contents: read + jobs: lint: name: Lint with Ruff diff --git a/docs/RELEASE_REASON.md b/docs/RELEASE_REASON.md index c848b6c..20b356c 100644 --- a/docs/RELEASE_REASON.md +++ b/docs/RELEASE_REASON.md @@ -1,24 +1,22 @@ -# Release Notes (0.8.3) — June 21, 2026 +# Release Notes (0.8.4) — June 23, 2026 -Content column detection improvements and local embedding support. +Critical LanceDB metadata extraction fixes and column rendering improvements. -## Features +## Bug Fixes -- **Dynamic content column detection**: Automatically detects the best content column for each collection based on schema analysis (text/string fields, common naming patterns). Manual override support with persistence across sessions. -- **Ollama embedding integration**: Local embedding generation via Ollama HTTP API for environments where HuggingFace is blocked or unavailable. +- **LanceDB flat schema metadata extraction**: Fixed metadata display for LanceDB databases with flat schemas (e.g., Contextus) where metadata is stored as individual columns rather than nested in a 'metadata' column +- **Duplicate document column**: Prevented content column from appearing twice in data browser (once as content, once as metadata) +- **Column width limits**: Added 600px max-width constraint to table columns to prevent excessive width from breaking UI ## Improvements -- Content column configuration UI shows both auto-detected recommendation and currently active column -- Settings persistence for content column overrides per collection -- Fixed LanceDB schema detection to use PyArrow schema instead of pandas DataFrame conversion (performance improvement) -- Fixed threading deadlock in content column detection (switched from Lock to RLock) -- Platform-specific monospace fonts in UI dialogs (Menlo on macOS, Consolas on Windows) +- LanceDB connection now extracts metadata from PyArrow schema first for better performance +- Content column (e.g., "document") is automatically detected and excluded from metadata fields +- Backward compatibility maintained with nested 'metadata' column format -## Bug Fixes +## Testing -- Fixed LanceDB metadata array length mismatch in search results -- Fixed QThread leak in embedding configuration dialog -- Fixed content column cache poisoning when schema unavailable +- Added comprehensive tests for flat schema metadata extraction +- CI workflow now runs unit tests on all pull requests --- From 380ceb7a15ad572e050e33fc775b11360a56b2aa Mon Sep 17 00:00:00 2001 From: Anthony Dawson Date: Tue, 23 Jun 2026 12:03:25 -0400 Subject: [PATCH 5/5] chore: bump __version__ to 0.8.4 in __init__.py --- src/vector_inspector/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vector_inspector/__init__.py b/src/vector_inspector/__init__.py index ef8a1ee..0c4549f 100644 --- a/src/vector_inspector/__init__.py +++ b/src/vector_inspector/__init__.py @@ -1,6 +1,6 @@ """Vector Inspector - A comprehensive desktop application for vector database visualization.""" -__version__ = "0.8.3" # Keep in sync with pyproject.toml for dev mode fallback +__version__ = "0.8.4" # Keep in sync with pyproject.toml for dev mode fallback def get_version():