diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index bd7a75c..fba7fd2 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -12,12 +12,22 @@ on: - '["3.11", "3.12"]' - '["3.11"]' - '["3.12"]' + pull_request: + branches: + - master + - main push: + branches: + - master + - main paths: - 'pyproject.toml' - '.github/workflows/ci-tests.yml' - 'tests/**' +permissions: + contents: read + jobs: test: name: Test on Python ${{ matrix.python-version }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..8b96752 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,38 @@ +name: Lint + +on: + pull_request: + branches: + - master + - main + push: + branches: + - master + - main + +permissions: + contents: read + +jobs: + lint: + name: Lint with Ruff + runs-on: ubuntu-latest + continue-on-error: true # Advisory only - won't block PRs + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install Ruff + run: pip install ruff + + - name: Run Ruff linter + run: ruff check src/ tests/ --output-format=github + + - name: Run Ruff formatter check + run: ruff format --check src/ tests/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f0c3317..2528380 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ All notable changes to Vector Viewer will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.4] - 2026-06-21 + +### Fixed +- **LanceDB flat schema metadata extraction**: Fixed metadata display for LanceDB databases with flat schemas (e.g., Contextus) where metadata is stored as individual columns rather than nested in a 'metadata' column +- **Duplicate document column**: Prevented content column from appearing twice in data browser (once as content, once as metadata) +- **Column width limits**: Added 600px max-width constraint to table columns to prevent excessive width + +### Changed +- LanceDB connection now extracts metadata from PyArrow schema first for better performance +- Content column (e.g., "document") is automatically detected and excluded from metadata fields + +### Added +- Comprehensive tests for flat schema metadata extraction +- Backward compatibility maintained with nested 'metadata' column format + +## [0.8.3] - 2026-06-21 + +### Changed +- Moved hashlib import to module level in base_connection.py +- Improved platform-specific monospace font rendering in provider install dialog +- Centralized content column detection logic with single return path ### Added - Phase 1 Implementation diff --git a/docs/RELEASE_REASON.md b/docs/RELEASE_REASON.md index c848b6c..20b356c 100644 --- a/docs/RELEASE_REASON.md +++ b/docs/RELEASE_REASON.md @@ -1,24 +1,22 @@ -# Release Notes (0.8.3) — June 21, 2026 +# Release Notes (0.8.4) — June 23, 2026 -Content column detection improvements and local embedding support. +Critical LanceDB metadata extraction fixes and column rendering improvements. -## Features +## Bug Fixes -- **Dynamic content column detection**: Automatically detects the best content column for each collection based on schema analysis (text/string fields, common naming patterns). Manual override support with persistence across sessions. -- **Ollama embedding integration**: Local embedding generation via Ollama HTTP API for environments where HuggingFace is blocked or unavailable. +- **LanceDB flat schema metadata extraction**: Fixed metadata display for LanceDB databases with flat schemas (e.g., Contextus) where metadata is stored as individual columns rather than nested in a 'metadata' column +- **Duplicate document column**: Prevented content column from appearing twice in data browser (once as content, once as metadata) +- **Column width limits**: Added 600px max-width constraint to table columns to prevent excessive width from breaking UI ## Improvements -- Content column configuration UI shows both auto-detected recommendation and currently active column -- Settings persistence for content column overrides per collection -- Fixed LanceDB schema detection to use PyArrow schema instead of pandas DataFrame conversion (performance improvement) -- Fixed threading deadlock in content column detection (switched from Lock to RLock) -- Platform-specific monospace fonts in UI dialogs (Menlo on macOS, Consolas on Windows) +- LanceDB connection now extracts metadata from PyArrow schema first for better performance +- Content column (e.g., "document") is automatically detected and excluded from metadata fields +- Backward compatibility maintained with nested 'metadata' column format -## Bug Fixes +## Testing -- Fixed LanceDB metadata array length mismatch in search results -- Fixed QThread leak in embedding configuration dialog -- Fixed content column cache poisoning when schema unavailable +- Added comprehensive tests for flat schema metadata extraction +- CI workflow now runs unit tests on all pull requests --- diff --git a/pyproject.toml b/pyproject.toml index c57267e..ba95bf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "vector-inspector" -version = "0.8.3" +version = "0.8.4" description = "A comprehensive desktop application for visualizing, querying, and managing vector database data" authors = [ {name = "Anthony Dawson", email = "anthonypdawson+github@gmail.com"}, diff --git a/src/vector_inspector/__init__.py b/src/vector_inspector/__init__.py index ef8a1ee..0c4549f 100644 --- a/src/vector_inspector/__init__.py +++ b/src/vector_inspector/__init__.py @@ -1,6 +1,6 @@ """Vector Inspector - A comprehensive desktop application for vector database visualization.""" -__version__ = "0.8.3" # Keep in sync with pyproject.toml for dev mode fallback +__version__ = "0.8.4" # Keep in sync with pyproject.toml for dev mode fallback def get_version(): diff --git a/src/vector_inspector/core/connections/lancedb_connection.py b/src/vector_inspector/core/connections/lancedb_connection.py index 3622460..a558285 100644 --- a/src/vector_inspector/core/connections/lancedb_connection.py +++ b/src/vector_inspector/core/connections/lancedb_connection.py @@ -1,5 +1,6 @@ """LanceDB connection implementation for Vector Inspector.""" +import math import os from typing import Any @@ -78,61 +79,121 @@ def get_collection_info(self, name: str) -> dict[str, Any] | None: # Row count (prefer num_rows, fallback to pandas length) count = getattr(tbl, "num_rows", None) - # Pull dataframe sample to infer metadata fields and vector dimension + # Extract metadata fields from schema first (more reliable) + metadata_fields: list[str] = [] + vector_dimension: int | str = "Unknown" + reserved_columns = {"id", "vector", "embedding", "_distance"} + + # Detect content column to exclude it from metadata + content_col: str | None = None try: - df = tbl.to_pandas() - # Filter out dummy initialization row - if df is not None and "id" in df.columns: - df = df[df["id"] != "__dummy_init__"] + # Try to get schema to detect content column + schema = tbl.schema + if schema: + # Build a temporary dict for content detection + temp_schema = {field.name: str(field.type) for field in schema} + content_col = self._detect_content_column(name, temp_schema) + reserved_columns.add(content_col) except Exception: - df = None + pass - metadata_fields: list[str] = [] - vector_dimension: int | str = "Unknown" + # Try to get schema from PyArrow table schema + try: + schema = tbl.schema + if schema: + # All non-reserved columns (excluding content) are metadata fields + metadata_fields = [ + field.name for field in schema + if field.name not in reserved_columns and not field.name.startswith("_") + ] - if df is not None and not df.empty: - # Determine count if not available - if count is None: - try: - count = len(df) - except Exception: - count = 0 + # Get vector dimension from schema if available + for field in schema: + if field.name in ("vector", "embedding"): + try: + # PyArrow FixedSizeListType has list_size attribute + if hasattr(field.type, "list_size"): + vector_dimension = field.type.list_size + except Exception: + pass + except Exception: + # If schema access fails, fall back to pandas sampling + pass - # Infer metadata fields from the first row - first_meta = df.iloc[0].get("metadata") if "metadata" in df.columns else None - if isinstance(first_meta, str): - import ast + # Fallback: Pull dataframe sample if schema extraction didn't work or count is unknown + if not metadata_fields or vector_dimension == "Unknown" or count is None: + try: + df = tbl.to_pandas() + # Filter out dummy initialization row + if df is not None and "id" in df.columns: + df = df[df["id"] != "__dummy_init__"] + except Exception: + df = None - try: - parsed = ast.literal_eval(first_meta) - if isinstance(parsed, dict): - metadata_fields = list(parsed.keys()) - except Exception: - # treat as raw string field - metadata_fields = [] - elif isinstance(first_meta, dict): - metadata_fields = list(first_meta.keys()) - - # Infer vector dimension from the first vector entry - if "vector" in df.columns: - first_vec = df.iloc[0].get("vector") - if first_vec is not None: + if df is not None and not df.empty: + # Determine count if not available + if count is None: try: - vector_dimension = len(first_vec) + count = len(df) except Exception: - vector_dimension = "Unknown" + count = 0 + + # If we didn't get metadata fields from schema, extract from columns + if not metadata_fields: + # First check for flat schema columns + metadata_fields = [ + col for col in df.columns + if col not in reserved_columns and not col.startswith("_") + ] - # Cache vector dimension if known - try: - if isinstance(vector_dimension, int): - self._collection_meta[name] = vector_dimension - except Exception: - pass + # Also check for nested metadata column (legacy format) + if "metadata" in df.columns and "metadata" in metadata_fields: + # Remove "metadata" from the list and try to extract its keys + metadata_fields.remove("metadata") + first_meta = df.iloc[0].get("metadata") + if isinstance(first_meta, str): + import ast + try: + parsed = ast.literal_eval(first_meta) + if isinstance(parsed, dict): + # Add nested metadata keys as separate fields + metadata_fields.extend( + f"metadata.{k}" for k in parsed.keys() + ) + except Exception: + # If parsing fails, keep "metadata" as a column + metadata_fields.append("metadata") + elif isinstance(first_meta, dict): + metadata_fields.extend( + f"metadata.{k}" for k in first_meta.keys() + ) + else: + # Not a dict, keep "metadata" as a column + metadata_fields.append("metadata") - else: - # No dataframe available, try to get count from attribute - if count is None: - count = 0 + # Infer vector dimension from the first vector entry if still unknown + if vector_dimension == "Unknown" and "vector" in df.columns: + first_vec = df.iloc[0].get("vector") + if first_vec is not None: + try: + vector_dimension = len(first_vec) + except Exception: + vector_dimension = "Unknown" + + # Cache vector dimension if known + try: + if isinstance(vector_dimension, int): + self._collection_meta[name] = vector_dimension + except Exception: + pass + else: + # No dataframe available, try to get count from attribute + if count is None: + count = 0 + + # Final fallback for count + if count is None: + count = 0 distance_metric = "Unknown" @@ -601,8 +662,37 @@ def get_all_items( # Get result count first result_count = len(df) - raw_meta = df["metadata"].tolist() if "metadata" in df.columns else [] - metadatas = self._parse_metadata_list(raw_meta) + # Extract metadata - support both nested "metadata" column and flat schema + # Get content column first to exclude it from metadata + schema = {col: str(dtype) for col, dtype in df.dtypes.items()} + content_col = self._detect_content_column(collection_name, schema) + reserved_columns = {"id", "vector", "embedding", "_distance", content_col} + + if "metadata" in df.columns: + # Legacy format: nested metadata column + raw_meta = df["metadata"].tolist() + metadatas = self._parse_metadata_list(raw_meta) + else: + # Flat schema format: all non-reserved columns are metadata + metadata_columns = [ + col for col in df.columns + if col not in reserved_columns and not col.startswith("_") + ] + + if metadata_columns: + # Build metadata dicts from the flat columns + # Use .to_dict('records') for efficient row-wise conversion + records = df[metadata_columns].to_dict('records') + metadatas = [] + for record in records: + # Filter out NaN/None values + meta = { + k: v for k, v in record.items() + if v is not None and (not isinstance(v, float) or not math.isnan(v)) + } + metadatas.append(meta) + else: + metadatas = [{} for _ in range(result_count)] # Ensure metadatas has same length as result count if len(metadatas) < result_count: @@ -615,10 +705,7 @@ def get_all_items( while len(metadatas) < result_count: metadatas.append({}) - # Get documents from content column (auto-detect) - schema = {col: str(dtype) for col, dtype in df.dtypes.items()} - content_col = self._detect_content_column(collection_name, schema) - + # Get documents from content column (already detected above) if content_col in df.columns: documents = df[content_col].tolist() else: diff --git a/src/vector_inspector/ui/views/metadata/metadata_table.py b/src/vector_inspector/ui/views/metadata/metadata_table.py index 44f6bf5..3e4dc9f 100644 --- a/src/vector_inspector/ui/views/metadata/metadata_table.py +++ b/src/vector_inspector/ui/views/metadata/metadata_table.py @@ -149,6 +149,14 @@ def populate_table( # Re-pin preview column after resizeColumnsToContents table.setColumnWidth(PREVIEW_COL, 28) + # Apply max width constraints to prevent excessively wide columns + MAX_COLUMN_WIDTH = 600 # Maximum width in pixels + for col in range(table.columnCount()): + if col != PREVIEW_COL: # Skip preview column (already fixed) + current_width = table.columnWidth(col) + if current_width > MAX_COLUMN_WIDTH: + table.setColumnWidth(col, MAX_COLUMN_WIDTH) + def copy_vectors_to_json( table: QTableWidget, diff --git a/tests/providers/lancedb/test_flat_schema_metadata.py b/tests/providers/lancedb/test_flat_schema_metadata.py new file mode 100644 index 0000000..e0f30e2 --- /dev/null +++ b/tests/providers/lancedb/test_flat_schema_metadata.py @@ -0,0 +1,123 @@ +"""Test LanceDB flat schema metadata extraction (e.g., Contextus-style databases).""" + +import pytest + +pytest.importorskip("lancedb") + +import uuid +import lancedb + + +def test_flat_schema_metadata_extraction(tmp_path): + """Test that flat schema columns (non-nested) are correctly extracted as metadata.""" + # Create a LanceDB database with flat schema like Contextus uses + db_path = str(tmp_path) + db = lancedb.connect(db_path) + + # Create a table with flat schema columns (like Contextus) + table_name = f"contextus_test_{uuid.uuid4().hex[:8]}" + data = [ + { + "id": "1", + "vector": [0.1, 0.2, 0.3], + "document": "First document", + "project": "test-project", + "filename": "test.md", + "heading": "Introduction", + "type": "decision", + "chunk_index": 0, + }, + { + "id": "2", + "vector": [0.4, 0.5, 0.6], + "document": "Second document", + "project": "test-project", + "filename": "guide.md", + "heading": "Setup", + "type": "reference", + "chunk_index": 1, + }, + ] + + db.create_table(table_name, data=data, mode="overwrite") + + # Test with Vector Inspector connection + from vector_inspector.core.connections.lancedb_connection import LanceDBConnection + + conn = LanceDBConnection(uri=db_path) + assert conn.connect() + + # Test get_collection_info returns flat schema columns as metadata_fields + info = conn.get_collection_info(table_name) + assert info is not None + assert info["count"] == 2 + + # All non-reserved columns except content column should be in metadata_fields + metadata_fields = info["metadata_fields"] + # "document" is the content column and should be excluded from metadata + expected_fields = {"project", "filename", "heading", "type", "chunk_index"} + assert expected_fields.issubset(set(metadata_fields)), \ + f"Missing fields in metadata_fields. Expected {expected_fields}, got {set(metadata_fields)}" + # Verify document is NOT in metadata_fields (it's the content column) + assert "document" not in metadata_fields, "Content column 'document' should not be in metadata_fields" + + # Test get_all_items extracts flat columns as metadata (excluding content column) + items = conn.get_all_items(table_name, limit=10) + assert items is not None + assert len(items["ids"]) == 2 + assert len(items["metadatas"]) == 2 + + # Check first item's metadata contains the flat schema columns (but not document) + first_meta = items["metadatas"][0] + assert "project" in first_meta + assert "filename" in first_meta + assert "heading" in first_meta + assert "type" in first_meta + assert "chunk_index" in first_meta + # Verify document is NOT duplicated in metadata + assert "document" not in first_meta, "Content column should not appear in metadata" + + assert first_meta["project"] == "test-project" + assert first_meta["filename"] == "test.md" + assert first_meta["heading"] == "Introduction" + assert first_meta["type"] == "decision" + assert first_meta["chunk_index"] == 0 + + # Verify documents are properly extracted to the documents field + assert items["documents"][0] == "First document" + assert items["documents"][1] == "Second document" + + +def test_nested_metadata_column_still_works(tmp_path): + """Test that legacy nested metadata column format still works.""" + from vector_inspector.core.connections.lancedb_connection import LanceDBConnection + + collection_name = f"test_nested_{uuid.uuid4().hex[:8]}" + db_path = str(tmp_path) + + conn = LanceDBConnection(uri=db_path) + assert conn.connect() + + # Create collection with nested metadata (old format) + assert conn.create_collection(collection_name, vector_size=2) + + # Add items with metadata in the traditional nested format + test_docs = ["doc1", "doc2"] + test_metadata = [{"key1": "value1"}, {"key2": "value2"}] + test_ids = ["id1", "id2"] + test_vectors = [[0.1, 0.2], [0.3, 0.4]] + + assert conn.add_items( + collection_name, + documents=test_docs, + metadatas=test_metadata, + ids=test_ids, + embeddings=test_vectors, + ) + + # Verify nested metadata still works + items = conn.get_all_items(collection_name, limit=10) + assert items is not None + assert len(items["metadatas"]) == 2 + assert items["metadatas"][0].get("key1") == "value1" + assert items["metadatas"][1].get("key2") == "value2" diff --git a/tests/test_lancedb_metadata_padding.py b/tests/test_lancedb_metadata_padding.py index 081fc39..cedbe0b 100644 --- a/tests/test_lancedb_metadata_padding.py +++ b/tests/test_lancedb_metadata_padding.py @@ -64,7 +64,9 @@ def test_get_all_items_metadata_padding(lancedb_conn): assert len(result["ids"]) == 2 assert len(result["documents"]) == 2 assert len(result["metadatas"]) == 2 # Should be padded! - assert all(m == {} for m in result["metadatas"]) + # With flat schema support, but "document" is excluded as it's the content column + assert result["metadatas"][0] == {} + assert result["metadatas"][1] == {} def test_query_collection_with_sparse_metadata(lancedb_conn):