Support local Qwen VLM reranking for search results

README.md

@@ -154,19 +154,19 @@ No confident match found (best score: 0.28). Show results anyway? [y/N]:
 
 With `--no-trim`, low-confidence results are shown with a note instead of a prompt.
 
-Options: `--results N`, `--output-dir DIR`, `--no-trim` to skip auto-trimming, `--threshold 0.5` to adjust the confidence cutoff, `--save-top N` to save the top N clips instead of just the best match, `--dedupe` to drop results too similar to a higher-ranked pick (prevents near-duplicate chunks of the same event from filling the list), and `--rerank` to ask Gemini Flash to re-rank the returned candidates before trimming. Backend and model are auto-detected from the index — pass `--backend` or `--model` only to override.
+Options: `--results N`, `--output-dir DIR`, `--no-trim` to skip auto-trimming, `--threshold 0.5` to adjust the confidence cutoff, `--save-top N` to save the top N clips instead of just the best match, `--dedupe` to drop results too similar to a higher-ranked pick (prevents near-duplicate chunks of the same event from filling the list), and `--rerank` to ask a VLM to re-rank the returned candidates before trimming. Backend and model are auto-detected from the index — pass `--backend` or `--model` only to override.
 
 ```bash
 # Save top 5 clips, dropping near-duplicates
 sentrysearch search "red truck" --save-top 5 --dedupe 0.9
 
-# Re-rank the top 10 embedding matches with Gemini Flash before trimming
+# Re-rank the top 10 embedding matches with a VLM before trimming
 sentrysearch search "pedestrian crossing behind the car" --rerank --results 10
 ```
 
 The `--dedupe` value is a cosine similarity ceiling (0–1). Any result whose similarity to an already-kept higher-ranked result exceeds this value is dropped. Lower values are stricter: `0.8` requires results to be very distinct, `0.95` only removes near-identical chunks. `0.9` is a good default.
 
-`--rerank` extracts each returned candidate clip, sends it to Gemini 2.5 Flash with the query, and sorts likely visual matches ahead of embedding-only results. If reranking cannot run or a candidate cannot be scored, SentrySearch keeps the embedding-ranked results instead of failing the search.
+`--rerank` extracts each returned candidate clip, sends it to a VLM with the query, and sorts likely visual matches ahead of embedding-only results. Gemini and qwen-cloud searches use Gemini 2.5 Flash for reranking; local searches use a local Qwen3-VL Instruct reranker. If reranking cannot run or a candidate cannot be scored, SentrySearch keeps the embedding-ranked results instead of failing the search.
 
 ### Search by image
 
@@ -279,6 +279,7 @@ Options:
 
 Notes:
 - First run downloads the model (~16 GB for 8B, ~4 GB for 2B).
+- Local `--rerank` downloads a separate Qwen3-VL Instruct model (`Qwen/Qwen3-VL-8B-Instruct` or `Qwen/Qwen3-VL-2B-Instruct`) in addition to the embedding model.
 - Embeddings from different backends and models are **not compatible**. Each backend/model combination gets its own isolated index, so they can't accidentally mix. If you search with a model that has no indexed data, you'll be told which model was actually used.
 - Speed varies by GPU core count — base M-series chips are slower than Pro/Max but produce identical results.
 

sentrysearch/cli.py

@@ -98,6 +98,30 @@ def _strip_private_result_fields(results: list[dict]) -> list[dict]:
     ]
 
 
+def _get_search_reranker(
+    backend: str,
+    model: str | None,
+    quantize: bool | None,
+):
+    if backend == "local":
+        from .embedder import reset_embedder
+        from .qwen_reranker import QwenReranker
+
+        reset_embedder()
+        return QwenReranker(
+            model_name=model or "qwen8b",
+            quantize=quantize,
+        ).load()
+
+    from .gemini_reranker import GeminiReranker
+    return GeminiReranker()
+
+
+def _first_error_line(exc: Exception) -> str:
+    lines = str(exc).splitlines()
+    return lines[0] if lines else exc.__class__.__name__
+
+
 def _open_file(path: str) -> None:
     """Open a file with the system's default application."""
     try:
@@ -659,7 +683,7 @@ def index(directory, chunk_duration, overlap, preprocess, target_resolution,
               help="Drop results whose cosine similarity to a higher-ranked "
                    "result exceeds this (e.g. 0.9).")
 @click.option("--rerank", is_flag=True,
-              help="Use Gemini Flash to rerank candidates before trimming.")
+              help="Use a VLM to rerank candidates before trimming.")
 @click.option("--verbose", is_flag=True, help="Show debug info.")
 def search(query, n_results, output_dir, trim, save_top, threshold, overlay, backend, model, dashscope_model, quantize, dedupe_threshold, rerank, verbose):
     """Search indexed footage with a natural language QUERY."""
@@ -740,14 +764,14 @@ def search(query, n_results, output_dir, trim, save_top, threshold, overlay, bac
         rerank_enabled = rerank
         if rerank and results:
             from .gemini_embedder import GeminiAPIKeyError
-            from .gemini_reranker import GeminiReranker
+            from .local_embedder import LocalModelError
             from .reranker import rerank_results
 
             try:
-                reranker = GeminiReranker()
-            except GeminiAPIKeyError:
+                reranker = _get_search_reranker(backend, model, quantize)
+            except (GeminiAPIKeyError, LocalModelError) as exc:
                 click.secho(
-                    "--rerank skipped: GEMINI_API_KEY is not set; "
+                    f"--rerank skipped: {_first_error_line(exc)}; "
                     "showing embedding results.",
                     fg="yellow",
                     err=True,

sentrysearch/gemini_reranker.py

@@ -14,37 +14,17 @@
     _RateLimiter,
     _retry,
 )
-from .reranker import RerankScore, parse_rerank_response
+from .reranker import (
+    RERANK_SCHEMA,
+    RerankScore,
+    build_rerank_prompt,
+    parse_rerank_response,
+)
 
 load_dotenv()
 
 RERANK_MODEL = "gemini-2.5-flash"
 
-_RERANK_SCHEMA = {
-    "type": "object",
-    "properties": {
-        "rerank_match": {"type": "boolean"},
-        "rerank_confidence": {
-            "type": "number",
-            "minimum": 0.0,
-            "maximum": 1.0,
-        },
-    },
-    "required": ["rerank_match", "rerank_confidence"],
-}
-
-
-def _prompt(query: str) -> str:
-    return (
-        "You are reranking video search candidates.\n"
-        "Look at the clip and decide whether it visually matches this search "
-        f"query: {query!r}\n\n"
-        "Return JSON only with this exact shape:\n"
-        '{"rerank_match": true, "rerank_confidence": 0.0}\n'
-        "Use rerank_match=true only when the clip contains the requested event. "
-        "Use rerank_confidence as your confidence from 0.0 to 1.0."
-    )
-
 
 class GeminiReranker:
     """Gemini Flash reranker for candidate video clips."""
@@ -74,10 +54,10 @@ def score(
         from google.genai import types
 
         video_part = GeminiEmbedder._make_video_part(clip_path, types)
-        prompt_part = types.Part(text=_prompt(query))
+        prompt_part = types.Part(text=build_rerank_prompt(query))
         config = types.GenerateContentConfig(
             response_mime_type="application/json",
-            response_json_schema=_RERANK_SCHEMA,
+            response_json_schema=RERANK_SCHEMA,
             temperature=0.0,
         )