From 8951748b697a0c372e2e6b16c309873389b2b49a Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Wed, 2 Sep 2026 06:58:55 +0800 Subject: [PATCH] Add Ollama embeddings to duplicate detector --- .../n8n/duplicate-issue-detector/README.md | 28 ++++++++-- .../duplicate-issue-detector/workflow.json | 52 +++++++++++++++-- .../test_duplicate_issue_detector_workflow.py | 56 +++++++++++++++++++ 3 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 tests/test_duplicate_issue_detector_workflow.py diff --git a/implementations/n8n/duplicate-issue-detector/README.md b/implementations/n8n/duplicate-issue-detector/README.md index 6387f63..ad59b9f 100644 --- a/implementations/n8n/duplicate-issue-detector/README.md +++ b/implementations/n8n/duplicate-issue-detector/README.md @@ -21,12 +21,12 @@ This workflow ships as **two separate flows in one file**: **1. Backfill (manual trigger, run once)** ``` Manual Trigger → Get All Open Issues → Loop (1 at a time) - → Embed (OpenAI) → Index into Qdrant → Wait 1s → next issue + → Embed (OpenAI or Ollama) → Index into Qdrant → Wait 1s → next issue ``` **2. Live Detection (GitHub trigger, runs forever after)** ``` -New Issue Opened → filter action=opened → Embed (OpenAI) +New Issue Opened → filter action=opened → Embed (OpenAI or Ollama) → Search Qdrant for closest match → score ≥ 0.87? → yes: comment with matched issue # and score → either way: index this issue into Qdrant @@ -41,26 +41,41 @@ The backfill flow exists because duplicate detection is useless with an empty ve ### Prerequisites - **n8n instance** — self-hosted or Cloud - **GitHub account** — fine-grained Personal Access Token scoped to one repo, with **Issues: Read & Write** permission -- **OpenAI API key** — used only for embeddings (`text-embedding-3-small`), which cost a fraction of a cent per issue — not the same cost tier as chat completion models +- **Embedding provider** — OpenAI (`text-embedding-3-small`) by default, or a self-hosted Ollama server running `nomic-embed-text` - **Qdrant instance** — free to self-host via Docker, or use Qdrant Cloud's free tier ### Installation 1. **Import the workflow** — n8n: **Workflows → Import from File**, select `workflow.json` -2. **Set credentials** — GitHub PAT, OpenAI API key +2. **Set credentials** — GitHub PAT and, when using the default provider, an OpenAI API key 3. **Fill in the placeholders** (table below) +### Local Ollama option + +Both flows include a disabled Ollama HTTP Request node alongside the enabled OpenAI node. To use local embeddings: + +1. Install [Ollama](https://ollama.com), then pull the default model: + ```bash + ollama pull nomic-embed-text + ``` +2. Set `REPLACE_OLLAMA_BASE_URL` in both Ollama nodes to the URL reachable from n8n (for example, `http://localhost:11434`). n8n Cloud cannot reach a service on your computer's `localhost`; use a self-hosted n8n instance or an appropriately exposed Ollama endpoint. +3. In both the live and backfill paths, disable the OpenAI node and enable the corresponding Ollama node. Do not leave both providers enabled, or each issue will be processed twice. +4. Recreate the Qdrant collection before switching providers. `nomic-embed-text` returns 768-dimensional vectors, so the collection must use size `768`; do not mix them with the existing 1536-dimensional OpenAI vectors. Run the backfill flow again after recreating the collection. + +To use another Ollama embedding model, change the model name in both Ollama nodes and create the Qdrant collection with that model's actual output dimension. + --- ## ⚠️ Before You Trust It Running Unattended Skipping any of these three steps means the workflow will either fail outright or silently do nothing useful — do them in order, before turning on the live trigger: -1. **Create the Qdrant collection first.** It needs to exist with the correct vector size before either flow will work — `text-embedding-3-small` produces 1536-dimensional vectors: +1. **Create the Qdrant collection first.** It needs to exist with the correct vector size before either flow will work. The default OpenAI path uses 1536-dimensional `text-embedding-3-small` vectors: ```bash curl -X PUT "{qdrant_url}/collections/{collection_name}" \ -H "Content-Type: application/json" \ -d '{"vectors": {"size": 1536, "distance": "Cosine"}}' ``` + If you switch to Ollama, use the local model's output dimension instead (768 for `nomic-embed-text`; see the local setup above). 2. **Run the backfill flow once, manually, before enabling the live trigger.** This is the step most duplicate-detector tutorials skip entirely — without it, the live trigger has nothing to compare new issues against, and will run for weeks appearing to work while actually catching nothing. @@ -75,6 +90,7 @@ Skipping any of these three steps means the workflow will either fail outright o | `REPLACE_OWNER` / `REPLACE_REPO` | 3 GitHub nodes | Your repo owner/name | | GitHub credential | 3 GitHub nodes | Fine-grained PAT, Issues read+write only | | OpenAI credential | 2 embedding nodes | Your OpenAI API key | +| `REPLACE_OLLAMA_BASE_URL` | 2 disabled Ollama nodes | Ollama base URL reachable from n8n, without the `/api` suffix | | `REPLACE_QDRANT_URL` | 4 HTTP Request nodes | Your Qdrant instance URL | | `REPLACE_COLLECTION_NAME` | Same 4 nodes | Any name, e.g. `github-issues` | | `REPLACE_QDRANT_API_KEY` | Same 4 nodes | Your Qdrant API key (Qdrant Cloud) or leave as-is for local Qdrant | @@ -93,7 +109,7 @@ Skipping any of these three steps means the workflow will either fail outright o - [ ] Auto-label flagged issues (e.g. `possible-duplicate`) in addition to commenting - [ ] Support closed issues in the similarity search, not just open ones -- [ ] Local embedding option (Ollama `nomic-embed-text`) as a free, private alternative to OpenAI — documented as a swap-in, same pattern as this collection's other workflows +- [x] Local embedding option (Ollama `nomic-embed-text`) as a free, private alternative to OpenAI — documented as a swap-in, same pattern as this collection's other workflows - [ ] Auto re-tune threshold suggestion based on maintainer feedback (thumbs up/down on the comment) --- diff --git a/implementations/n8n/duplicate-issue-detector/workflow.json b/implementations/n8n/duplicate-issue-detector/workflow.json index afc85c6..d1f0fe2 100644 --- a/implementations/n8n/duplicate-issue-detector/workflow.json +++ b/implementations/n8n/duplicate-issue-detector/workflow.json @@ -73,7 +73,23 @@ }, { "parameters": { - "jsCode": "const vector = $input.first().json.data[0].embedding;\nconst prev = $('Prepare Issue Text').first().json;\nreturn [{ json: { ...prev, vector } }];" + "method": "POST", + "url": "=REPLACE_OLLAMA_BASE_URL/api/embed", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ model: \"nomic-embed-text\", input: $json.combinedText }) }}", + "options": {} + }, + "id": "embed-ollama-live", + "name": "Embed New Issue (Ollama)", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [-1040, -80], + "disabled": true + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\nconst vector = response.data?.[0]?.embedding || response.embeddings?.[0];\nif (!Array.isArray(vector)) throw new Error('Embedding response did not contain a vector');\nconst prev = $('Prepare Issue Text').first().json;\nreturn [{ json: { ...prev, vector } }];" }, "id": "extract-vector-live", "name": "Extract Vector", @@ -237,7 +253,23 @@ }, { "parameters": { - "jsCode": "const vector = $input.first().json.data[0].embedding;\nconst prev = $('Prepare Issue Text (Backfill)').first().json;\nreturn [{ json: { ...prev, vector } }];" + "method": "POST", + "url": "=REPLACE_OLLAMA_BASE_URL/api/embed", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ model: \"nomic-embed-text\", input: $json.combinedText }) }}", + "options": {} + }, + "id": "embed-ollama-backfill", + "name": "Embed Issue (Ollama)", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [-820, 520], + "disabled": true + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\nconst vector = response.data?.[0]?.embedding || response.embeddings?.[0];\nif (!Array.isArray(vector)) throw new Error('Embedding response did not contain a vector');\nconst prev = $('Prepare Issue Text (Backfill)').first().json;\nreturn [{ json: { ...prev, vector } }];" }, "id": "extract-vector-backfill", "name": "Extract Vector (Backfill)", @@ -281,11 +313,17 @@ "main": [[{ "node": "Prepare Issue Text", "type": "main", "index": 0 }]] }, "Prepare Issue Text": { - "main": [[{ "node": "Embed New Issue (OpenAI)", "type": "main", "index": 0 }]] + "main": [[ + { "node": "Embed New Issue (OpenAI)", "type": "main", "index": 0 }, + { "node": "Embed New Issue (Ollama)", "type": "main", "index": 0 } + ]] }, "Embed New Issue (OpenAI)": { "main": [[{ "node": "Extract Vector", "type": "main", "index": 0 }]] }, + "Embed New Issue (Ollama)": { + "main": [[{ "node": "Extract Vector", "type": "main", "index": 0 }]] + }, "Extract Vector": { "main": [[{ "node": "Search Similar Issues (Qdrant)", "type": "main", "index": 0 }]] }, @@ -317,11 +355,17 @@ ] }, "Prepare Issue Text (Backfill)": { - "main": [[{ "node": "Embed Issue (OpenAI)", "type": "main", "index": 0 }]] + "main": [[ + { "node": "Embed Issue (OpenAI)", "type": "main", "index": 0 }, + { "node": "Embed Issue (Ollama)", "type": "main", "index": 0 } + ]] }, "Embed Issue (OpenAI)": { "main": [[{ "node": "Extract Vector (Backfill)", "type": "main", "index": 0 }]] }, + "Embed Issue (Ollama)": { + "main": [[{ "node": "Extract Vector (Backfill)", "type": "main", "index": 0 }]] + }, "Extract Vector (Backfill)": { "main": [[{ "node": "Index Issue (Qdrant Upsert)", "type": "main", "index": 0 }]] }, diff --git a/tests/test_duplicate_issue_detector_workflow.py b/tests/test_duplicate_issue_detector_workflow.py new file mode 100644 index 0000000..56ee855 --- /dev/null +++ b/tests/test_duplicate_issue_detector_workflow.py @@ -0,0 +1,56 @@ +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW_PATH = ROOT / "implementations" / "n8n" / "duplicate-issue-detector" / "workflow.json" +README_PATH = ROOT / "implementations" / "n8n" / "duplicate-issue-detector" / "README.md" + + +def _workflow() -> dict: + return json.loads(WORKFLOW_PATH.read_text(encoding="utf-8")) + + +def _node(workflow: dict, name: str) -> dict: + return next(node for node in workflow["nodes"] if node["name"] == name) + + +def _targets(workflow: dict, source: str) -> set[str]: + return { + edge["node"] + for output in workflow["connections"][source]["main"] + for edge in output + } + + +def test_duplicate_detector_supports_swappable_ollama_embeddings(): + workflow = _workflow() + live_ollama = _node(workflow, "Embed New Issue (Ollama)") + backfill_ollama = _node(workflow, "Embed Issue (Ollama)") + + for node in (live_ollama, backfill_ollama): + assert node["type"] == "n8n-nodes-base.httpRequest" + assert node["disabled"] is True + assert node["parameters"]["url"] == "=REPLACE_OLLAMA_BASE_URL/api/embed" + assert 'model: "nomic-embed-text"' in node["parameters"]["jsonBody"] + + assert _targets(workflow, "Prepare Issue Text") == { + "Embed New Issue (OpenAI)", + "Embed New Issue (Ollama)", + } + assert _targets(workflow, "Prepare Issue Text (Backfill)") == { + "Embed Issue (OpenAI)", + "Embed Issue (Ollama)", + } + assert _targets(workflow, "Embed New Issue (Ollama)") == {"Extract Vector"} + assert _targets(workflow, "Embed Issue (Ollama)") == {"Extract Vector (Backfill)"} + + for name in ("Extract Vector", "Extract Vector (Backfill)"): + code = _node(workflow, name)["parameters"]["jsCode"] + assert "data?.[0]?.embedding" in code + assert "embeddings?.[0]" in code + assert "Embedding response did not contain a vector" in code + + readme = README_PATH.read_text(encoding="utf-8") + assert "REPLACE_OLLAMA_BASE_URL" in readme + assert "768" in readme + assert "Do not leave both providers enabled" in readme