diff --git a/JS/edgechains/arakoodev/README.md b/JS/edgechains/arakoodev/README.md index 81237e662..57c7d963b 100644 --- a/JS/edgechains/arakoodev/README.md +++ b/JS/edgechains/arakoodev/README.md @@ -3,3 +3,37 @@ Installation ``` npm install arakoodev ``` + +## Vector database clients + +The vector-db package exports Supabase and Qdrant helpers. + +```ts +import { Qdrant } from "@arakoodev/edgechains.js/vector-db"; + +const qdrant = new Qdrant(process.env.QDRANT_URL, process.env.QDRANT_API_KEY); +const client = qdrant.createClient(); + +await qdrant.createCollection({ + client, + collectionName: "documents", + vectorSize: 1536, +}); + +await qdrant.insertVectorData({ + client, + collectionName: "documents", + id: "doc-1", + embedding: [0.1, 0.2, 0.3], + content: "hello qdrant", + namespace: "docs", +}); + +const matches = await qdrant.getDataFromQuery({ + client, + collectionName: "documents", + embedding: [0.1, 0.2, 0.3], + filter: { must: [{ key: "namespace", match: { value: "docs" } }] }, + limit: 5, +}); +``` diff --git a/JS/edgechains/arakoodev/src/vector-db/src/index.ts b/JS/edgechains/arakoodev/src/vector-db/src/index.ts index 557104a14..60b00b132 100644 --- a/JS/edgechains/arakoodev/src/vector-db/src/index.ts +++ b/JS/edgechains/arakoodev/src/vector-db/src/index.ts @@ -1 +1,7 @@ export { Supabase } from "./lib/supabase/supabase.js"; +export { + Qdrant, + QdrantRestClient, + QdrantVectorClient, +} from "./lib/qdrant/qdrant.js"; +export { QdrantDistanceMetric } from "./lib/qdrant/qdrant.js"; diff --git a/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts b/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts new file mode 100644 index 000000000..304df17a0 --- /dev/null +++ b/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts @@ -0,0 +1,548 @@ +import { config } from "dotenv"; +config(); + +type QdrantPointId = string | number; + +interface QdrantPoint { + id: QdrantPointId; + vector?: number[] | Record; + payload?: Record; +} + +interface QdrantRequestOptions extends RequestInit { + headers?: Record; +} + +interface QdrantQueryOptions { + query?: number[] | Record; + vector?: number[]; + embedding?: number[]; + filter?: Record; + limit?: number; + withPayload?: boolean | string[] | Record; + withVector?: boolean | string[] | Record; + scoreThreshold?: number; + using?: string; +} + +interface InsertVectorDataArgs { + client: QdrantRestClient; + collectionName: string; + points?: QdrantPoint[]; + id?: QdrantPointId; + vector?: number[] | Record; + embedding?: number[]; + payload?: Record; + wait?: boolean; + [key: string]: any; +} + +interface CreateCollectionArgs { + client: QdrantRestClient; + collectionName: string; + vectorSize?: number; + distance?: QdrantDistanceMetric | string; + vectors?: Record; + wait?: boolean; + [key: string]: any; +} + +interface CollectionArgs { + client: QdrantRestClient; + collectionName: string; +} + +interface GetDataFromQueryArgs extends QdrantQueryOptions { + client: QdrantRestClient; + collectionName: string; +} + +interface GetDataArgs { + client: QdrantRestClient; + collectionName: string; + filter?: Record; + limit?: number; + offset?: QdrantPointId; + withPayload?: boolean | string[] | Record; + withVector?: boolean | string[] | Record; +} + +interface UpdateByIdArgs { + client: QdrantRestClient; + collectionName: string; + id: QdrantPointId; + updatedContent: Record; + wait?: boolean; +} + +interface DeleteByIdArgs { + client: QdrantRestClient; + collectionName: string; + id: QdrantPointId; + wait?: boolean; +} + +interface QdrantVectorClientArgs { + wordEmbeddings: number[][]; + topK: number; + collectionName: string; + namespace?: string; + arkRequest?: any; + upperLimit?: number; + client?: QdrantRestClient; + qdrantUrl?: string; + qdrantApiKey?: string; +} + +export enum QdrantDistanceMetric { + COSINE = "Cosine", + DOT = "Dot", + EUCLID = "Euclid", + MANHATTAN = "Manhattan", +} + +export class QdrantRestClient { + baseUrl: string; + apiKey?: string; + + constructor(baseUrl: string, apiKey?: string) { + if (!baseUrl) { + throw new Error("Qdrant URL is required"); + } + + this.baseUrl = baseUrl.replace(/\/+$/, ""); + this.apiKey = apiKey; + } + + async request( + path: string, + options: QdrantRequestOptions = {}, + ): Promise { + const headers: Record = { + "Content-Type": "application/json", + ...(this.apiKey ? { "api-key": this.apiKey } : {}), + ...(options.headers || {}), + }; + + const response = await fetch(`${this.baseUrl}${path}`, { + ...options, + headers, + }); + + const rawBody = await response.text(); + const body = rawBody ? this.parseResponseBody(rawBody) : null; + + if (!response.ok) { + const message = + typeof body === "string" + ? body + : JSON.stringify(body || response.statusText); + throw new Error( + `Qdrant request failed with ${response.status} ${response.statusText}: ${message}`, + ); + } + + return body; + } + + async upsertPoints( + collectionName: string, + points: QdrantPoint[], + wait = true, + ): Promise { + return this.request( + `/collections/${encodeURIComponent(collectionName)}/points?wait=${wait}`, + { + method: "PUT", + body: JSON.stringify({ points }), + }, + ); + } + + async createCollection( + collectionName: string, + body: Record, + wait = true, + ): Promise { + return this.request( + `/collections/${encodeURIComponent(collectionName)}?wait=${wait}`, + { + method: "PUT", + body: JSON.stringify(body), + }, + ); + } + + async getCollection(collectionName: string): Promise { + return this.request(`/collections/${encodeURIComponent(collectionName)}`); + } + + async deleteCollection(collectionName: string, wait = true): Promise { + return this.request( + `/collections/${encodeURIComponent(collectionName)}?wait=${wait}`, + { method: "DELETE" }, + ); + } + + async queryPoints( + collectionName: string, + query: Record, + ): Promise { + return this.request( + `/collections/${encodeURIComponent(collectionName)}/points/query`, + { + method: "POST", + body: JSON.stringify(query), + }, + ); + } + + async retrievePoint( + collectionName: string, + id: QdrantPointId, + withPayload: boolean | string[] | Record = true, + withVector: boolean | string[] | Record = true, + ): Promise { + const params = new URLSearchParams({ + with_payload: JSON.stringify(withPayload), + with_vector: JSON.stringify(withVector), + }); + + return this.request( + `/collections/${encodeURIComponent(collectionName)}/points/${encodeURIComponent( + String(id), + )}?${params.toString()}`, + ); + } + + async scrollPoints( + collectionName: string, + body: Record, + ): Promise { + return this.request( + `/collections/${encodeURIComponent(collectionName)}/points/scroll`, + { + method: "POST", + body: JSON.stringify(body), + }, + ); + } + + async setPayload( + collectionName: string, + points: QdrantPointId[], + payload: Record, + wait = true, + ): Promise { + return this.request( + `/collections/${encodeURIComponent(collectionName)}/points/payload?wait=${wait}`, + { + method: "POST", + body: JSON.stringify({ points, payload }), + }, + ); + } + + async deletePoints( + collectionName: string, + points: QdrantPointId[], + wait = true, + ): Promise { + return this.request( + `/collections/${encodeURIComponent(collectionName)}/points/delete?wait=${wait}`, + { + method: "POST", + body: JSON.stringify({ points }), + }, + ); + } + + private parseResponseBody(rawBody: string): any { + try { + return JSON.parse(rawBody); + } catch { + return rawBody; + } + } +} + +export class Qdrant { + QDRANT_URL: string; + QDRANT_API_KEY?: string; + + constructor(QDRANT_URL?: string, QDRANT_API_KEY?: string) { + this.QDRANT_URL = QDRANT_URL || process.env.QDRANT_URL || ""; + this.QDRANT_API_KEY = QDRANT_API_KEY || process.env.QDRANT_API_KEY; + } + + createClient() { + return new QdrantRestClient(this.QDRANT_URL, this.QDRANT_API_KEY); + } + + async createCollection({ + client, + collectionName, + vectorSize, + distance = QdrantDistanceMetric.COSINE, + vectors, + wait = true, + ...config + }: CreateCollectionArgs): Promise { + if (!vectors && !vectorSize) { + throw new Error("Qdrant vectorSize or vectors config is required"); + } + + const body = { + ...config, + vectors: vectors || { + size: vectorSize, + distance, + }, + }; + + return client.createCollection(collectionName, body, wait); + } + + async getCollection({ + client, + collectionName, + }: CollectionArgs): Promise { + const response = await client.getCollection(collectionName); + return response?.result || response; + } + + async deleteCollection({ + client, + collectionName, + wait = true, + }: CollectionArgs & { wait?: boolean }): Promise { + return client.deleteCollection(collectionName, wait); + } + + async insertVectorData({ + client, + collectionName, + points, + id, + vector, + embedding, + payload, + wait = true, + ...args + }: InsertVectorDataArgs): Promise { + const qdrantPoints = points || [ + { + id: this.requirePointId(id), + vector: vector || embedding, + payload: payload || args, + }, + ]; + + for (const point of qdrantPoints) { + if (!point.vector) { + throw new Error("Qdrant point vector or embedding is required"); + } + } + + return client.upsertPoints(collectionName, qdrantPoints, wait); + } + + async getDataFromQuery({ + client, + collectionName, + query, + vector, + embedding, + filter, + limit = 10, + withPayload = true, + withVector = false, + scoreThreshold, + using, + }: GetDataFromQueryArgs): Promise { + const queryBody: Record = { + query: query || vector || embedding, + limit, + with_payload: withPayload, + with_vector: withVector, + }; + + if (!queryBody.query) { + throw new Error("Qdrant query, vector, or embedding is required"); + } + + if (filter) queryBody.filter = filter; + if (scoreThreshold !== undefined) + queryBody.score_threshold = scoreThreshold; + if (using) queryBody.using = using; + + const response = await client.queryPoints(collectionName, queryBody); + return response?.result?.points || response?.result || response; + } + + async getData({ + client, + collectionName, + filter, + limit = 10, + offset, + withPayload = true, + withVector = false, + }: GetDataArgs): Promise { + const body: Record = { + limit, + with_payload: withPayload, + with_vector: withVector, + }; + + if (filter) body.filter = filter; + if (offset !== undefined) body.offset = offset; + + const response = await client.scrollPoints(collectionName, body); + return response?.result || response; + } + + async getDataById({ + client, + collectionName, + id, + }: { + client: QdrantRestClient; + collectionName: string; + id: QdrantPointId; + }): Promise { + const response = await client.retrievePoint(collectionName, id); + return response?.result || response; + } + + async updateById({ + client, + collectionName, + id, + updatedContent, + wait = true, + }: UpdateByIdArgs): Promise { + return client.setPayload(collectionName, [id], updatedContent, wait); + } + + async deleteById({ + client, + collectionName, + id, + wait = true, + }: DeleteByIdArgs): Promise { + return client.deletePoints(collectionName, [id], wait); + } + + private requirePointId(id?: QdrantPointId): QdrantPointId { + if (id === undefined || id === null) { + throw new Error("Qdrant point id is required"); + } + + return id; + } +} + +export class QdrantVectorClient { + wordEmbeddings: number[][]; + topK: number; + collectionName: string; + namespace?: string; + arkRequest?: any; + upperLimit: number; + client?: QdrantRestClient; + qdrantUrl?: string; + qdrantApiKey?: string; + + constructor({ + wordEmbeddings, + topK, + collectionName, + namespace, + arkRequest, + upperLimit, + client, + qdrantUrl, + qdrantApiKey, + }: QdrantVectorClientArgs) { + this.wordEmbeddings = wordEmbeddings; + this.topK = topK; + this.collectionName = collectionName; + this.namespace = namespace; + this.arkRequest = arkRequest; + this.upperLimit = upperLimit || topK; + this.client = client; + this.qdrantUrl = qdrantUrl; + this.qdrantApiKey = qdrantApiKey; + } + + async dbQuery(): Promise { + const client = + this.client || + new Qdrant(this.qdrantUrl, this.qdrantApiKey).createClient(); + const filter = this.buildFilter(); + const pointMap = new Map(); + + for (const embedding of this.wordEmbeddings) { + const response = await client.queryPoints(this.collectionName, { + query: embedding, + filter, + limit: this.topK, + with_payload: true, + with_vector: false, + }); + + const points = response?.result?.points || response?.result || []; + + for (const point of points) { + const normalized = this.normalizePoint(point); + const previous = pointMap.get(String(normalized.id)); + + if (!previous || normalized.score > previous.score) { + pointMap.set(String(normalized.id), normalized); + } + } + } + + return Array.from(pointMap.values()) + .sort((left, right) => (right.score || 0) - (left.score || 0)) + .slice(0, this.upperLimit); + } + + private buildFilter(): Record | undefined { + const filters: Record[] = []; + + if (this.namespace) { + filters.push({ + key: "namespace", + match: { value: this.namespace }, + }); + } + + if (this.arkRequest?.qdrantFilter?.must) { + filters.push(...this.arkRequest.qdrantFilter.must); + } + + if (!filters.length) { + return undefined; + } + + return { must: filters }; + } + + private normalizePoint(point: any): Record { + const payload = point.payload || {}; + + return { + id: point.id, + score: point.score, + raw_text: payload.raw_text || payload.content || payload.text || "", + document_date: payload.document_date, + metadata: payload.metadata, + namespace: payload.namespace, + filename: payload.filename, + timestamp: payload.timestamp, + payload, + }; + } +} diff --git a/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts b/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts new file mode 100644 index 000000000..cbcb07fdf --- /dev/null +++ b/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts @@ -0,0 +1,234 @@ +import { + Qdrant, + QdrantVectorClient, +} from "../../../../../dist/vector-db/src/lib/qdrant/qdrant.js"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const MOCK_QDRANT_URL = "https://mock-qdrant.local"; +const MOCK_QDRANT_API_KEY = "mock-api-key"; + +function mockFetchResponse(body: unknown) { + return Promise.resolve({ + ok: true, + status: 200, + statusText: "OK", + text: () => Promise.resolve(JSON.stringify(body)), + } as Response); +} + +describe("Qdrant", () => { + beforeEach(() => { + global.fetch = vi.fn(); + }); + + it("creates a Qdrant collection with vector configuration", async () => { + (global.fetch as any).mockImplementation(() => + mockFetchResponse({ + result: { operation_id: 1, status: "acknowledged" }, + }), + ); + + const qdrant = new Qdrant(MOCK_QDRANT_URL, MOCK_QDRANT_API_KEY); + const client = qdrant.createClient(); + + await qdrant.createCollection({ + client, + collectionName: "documents", + vectorSize: 1536, + }); + + expect(global.fetch).toHaveBeenCalledWith( + "https://mock-qdrant.local/collections/documents?wait=true", + expect.objectContaining({ + method: "PUT", + headers: expect.objectContaining({ + "api-key": MOCK_QDRANT_API_KEY, + "Content-Type": "application/json", + }), + body: JSON.stringify({ + vectors: { + size: 1536, + distance: "Cosine", + }, + }), + }), + ); + }); + + it("upserts embedding data into a Qdrant collection", async () => { + (global.fetch as any).mockImplementation(() => + mockFetchResponse({ + result: { operation_id: 1, status: "acknowledged" }, + }), + ); + + const qdrant = new Qdrant(MOCK_QDRANT_URL, MOCK_QDRANT_API_KEY); + const client = qdrant.createClient(); + + await qdrant.insertVectorData({ + client, + collectionName: "documents", + id: "doc-1", + embedding: [0.1, 0.2, 0.3], + content: "hello qdrant", + namespace: "docs", + }); + + expect(global.fetch).toHaveBeenCalledWith( + "https://mock-qdrant.local/collections/documents/points?wait=true", + expect.objectContaining({ + method: "PUT", + headers: expect.objectContaining({ + "api-key": MOCK_QDRANT_API_KEY, + "Content-Type": "application/json", + }), + body: JSON.stringify({ + points: [ + { + id: "doc-1", + vector: [0.1, 0.2, 0.3], + payload: { + content: "hello qdrant", + namespace: "docs", + }, + }, + ], + }), + }), + ); + }); + + it("queries points with filters and returns the result points", async () => { + const points = [ + { id: "doc-1", score: 0.91, payload: { raw_text: "hello" } }, + ]; + (global.fetch as any).mockImplementation(() => + mockFetchResponse({ result: { points } }), + ); + + const qdrant = new Qdrant(MOCK_QDRANT_URL, MOCK_QDRANT_API_KEY); + const client = qdrant.createClient(); + const result = await qdrant.getDataFromQuery({ + client, + collectionName: "documents", + embedding: [0.1, 0.2, 0.3], + filter: { must: [{ key: "namespace", match: { value: "docs" } }] }, + limit: 3, + }); + + expect(result).toEqual(points); + expect(global.fetch).toHaveBeenCalledWith( + "https://mock-qdrant.local/collections/documents/points/query", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ + query: [0.1, 0.2, 0.3], + limit: 3, + with_payload: true, + with_vector: false, + filter: { must: [{ key: "namespace", match: { value: "docs" } }] }, + }), + }), + ); + }); + + it("deletes a point by id", async () => { + (global.fetch as any).mockImplementation(() => + mockFetchResponse({ result: { status: "acknowledged" } }), + ); + + const qdrant = new Qdrant(MOCK_QDRANT_URL, MOCK_QDRANT_API_KEY); + const client = qdrant.createClient(); + + await qdrant.deleteById({ + client, + collectionName: "documents", + id: "doc-1", + }); + + expect(global.fetch).toHaveBeenCalledWith( + "https://mock-qdrant.local/collections/documents/points/delete?wait=true", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ points: ["doc-1"] }), + }), + ); + }); + + it("deletes a collection by name", async () => { + (global.fetch as any).mockImplementation(() => + mockFetchResponse({ result: true }), + ); + + const qdrant = new Qdrant(MOCK_QDRANT_URL, MOCK_QDRANT_API_KEY); + const client = qdrant.createClient(); + + await qdrant.deleteCollection({ + client, + collectionName: "documents", + }); + + expect(global.fetch).toHaveBeenCalledWith( + "https://mock-qdrant.local/collections/documents?wait=true", + expect.objectContaining({ + method: "DELETE", + }), + ); + }); +}); + +describe("QdrantVectorClient", () => { + it("normalizes query results for HydeSearch style retrieval", async () => { + const queryPoints = vi.fn().mockResolvedValue({ + result: { + points: [ + { + id: "doc-1", + score: 0.91, + payload: { + raw_text: "retrieved text", + namespace: "docs", + filename: "guide.md", + }, + }, + ], + }, + }); + + const vectorClient = new QdrantVectorClient({ + wordEmbeddings: [[0.1, 0.2, 0.3]], + topK: 5, + upperLimit: 2, + collectionName: "documents", + namespace: "docs", + client: { queryPoints } as any, + }); + + const result = await vectorClient.dbQuery(); + + expect(queryPoints).toHaveBeenCalledWith("documents", { + query: [0.1, 0.2, 0.3], + filter: { must: [{ key: "namespace", match: { value: "docs" } }] }, + limit: 5, + with_payload: true, + with_vector: false, + }); + expect(result).toEqual([ + { + id: "doc-1", + score: 0.91, + raw_text: "retrieved text", + document_date: undefined, + metadata: undefined, + namespace: "docs", + filename: "guide.md", + timestamp: undefined, + payload: { + raw_text: "retrieved text", + namespace: "docs", + filename: "guide.md", + }, + }, + ]); + }); +});