diff --git a/src/cli/index.ts b/src/cli/index.ts index 9a5bdf42..476d9f79 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,16 +1,15 @@ #!/usr/bin/env node import { Command } from "commander"; -import { loadConfig, saveUserConfig, type LibScopeConfig } from "../config.js"; +import { loadConfig, saveUserConfig } from "../config.js"; import { getDatabase, runMigrations, createVectorTable, closeDatabase } from "../db/index.js"; -import { createEmbeddingProvider, type EmbeddingProvider } from "../providers/index.js"; +import { createEmbeddingProvider } from "../providers/index.js"; import { indexDocument } from "../core/indexing.js"; import { searchDocuments } from "../core/search.js"; import { getDocumentRatings, listRatings } from "../core/ratings.js"; import { createTopic, listTopics } from "../core/topics.js"; import { getDocument, listDocuments, deleteDocument } from "../core/documents.js"; import { initLogger, type LogLevel } from "../logger.js"; -import type Database from "better-sqlite3"; import { readFileSync, readdirSync, statSync } from "node:fs"; import { join, extname, basename } from "node:path"; import { fetchAndConvert } from "../core/url-fetcher.js"; @@ -370,7 +369,7 @@ function setupLogging(opts: ProgramOpts): void { } /** Shared CLI initialization: loadConfig → setupLogging → getDatabase → runMigrations. */ -function initializeApp(): { config: LibScopeConfig; db: Database.Database } { +function initializeApp() { const config = loadConfig(); setupLogging(program.opts()); const db = getDatabase(config.database.path); @@ -379,11 +378,7 @@ function initializeApp(): { config: LibScopeConfig; db: Database.Database } { } /** Initialization with an embedding provider and vector table. */ -function initializeAppWithEmbedding(): { - config: LibScopeConfig; - db: Database.Database; - provider: EmbeddingProvider; -} { +function initializeAppWithEmbedding() { const { config, db } = initializeApp(); const provider = createEmbeddingProvider(config); createVectorTable(db, provider.dimensions); diff --git a/src/mcp/server.ts b/src/mcp/server.ts index d9b839d3..3ad0a297 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -13,289 +13,6 @@ import { fetchAndConvert } from "../core/url-fetcher.js"; import { initLogger, getLogger } from "../logger.js"; import { LibScopeError, ValidationError } from "../errors.js"; -const config = loadConfig(); -initLogger(config.logging.level); - -const db = getDatabase(config.database.path); -runMigrations(db); - -const provider = createEmbeddingProvider(config); -createVectorTable(db, provider.dimensions); - -const server = new McpServer({ - name: "libscope", - version: "0.1.0", -}); - -// Tool: search-docs -server.tool( - "search-docs", - "Semantic search across all indexed documentation, library docs, and topics", - { - query: z.string().describe("The search query"), - topic: z.string().optional().describe("Filter by topic ID"), - library: z.string().optional().describe("Filter by library name"), - version: z.string().optional().describe("Filter by library version"), - minRating: z.number().min(1).max(5).optional().describe("Minimum average rating filter"), - limit: z.number().min(1).max(50).optional().describe("Maximum results to return (default: 10)"), - }, - async (params) => { - try { - const results = await searchDocuments(db, provider, { - query: params.query, - topic: params.topic, - library: params.library, - version: params.version, - minRating: params.minRating, - limit: params.limit, - }); - - if (results.length === 0) { - return { - content: [{ type: "text" as const, text: "No documents found matching your query." }], - }; - } - - const text = results - .map( - (r, i) => - `## Result ${i + 1}: ${r.title}\n` + - (r.library ? `**Library:** ${r.library}${r.version ? ` v${r.version}` : ""}\n` : "") + - (r.url ? `**Source:** ${r.url}\n` : "") + - (r.avgRating ? `**Rating:** ${r.avgRating.toFixed(1)}/5\n` : "") + - `\n${r.content}\n`, - ) - .join("\n---\n\n"); - - return { content: [{ type: "text" as const, text }] }; - } catch (err) { - return errorResponse(err); - } - }, -); - -// Tool: get-document -server.tool( - "get-document", - "Retrieve a specific document by its ID", - { - documentId: z.string().describe("The document ID"), - }, - (params) => { - try { - const doc = getDocument(db, params.documentId); - const ratings = getDocumentRatings(db, params.documentId); - - const text = - `# ${doc.title}\n\n` + - `**Type:** ${doc.sourceType}\n` + - (doc.library - ? `**Library:** ${doc.library}${doc.version ? ` v${doc.version}` : ""}\n` - : "") + - (doc.url ? `**Source:** ${doc.url}\n` : "") + - `**Rating:** ${ratings.averageRating.toFixed(1)}/5 (${ratings.totalRatings} ratings)\n\n` + - doc.content; - - return { content: [{ type: "text" as const, text }] }; - } catch (err) { - return errorResponse(err); - } - }, -); - -// Tool: rate-document -server.tool( - "rate-document", - "Rate a document or suggest corrections. Use this when documentation appears outdated, incorrect, or particularly helpful.", - { - documentId: z.string().describe("The document ID to rate"), - chunkId: z.string().optional().describe("Optional specific chunk ID to rate"), - rating: z.number().min(1).max(5).describe("Rating from 1 (poor) to 5 (excellent)"), - feedback: z.string().optional().describe("Text feedback about the document"), - suggestedCorrection: z - .string() - .optional() - .describe("Suggested replacement content if the doc is wrong"), - }, - (params) => { - try { - const result = rateDocument(db, { - documentId: params.documentId, - chunkId: params.chunkId, - rating: params.rating, - feedback: params.feedback, - suggestedCorrection: params.suggestedCorrection, - ratedBy: "model", - }); - - return { - content: [ - { - type: "text" as const, - text: - `Rating submitted: ${result.rating}/5 for document ${result.documentId}` + - (result.feedback ? `\nFeedback: ${result.feedback}` : "") + - (result.suggestedCorrection ? `\nCorrection suggested.` : ""), - }, - ], - }; - } catch (err) { - return errorResponse(err); - } - }, -); - -// Tool: submit-document -server.tool( - "submit-document", - "Submit a new document for indexing into the knowledge base. You can provide content directly, or provide a URL to fetch and index automatically.", - { - title: z - .string() - .optional() - .describe("Document title (auto-detected from URL if not provided)"), - content: z - .string() - .optional() - .describe("Document content in markdown (omit if providing a URL to fetch)"), - url: z - .string() - .optional() - .describe( - "URL to fetch and index. When provided, content is fetched automatically. Title is auto-detected if not specified.", - ), - sourceType: z - .enum(["library", "topic", "manual", "model-generated"]) - .optional() - .describe("Type of document (default: 'manual', or 'library' if library name is given)"), - topic: z.string().optional().describe("Topic ID to categorize under"), - library: z.string().optional().describe("Library name (for library docs)"), - version: z.string().optional().describe("Library version"), - }, - async (params) => { - try { - let { title, content } = params; - const { url, library, version, topic } = params; - - // If URL is provided and no content, fetch it - if (url && !content) { - const fetched = await fetchAndConvert(url); - content = fetched.content; - title ??= fetched.title; - } - - if (!title) { - return errorResponse(new ValidationError("A title is required when not providing a URL")); - } - if (!content) { - return errorResponse(new ValidationError("Either content or a URL must be provided")); - } - - const sourceType = params.sourceType ?? (library ? "library" : "manual"); - - const result = await indexDocument(db, provider, { - title, - content, - sourceType, - library, - version, - topicId: topic, - url, - submittedBy: "model", - }); - - return { - content: [ - { - type: "text" as const, - text: - `Document indexed successfully.\n` + - `Title: ${title}\n` + - `ID: ${result.id}\n` + - `Chunks: ${result.chunkCount}` + - (url ? `\nSource: ${url}` : ""), - }, - ], - }; - } catch (err) { - return errorResponse(err); - } - }, -); - -// Tool: list-topics -server.tool( - "list-topics", - "List available documentation topics", - { - parentId: z.string().optional().describe("Filter by parent topic ID for subtopics"), - }, - (params) => { - try { - const topics = listTopics(db, params.parentId); - - if (topics.length === 0) { - return { - content: [{ type: "text" as const, text: "No topics found." }], - }; - } - - const text = topics - .map((t) => `- **${t.name}** (\`${t.id}\`)${t.description ? `: ${t.description}` : ""}`) - .join("\n"); - - return { content: [{ type: "text" as const, text: `## Topics\n\n${text}` }] }; - } catch (err) { - return errorResponse(err); - } - }, -); - -// Tool: list-documents -server.tool( - "list-documents", - "List all indexed documents with optional filters", - { - library: z.string().optional().describe("Filter by library name"), - topic: z.string().optional().describe("Filter by topic ID"), - sourceType: z - .enum(["library", "topic", "manual", "model-generated"]) - .optional() - .describe("Filter by source type"), - limit: z.number().min(1).max(100).optional().describe("Maximum results (default: 50)"), - }, - (params) => { - try { - const docs = listDocuments(db, { - library: params.library, - topicId: params.topic, - sourceType: params.sourceType, - limit: params.limit, - }); - - if (docs.length === 0) { - return { content: [{ type: "text" as const, text: "No documents found." }] }; - } - - const text = docs - .map( - (d) => - `- **${d.title}** (\`${d.id}\`)` + - (d.library ? ` — ${d.library}${d.version ? ` v${d.version}` : ""}` : "") + - (d.url ? ` — [source](${d.url})` : "") + - ` (${d.sourceType})`, - ) - .join("\n"); - - return { - content: [{ type: "text" as const, text: `## Documents (${docs.length})\n\n${text}` }], - }; - } catch (err) { - return errorResponse(err); - } - }, -); - function errorResponse(err: unknown): { content: Array<{ type: "text"; text: string }>; isError: true; @@ -320,19 +37,330 @@ function errorResponse(err: unknown): { // Start the server async function main(): Promise { + let config; + try { + config = loadConfig(); + } catch (err) { + console.error("Failed to load configuration:", err instanceof Error ? err.message : err); + process.exit(1); + } + + initLogger(config.logging.level); + + let db; + try { + db = getDatabase(config.database.path); + runMigrations(db); + } catch (err) { + console.error("Failed to initialize database:", err instanceof Error ? err.message : err); + process.exit(1); + } + + let provider; + try { + provider = createEmbeddingProvider(config); + createVectorTable(db, provider.dimensions); + } catch (err) { + console.error( + "Failed to initialize embedding provider:", + err instanceof Error ? err.message : err, + ); + db.close(); + process.exit(1); + } + + process.on("SIGINT", () => { + db.close(); + process.exit(0); + }); + process.on("SIGTERM", () => { + db.close(); + process.exit(0); + }); + + const server = new McpServer({ + name: "libscope", + version: "0.1.0", + }); + + // Tool: search-docs + server.tool( + "search-docs", + "Semantic search across all indexed documentation, library docs, and topics", + { + query: z.string().describe("The search query"), + topic: z.string().optional().describe("Filter by topic ID"), + library: z.string().optional().describe("Filter by library name"), + version: z.string().optional().describe("Filter by library version"), + minRating: z.number().min(1).max(5).optional().describe("Minimum average rating filter"), + limit: z + .number() + .min(1) + .max(50) + .optional() + .describe("Maximum results to return (default: 10)"), + }, + async (params) => { + try { + const results = await searchDocuments(db, provider, { + query: params.query, + topic: params.topic, + library: params.library, + version: params.version, + minRating: params.minRating, + limit: params.limit, + }); + + if (results.length === 0) { + return { + content: [{ type: "text" as const, text: "No documents found matching your query." }], + }; + } + + const text = results + .map( + (r, i) => + `## Result ${i + 1}: ${r.title}\n` + + (r.library ? `**Library:** ${r.library}${r.version ? ` v${r.version}` : ""}\n` : "") + + (r.url ? `**Source:** ${r.url}\n` : "") + + (r.avgRating ? `**Rating:** ${r.avgRating.toFixed(1)}/5\n` : "") + + `\n${r.content}\n`, + ) + .join("\n---\n\n"); + + return { content: [{ type: "text" as const, text }] }; + } catch (err) { + return errorResponse(err); + } + }, + ); + + // Tool: get-document + server.tool( + "get-document", + "Retrieve a specific document by its ID", + { + documentId: z.string().describe("The document ID"), + }, + (params) => { + try { + const doc = getDocument(db, params.documentId); + const ratings = getDocumentRatings(db, params.documentId); + + const text = + `# ${doc.title}\n\n` + + `**Type:** ${doc.sourceType}\n` + + (doc.library + ? `**Library:** ${doc.library}${doc.version ? ` v${doc.version}` : ""}\n` + : "") + + (doc.url ? `**Source:** ${doc.url}\n` : "") + + `**Rating:** ${ratings.averageRating.toFixed(1)}/5 (${ratings.totalRatings} ratings)\n\n` + + doc.content; + + return { content: [{ type: "text" as const, text }] }; + } catch (err) { + return errorResponse(err); + } + }, + ); + + // Tool: rate-document + server.tool( + "rate-document", + "Rate a document or suggest corrections. Use this when documentation appears outdated, incorrect, or particularly helpful.", + { + documentId: z.string().describe("The document ID to rate"), + chunkId: z.string().optional().describe("Optional specific chunk ID to rate"), + rating: z.number().min(1).max(5).describe("Rating from 1 (poor) to 5 (excellent)"), + feedback: z.string().optional().describe("Text feedback about the document"), + suggestedCorrection: z + .string() + .optional() + .describe("Suggested replacement content if the doc is wrong"), + }, + (params) => { + try { + const result = rateDocument(db, { + documentId: params.documentId, + chunkId: params.chunkId, + rating: params.rating, + feedback: params.feedback, + suggestedCorrection: params.suggestedCorrection, + ratedBy: "model", + }); + + return { + content: [ + { + type: "text" as const, + text: + `Rating submitted: ${result.rating}/5 for document ${result.documentId}` + + (result.feedback ? `\nFeedback: ${result.feedback}` : "") + + (result.suggestedCorrection ? `\nCorrection suggested.` : ""), + }, + ], + }; + } catch (err) { + return errorResponse(err); + } + }, + ); + + // Tool: submit-document + server.tool( + "submit-document", + "Submit a new document for indexing into the knowledge base. You can provide content directly, or provide a URL to fetch and index automatically.", + { + title: z + .string() + .optional() + .describe("Document title (auto-detected from URL if not provided)"), + content: z + .string() + .optional() + .describe("Document content in markdown (omit if providing a URL to fetch)"), + url: z + .string() + .optional() + .describe( + "URL to fetch and index. When provided, content is fetched automatically. Title is auto-detected if not specified.", + ), + sourceType: z + .enum(["library", "topic", "manual", "model-generated"]) + .optional() + .describe("Type of document (default: 'manual', or 'library' if library name is given)"), + topic: z.string().optional().describe("Topic ID to categorize under"), + library: z.string().optional().describe("Library name (for library docs)"), + version: z.string().optional().describe("Library version"), + }, + async (params) => { + try { + let { title, content } = params; + const { url, library, version, topic } = params; + + // If URL is provided and no content, fetch it + if (url && !content) { + const fetched = await fetchAndConvert(url); + content = fetched.content; + title ??= fetched.title; + } + + if (!title) { + return errorResponse(new ValidationError("A title is required when not providing a URL")); + } + if (!content) { + return errorResponse(new ValidationError("Either content or a URL must be provided")); + } + + const sourceType = params.sourceType ?? (library ? "library" : "manual"); + + const result = await indexDocument(db, provider, { + title, + content, + sourceType, + library, + version, + topicId: topic, + url, + submittedBy: "model", + }); + + return { + content: [ + { + type: "text" as const, + text: + `Document indexed successfully.\n` + + `Title: ${title}\n` + + `ID: ${result.id}\n` + + `Chunks: ${result.chunkCount}` + + (url ? `\nSource: ${url}` : ""), + }, + ], + }; + } catch (err) { + return errorResponse(err); + } + }, + ); + + // Tool: list-topics + server.tool( + "list-topics", + "List available documentation topics", + { + parentId: z.string().optional().describe("Filter by parent topic ID for subtopics"), + }, + (params) => { + try { + const topics = listTopics(db, params.parentId); + + if (topics.length === 0) { + return { + content: [{ type: "text" as const, text: "No topics found." }], + }; + } + + const text = topics + .map((t) => `- **${t.name}** (\`${t.id}\`)${t.description ? `: ${t.description}` : ""}`) + .join("\n"); + + return { content: [{ type: "text" as const, text: `## Topics\n\n${text}` }] }; + } catch (err) { + return errorResponse(err); + } + }, + ); + + // Tool: list-documents + server.tool( + "list-documents", + "List all indexed documents with optional filters", + { + library: z.string().optional().describe("Filter by library name"), + topic: z.string().optional().describe("Filter by topic ID"), + sourceType: z + .enum(["library", "topic", "manual", "model-generated"]) + .optional() + .describe("Filter by source type"), + limit: z.number().min(1).max(100).optional().describe("Maximum results (default: 50)"), + }, + (params) => { + try { + const docs = listDocuments(db, { + library: params.library, + topicId: params.topic, + sourceType: params.sourceType, + limit: params.limit, + }); + + if (docs.length === 0) { + return { content: [{ type: "text" as const, text: "No documents found." }] }; + } + + const text = docs + .map( + (d) => + `- **${d.title}** (\`${d.id}\`)` + + (d.library ? ` — ${d.library}${d.version ? ` v${d.version}` : ""}` : "") + + (d.url ? ` — [source](${d.url})` : "") + + ` (${d.sourceType})`, + ) + .join("\n"); + + return { + content: [{ type: "text" as const, text: `## Documents (${docs.length})\n\n${text}` }], + }; + } catch (err) { + return errorResponse(err); + } + }, + ); + const transport = new StdioServerTransport(); await server.connect(transport); } -process.on("SIGINT", () => { - db.close(); - process.exit(0); -}); -process.on("SIGTERM", () => { - db.close(); - process.exit(0); -}); - main().catch((err: unknown) => { console.error("Fatal error starting LibScope MCP server:", err); process.exit(1); diff --git a/src/providers/local.ts b/src/providers/local.ts index d9e32568..10e4a7b4 100644 --- a/src/providers/local.ts +++ b/src/providers/local.ts @@ -33,18 +33,35 @@ export class LocalEmbeddingProvider implements EmbeddingProvider { } async embed(text: string): Promise { + if (!text.trim()) { + throw new EmbeddingError("Input text must not be empty"); + } await this.ensureInitialized(); try { // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment const output = await (this.pipeline as any)(text, { pooling: "mean", normalize: true }); // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - return Array.from(output.data as Float32Array); + const embedding = Array.from(output.data as Float32Array); + if (embedding.length !== this.dimensions) { + throw new EmbeddingError( + `Expected embedding dimension ${this.dimensions}, got ${embedding.length}`, + ); + } + return embedding; } catch (err) { throw new EmbeddingError(`Failed to generate embedding: ${String(err)}`, err); } } async embedBatch(texts: string[]): Promise { + if (texts.length === 0) { + throw new EmbeddingError("Input texts array must not be empty"); + } + for (const t of texts) { + if (!t.trim()) { + throw new EmbeddingError("Input text must not be empty"); + } + } // Process sequentially to avoid memory issues with local model const results: number[][] = []; for (const text of texts) { diff --git a/src/providers/ollama.ts b/src/providers/ollama.ts index bbd0c6c8..92de3e99 100644 --- a/src/providers/ollama.ts +++ b/src/providers/ollama.ts @@ -18,6 +18,9 @@ export class OllamaEmbeddingProvider implements EmbeddingProvider { } async embed(text: string): Promise { + if (!text.trim()) { + throw new EmbeddingError("Input text must not be empty"); + } try { const response = await fetch(`${this.baseUrl}/api/embed`, { method: "POST", @@ -34,6 +37,11 @@ export class OllamaEmbeddingProvider implements EmbeddingProvider { if (!embedding) { throw new Error("Ollama returned empty embeddings"); } + if (embedding.length !== this.dimensions) { + throw new EmbeddingError( + `Expected embedding dimension ${this.dimensions}, got ${embedding.length}`, + ); + } return embedding; } catch (err) { if (err instanceof EmbeddingError) throw err; @@ -45,6 +53,14 @@ export class OllamaEmbeddingProvider implements EmbeddingProvider { } async embedBatch(texts: string[]): Promise { + if (texts.length === 0) { + throw new EmbeddingError("Input texts array must not be empty"); + } + for (const t of texts) { + if (!t.trim()) { + throw new EmbeddingError("Input text must not be empty"); + } + } // Ollama supports batch input try { const response = await fetch(`${this.baseUrl}/api/embed`, { @@ -63,6 +79,13 @@ export class OllamaEmbeddingProvider implements EmbeddingProvider { `Ollama returned ${data.embeddings.length} embeddings for ${texts.length} inputs`, ); } + for (const emb of data.embeddings) { + if (emb.length !== this.dimensions) { + throw new EmbeddingError( + `Expected embedding dimension ${this.dimensions}, got ${emb.length}`, + ); + } + } return data.embeddings; } catch (err) { if (err instanceof EmbeddingError) throw err; diff --git a/src/providers/openai.ts b/src/providers/openai.ts index 9f7125b0..87f72ecf 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -20,6 +20,9 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { } async embed(text: string): Promise { + if (!text.trim()) { + throw new EmbeddingError("Input text must not be empty"); + } try { const response = await this.client.embeddings.create({ model: this.model, @@ -29,6 +32,11 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { if (!embedding) { throw new Error("OpenAI returned empty embedding"); } + if (embedding.length !== this.dimensions) { + throw new EmbeddingError( + `Expected embedding dimension ${this.dimensions}, got ${embedding.length}`, + ); + } return embedding; } catch (err) { if (err instanceof EmbeddingError) throw err; @@ -40,6 +48,14 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { } async embedBatch(texts: string[]): Promise { + if (texts.length === 0) { + throw new EmbeddingError("Input texts array must not be empty"); + } + for (const t of texts) { + if (!t.trim()) { + throw new EmbeddingError("Input text must not be empty"); + } + } try { const response = await this.client.embeddings.create({ model: this.model, @@ -50,7 +66,15 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { `OpenAI returned ${response.data.length} embeddings for ${texts.length} inputs`, ); } - return response.data.map((d) => d.embedding); + const embeddings = response.data.map((d) => d.embedding); + for (const emb of embeddings) { + if (emb.length !== this.dimensions) { + throw new EmbeddingError( + `Expected embedding dimension ${this.dimensions}, got ${emb.length}`, + ); + } + } + return embeddings; } catch (err) { if (err instanceof EmbeddingError) throw err; throw new EmbeddingError(