From 6dbceb7ecde0bbb638da3150f201911efcc2598d Mon Sep 17 00:00:00 2001 From: Jung Date: Tue, 2 Jun 2026 22:27:18 -0400 Subject: [PATCH 01/11] test embedding --- .env.example | 10 + Dockerfile | 41 +- docker-compose.yml | 27 + package.json | 8 +- .../embedding-course-indexer.service.ts | 208 +++ src/embedding/embedding-course.mapper.ts | 192 +++ src/embedding/embedding-worker.client.ts | 177 +++ src/embedding/embedding.module.ts | 15 +- src/embedding/qdrant-course-index.service.ts | 245 +++ src/embedding/workers/bge-m3.worker.ts | 278 ++++ src/jobs/jobs-embeddings.ts | 40 + yarn.lock | 1312 ++++++----------- 12 files changed, 1673 insertions(+), 880 deletions(-) create mode 100644 src/embedding/embedding-course-indexer.service.ts create mode 100644 src/embedding/embedding-course.mapper.ts create mode 100644 src/embedding/embedding-worker.client.ts create mode 100644 src/embedding/qdrant-course-index.service.ts create mode 100644 src/embedding/workers/bge-m3.worker.ts create mode 100644 src/jobs/jobs-embeddings.ts diff --git a/.env.example b/.env.example index 1d231d8..fd26791 100644 --- a/.env.example +++ b/.env.example @@ -16,3 +16,13 @@ APP_GIT_SHORT_SHA=localdev # Monitoring POSTHOG_API_KEY="" POSTHOG_HOST="https://us.i.posthog.com" + +# Qdrant +QDRANT_URL=http://qdrant:6333 +QDRANT_COLLECTION=courses_bge_m3 + +# Embedding +EMBEDDING_MODEL=Xenova/bge-m3 +EMBEDDING_DTYPE=q4 +EMBEDDING_BATCH_SIZE=50 +TRANSFORMERS_CACHE_DIR=/app/.cache/transformers diff --git a/Dockerfile b/Dockerfile index 5ba9471..e9f7cbb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,23 @@ # syntax=docker/dockerfile:1 -# Base dependencies -FROM node:22.22.3-alpine3.22 AS base +FROM node:22-bookworm-slim AS base WORKDIR /app -RUN apk add --no-cache openssl + +RUN apt-get update && apt-get install -y --no-install-recommends \ + openssl \ + ca-certificates \ + tzdata \ + python3 \ + make \ + g++ \ + && rm -rf /var/lib/apt/lists/* + COPY package.json yarn.lock ./ RUN yarn install --frozen-lockfile --ignore-scripts -# Build FROM base AS build + WORKDIR /app COPY . ./ @@ -17,34 +25,45 @@ ENV NODE_ENV=production RUN yarn build -# Development FROM base AS dev + WORKDIR /app COPY prisma ./prisma + ENV NODE_ENV=development ENV APP_ENV=development +ENV TZ=America/Toronto + EXPOSE 3001 + CMD ["sh", "-c", "yarn prisma:generate && yarn start:dev"] -# Production -FROM node:22.22.3-alpine3.22 AS production +FROM node:22-bookworm-slim AS production ARG APP_GIT_SHORT_SHA + ENV APP_GIT_SHORT_SHA=${APP_GIT_SHORT_SHA} ENV APP_ENV=production ENV TZ=America/Toronto WORKDIR /app -RUN apk add --no-cache tzdata openssl && \ - cp /usr/share/zoneinfo/${TZ} /etc/localtime && \ - echo "${TZ}" > /etc/timezone + +RUN apt-get update && apt-get install -y --no-install-recommends \ + openssl \ + ca-certificates \ + tzdata \ + python3 \ + make \ + g++ \ + && rm -rf /var/lib/apt/lists/* + COPY package.json yarn.lock ./ COPY prisma ./prisma + RUN yarn install --production --frozen-lockfile --ignore-scripts COPY --from=build /app/dist ./dist -# Generate Prisma Client RUN yarn prisma:generate EXPOSE 3001 diff --git a/docker-compose.yml b/docker-compose.yml index 141cb60..0a1389e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,15 +39,26 @@ services: volumes: - .:/app - /app/node_modules + - transformers-cache:/app/.cache/transformers + mem_limit: 4g + memswap_limit: 4g environment: DATABASE_URL: ${DATABASE_URL} APP_ENV: development PORT: ${PORT:-3001} LOG_LEVELS: ${LOG_LEVELS:-log,error,warn} TZ: America/Toronto + QDRANT_URL: ${QDRANT_URL:-http://qdrant:6333} + QDRANT_COLLECTION: ${QDRANT_COLLECTION:-courses_bge_m3} + EMBEDDING_MODEL: ${EMBEDDING_MODEL:-Xenova/bge-m3} + EMBEDDING_DTYPE: ${EMBEDDING_DTYPE:-q4} + EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-50} + TRANSFORMERS_CACHE_DIR: ${TRANSFORMERS_CACHE_DIR:-/app/.cache/transformers} depends_on: db: condition: service_healthy + qdrant: + condition: service_started networks: - app-network profiles: @@ -72,9 +83,25 @@ services: timeout: 5s retries: 5 + qdrant: + image: qdrant/qdrant:latest + ports: + - "6333:6333" + - "6334:6334" + volumes: + - qdrant-data:/qdrant/storage + networks: + - app-network + profiles: + - dev + volumes: db-data: driver: local + qdrant-data: + driver: local + transformers-cache: + driver: local networks: app-network: diff --git a/package.json b/package.json index 5dbfde9..c4f5709 100644 --- a/package.json +++ b/package.json @@ -28,13 +28,15 @@ "test:all": "yarn test && yarn test:integration && yarn test:e2e", "test:e2e": "jest --config ./jest/e2e.config.mjs", "knip": "knip", - "unused": "knip --use-tsconfig-files" + "unused": "knip --use-tsconfig-files", + "job:index": "nest build && node dist/jobs/jobs-embeddings.js" }, "prisma": { "schema": "prisma/schema.prisma", "seed": "node dist/prisma/seeds/seed.js" }, "dependencies": { + "@huggingface/transformers": "^4.2.0", "@nestjs/axios": "^3.1.1", "@nestjs/common": "^10.4.10", "@nestjs/core": "^10.4.10", @@ -42,6 +44,7 @@ "@nestjs/schedule": "^4.1.1", "@nestjs/swagger": "^11.2.6", "@prisma/client": "^5.22.0", + "@qdrant/js-client-rest": "^1.18.0", "@types/unzipper": "^0.10.10", "axios": "^1.13.5", "cheerio": "^1.2.0", @@ -54,7 +57,8 @@ "posthog-node": "^5.34.2", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", - "unzipper": "^0.12.3" + "unzipper": "^0.12.3", + "uuid": "^14.0.0" }, "devDependencies": { "@nestjs/cli": "^11.0.17", diff --git a/src/embedding/embedding-course-indexer.service.ts b/src/embedding/embedding-course-indexer.service.ts new file mode 100644 index 0000000..607047c --- /dev/null +++ b/src/embedding/embedding-course-indexer.service.ts @@ -0,0 +1,208 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { EmbeddingViewDto } from './dtos/embedding-view.dto'; +import { EmbeddingService } from './embedding.service'; +import { + BGE_M3_VECTOR_SIZE, + prepareCourseEmbedding, + PreparedCourseEmbedding, +} from './embedding-course.mapper'; +import { EmbeddingWorkerClient } from './embedding-worker.client'; +import { + CourseQdrantPoint, + QdrantCourseIndexService, +} from './qdrant-course-index.service'; + +type IndexingCounters = { + indexedCount: number; + errorCount: number; +}; + +@Injectable() +export class CourseEmbeddingIndexerService { + private readonly logger = new Logger(CourseEmbeddingIndexerService.name); + + constructor( + private readonly embeddingService: EmbeddingService, + private readonly embeddingWorkerClient: EmbeddingWorkerClient, + private readonly qdrantCourseIndexService: QdrantCourseIndexService, + ) {} + + public async run(): Promise { + const startedAt = Date.now(); + const embeddingModel = process.env.EMBEDDING_MODEL ?? 'Xenova/bge-m3'; + const batchSize = parsePositiveInteger(process.env.EMBEDDING_BATCH_SIZE, 50); + + const counters: IndexingCounters = { + indexedCount: 0, + errorCount: 0, + }; + + this.logger.log( + `Starting course embedding indexation. Model: ${embeddingModel}. Batch size: ${batchSize}.`, + ); + + await this.qdrantCourseIndexService.ensureCollection(); + + const rows = await this.embeddingService.findAll(); + + this.logger.log(`Loaded ${rows.length} rows from v_courses_for_embedding.`); + + for (let offset = 0; offset < rows.length; offset += batchSize) { + const rowsBatch = rows.slice(offset, offset + batchSize); + + await this.processRowsBatch(rowsBatch, embeddingModel, counters, offset); + } + + const durationSeconds = ((Date.now() - startedAt) / 1000).toFixed(2); + + this.logger.log( + `${counters.indexedCount} cours indexés en ${durationSeconds} secondes, ${counters.errorCount} erreurs.`, + ); + } + + private async processRowsBatch( + rowsBatch: EmbeddingViewDto[], + embeddingModel: string, + counters: IndexingCounters, + offset: number, + ): Promise { + const preparedBatch: PreparedCourseEmbedding[] = []; + + for (const row of rowsBatch) { + try { + preparedBatch.push(prepareCourseEmbedding(row, embeddingModel)); + } catch (error) { + counters.errorCount += 1; + + this.logger.warn( + `Skipping ${row.code}/${row.program_id}: mapper failed: ${formatError(error)}`, + ); + } + } + + if (preparedBatch.length === 0) { + return; + } + + let points: CourseQdrantPoint[]; + + try { + points = await this.embedPreparedBatch(preparedBatch); + } catch (error) { + this.logger.warn( + `Embedding batch failed at offset ${offset}. Falling back to item-by-item. Error: ${formatError(error)}`, + ); + + await this.processPreparedBatchOneByOne(preparedBatch, counters); + return; + } + + try { + await this.qdrantCourseIndexService.upsertPoints(points); + counters.indexedCount += points.length; + } catch (error) { + this.logger.error( + `Qdrant upsert failed at offset ${offset}. Stopping job. Error: ${formatError(error)}`, + ); + + throw error; + } + } + + private async processPreparedBatchOneByOne( + preparedBatch: PreparedCourseEmbedding[], + counters: IndexingCounters, + ): Promise { + for (const prepared of preparedBatch) { + let point: CourseQdrantPoint; + + try { + const points = await this.embedPreparedBatch([prepared]); + const firstPoint = points.at(0); + + if (!firstPoint) { + throw new Error('No Qdrant point produced for single prepared course.'); + } + + point = firstPoint; + } catch (error) { + counters.errorCount += 1; + + this.logger.warn( + `Skipping ${prepared.payload.code}/${prepared.payload.program_id}: embedding failed: ${formatError(error)}`, + ); + + continue; + } + + try { + await this.qdrantCourseIndexService.upsertPoints([point]); + counters.indexedCount += 1; + } catch (error) { + this.logger.error( + `Qdrant upsert failed for ${prepared.payload.code}/${prepared.payload.program_id}. Stopping job. Error: ${formatError(error)}`, + ); + + throw error; + } + } + } + + private async embedPreparedBatch( + preparedBatch: PreparedCourseEmbedding[], + ): Promise { + const vectors = await this.embeddingWorkerClient.embed( + preparedBatch.map((prepared) => prepared.text), + ); + + if (vectors.length !== preparedBatch.length) { + throw new Error( + `Embedding count mismatch: got ${vectors.length}, expected ${preparedBatch.length}.`, + ); + } + + return preparedBatch.map((prepared, index) => { + const vector = vectors[index]; + + if (!vector) { + throw new Error( + `Missing vector for ${prepared.payload.code}/${prepared.payload.program_id}.`, + ); + } + + if (vector.length !== BGE_M3_VECTOR_SIZE) { + throw new Error( + `Invalid vector size for ${prepared.payload.code}/${prepared.payload.program_id}: got ${vector.length}, expected ${BGE_M3_VECTOR_SIZE}.`, + ); + } + + return { + id: prepared.id, + vector, + payload: prepared.payload, + }; + }); + } +} + +function parsePositiveInteger( + value: string | undefined, + fallback: number, +): number { + const parsedValue = Number(value); + + if (!Number.isInteger(parsedValue) || parsedValue <= 0) { + return fallback; + } + + return parsedValue; +} + +function formatError(error: unknown): string { + if (error instanceof Error) { + return error.stack ?? error.message; + } + + return String(error); +} diff --git a/src/embedding/embedding-course.mapper.ts b/src/embedding/embedding-course.mapper.ts new file mode 100644 index 0000000..fc35c5a --- /dev/null +++ b/src/embedding/embedding-course.mapper.ts @@ -0,0 +1,192 @@ +import { v5 as uuidv5 } from 'uuid'; + +import { EmbeddingViewDto } from './dtos/embedding-view.dto'; + +export const BGE_M3_VECTOR_SIZE = 1024; + +const QDRANT_ID_NAMESPACE = + process.env.QDRANT_ID_NAMESPACE ?? '5e7f1c4d-3d8a-45f1-87a4-9cf4de6f6b29'; + +const TYPE_LABELS: Record = { + CONCE: 'cours optionnel', + TRONC: 'tronc commun', + PROFI: 'cours de profil', +}; + +export interface CourseEmbeddingPayload { + embedding_id: string; + course_id: number; + program_id: number; + code: string; + title: string; + description: string; + cycle?: number; + program_title: string; + type?: string; + type_label?: string; + typical_session_index?: number; + unstructured_prerequisite?: string; + prerequisite_codes: string[]; + has_prerequisites: boolean; + availability: string[]; + sessions: string[]; + text: string; + embedding_model: string; + indexed_at: string; +} + +export interface PreparedCourseEmbedding { + id: string; + text: string; + payload: CourseEmbeddingPayload; +} + +export function toQdrantPointId(embeddingId: string): string { + return uuidv5(embeddingId, QDRANT_ID_NAMESPACE); +} + +export function getCourseTypeLabel(type: string | null | undefined): string | undefined { + if (!type) return undefined; + return TYPE_LABELS[type] ?? type; +} + +export function buildCourseEmbeddingText(row: EmbeddingViewDto): string { + const parts: string[] = []; + + parts.push(toSentence(`${row.code} — ${row.title}`)); + + const description = clean(row.description); + if (description) parts.push(toSentence(description)); + + const prerequisiteCodes = cleanStringArray(row.prerequisite_codes); + if (prerequisiteCodes.length > 0) { + parts.push(`Préalables : ${prerequisiteCodes.join(', ')}.`); + } + + const unstructuredPrerequisite = clean(row.unstructured_prerequisite); + if (unstructuredPrerequisite) { + parts.push(`Préalables non structurés : ${unstructuredPrerequisite}.`); + } + + const typeLabel = getCourseTypeLabel(row.type); + if (typeLabel) { + parts.push(`Type : ${typeLabel}.`); + } + + const programTitle = clean(row.program_title); + if (programTitle) { + parts.push(`Programme : ${programTitle}.`); + } + + if (row.cycle !== null && row.cycle !== undefined) { + parts.push(`Cycle : ${row.cycle}.`); + } + + if (row.typical_session_index !== null && row.typical_session_index !== undefined) { + parts.push(`Session typique : ${row.typical_session_index}.`); + } + + const availability = cleanStringArray(row.availability); + if (availability.length > 0) { + parts.push(`Disponibilité : ${availability.join(', ')}.`); + } + + const sessions = cleanStringArray(row.sessions); + if (sessions.length > 0) { + parts.push(`Sessions : ${sessions.join(', ')}.`); + } + + return normalizeWhitespace(parts.join(' ')); +} + +export function buildCourseEmbeddingPayload( + row: EmbeddingViewDto, + text: string, + embeddingModel: string, + indexedAt = new Date().toISOString(), +): CourseEmbeddingPayload { + const prerequisiteCodes = cleanStringArray(row.prerequisite_codes); + const availability = cleanStringArray(row.availability); + const sessions = cleanStringArray(row.sessions); + const typeLabel = getCourseTypeLabel(row.type); + + const payload: CourseEmbeddingPayload = { + embedding_id: row.embedding_id, + course_id: row.course_id, + program_id: row.program_id, + code: row.code, + title: row.title, + description: row.description ?? '', + program_title: row.program_title, + prerequisite_codes: prerequisiteCodes, + has_prerequisites: Boolean(row.has_prerequisites), + availability, + sessions, + text, + embedding_model: embeddingModel, + indexed_at: indexedAt, + }; + + if (row.cycle !== null && row.cycle !== undefined) { + payload.cycle = row.cycle; + } + + if (row.type) { + payload.type = row.type; + } + + if (typeLabel) { + payload.type_label = typeLabel; + } + + if (row.typical_session_index !== null && row.typical_session_index !== undefined) { + payload.typical_session_index = row.typical_session_index; + } + + const unstructuredPrerequisite = clean(row.unstructured_prerequisite); + if (unstructuredPrerequisite) { + payload.unstructured_prerequisite = unstructuredPrerequisite; + } + + return payload; +} + +export function prepareCourseEmbedding( + row: EmbeddingViewDto, + embeddingModel: string, +): PreparedCourseEmbedding { + const text = buildCourseEmbeddingText(row); + + return { + id: toQdrantPointId(row.embedding_id), + text, + payload: buildCourseEmbeddingPayload(row, text, embeddingModel), + }; +} + +function clean(value: string | null | undefined): string { + return normalizeWhitespace(value ?? '').trim(); +} + +function normalizeWhitespace(value: string): string { + return value.replace(/\s+/g, ' ').trim(); +} + +function toSentence(value: string): string { + const text = clean(value); + if (!text) return ''; + return /[.!?]$/.test(text) ? text : `${text}.`; +} + +function cleanStringArray(values: unknown): string[] { + if (!Array.isArray(values)) return []; + + return Array.from( + new Set( + values + .filter((value): value is string => typeof value === 'string') + .map((value) => clean(value)) + .filter(Boolean), + ), + ).sort((a, b) => a.localeCompare(b, 'fr-CA')); +} diff --git a/src/embedding/embedding-worker.client.ts b/src/embedding/embedding-worker.client.ts new file mode 100644 index 0000000..b02fae2 --- /dev/null +++ b/src/embedding/embedding-worker.client.ts @@ -0,0 +1,177 @@ +import * as path from 'node:path'; +import { Worker } from 'node:worker_threads'; + +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; + +type PendingRequest = { + resolve: (vectors: number[][]) => void; + reject: (error: Error) => void; +}; + +type EmbedWorkerSuccessMessage = { + id: number; + ok: true; + vectors: number[][]; +}; + +type EmbedWorkerFailureMessage = { + id: number; + ok: false; + error: string; +}; + +type EmbedWorkerMessage = EmbedWorkerSuccessMessage | EmbedWorkerFailureMessage; + +@Injectable() +export class EmbeddingWorkerClient implements OnModuleDestroy { + private readonly logger = new Logger(EmbeddingWorkerClient.name); + private readonly worker: Worker; + private readonly pending = new Map(); + private nextRequestId = 1; + + constructor() { + const workerPath = path.join(__dirname, 'workers', 'bge-m3.worker.js'); + + this.worker = new Worker(workerPath, { + env: process.env, + }); + + this.worker.on('message', (message: unknown) => { + this.handleWorkerMessage(message); + }); + + this.worker.on('error', (error: Error) => { + this.logger.error(`Embedding worker error: ${error.message}`, error.stack); + this.rejectAll(error); + }); + + this.worker.on('exit', (code: number) => { + if (code !== 0) { + const error = new Error(`Embedding worker exited with code ${code}`); + this.logger.error(error.message); + this.rejectAll(error); + } + }); + } + + public embed(texts: string[]): Promise { + if (texts.length === 0) { + return Promise.resolve([]); + } + + const id = this.nextRequestId++; + const model = process.env.EMBEDDING_MODEL ?? 'Xenova/bge-m3'; + const dtype = process.env.EMBEDDING_DTYPE ?? 'q4'; + + return new Promise((resolve, reject) => { + this.pending.set(id, { + resolve, + reject, + }); + + this.worker.postMessage({ + id, + texts, + model, + dtype, + }); + }); + } + + public async onModuleDestroy(): Promise { + await this.worker.terminate(); + } + + private handleWorkerMessage(message: unknown): void { + const parsedMessage = parseEmbedWorkerMessage(message); + + if (!parsedMessage) { + this.logger.warn('Received invalid message from embedding worker.'); + return; + } + + const pendingRequest = this.pending.get(parsedMessage.id); + + if (!pendingRequest) { + this.logger.warn( + `Received embedding worker response for unknown request id: ${parsedMessage.id}`, + ); + return; + } + + this.pending.delete(parsedMessage.id); + + if (parsedMessage.ok) { + pendingRequest.resolve(parsedMessage.vectors); + return; + } + + pendingRequest.reject(new Error(parsedMessage.error)); + } + + private rejectAll(error: Error): void { + for (const request of this.pending.values()) { + request.reject(error); + } + + this.pending.clear(); + } +} + +function parseEmbedWorkerMessage(message: unknown): EmbedWorkerMessage | null { + if (!isRecord(message)) { + return null; + } + + const id = message.id; + const ok = message.ok; + + if (typeof id !== 'number' || !Number.isInteger(id)) { + return null; + } + + if (ok === true) { + const vectors = message.vectors; + + if (!isNumberMatrix(vectors)) { + return null; + } + + return { + id, + ok: true, + vectors, + }; + } + + if (ok === false) { + const error = message.error; + + if (typeof error !== 'string') { + return null; + } + + return { + id, + ok: false, + error, + }; + } + + return null; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function isNumberMatrix(value: unknown): value is number[][] { + return Array.isArray(value) && value.every(isNumberVector); +} + +function isNumberVector(value: unknown): value is number[] { + return ( + Array.isArray(value) && + value.every((item) => typeof item === 'number' && Number.isFinite(item)) + ); +} diff --git a/src/embedding/embedding.module.ts b/src/embedding/embedding.module.ts index e4ddaef..cc14ab9 100644 --- a/src/embedding/embedding.module.ts +++ b/src/embedding/embedding.module.ts @@ -3,11 +3,22 @@ import { Module } from '@nestjs/common'; import { PrismaModule } from '../prisma/prisma.module'; import { EmbeddingController } from './embedding.controller'; import { EmbeddingService } from './embedding.service'; +import { CourseEmbeddingIndexerService } from './embedding-course-indexer.service'; +import { EmbeddingWorkerClient } from './embedding-worker.client'; +import { QdrantCourseIndexService } from './qdrant-course-index.service'; @Module({ imports: [PrismaModule], controllers: [EmbeddingController], - providers: [EmbeddingService], - exports: [EmbeddingService], + providers: [ + EmbeddingService, + CourseEmbeddingIndexerService, + EmbeddingWorkerClient, + QdrantCourseIndexService, + ], + exports: [ + EmbeddingService, + CourseEmbeddingIndexerService, + ], }) export class EmbeddingModule {} diff --git a/src/embedding/qdrant-course-index.service.ts b/src/embedding/qdrant-course-index.service.ts new file mode 100644 index 0000000..13278dc --- /dev/null +++ b/src/embedding/qdrant-course-index.service.ts @@ -0,0 +1,245 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { QdrantClient } from '@qdrant/js-client-rest'; + +import { + BGE_M3_VECTOR_SIZE, + CourseEmbeddingPayload, +} from './embedding-course.mapper'; + +export interface CourseQdrantPoint { + id: string; + vector: number[]; + payload: CourseEmbeddingPayload; +} + +type QdrantUpsertPoint = { + id: string; + vector: number[]; + payload: Record; +}; + +type VectorConfig = { + size: number; + distance: string; +}; + +@Injectable() +export class QdrantCourseIndexService { + private readonly logger = new Logger(QdrantCourseIndexService.name); + private readonly client: QdrantClient; + private readonly collectionName: string; + + constructor() { + this.collectionName = process.env.QDRANT_COLLECTION ?? 'courses_bge_m3'; + + const url = process.env.QDRANT_URL ?? 'http://localhost:6333'; + const apiKey = process.env.QDRANT_API_KEY; + + this.client = apiKey + ? new QdrantClient({ url, apiKey }) + : new QdrantClient({ url }); + } + + public async ensureCollection(): Promise { + try { + const info = await this.client.getCollection(this.collectionName); + + this.validateCollection(info); + + this.logger.log(`Qdrant collection already exists: ${this.collectionName}`); + } catch (error) { + if (!isNotFoundError(error)) { + throw error; + } + + this.logger.log(`Creating Qdrant collection: ${this.collectionName}`); + + await this.client.createCollection(this.collectionName, { + vectors: { + size: BGE_M3_VECTOR_SIZE, + distance: 'Cosine', + }, + }); + } + } + + public async upsertPoints(points: CourseQdrantPoint[]): Promise { + if (points.length === 0) { + return; + } + + const qdrantPoints = points.map(toQdrantUpsertPoint); + + await retryTransient( + () => + this.client.upsert(this.collectionName, { + wait: true, + points: qdrantPoints, + }), + 3, + 1000, + ); + } + + private validateCollection(info: unknown): void { + const vectors = extractVectorsConfig(info); + + if (!vectors) { + throw new Error( + `Cannot read vector configuration for collection ${this.collectionName}.`, + ); + } + + if (vectors.size !== BGE_M3_VECTOR_SIZE) { + throw new Error( + `Invalid Qdrant vector size for ${this.collectionName}: got ${vectors.size}, expected ${BGE_M3_VECTOR_SIZE}.`, + ); + } + + if (vectors.distance.toLowerCase() !== 'cosine') { + throw new Error( + `Invalid Qdrant distance for ${this.collectionName}: got ${vectors.distance}, expected Cosine.`, + ); + } + } +} + +function toQdrantUpsertPoint(point: CourseQdrantPoint): QdrantUpsertPoint { + return { + id: point.id, + vector: point.vector, + payload: { + ...point.payload, + }, + }; +} + +function extractVectorsConfig(info: unknown): VectorConfig | undefined { + const vectors = + getNestedProperty(info, ['config', 'params', 'vectors']) ?? + getNestedProperty(info, ['result', 'config', 'params', 'vectors']); + + return parseVectorConfig(vectors); +} + +function parseVectorConfig(value: unknown): VectorConfig | undefined { + if (!isRecord(value)) { + return undefined; + } + + const size = value.size; + const distance = value.distance; + + if (typeof size !== 'number') { + return undefined; + } + + if (typeof distance !== 'string') { + return undefined; + } + + return { + size, + distance, + }; +} + +async function retryTransient( + operation: () => Promise, + maxAttempts: number, + delayMs: number, +): Promise { + let lastError: unknown; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await operation(); + } catch (error) { + lastError = error; + + if (!isTransientError(error) || attempt === maxAttempts) { + throw error; + } + + await sleep(delayMs * attempt); + } + } + + throw lastError; +} + +function isNotFoundError(error: unknown): boolean { + return getStatusCode(error) === 404; +} + +function isTransientError(error: unknown): boolean { + const status = getStatusCode(error); + const code = getErrorCode(error); + + return ( + status === 408 || + status === 429 || + isServerErrorStatus(status) || + code === 'ECONNRESET' || + code === 'ECONNREFUSED' || + code === 'ETIMEDOUT' + ); +} + +function isServerErrorStatus(status: number | undefined): boolean { + return typeof status === 'number' && status >= 500; +} + +function getStatusCode(error: unknown): number | undefined { + if (!isRecord(error)) { + return undefined; + } + + const status = error.status; + + if (typeof status === 'number') { + return status; + } + + const statusCode = error.statusCode; + + if (typeof statusCode === 'number') { + return statusCode; + } + + const response = error.response; + + if (isRecord(response) && typeof response.status === 'number') { + return response.status; + } + + return undefined; +} + +function getErrorCode(error: unknown): string | undefined { + if (!isRecord(error)) { + return undefined; + } + + return typeof error.code === 'string' ? error.code : undefined; +} + +function getNestedProperty(value: unknown, path: string[]): unknown { + return path.reduce((currentValue, key) => { + if (!isRecord(currentValue)) { + return undefined; + } + + return currentValue[key]; + }, value); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} diff --git a/src/embedding/workers/bge-m3.worker.ts b/src/embedding/workers/bge-m3.worker.ts new file mode 100644 index 0000000..7c22817 --- /dev/null +++ b/src/embedding/workers/bge-m3.worker.ts @@ -0,0 +1,278 @@ +import { parentPort } from 'node:worker_threads'; + +type EmbedRequest = { + id: number; + texts: string[]; + model: string; + dtype?: string; +}; + +type EmbedResponse = + | { + id: number; + ok: true; + vectors: number[][]; + } + | { + id: number; + ok: false; + error: string; + }; + +type TensorLike = { + data: ArrayLike | Iterable; + dims?: number[]; +}; + +type FeatureExtractionOptions = { + pooling: 'mean'; + normalize: boolean; + truncation?: boolean; + max_length?: number; +}; + +type FeatureExtractor = ( + texts: string | string[], + options: FeatureExtractionOptions, +) => Promise | TensorLike; + +type PipelineOptions = { + dtype?: string; +}; + +type PipelineFactory = ( + task: 'feature-extraction', + model: string, + options?: PipelineOptions, +) => Promise | FeatureExtractor; + +type TransformersModule = { + pipeline: PipelineFactory; + env: { + cacheDir?: string; + }; +}; + +type ExtractorState = { + key: string; + promise: Promise; +}; + +let extractorState: ExtractorState | null = null; + +if (parentPort === null) { + throw new Error('bge-m3.worker.ts must be executed as a worker thread.'); +} + +const port = parentPort; + +port.on('message', (message: unknown) => { + void handleMessage(message); +}); + +async function handleMessage(message: unknown): Promise { + let requestId = extractRequestId(message); + + try { + const request = parseEmbedRequest(message); + requestId = request.id; + + if (request.texts.length === 0) { + const response: EmbedResponse = { + id: request.id, + ok: true, + vectors: [], + }; + + port.postMessage(response); + return; + } + + const extractor = await getExtractor(request.model, request.dtype); + + const output = await extractor(request.texts, { + pooling: 'mean', + normalize: true, + truncation: true, + max_length: 8192, + }); + + const vectors = tensorToVectors(output, request.texts.length); + + const response: EmbedResponse = { + id: request.id, + ok: true, + vectors, + }; + + port.postMessage(response); + } catch (error) { + const response: EmbedResponse = { + id: requestId, + ok: false, + error: formatError(error), + }; + + port.postMessage(response); + } +} + +async function getExtractor( + model: string, + dtype?: string, +): Promise { + const key = `${model}:${dtype ?? 'default'}`; + + if (!extractorState || extractorState.key !== key) { + extractorState = { + key, + promise: createExtractor(model, dtype), + }; + } + + return extractorState.promise; +} + +async function createExtractor( + model: string, + dtype?: string, +): Promise { + const transformersModule = (await import( + '@huggingface/transformers' + )) as unknown as TransformersModule; + + if (process.env.TRANSFORMERS_CACHE_DIR) { + transformersModule.env.cacheDir = process.env.TRANSFORMERS_CACHE_DIR; + } + + const options: PipelineOptions = {}; + + if (dtype) { + options.dtype = dtype; + } + + return transformersModule.pipeline('feature-extraction', model, options); +} + +function parseEmbedRequest(message: unknown): EmbedRequest { + if (!isRecord(message)) { + throw new Error('Invalid worker message: expected an object.'); + } + + const id = message.id; + const texts = message.texts; + const model = message.model; + const dtype = message.dtype; + + if (typeof id !== 'number' || !Number.isInteger(id)) { + throw new Error('Invalid worker message: "id" must be an integer.'); + } + + if (!Array.isArray(texts) || !texts.every((text) => typeof text === 'string')) { + throw new Error('Invalid worker message: "texts" must be a string array.'); + } + + if (typeof model !== 'string' || model.trim().length === 0) { + throw new Error('Invalid worker message: "model" must be a non-empty string.'); + } + + if (dtype !== undefined && typeof dtype !== 'string') { + throw new Error('Invalid worker message: "dtype" must be a string when provided.'); + } + + const request: EmbedRequest = { + id, + texts, + model, + }; + + if (typeof dtype === 'string' && dtype.trim().length > 0) { + request.dtype = dtype; + } + + return request; +} + +function tensorToVectors( + output: TensorLike, + expectedBatchSize: number, +): number[][] { + const data = Array.from(output.data); + + if (data.length === 0) { + throw new Error('Embedding output is empty.'); + } + + if (output.dims && output.dims.length === 2) { + const [batchSize, vectorSize] = output.dims; + + if (!Number.isInteger(batchSize) || !Number.isInteger(vectorSize)) { + throw new Error(`Invalid embedding tensor dims: ${output.dims.join(', ')}.`); + } + + if (batchSize !== expectedBatchSize) { + throw new Error( + `Unexpected embedding batch size: got ${batchSize}, expected ${expectedBatchSize}.`, + ); + } + + return splitVectorData(data, batchSize, vectorSize); + } + + if (expectedBatchSize === 1) { + return [data]; + } + + if (data.length % expectedBatchSize !== 0) { + throw new Error( + `Cannot split embedding tensor: data length ${data.length}, batch size ${expectedBatchSize}.`, + ); + } + + return splitVectorData( + data, + expectedBatchSize, + data.length / expectedBatchSize, + ); +} + +function splitVectorData( + data: number[], + batchSize: number, + vectorSize: number, +): number[][] { + const vectors: number[][] = []; + + for (let index = 0; index < batchSize; index++) { + const start = index * vectorSize; + const end = start + vectorSize; + + vectors.push(data.slice(start, end)); + } + + return vectors; +} + +function extractRequestId(message: unknown): number { + if ( + isRecord(message) && + typeof message.id === 'number' && + Number.isInteger(message.id) + ) { + return message.id; + } + + return -1; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function formatError(error: unknown): string { + if (error instanceof Error) { + return error.stack ?? error.message; + } + + return String(error); +} diff --git a/src/jobs/jobs-embeddings.ts b/src/jobs/jobs-embeddings.ts new file mode 100644 index 0000000..ff5616c --- /dev/null +++ b/src/jobs/jobs-embeddings.ts @@ -0,0 +1,40 @@ +import { INestApplicationContext, Logger } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; + +import { AppModule } from '../app.module'; +import { CourseEmbeddingIndexerService } from '../embedding/embedding-course-indexer.service'; + +const logger = new Logger('IndexCourseEmbeddingsJob'); + +async function bootstrap(): Promise { + let app: INestApplicationContext | undefined; + + try { + app = await NestFactory.createApplicationContext(AppModule, { + logger: ['log', 'warn', 'error'], + }); + + const indexer = app.get(CourseEmbeddingIndexerService, { + strict: false, + }); + + await indexer.run(); + } catch (error) { + logger.error(`Course embedding indexation failed: ${formatError(error)}`); + process.exitCode = 1; + } finally { + if (app) { + await app.close(); + } + } +} + +function formatError(error: unknown): string { + if (error instanceof Error) { + return error.stack ?? error.message; + } + + return String(error); +} + +void bootstrap(); diff --git a/yarn.lock b/yarn.lock index 4cb0981..1900943 100644 --- a/yarn.lock +++ b/yarn.lock @@ -60,20 +60,6 @@ ora "5.4.1" rxjs "7.8.1" -"@apm-js-collab/code-transformer@^0.8.0": - version "0.8.2" - resolved "https://registry.yarnpkg.com/@apm-js-collab/code-transformer/-/code-transformer-0.8.2.tgz#a3160f16d1c4df9cb81303527287ad18d00994d1" - integrity sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA== - -"@apm-js-collab/tracing-hooks@^0.3.1": - version "0.3.1" - resolved "https://registry.yarnpkg.com/@apm-js-collab/tracing-hooks/-/tracing-hooks-0.3.1.tgz#414d3a93c3a15d8be543a3fac561f7c602b6a588" - integrity sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw== - dependencies: - "@apm-js-collab/code-transformer" "^0.8.0" - debug "^4.4.1" - module-details-from-path "^1.0.4" - "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.16.7", "@babel/code-frame@^7.27.1", "@babel/code-frame@^7.28.6", "@babel/code-frame@^7.29.0": version "7.29.0" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.29.0.tgz#7cd7a59f15b3cc0dcd803038f7792712a7d0b15c" @@ -374,6 +360,13 @@ dependencies: tslib "^2.4.0" +"@emnapi/runtime@^1.7.0": + version "1.10.0" + resolved "https://registry.yarnpkg.com/@emnapi/runtime/-/runtime-1.10.0.tgz#4b260c0d3534204e98c6110b8db1a987d26ec87c" + integrity sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA== + dependencies: + tslib "^2.4.0" + "@emnapi/wasi-threads@1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@emnapi/wasi-threads/-/wasi-threads-1.2.0.tgz#a19d9772cc3d195370bf6e2a805eec40aa75e18e" @@ -413,15 +406,26 @@ resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.1.tgz#de633db3ec2ef6a3c89e2f19038063e8a122e2c2" integrity sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q== -"@fastify/otel@0.16.0": - version "0.16.0" - resolved "https://registry.yarnpkg.com/@fastify/otel/-/otel-0.16.0.tgz#e003c9b81039490af9141a7f1397de6b05baa768" - integrity sha512-2304BdM5Q/kUvQC9qJO1KZq3Zn1WWsw+WWkVmFEaj1UE2hEIiuFqrPeglQOwEtw/ftngisqfQ3v70TWMmwhhHA== +"@huggingface/jinja@^0.5.6": + version "0.5.9" + resolved "https://registry.yarnpkg.com/@huggingface/jinja/-/jinja-0.5.9.tgz#294f741fa098c2b3173788b44ca651958bffa36d" + integrity sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw== + +"@huggingface/tokenizers@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@huggingface/tokenizers/-/tokenizers-0.1.3.tgz#d1bb2b25375e550c826e4c7151d5f764a14a6a69" + integrity sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA== + +"@huggingface/transformers@^4.2.0": + version "4.2.0" + resolved "https://registry.yarnpkg.com/@huggingface/transformers/-/transformers-4.2.0.tgz#5a5342a1a148c5d297ed8384d21cee1824fab031" + integrity sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ== dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.28.0" - minimatch "^10.0.3" + "@huggingface/jinja" "^0.5.6" + "@huggingface/tokenizers" "^0.1.3" + onnxruntime-node "1.24.3" + onnxruntime-web "1.26.0-dev.20260416-b7804b056c" + sharp "^0.34.5" "@humanwhocodes/config-array@^0.13.0": version "0.13.0" @@ -442,6 +446,153 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz#4a2868d75d6d6963e423bcf90b7fd1be343409d3" integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA== +"@img/colour@^1.0.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@img/colour/-/colour-1.1.0.tgz#b0c2c2fa661adf75effd6b4964497cd80010bb9d" + integrity sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ== + +"@img/sharp-darwin-arm64@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz#6e0732dcade126b6670af7aa17060b926835ea86" + integrity sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w== + optionalDependencies: + "@img/sharp-libvips-darwin-arm64" "1.2.4" + +"@img/sharp-darwin-x64@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz#19bc1dd6eba6d5a96283498b9c9f401180ee9c7b" + integrity sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw== + optionalDependencies: + "@img/sharp-libvips-darwin-x64" "1.2.4" + +"@img/sharp-libvips-darwin-arm64@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz#2894c0cb87d42276c3889942e8e2db517a492c43" + integrity sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g== + +"@img/sharp-libvips-darwin-x64@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz#e63681f4539a94af9cd17246ed8881734386f8cc" + integrity sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg== + +"@img/sharp-libvips-linux-arm64@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz#b1b288b36864b3bce545ad91fa6dadcf1a4ad318" + integrity sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw== + +"@img/sharp-libvips-linux-arm@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz#b9260dd1ebe6f9e3bdbcbdcac9d2ac125f35852d" + integrity sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A== + +"@img/sharp-libvips-linux-ppc64@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz#4b83ecf2a829057222b38848c7b022e7b4d07aa7" + integrity sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA== + +"@img/sharp-libvips-linux-riscv64@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz#880b4678009e5a2080af192332b00b0aaf8a48de" + integrity sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA== + +"@img/sharp-libvips-linux-s390x@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz#74f343c8e10fad821b38f75ced30488939dc59ec" + integrity sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ== + +"@img/sharp-libvips-linux-x64@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz#df4183e8bd8410f7d61b66859a35edeab0a531ce" + integrity sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw== + +"@img/sharp-libvips-linuxmusl-arm64@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz#c8d6b48211df67137541007ee8d1b7b1f8ca8e06" + integrity sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw== + +"@img/sharp-libvips-linuxmusl-x64@1.2.4": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz#be11c75bee5b080cbee31a153a8779448f919f75" + integrity sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg== + +"@img/sharp-linux-arm64@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz#7aa7764ef9c001f15e610546d42fce56911790cc" + integrity sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg== + optionalDependencies: + "@img/sharp-libvips-linux-arm64" "1.2.4" + +"@img/sharp-linux-arm@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz#5fb0c3695dd12522d39c3ff7a6bc816461780a0d" + integrity sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw== + optionalDependencies: + "@img/sharp-libvips-linux-arm" "1.2.4" + +"@img/sharp-linux-ppc64@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz#9c213a81520a20caf66978f3d4c07456ff2e0813" + integrity sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA== + optionalDependencies: + "@img/sharp-libvips-linux-ppc64" "1.2.4" + +"@img/sharp-linux-riscv64@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz#cdd28182774eadbe04f62675a16aabbccb833f60" + integrity sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw== + optionalDependencies: + "@img/sharp-libvips-linux-riscv64" "1.2.4" + +"@img/sharp-linux-s390x@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz#93eac601b9f329bb27917e0e19098c722d630df7" + integrity sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg== + optionalDependencies: + "@img/sharp-libvips-linux-s390x" "1.2.4" + +"@img/sharp-linux-x64@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz#55abc7cd754ffca5002b6c2b719abdfc846819a8" + integrity sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ== + optionalDependencies: + "@img/sharp-libvips-linux-x64" "1.2.4" + +"@img/sharp-linuxmusl-arm64@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz#d6515ee971bb62f73001a4829b9d865a11b77086" + integrity sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg== + optionalDependencies: + "@img/sharp-libvips-linuxmusl-arm64" "1.2.4" + +"@img/sharp-linuxmusl-x64@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz#d97978aec7c5212f999714f2f5b736457e12ee9f" + integrity sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q== + optionalDependencies: + "@img/sharp-libvips-linuxmusl-x64" "1.2.4" + +"@img/sharp-wasm32@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz#2f15803aa626f8c59dd7c9d0bbc766f1ab52cfa0" + integrity sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw== + dependencies: + "@emnapi/runtime" "^1.7.0" + +"@img/sharp-win32-arm64@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz#3706e9e3ac35fddfc1c87f94e849f1b75307ce0a" + integrity sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g== + +"@img/sharp-win32-ia32@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz#0b71166599b049e032f085fb9263e02f4e4788de" + integrity sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg== + +"@img/sharp-win32-x64@0.34.5": + version "0.34.5" + resolved "https://registry.yarnpkg.com/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz#a81ffb00e69267cd0a1d626eaedb8a8430b2b2f8" + integrity sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw== + "@inquirer/ansi@^1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@inquirer/ansi/-/ansi-1.0.2.tgz#674a4c4d81ad460695cb2a1fc69d78cd187f337e" @@ -1118,530 +1269,6 @@ consola "^2.15.0" node-fetch "^2.6.1" -"@opentelemetry/api-logs@0.207.0": - version "0.207.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/api-logs/-/api-logs-0.207.0.tgz#ae991c51eedda55af037a3e6fc1ebdb12b289f49" - integrity sha512-lAb0jQRVyleQQGiuuvCOTDVspc14nx6XJjP4FspJ1sNARo3Regq4ZZbrc3rN4b1TYSuUCvgH+UXUPug4SLOqEQ== - dependencies: - "@opentelemetry/api" "^1.3.0" - -"@opentelemetry/api-logs@0.208.0": - version "0.208.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz#56d3891010a1fa1cf600ba8899ed61b43ace511c" - integrity sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg== - dependencies: - "@opentelemetry/api" "^1.3.0" - -"@opentelemetry/api-logs@0.211.0": - version "0.211.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/api-logs/-/api-logs-0.211.0.tgz#32d9ed98939956a84d4e2ff5e01598cb9d28d744" - integrity sha512-swFdZq8MCdmdR22jTVGQDhwqDzcI4M10nhjXkLr1EsIzXgZBqm4ZlmmcWsg3TSNf+3mzgOiqveXmBLZuDi2Lgg== - dependencies: - "@opentelemetry/api" "^1.3.0" - -"@opentelemetry/api@^1.3.0", "@opentelemetry/api@^1.9.0": - version "1.9.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" - integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== - -"@opentelemetry/context-async-hooks@^2.2.0": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/context-async-hooks/-/context-async-hooks-2.2.0.tgz#5465f6fad6350f52cf9d95a92907a3a464d50644" - integrity sha512-qRkLWiUEZNAmYapZ7KGS5C4OmBLcP/H2foXeOEaowYCR0wi89fHejrfYfbuLVCMLp/dWZXKvQusdbUEZjERfwQ== - -"@opentelemetry/context-async-hooks@^2.5.1": - version "2.6.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/context-async-hooks/-/context-async-hooks-2.6.0.tgz#6c824e900630b378233c1a78ca7f0dc5a3b460b2" - integrity sha512-L8UyDwqpTcbkIK5cgwDRDYDoEhQoj8wp8BwsO19w3LB1Z41yEQm2VJyNfAi9DrLP/YTqXqWpKHyZfR9/tFYo1Q== - -"@opentelemetry/core@2.2.0", "@opentelemetry/core@^2.0.0", "@opentelemetry/core@^2.2.0": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-2.2.0.tgz#2f857d7790ff160a97db3820889b5f4cade6eaee" - integrity sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw== - dependencies: - "@opentelemetry/semantic-conventions" "^1.29.0" - -"@opentelemetry/core@2.5.0": - version "2.5.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-2.5.0.tgz#3b2ac6cf471ed9a85eea836048a4de77a2e549d3" - integrity sha512-ka4H8OM6+DlUhSAZpONu0cPBtPPTQKxbxVzC4CzVx5+K4JnroJVBtDzLAMx4/3CDTJXRvVFhpFjtl4SaiTNoyQ== - dependencies: - "@opentelemetry/semantic-conventions" "^1.29.0" - -"@opentelemetry/core@2.6.0", "@opentelemetry/core@^2.5.1": - version "2.6.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-2.6.0.tgz#719c829ed98bd7af808a2d2c83374df1fd1f3c66" - integrity sha512-HLM1v2cbZ4TgYN6KEOj+Bbj8rAKriOdkF9Ed3tG25FoprSiQl7kYc+RRT6fUZGOvx0oMi5U67GoFdT+XUn8zEg== - dependencies: - "@opentelemetry/semantic-conventions" "^1.29.0" - -"@opentelemetry/instrumentation-amqplib@0.55.0": - version "0.55.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.55.0.tgz#4d1afc47e7690693efa690ed06fbda3acc585a2f" - integrity sha512-5ULoU8p+tWcQw5PDYZn8rySptGSLZHNX/7srqo2TioPnAAcvTy6sQFQXsNPrAnyRRtYGMetXVyZUy5OaX1+IfA== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.208.0" - -"@opentelemetry/instrumentation-amqplib@0.58.0": - version "0.58.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.58.0.tgz#e3dc86ebfa7d72fe861a63b1c24a062faeb64a8c" - integrity sha512-fjpQtH18J6GxzUZ+cwNhWUpb71u+DzT7rFkg5pLssDGaEber91Y2WNGdpVpwGivfEluMlNMZumzjEqfg8DeKXQ== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.33.0" - -"@opentelemetry/instrumentation-connect@0.52.0": - version "0.52.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.52.0.tgz#60cde91c548e9da4528ae47fe69af41d05eeb485" - integrity sha512-GXPxfNB5szMbV3I9b7kNWSmQBoBzw7MT0ui6iU/p+NIzVx3a06Ri2cdQO7tG9EKb4aKSLmfX9Cw5cKxXqX6Ohg== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.27.0" - "@types/connect" "3.4.38" - -"@opentelemetry/instrumentation-connect@0.54.0": - version "0.54.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.54.0.tgz#87312850844b6c57976d00bd3256d55650543772" - integrity sha512-43RmbhUhqt3uuPnc16cX6NsxEASEtn8z/cYV8Zpt6EP4p2h9s4FNuJ4Q9BbEQ2C0YlCCB/2crO1ruVz/hWt8fA== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.27.0" - "@types/connect" "3.4.38" - -"@opentelemetry/instrumentation-dataloader@0.26.0": - version "0.26.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.26.0.tgz#d10d22854ee8eac4471c82b8862b177a40f3bf8e" - integrity sha512-P2BgnFfTOarZ5OKPmYfbXfDFjQ4P9WkQ1Jji7yH5/WwB6Wm/knynAoA1rxbjWcDlYupFkyT0M1j6XLzDzy0aCA== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - -"@opentelemetry/instrumentation-dataloader@0.28.0": - version "0.28.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.28.0.tgz#b857bb038e4a2a3b7278f3da89a1e210bb15339e" - integrity sha512-ExXGBp0sUj8yhm6Znhf9jmuOaGDsYfDES3gswZnKr4MCqoBWQdEFn6EoDdt5u+RdbxQER+t43FoUihEfTSqsjA== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - -"@opentelemetry/instrumentation-express@0.57.0": - version "0.57.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-express/-/instrumentation-express-0.57.0.tgz#7a2a7e90a84ad6c109f42c15acabdc7f6646a412" - integrity sha512-HAdx/o58+8tSR5iW+ru4PHnEejyKrAy9fYFhlEI81o10nYxrGahnMAHWiSjhDC7UQSY3I4gjcPgSKQz4rm/asg== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.27.0" - -"@opentelemetry/instrumentation-express@0.59.0": - version "0.59.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-express/-/instrumentation-express-0.59.0.tgz#c2ac7dcb4f9904926518408cdf4efb046e724382" - integrity sha512-pMKV/qnHiW/Q6pmbKkxt0eIhuNEtvJ7sUAyee192HErlr+a1Jx+FZ3WjfmzhQL1geewyGEiPGkmjjAgNY8TgDA== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.27.0" - -"@opentelemetry/instrumentation-fs@0.28.0": - version "0.28.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.28.0.tgz#6387fb7c19213afa31a2eb1b646d6356b95176bf" - integrity sha512-FFvg8fq53RRXVBRHZViP+EMxMR03tqzEGpuq55lHNbVPyFklSVfQBN50syPhK5UYYwaStx0eyCtHtbRreusc5g== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.208.0" - -"@opentelemetry/instrumentation-fs@0.30.0": - version "0.30.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.30.0.tgz#5e28edde0591dc4ffa471a86a68f91e737fe31fb" - integrity sha512-n3Cf8YhG7reaj5dncGlRIU7iT40bxPOjsBEA5Bc1a1g6e9Qvb+JFJ7SEiMlPbUw4PBmxE3h40ltE8LZ3zVt6OA== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.211.0" - -"@opentelemetry/instrumentation-generic-pool@0.52.0": - version "0.52.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.52.0.tgz#12b57774ca3664edb9649687674320955e025906" - integrity sha512-ISkNcv5CM2IwvsMVL31Tl61/p2Zm2I2NAsYq5SSBgOsOndT0TjnptjufYVScCnD5ZLD1tpl4T3GEYULLYOdIdQ== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - -"@opentelemetry/instrumentation-generic-pool@0.54.0": - version "0.54.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.54.0.tgz#9f3ad0cedbfe5011efe4ebdc76c85a73a0b967a6" - integrity sha512-8dXMBzzmEdXfH/wjuRvcJnUFeWzZHUnExkmFJ2uPfa31wmpyBCMxO59yr8f/OXXgSogNgi/uPo9KW9H7LMIZ+g== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - -"@opentelemetry/instrumentation-graphql@0.56.0": - version "0.56.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.56.0.tgz#77464dec65efe5aa53d8787d0760534cf2e2a88f" - integrity sha512-IPvNk8AFoVzTAM0Z399t34VDmGDgwT6rIqCUug8P9oAGerl2/PEIYMPOl/rerPGu+q8gSWdmbFSjgg7PDVRd3Q== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - -"@opentelemetry/instrumentation-graphql@0.58.0": - version "0.58.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.58.0.tgz#3ca294ba410e04c920dc82ab4caa23ec1c2e1a2e" - integrity sha512-+yWVVY7fxOs3j2RixCbvue8vUuJ1inHxN2q1sduqDB0Wnkr4vOzVKRYl/Zy7B31/dcPS72D9lo/kltdOTBM3bQ== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - -"@opentelemetry/instrumentation-hapi@0.55.0": - version "0.55.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.55.0.tgz#a687b9bddfcc484f2cc85f022c123f83c19883a4" - integrity sha512-prqAkRf9e4eEpy4G3UcR32prKE8NLNlA90TdEU1UsghOTg0jUvs40Jz8LQWFEs5NbLbXHYGzB4CYVkCI8eWEVQ== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.27.0" - -"@opentelemetry/instrumentation-hapi@0.57.0": - version "0.57.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.57.0.tgz#27b3a44a51444af3100a321f2e40623e89e5bb75" - integrity sha512-Os4THbvls8cTQTVA8ApLfZZztuuqGEeqog0XUnyRW7QVF0d/vOVBEcBCk1pazPFmllXGEdNbbat8e2fYIWdFbw== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.27.0" - -"@opentelemetry/instrumentation-http@0.208.0": - version "0.208.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-http/-/instrumentation-http-0.208.0.tgz#64fcc02bfbc80eb3bbb91cd3c7e0e24c695f2bef" - integrity sha512-rhmK46DRWEbQQB77RxmVXGyjs6783crXCnFjYQj+4tDH/Kpv9Rbg3h2kaNyp5Vz2emF1f9HOQQvZoHzwMWOFZQ== - dependencies: - "@opentelemetry/core" "2.2.0" - "@opentelemetry/instrumentation" "0.208.0" - "@opentelemetry/semantic-conventions" "^1.29.0" - forwarded-parse "2.1.2" - -"@opentelemetry/instrumentation-http@0.211.0": - version "0.211.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-http/-/instrumentation-http-0.211.0.tgz#2f12f83f0c21d37917fd9710fb5b755f28858cf6" - integrity sha512-n0IaQ6oVll9PP84SjbOCwDjaJasWRHi6BLsbMLiT6tNj7QbVOkuA5sk/EfZczwI0j5uTKl1awQPivO/ldVtsqA== - dependencies: - "@opentelemetry/core" "2.5.0" - "@opentelemetry/instrumentation" "0.211.0" - "@opentelemetry/semantic-conventions" "^1.29.0" - forwarded-parse "2.1.2" - -"@opentelemetry/instrumentation-ioredis@0.56.0": - version "0.56.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.56.0.tgz#9b89cca6c3e440ae9e896f81dc6d2ab1dfee2581" - integrity sha512-XSWeqsd3rKSsT3WBz/JKJDcZD4QYElZEa0xVdX8f9dh4h4QgXhKRLorVsVkK3uXFbC2sZKAS2Ds+YolGwD83Dg== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/redis-common" "^0.38.2" - -"@opentelemetry/instrumentation-ioredis@0.59.0": - version "0.59.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.59.0.tgz#530d06aa67b73ea732414557adebe1dde7de430f" - integrity sha512-875UxzBHWkW+P4Y45SoFM2AR8f8TzBMD8eO7QXGCyFSCUMP5s9vtt/BS8b/r2kqLyaRPK6mLbdnZznK3XzQWvw== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/redis-common" "^0.38.2" - "@opentelemetry/semantic-conventions" "^1.33.0" - -"@opentelemetry/instrumentation-kafkajs@0.18.0": - version "0.18.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.18.0.tgz#b836e6883afb7ca6df9fd3b6e024408dcc5e584b" - integrity sha512-KCL/1HnZN5zkUMgPyOxfGjLjbXjpd4odDToy+7c+UsthIzVLFf99LnfIBE8YSSrYE4+uS7OwJMhvhg3tWjqMBg== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.30.0" - -"@opentelemetry/instrumentation-kafkajs@0.20.0": - version "0.20.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.20.0.tgz#521db06d10d39f42e842ce336e5c1e48b3da2956" - integrity sha512-yJXOuWZROzj7WmYCUiyT27tIfqBrVtl1/TwVbQyWPz7rL0r1Lu7kWjD0PiVeTCIL6CrIZ7M2s8eBxsTAOxbNvw== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.30.0" - -"@opentelemetry/instrumentation-knex@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.53.0.tgz#c2158c9259ff6789f6c2849bfd3c319edc0fcdf6" - integrity sha512-xngn5cH2mVXFmiT1XfQ1aHqq1m4xb5wvU6j9lSgLlihJ1bXzsO543cpDwjrZm2nMrlpddBf55w8+bfS4qDh60g== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.33.1" - -"@opentelemetry/instrumentation-knex@0.55.0": - version "0.55.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.55.0.tgz#fefc17d854a107d99ab0dbc8933d5897efce1abd" - integrity sha512-FtTL5DUx5Ka/8VK6P1VwnlUXPa3nrb7REvm5ddLUIeXXq4tb9pKd+/ThB1xM/IjefkRSN3z8a5t7epYw1JLBJQ== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.33.1" - -"@opentelemetry/instrumentation-koa@0.57.0": - version "0.57.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.57.0.tgz#9a9edcde7de472f7f03904c00d31d87c6ee0ee42" - integrity sha512-3JS8PU/D5E3q295mwloU2v7c7/m+DyCqdu62BIzWt+3u9utjxC9QS7v6WmUNuoDN3RM+Q+D1Gpj13ERo+m7CGg== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.36.0" - -"@opentelemetry/instrumentation-koa@0.59.0": - version "0.59.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.59.0.tgz#7df8850fa193a8f590e3fbcab00016e25db27041" - integrity sha512-K9o2skADV20Skdu5tG2bogPKiSpXh4KxfLjz6FuqIVvDJNibwSdu5UvyyBzRVp1rQMV6UmoIk6d3PyPtJbaGSg== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.36.0" - -"@opentelemetry/instrumentation-lru-memoizer@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.53.0.tgz#936c05263b719ee66999a9240b82fded044ebd2c" - integrity sha512-LDwWz5cPkWWr0HBIuZUjslyvijljTwmwiItpMTHujaULZCxcYE9eU44Qf/pbVC8TulT0IhZi+RoGvHKXvNhysw== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - -"@opentelemetry/instrumentation-lru-memoizer@0.55.0": - version "0.55.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.55.0.tgz#776d5f10178adfbda7286b4f31adde8bb518d55a" - integrity sha512-FDBfT7yDGcspN0Cxbu/k8A0Pp1Jhv/m7BMTzXGpcb8ENl3tDj/51U65R5lWzUH15GaZA15HQ5A5wtafklxYj7g== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - -"@opentelemetry/instrumentation-mongodb@0.61.0": - version "0.61.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.61.0.tgz#4db130d537d630c3089115d2d214d29bcfb49f41" - integrity sha512-OV3i2DSoY5M/pmLk+68xr5RvkHU8DRB3DKMzYJdwDdcxeLs62tLbkmRyqJZsYf3Ht7j11rq35pHOWLuLzXL7pQ== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - -"@opentelemetry/instrumentation-mongodb@0.64.0": - version "0.64.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.64.0.tgz#0027c13fdd7506eb1f618998245edd244cc23cc7" - integrity sha512-pFlCJjweTqVp7B220mCvCld1c1eYKZfQt1p3bxSbcReypKLJTwat+wbL2YZoX9jPi5X2O8tTKFEOahO5ehQGsA== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.33.0" - -"@opentelemetry/instrumentation-mongoose@0.55.0": - version "0.55.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.55.0.tgz#e6851aba996b23b9709143c2b640084e92313dea" - integrity sha512-5afj0HfF6aM6Nlqgu6/PPHFk8QBfIe3+zF9FGpX76jWPS0/dujoEYn82/XcLSaW5LPUDW8sni+YeK0vTBNri+w== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.208.0" - -"@opentelemetry/instrumentation-mongoose@0.57.0": - version "0.57.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.57.0.tgz#2ce3f3bbf66a255958c3a112a92079898d69f624" - integrity sha512-MthiekrU/BAJc5JZoZeJmo0OTX6ycJMiP6sMOSRTkvz5BrPMYDqaJos0OgsLPL/HpcgHP7eo5pduETuLguOqcg== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.33.0" - -"@opentelemetry/instrumentation-mysql2@0.55.0": - version "0.55.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.55.0.tgz#a0957590aa8d402d1debd10e42d7b5da359164ec" - integrity sha512-0cs8whQG55aIi20gnK8B7cco6OK6N+enNhW0p5284MvqJ5EPi+I1YlWsWXgzv/V2HFirEejkvKiI4Iw21OqDWg== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.33.0" - "@opentelemetry/sql-common" "^0.41.2" - -"@opentelemetry/instrumentation-mysql2@0.57.0": - version "0.57.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.57.0.tgz#928eda47c6f4ab193d3363fcab01d81a70adc46b" - integrity sha512-nHSrYAwF7+aV1E1V9yOOP9TchOodb6fjn4gFvdrdQXiRE7cMuffyLLbCZlZd4wsspBzVwOXX8mpURdRserAhNA== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.33.0" - "@opentelemetry/sql-common" "^0.41.2" - -"@opentelemetry/instrumentation-mysql@0.54.0": - version "0.54.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.54.0.tgz#6181ae097a2b5501049c518fe90393e1f136341d" - integrity sha512-bqC1YhnwAeWmRzy1/Xf9cDqxNG2d/JDkaxnqF5N6iJKN1eVWI+vg7NfDkf52/Nggp3tl1jcC++ptC61BD6738A== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - "@types/mysql" "2.15.27" - -"@opentelemetry/instrumentation-mysql@0.57.0": - version "0.57.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.57.0.tgz#74d42a1c6d20aee93996f8b6f6b7b69469748754" - integrity sha512-HFS/+FcZ6Q7piM7Il7CzQ4VHhJvGMJWjx7EgCkP5AnTntSN5rb5Xi3TkYJHBKeR27A0QqPlGaCITi93fUDs++Q== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.33.0" - "@types/mysql" "2.15.27" - -"@opentelemetry/instrumentation-nestjs-core@0.55.0": - version "0.55.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-nestjs-core/-/instrumentation-nestjs-core-0.55.0.tgz#820391be7ed2b699b49fef55b78619832ac0e0ae" - integrity sha512-JFLNhbbEGnnQrMKOYoXx0nNk5N9cPeghu4xP/oup40a7VaSeYruyOiFbg9nkbS4ZQiI8aMuRqUT3Mo4lQjKEKg== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.30.0" - -"@opentelemetry/instrumentation-pg@0.61.0": - version "0.61.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.61.0.tgz#c755d00dba640e229fe50f817423dcf3376957ab" - integrity sha512-UeV7KeTnRSM7ECHa3YscoklhUtTQPs6V6qYpG283AB7xpnPGCUCUfECFT9jFg6/iZOQTt3FHkB1wGTJCNZEvPw== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.34.0" - "@opentelemetry/sql-common" "^0.41.2" - "@types/pg" "8.15.6" - "@types/pg-pool" "2.0.6" - -"@opentelemetry/instrumentation-pg@0.63.0": - version "0.63.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.63.0.tgz#852ca5519d756c613bb9f3153a5e70c2b805e5cf" - integrity sha512-dKm/ODNN3GgIQVlbD6ZPxwRc3kleLf95hrRWXM+l8wYo+vSeXtEpQPT53afEf6VFWDVzJK55VGn8KMLtSve/cg== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.34.0" - "@opentelemetry/sql-common" "^0.41.2" - "@types/pg" "8.15.6" - "@types/pg-pool" "2.0.7" - -"@opentelemetry/instrumentation-redis@0.57.0": - version "0.57.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.57.0.tgz#c6996eb8ace9cb16cf5be3db3a6b0fb599f47fab" - integrity sha512-bCxTHQFXzrU3eU1LZnOZQ3s5LURxQPDlU3/upBzlWY77qOI1GZuGofazj3jtzjctMJeBEJhNwIFEgRPBX1kp/Q== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/redis-common" "^0.38.2" - "@opentelemetry/semantic-conventions" "^1.27.0" - -"@opentelemetry/instrumentation-redis@0.59.0": - version "0.59.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.59.0.tgz#44c1bd7852cdadbe77c1bdfa94185528012558cf" - integrity sha512-JKv1KDDYA2chJ1PC3pLP+Q9ISMQk6h5ey+99mB57/ARk0vQPGZTTEb4h4/JlcEpy7AYT8HIGv7X6l+br03Neeg== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/redis-common" "^0.38.2" - "@opentelemetry/semantic-conventions" "^1.27.0" - -"@opentelemetry/instrumentation-tedious@0.27.0": - version "0.27.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.27.0.tgz#f4ba662fd17edde80f1b14d0dc4c42c7fa4a3139" - integrity sha512-jRtyUJNZppPBjPae4ZjIQ2eqJbcRaRfJkr0lQLHFmOU/no5A6e9s1OHLd5XZyZoBJ/ymngZitanyRRA5cniseA== - dependencies: - "@opentelemetry/instrumentation" "^0.208.0" - "@types/tedious" "^4.0.14" - -"@opentelemetry/instrumentation-tedious@0.30.0": - version "0.30.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.30.0.tgz#4a8906b5322c4add4132e6e086c23e17bc23626b" - integrity sha512-bZy9Q8jFdycKQ2pAsyuHYUHNmCxCOGdG6eg1Mn75RvQDccq832sU5OWOBnc12EFUELI6icJkhR7+EQKMBam2GA== - dependencies: - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.33.0" - "@types/tedious" "^4.0.14" - -"@opentelemetry/instrumentation-undici@0.19.0": - version "0.19.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.19.0.tgz#a9db59a7630261269239d17d2990d406e2ecddf8" - integrity sha512-Pst/RhR61A2OoZQZkn6OLpdVpXp6qn3Y92wXa6umfJe9rV640r4bc6SWvw4pPN6DiQqPu2c8gnSSZPDtC6JlpQ== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/semantic-conventions" "^1.24.0" - -"@opentelemetry/instrumentation-undici@0.21.0": - version "0.21.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.21.0.tgz#dcb43a364c39e78217946aeb7aa09156e55f4c6c" - integrity sha512-gok0LPUOTz2FQ1YJMZzaHcOzDFyT64XJ8M9rNkugk923/p6lDGms/cRW1cqgqp6N6qcd6K6YdVHwPEhnx9BWbw== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/semantic-conventions" "^1.24.0" - -"@opentelemetry/instrumentation@0.208.0", "@opentelemetry/instrumentation@>=0.52.0 <1", "@opentelemetry/instrumentation@^0.208.0": - version "0.208.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.208.0.tgz#d764f8e4329dad50804e2e98f010170c14c4ce8f" - integrity sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA== - dependencies: - "@opentelemetry/api-logs" "0.208.0" - import-in-the-middle "^2.0.0" - require-in-the-middle "^8.0.0" - -"@opentelemetry/instrumentation@0.211.0", "@opentelemetry/instrumentation@^0.211.0": - version "0.211.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.211.0.tgz#d45e20eafa75b5d3e8a9745a6205332893c55f37" - integrity sha512-h0nrZEC/zvI994nhg7EgQ8URIHt0uDTwN90r3qQUdZORS455bbx+YebnGeEuFghUT0HlJSrLF4iHw67f+odY+Q== - dependencies: - "@opentelemetry/api-logs" "0.211.0" - import-in-the-middle "^2.0.0" - require-in-the-middle "^8.0.0" - -"@opentelemetry/instrumentation@^0.207.0": - version "0.207.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.207.0.tgz#1a5a921c04f171ff28096fa320af713f3c87ec14" - integrity sha512-y6eeli9+TLKnznrR8AZlQMSJT7wILpXH+6EYq5Vf/4Ao+huI7EedxQHwRgVUOMLFbe7VFDvHJrX9/f4lcwnJsA== - dependencies: - "@opentelemetry/api-logs" "0.207.0" - import-in-the-middle "^2.0.0" - require-in-the-middle "^8.0.0" - -"@opentelemetry/redis-common@^0.38.2": - version "0.38.2" - resolved "https://registry.yarnpkg.com/@opentelemetry/redis-common/-/redis-common-0.38.2.tgz#cefa4f3e79db1cd54f19e233b7dfb56621143955" - integrity sha512-1BCcU93iwSRZvDAgwUxC/DV4T/406SkMfxGqu5ojc3AvNI+I9GhV7v0J1HljsczuuhcnFLYqD5VmwVXfCGHzxA== - -"@opentelemetry/resources@2.2.0", "@opentelemetry/resources@^2.2.0": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-2.2.0.tgz#b90a950ad98551295b76ea8a0e7efe45a179badf" - integrity sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A== - dependencies: - "@opentelemetry/core" "2.2.0" - "@opentelemetry/semantic-conventions" "^1.29.0" - -"@opentelemetry/resources@2.6.0", "@opentelemetry/resources@^2.5.1": - version "2.6.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-2.6.0.tgz#1a945dbb8986043d8b593c358d5d8e3de6becf5a" - integrity sha512-D4y/+OGe3JSuYUCBxtH5T9DSAWNcvCb/nQWIga8HNtXTVPQn59j0nTBAgaAXxUVBDl40mG3Tc76b46wPlZaiJQ== - dependencies: - "@opentelemetry/core" "2.6.0" - "@opentelemetry/semantic-conventions" "^1.29.0" - -"@opentelemetry/sdk-trace-base@^2.2.0": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.2.0.tgz#ddef9a0afd01a623d8625a3529f2137b05e67d0b" - integrity sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw== - dependencies: - "@opentelemetry/core" "2.2.0" - "@opentelemetry/resources" "2.2.0" - "@opentelemetry/semantic-conventions" "^1.29.0" - -"@opentelemetry/sdk-trace-base@^2.5.1": - version "2.6.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.6.0.tgz#d7e752a0906f2bcae3c1261e224aef3e3b3746f9" - integrity sha512-g/OZVkqlxllgFM7qMKqbPV9c1DUPhQ7d4n3pgZFcrnrNft9eJXZM2TNHTPYREJBrtNdRytYyvwjgL5geDKl3EQ== - dependencies: - "@opentelemetry/core" "2.6.0" - "@opentelemetry/resources" "2.6.0" - "@opentelemetry/semantic-conventions" "^1.29.0" - -"@opentelemetry/semantic-conventions@^1.24.0", "@opentelemetry/semantic-conventions@^1.27.0", "@opentelemetry/semantic-conventions@^1.29.0", "@opentelemetry/semantic-conventions@^1.30.0", "@opentelemetry/semantic-conventions@^1.33.0", "@opentelemetry/semantic-conventions@^1.33.1", "@opentelemetry/semantic-conventions@^1.34.0", "@opentelemetry/semantic-conventions@^1.36.0", "@opentelemetry/semantic-conventions@^1.37.0": - version "1.38.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.38.0.tgz#8b5f415395a7ddb7c8e0c7932171deb9278df1a3" - integrity sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg== - -"@opentelemetry/semantic-conventions@^1.28.0", "@opentelemetry/semantic-conventions@^1.39.0": - version "1.40.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.40.0.tgz#10b2944ca559386590683392022a897eefd011d3" - integrity sha512-cifvXDhcqMwwTlTK04GBNeIe7yyo28Mfby85QXFe1Yk8nmi36Ab/5UQwptOx84SsoGNRg+EVSjwzfSZMy6pmlw== - -"@opentelemetry/sql-common@^0.41.2": - version "0.41.2" - resolved "https://registry.yarnpkg.com/@opentelemetry/sql-common/-/sql-common-0.41.2.tgz#7f4a14166cfd6c9ffe89096db1cc75eaf6443b19" - integrity sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ== - dependencies: - "@opentelemetry/core" "^2.0.0" - "@oxc-resolver/binding-android-arm-eabi@11.19.1": version "11.19.1" resolved "https://registry.yarnpkg.com/@oxc-resolver/binding-android-arm-eabi/-/binding-android-arm-eabi-11.19.1.tgz#c44120aa5104e991e4a9969bb0b816263a6f4bc1" @@ -1814,19 +1441,70 @@ dependencies: "@prisma/debug" "5.22.0" -"@prisma/instrumentation@6.19.0": - version "6.19.0" - resolved "https://registry.yarnpkg.com/@prisma/instrumentation/-/instrumentation-6.19.0.tgz#46d15adc8bc4a5a3167032eea6d0a7aa64fb7d93" - integrity sha512-QcuYy25pkXM8BJ37wVFBO7Zh34nyRV1GOb2n3lPkkbRYfl4hWl3PTcImP41P0KrzVXfa/45p6eVCos27x3exIg== +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.5.tgz#d9315ad7cf3f30aac70bda3c068443dc6f143659" + integrity sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g== + +"@protobufjs/eventemitter@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz#d512cb26c0ae026091ee2c1167f1be6faf5c842a" + integrity sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg== + +"@protobufjs/fetch@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.1.tgz#4d6fc00c8fb64016a5c81b469d549046350f1065" + integrity sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw== dependencies: - "@opentelemetry/instrumentation" ">=0.52.0 <1" + "@protobufjs/aspromise" "^1.1.1" -"@prisma/instrumentation@7.2.0": - version "7.2.0" - resolved "https://registry.yarnpkg.com/@prisma/instrumentation/-/instrumentation-7.2.0.tgz#9409a436d8f98e8950c8659aeeba045c4a07e891" - integrity sha512-Rh9Z4x5kEj1OdARd7U18AtVrnL6rmLSI0qYShaB4W7Wx5BKbgzndWF+QnuzMb7GLfVdlT5aYCXoPQVYuYtVu0g== +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== + +"@protobufjs/inquire@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.2.tgz#ae64fbc014ff44c8bfad03dd4c93cd2d6a4c82db" + integrity sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw== + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== + +"@protobufjs/utf8@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.1.tgz#eaee5900122c110a3dbcb728c0597014a2621774" + integrity sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg== + +"@qdrant/js-client-rest@^1.18.0": + version "1.18.0" + resolved "https://registry.yarnpkg.com/@qdrant/js-client-rest/-/js-client-rest-1.18.0.tgz#7595b3e4bbdee95bd1748467c02a20785e05133b" + integrity sha512-/0dqX5uV9chC1DnYSnU4gNMrDqse/pt6hHg3Rqqpl5isH7xl1xSNvffjzBoxycDD79luWn7Ho6Rh/61sOs5DNw== dependencies: - "@opentelemetry/instrumentation" "^0.207.0" + "@qdrant/openapi-typescript-fetch" "1.2.6" + undici "^6.24.0" + +"@qdrant/openapi-typescript-fetch@1.2.6": + version "1.2.6" + resolved "https://registry.yarnpkg.com/@qdrant/openapi-typescript-fetch/-/openapi-typescript-fetch-1.2.6.tgz#c2682a9fa26ded86384f421c991f6c461785af7e" + integrity sha512-oQG/FejNpItrxRHoyctYvT3rwGZOnK4jr3JdppO/c78ktDvkWiPXPHNsrDf33K9sZdRb6PR7gi4noIapu5q4HA== "@quramy/jest-prisma-core@^1.8.0", "@quramy/jest-prisma-core@^1.8.2": version "1.8.2" @@ -1857,161 +1535,6 @@ resolved "https://registry.yarnpkg.com/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz#60de891bb126abfdc5410fdc6166aca065f10a0c" integrity sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg== -"@sentry-internal/node-cpu-profiler@^2.2.0": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@sentry-internal/node-cpu-profiler/-/node-cpu-profiler-2.2.0.tgz#0640d4aebb4d36031658ccff83dc22b76f437ede" - integrity sha512-oLHVYurqZfADPh5hvmQYS5qx8t0UZzT2u6+/68VXsFruQEOnYJTODKgU3BVLmemRs3WE6kCJjPeFdHVYOQGSzQ== - dependencies: - detect-libc "^2.0.3" - node-abi "^3.73.0" - -"@sentry/core@10.32.1": - version "10.32.1" - resolved "https://registry.yarnpkg.com/@sentry/core/-/core-10.32.1.tgz#2a5c245b5e16063456cf44c7fe926c6ffb6116dd" - integrity sha512-PH2ldpSJlhqsMj2vCTyU0BI2Fx1oIDhm7Izo5xFALvjVCS0gmlqHt1udu6YlKn8BtpGH6bGzssvv5APrk+OdPQ== - -"@sentry/core@10.43.0": - version "10.43.0" - resolved "https://registry.yarnpkg.com/@sentry/core/-/core-10.43.0.tgz#48b7b2295f36097775b529c59712688c9087c7bc" - integrity sha512-l0SszQAPiQGWl/ferw8GP3ALyHXiGiRKJaOvNmhGO+PrTQyZTZ6OYyPnGijAFRg58dE1V3RCH/zw5d2xSUIiNg== - -"@sentry/nestjs@^10.27.0": - version "10.32.1" - resolved "https://registry.yarnpkg.com/@sentry/nestjs/-/nestjs-10.32.1.tgz#931052d6068fd81572e1ff898960c21b00920a6b" - integrity sha512-StgRg8AojiCbH+Q7uhO/9DOhfpjw6SxtsTWwNoioLfHIx968btdQPhALrHji0xXR8DYDBf+bk99P1KdqgDDh/w== - dependencies: - "@opentelemetry/api" "^1.9.0" - "@opentelemetry/core" "^2.2.0" - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/instrumentation-nestjs-core" "0.55.0" - "@opentelemetry/semantic-conventions" "^1.37.0" - "@sentry/core" "10.32.1" - "@sentry/node" "10.32.1" - -"@sentry/node-core@10.32.1": - version "10.32.1" - resolved "https://registry.yarnpkg.com/@sentry/node-core/-/node-core-10.32.1.tgz#252f327aa091db6924d4c5fdadfd21367df99ad9" - integrity sha512-w56rxdBanBKc832zuwnE+zNzUQ19fPxfHEtOhK8JGPu3aSwQYcIxwz9z52lOx3HN7k/8Fj5694qlT3x/PokhRw== - dependencies: - "@apm-js-collab/tracing-hooks" "^0.3.1" - "@sentry/core" "10.32.1" - "@sentry/opentelemetry" "10.32.1" - import-in-the-middle "^2" - -"@sentry/node-core@10.43.0": - version "10.43.0" - resolved "https://registry.yarnpkg.com/@sentry/node-core/-/node-core-10.43.0.tgz#f8575be3ad09e86e8e18e54074f3bcafea344234" - integrity sha512-w2H3NSkNMoYOS7o7mR55BM7+xL++dPxMSv1/XDfsra9FYHGppO+Mxk667Ee5k+uDi+wNIioICIh+5XOvZh4+HQ== - dependencies: - "@sentry/core" "10.43.0" - "@sentry/opentelemetry" "10.43.0" - import-in-the-middle "^2.0.6" - -"@sentry/node@10.32.1": - version "10.32.1" - resolved "https://registry.yarnpkg.com/@sentry/node/-/node-10.32.1.tgz#0d402f9c2ccd5e969a56d325645acd1acfb46b22" - integrity sha512-oxlybzt8QW0lx/QaEj1DcvZDRXkgouewFelu/10dyUwv5So3YvipfvWInda+yMLmn25OggbloDQ0gyScA2jU3g== - dependencies: - "@opentelemetry/api" "^1.9.0" - "@opentelemetry/context-async-hooks" "^2.2.0" - "@opentelemetry/core" "^2.2.0" - "@opentelemetry/instrumentation" "^0.208.0" - "@opentelemetry/instrumentation-amqplib" "0.55.0" - "@opentelemetry/instrumentation-connect" "0.52.0" - "@opentelemetry/instrumentation-dataloader" "0.26.0" - "@opentelemetry/instrumentation-express" "0.57.0" - "@opentelemetry/instrumentation-fs" "0.28.0" - "@opentelemetry/instrumentation-generic-pool" "0.52.0" - "@opentelemetry/instrumentation-graphql" "0.56.0" - "@opentelemetry/instrumentation-hapi" "0.55.0" - "@opentelemetry/instrumentation-http" "0.208.0" - "@opentelemetry/instrumentation-ioredis" "0.56.0" - "@opentelemetry/instrumentation-kafkajs" "0.18.0" - "@opentelemetry/instrumentation-knex" "0.53.0" - "@opentelemetry/instrumentation-koa" "0.57.0" - "@opentelemetry/instrumentation-lru-memoizer" "0.53.0" - "@opentelemetry/instrumentation-mongodb" "0.61.0" - "@opentelemetry/instrumentation-mongoose" "0.55.0" - "@opentelemetry/instrumentation-mysql" "0.54.0" - "@opentelemetry/instrumentation-mysql2" "0.55.0" - "@opentelemetry/instrumentation-pg" "0.61.0" - "@opentelemetry/instrumentation-redis" "0.57.0" - "@opentelemetry/instrumentation-tedious" "0.27.0" - "@opentelemetry/instrumentation-undici" "0.19.0" - "@opentelemetry/resources" "^2.2.0" - "@opentelemetry/sdk-trace-base" "^2.2.0" - "@opentelemetry/semantic-conventions" "^1.37.0" - "@prisma/instrumentation" "6.19.0" - "@sentry/core" "10.32.1" - "@sentry/node-core" "10.32.1" - "@sentry/opentelemetry" "10.32.1" - import-in-the-middle "^2" - minimatch "^9.0.0" - -"@sentry/node@^10.43.0": - version "10.43.0" - resolved "https://registry.yarnpkg.com/@sentry/node/-/node-10.43.0.tgz#466acf853565d92d972b7e012fab6558a32a739e" - integrity sha512-oNwXcuZUc4uTTr0WbHZBBIKsKwAKvNMTgbXwxfB37CfzV18wbTirbQABZ/Ir3WNxSgi6ZcnC6UE013jF5XWPqw== - dependencies: - "@fastify/otel" "0.16.0" - "@opentelemetry/api" "^1.9.0" - "@opentelemetry/context-async-hooks" "^2.5.1" - "@opentelemetry/core" "^2.5.1" - "@opentelemetry/instrumentation" "^0.211.0" - "@opentelemetry/instrumentation-amqplib" "0.58.0" - "@opentelemetry/instrumentation-connect" "0.54.0" - "@opentelemetry/instrumentation-dataloader" "0.28.0" - "@opentelemetry/instrumentation-express" "0.59.0" - "@opentelemetry/instrumentation-fs" "0.30.0" - "@opentelemetry/instrumentation-generic-pool" "0.54.0" - "@opentelemetry/instrumentation-graphql" "0.58.0" - "@opentelemetry/instrumentation-hapi" "0.57.0" - "@opentelemetry/instrumentation-http" "0.211.0" - "@opentelemetry/instrumentation-ioredis" "0.59.0" - "@opentelemetry/instrumentation-kafkajs" "0.20.0" - "@opentelemetry/instrumentation-knex" "0.55.0" - "@opentelemetry/instrumentation-koa" "0.59.0" - "@opentelemetry/instrumentation-lru-memoizer" "0.55.0" - "@opentelemetry/instrumentation-mongodb" "0.64.0" - "@opentelemetry/instrumentation-mongoose" "0.57.0" - "@opentelemetry/instrumentation-mysql" "0.57.0" - "@opentelemetry/instrumentation-mysql2" "0.57.0" - "@opentelemetry/instrumentation-pg" "0.63.0" - "@opentelemetry/instrumentation-redis" "0.59.0" - "@opentelemetry/instrumentation-tedious" "0.30.0" - "@opentelemetry/instrumentation-undici" "0.21.0" - "@opentelemetry/resources" "^2.5.1" - "@opentelemetry/sdk-trace-base" "^2.5.1" - "@opentelemetry/semantic-conventions" "^1.39.0" - "@prisma/instrumentation" "7.2.0" - "@sentry/core" "10.43.0" - "@sentry/node-core" "10.43.0" - "@sentry/opentelemetry" "10.43.0" - import-in-the-middle "^2.0.6" - -"@sentry/opentelemetry@10.32.1": - version "10.32.1" - resolved "https://registry.yarnpkg.com/@sentry/opentelemetry/-/opentelemetry-10.32.1.tgz#c82955c914875ce5d81e6737dc2c05438eec8a2d" - integrity sha512-YLssSz5Y+qPvufrh2cDaTXDoXU8aceOhB+YTjT8/DLF6SOj7Tzen52aAcjNaifawaxEsLCC8O+B+A2iA+BllvA== - dependencies: - "@sentry/core" "10.32.1" - -"@sentry/opentelemetry@10.43.0": - version "10.43.0" - resolved "https://registry.yarnpkg.com/@sentry/opentelemetry/-/opentelemetry-10.43.0.tgz#c2d3d58e943541a7ae81e79cf264b61c7fa20f1b" - integrity sha512-+fIcnnLdvBHdq4nKq23t9v/B9D4L97fPWEDksXbpGs11o6BsqY4Tlzmce6cP95iiQhPckCEag3FthSND+BYtYQ== - dependencies: - "@sentry/core" "10.43.0" - -"@sentry/profiling-node@^10.27.0": - version "10.32.1" - resolved "https://registry.yarnpkg.com/@sentry/profiling-node/-/profiling-node-10.32.1.tgz#5c8ff3a48ffe386087714395dd83b6ff1f7b1239" - integrity sha512-UDSZayQw4K5wv/XNHoB+i+KrnlCStLb0H2lsypF4dgQFCrHXmbwhMh9ieofVGk5bxdmXoL3lSE+3W9cJbpqy2A== - dependencies: - "@sentry-internal/node-cpu-profiler" "^2.2.0" - "@sentry/core" "10.32.1" - "@sentry/node" "10.32.1" - "@sinclair/typebox@^0.27.8": version "0.27.10" resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.10.tgz#beefe675f1853f73676aecc915b2bd2ac98c4fc6" @@ -2247,7 +1770,7 @@ "@types/node" "*" "@types/responselike" "^1.0.0" -"@types/connect@*", "@types/connect@3.4.38": +"@types/connect@*": version "3.4.38" resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.38.tgz#5ba7f3bc4fbbdeaff8dded952e5ff2cc53f8d858" integrity sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug== @@ -2395,13 +1918,6 @@ dependencies: "@types/express" "*" -"@types/mysql@2.15.27": - version "2.15.27" - resolved "https://registry.yarnpkg.com/@types/mysql/-/mysql-2.15.27.tgz#fb13b0e8614d39d42f40f381217ec3215915f1e9" - integrity sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA== - dependencies: - "@types/node" "*" - "@types/node@*": version "25.5.0" resolved "https://registry.yarnpkg.com/@types/node/-/node-25.5.0.tgz#5c99f37c443d9ccc4985866913f1ed364217da31" @@ -2416,37 +1932,12 @@ dependencies: undici-types "~6.19.8" -"@types/pg-pool@2.0.6": - version "2.0.6" - resolved "https://registry.yarnpkg.com/@types/pg-pool/-/pg-pool-2.0.6.tgz#1376d9dc5aec4bb2ec67ce28d7e9858227403c77" - integrity sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ== - dependencies: - "@types/pg" "*" - -"@types/pg-pool@2.0.7": - version "2.0.7" - resolved "https://registry.yarnpkg.com/@types/pg-pool/-/pg-pool-2.0.7.tgz#c17945a74472d9a3beaf8e66d5aa6fc938328734" - integrity sha512-U4CwmGVQcbEuqpyju8/ptOKg6gEC+Tqsvj2xS9o1g71bUh8twxnC6ZL5rZKCsGN0iyH0CwgUyc9VR5owNQF9Ng== - dependencies: - "@types/pg" "*" - -"@types/pg@*": - version "8.16.0" - resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.16.0.tgz#b7af0d642752340b7c9de1c33afd9bc5c5f0ebeb" - integrity sha512-RmhMd/wD+CF8Dfo+cVIy3RR5cl8CyfXQ0tGgW6XBL8L4LM/UTEbNXYRbLwU6w+CgrKBNbrQWt4FUtTfaU5jSYQ== - dependencies: - "@types/node" "*" - pg-protocol "*" - pg-types "^2.2.0" - -"@types/pg@8.15.6": - version "8.15.6" - resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.15.6.tgz#4df7590b9ac557cbe5479e0074ec1540cbddad9b" - integrity sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ== +"@types/node@>=13.7.0": + version "25.9.1" + resolved "https://registry.yarnpkg.com/@types/node/-/node-25.9.1.tgz#3bda556db500ae4319c08e7fc9ab94f19013ba0b" + integrity sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg== dependencies: - "@types/node" "*" - pg-protocol "*" - pg-types "^2.2.0" + undici-types ">=7.24.0 <7.24.7" "@types/qs@*": version "6.14.0" @@ -2520,13 +2011,6 @@ "@types/methods" "^1.1.4" "@types/superagent" "^8.1.0" -"@types/tedious@^4.0.14": - version "4.0.14" - resolved "https://registry.yarnpkg.com/@types/tedious/-/tedious-4.0.14.tgz#868118e7a67808258c05158e9cad89ca58a2aec1" - integrity sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw== - dependencies: - "@types/node" "*" - "@types/unzipper@^0.10.10": version "0.10.11" resolved "https://registry.yarnpkg.com/@types/unzipper/-/unzipper-0.10.11.tgz#2a605ae639fc20ee6886be0f7d28dc61c1e6d3d3" @@ -2934,11 +2418,6 @@ accepts@~1.3.8: mime-types "~2.1.34" negotiator "0.6.3" -acorn-import-attributes@^1.9.5: - version "1.9.5" - resolved "https://registry.yarnpkg.com/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz#7eb1557b1ba05ef18b5ed0ec67591bfab04688ef" - integrity sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ== - acorn-import-phases@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/acorn-import-phases/-/acorn-import-phases-1.0.4.tgz#16eb850ba99a056cb7cbfe872ffb8972e18c8bd7" @@ -2956,7 +2435,7 @@ acorn-walk@^8.1.1: dependencies: acorn "^8.11.0" -acorn@^8.11.0, acorn@^8.14.0, acorn@^8.4.1: +acorn@^8.11.0, acorn@^8.4.1: version "8.15.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.15.0.tgz#a360898bc415edaac46c8241f6383975b930b816" integrity sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg== @@ -2966,6 +2445,11 @@ acorn@^8.15.0, acorn@^8.16.0, acorn@^8.9.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.16.0.tgz#4ce79c89be40afe7afe8f3adb902a1f1ce9ac08a" integrity sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw== +adm-zip@^0.5.16: + version "0.5.17" + resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.17.tgz#5c0b65f37aeec5c2a94995c024f931f62e4bbc5a" + integrity sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ== + ajv-formats@3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-3.0.1.tgz#3d5dc762bca17679c3c2ea7e90ad6b7532309578" @@ -3345,6 +2829,11 @@ boolbase@^1.0.0: resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== +boolean@^3.0.1: + version "3.2.0" + resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.2.0.tgz#9e5294af4e98314494cbb17979fa54ca159f116b" + integrity sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw== + brace-expansion@^5.0.2: version "5.0.5" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-5.0.5.tgz#dcc3a37116b79f3e1b46db994ced5d570e930fdb" @@ -3548,12 +3037,7 @@ ci-info@^4.2.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-4.4.0.tgz#7d54eff9f54b45b62401c26032696eb59c8bd18c" integrity sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg== -cjs-module-lexer@^1.2.2: - version "1.4.3" - resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz#0f79731eb8cfe1ec72acd4066efac9d61991b00d" - integrity sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q== - -cjs-module-lexer@^2.1.0, cjs-module-lexer@^2.2.0: +cjs-module-lexer@^2.1.0: version "2.2.0" resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz#b3ca5101843389259ade7d88c77bd06ce55849ca" integrity sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ== @@ -3835,7 +3319,7 @@ debug@^3.2.7: dependencies: ms "^2.1.1" -debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.5, debug@^4.3.7, debug@^4.4.1, debug@^4.4.3: +debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.7, debug@^4.4.3: version "4.4.3" resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.3.tgz#c6ae432d9bd9662582fce08709b038c58e9e3d6a" integrity sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA== @@ -3909,7 +3393,7 @@ destroy@1.2.0, destroy@~1.2.0: resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.2.0.tgz#4803735509ad8be552934c67df614f94e66fa015" integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== -detect-libc@^2.0.3: +detect-libc@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.1.2.tgz#689c5dcdc1900ef5583a4cb9f6d7b473742074ad" integrity sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ== @@ -3919,6 +3403,11 @@ detect-newline@^3.1.0: resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== +detect-node@^2.0.4: + version "2.1.0" + resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.1.0.tgz#c9c70775a49c3d03bc2c06d9a73be550f978f8b1" + integrity sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g== + dezalgo@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.4.tgz#751235260469084c132157dfa857f386d4c33d81" @@ -4217,6 +3706,11 @@ es-to-primitive@^1.3.0: is-date-object "^1.0.5" is-symbol "^1.0.4" +es6-error@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d" + integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg== + escalade@^3.1.1, escalade@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" @@ -4737,6 +4231,11 @@ flat-cache@^3.0.4: keyv "^4.5.3" rimraf "^3.0.2" +flatbuffers@^25.1.24: + version "25.9.23" + resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-25.9.23.tgz#346811557fe9312ab5647535e793c761e9c81eb1" + integrity sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ== + flatted@^3.2.9: version "3.4.2" resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.4.2.tgz#f5c23c107f0f37de8dbdf24f13722b3b98d52726" @@ -4807,11 +4306,6 @@ formidable@^3.5.2, formidable@^3.5.4: dezalgo "^1.0.4" once "^1.4.0" -forwarded-parse@2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/forwarded-parse/-/forwarded-parse-2.1.2.tgz#08511eddaaa2ddfd56ba11138eee7df117a09325" - integrity sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw== - forwarded@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" @@ -4981,6 +4475,18 @@ glob@13.0.6, glob@^10.5.0, glob@^7.1.3, glob@^7.1.4: package-json-from-dist "^1.0.0" path-scurry "^1.11.1" +global-agent@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/global-agent/-/global-agent-3.0.0.tgz#ae7cd31bd3583b93c5a16437a1afe27cc33a1ab6" + integrity sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q== + dependencies: + boolean "^3.0.1" + es6-error "^4.1.1" + matcher "^3.0.0" + roarr "^2.15.3" + semver "^7.3.2" + serialize-error "^7.0.1" + globals@^13.19.0: version "13.24.0" resolved "https://registry.yarnpkg.com/globals/-/globals-13.24.0.tgz#8432a19d78ce0c1e833949c36adb345400bb1171" @@ -4988,7 +4494,7 @@ globals@^13.19.0: dependencies: type-fest "^0.20.2" -globalthis@^1.0.4: +globalthis@^1.0.1, globalthis@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/globalthis/-/globalthis-1.0.4.tgz#7430ed3a975d97bfb59bcce41f5cabbafa651236" integrity sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ== @@ -5040,6 +4546,11 @@ graphemer@^1.4.0: resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== +guid-typescript@^1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/guid-typescript/-/guid-typescript-1.0.9.tgz#e35f77003535b0297ea08548f5ace6adb1480ddc" + integrity sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ== + handlebars@^4.7.8: version "4.7.9" resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.9.tgz#6f139082ab58dc4e5a0e51efe7db5ae890d56a0f" @@ -5195,26 +4706,6 @@ import-fresh@^3.2.1, import-fresh@^3.3.0: parent-module "^1.0.0" resolve-from "^4.0.0" -import-in-the-middle@^2, import-in-the-middle@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-2.0.1.tgz#8d1aa2db18374f2c811de2aa4756ebd6e9859243" - integrity sha512-bruMpJ7xz+9jwGzrwEhWgvRrlKRYCRDBrfU+ur3FcasYXLJDxTruJ//8g2Noj+QFyRBeqbpj8Bhn4Fbw6HjvhA== - dependencies: - acorn "^8.14.0" - acorn-import-attributes "^1.9.5" - cjs-module-lexer "^1.2.2" - module-details-from-path "^1.0.3" - -import-in-the-middle@^2.0.6: - version "2.0.6" - resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-2.0.6.tgz#1972337bfe020d05f6b5e020c13334567436324f" - integrity sha512-3vZV3jX0XRFW3EJDTwzWoZa+RH1b8eTTx6YOCjglrLyPuepwoBti1k3L2dKwdCUrnVEfc5CuRuGstaC/uQJJaw== - dependencies: - acorn "^8.15.0" - acorn-import-attributes "^1.9.5" - cjs-module-lexer "^2.2.0" - module-details-from-path "^1.0.4" - import-local@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/import-local/-/import-local-3.2.0.tgz#c3d5c745798c02a6f8b897726aba5100186ee260" @@ -6033,6 +5524,11 @@ json-stable-stringify-without-jsonify@^1.0.1: resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== +json-stringify-safe@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== + json5@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" @@ -6163,6 +5659,11 @@ loglevel@^1.4.1: resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.9.2.tgz#c2e028d6c757720107df4e64508530db6621ba08" integrity sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg== +long@^5.2.3, long@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/long/-/long-5.3.2.tgz#1d84463095999262d7d7b7f8bfd4a8cc55167f83" + integrity sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA== + lowercase-keys@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479" @@ -6211,6 +5712,13 @@ makeerror@1.0.12: dependencies: tmpl "1.0.5" +matcher@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/matcher/-/matcher-3.0.0.tgz#bd9060f4c5b70aa8041ccc6f80368760994f30ca" + integrity sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng== + dependencies: + escape-string-regexp "^4.0.0" + math-intrinsics@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" @@ -6298,7 +5806,7 @@ mimic-response@^3.1.0: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== -minimatch@9.0.3, minimatch@>=9.0.7, minimatch@^10.0.3, minimatch@^10.2.2, minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.2, minimatch@^9.0.0, minimatch@^9.0.3, minimatch@^9.0.4: +minimatch@9.0.3, minimatch@>=9.0.7, minimatch@^10.2.2, minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.2, minimatch@^9.0.3, minimatch@^9.0.4: version "10.2.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.2.4.tgz#465b3accbd0218b8281f5301e27cedc697f96fde" integrity sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg== @@ -6315,11 +5823,6 @@ minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6, minimist@^1.2.8: resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.3.tgz#79389b4eb1bb2d003a9bba87d492f2bd37bdc65b" integrity sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A== -module-details-from-path@^1.0.3, module-details-from-path@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.4.tgz#b662fdcd93f6c83d3f25289da0ce81c8d9685b94" - integrity sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w== - ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -6365,13 +5868,6 @@ neo-async@^2.6.2: resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== -node-abi@^3.73.0: - version "3.85.0" - resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.85.0.tgz#b115d575e52b2495ef08372b058e13d202875a7d" - integrity sha512-zsFhmbkAzwhTft6nd3VxcG0cvJsT70rL+BIGHWVq5fi6MwGrHwzqKaxXE+Hl2GmnGItnDKPPkO5/LQqjVkIdFg== - dependencies: - semver "^7.3.5" - node-abort-controller@^3.0.1: version "3.1.1" resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" @@ -6527,6 +6023,37 @@ onetime@^5.1.0, onetime@^5.1.2: dependencies: mimic-fn "^2.1.0" +onnxruntime-common@1.24.0-dev.20251116-b39e144322: + version "1.24.0-dev.20251116-b39e144322" + resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.24.0-dev.20251116-b39e144322.tgz#6edecf4e5178f9d76907d5abbe241c3635e528dd" + integrity sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw== + +onnxruntime-common@1.24.3: + version "1.24.3" + resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz#fb93319a2d041fc9fe2a209167ba6ef2f1a47fee" + integrity sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA== + +onnxruntime-node@1.24.3: + version "1.24.3" + resolved "https://registry.yarnpkg.com/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz#ec8da99c335aa5ea95559a3ac1ae915a89541d5e" + integrity sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg== + dependencies: + adm-zip "^0.5.16" + global-agent "^3.0.0" + onnxruntime-common "1.24.3" + +onnxruntime-web@1.26.0-dev.20260416-b7804b056c: + version "1.26.0-dev.20260416-b7804b056c" + resolved "https://registry.yarnpkg.com/onnxruntime-web/-/onnxruntime-web-1.26.0-dev.20260416-b7804b056c.tgz#453f29768836322a6b8b6d1d730a8426b6582ae2" + integrity sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw== + dependencies: + flatbuffers "^25.1.24" + guid-typescript "^1.0.9" + long "^5.2.3" + onnxruntime-common "1.24.0-dev.20251116-b39e144322" + platform "^1.3.6" + protobufjs "^7.2.4" + optionator@^0.9.3: version "0.9.4" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.4.tgz#7ea1c1a5d91d764fb282139c88fe11e182a3a734" @@ -6751,27 +6278,6 @@ pdf2json@^4.0.0: resolved "https://registry.yarnpkg.com/pdf2json/-/pdf2json-4.0.0.tgz#61aa5b109a3470dd37cd7c4ff9c9b26c375c76be" integrity sha512-WkezNsLK8sGpuFC7+PPP0DsXROwdoOxmXPBTtUWWkCwCi/Vi97MRC52Ly6FWIJjOKIywpm/L2oaUgSrmtU+7ZQ== -pg-int8@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" - integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== - -pg-protocol@*: - version "1.10.3" - resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.10.3.tgz#ac9e4778ad3f84d0c5670583bab976ea0a34f69f" - integrity sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ== - -pg-types@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" - integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== - dependencies: - pg-int8 "1.0.1" - postgres-array "~2.0.0" - postgres-bytea "~1.0.0" - postgres-date "~1.0.4" - postgres-interval "^1.1.0" - picocolors@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" @@ -6804,6 +6310,11 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +platform@^1.3.6: + version "1.3.6" + resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7" + integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg== + pluralize@8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1" @@ -6814,28 +6325,6 @@ possible-typed-array-names@^1.0.0: resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz#93e3582bc0e5426586d9d07b79ee40fc841de4ae" integrity sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg== -postgres-array@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" - integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA== - -postgres-bytea@~1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.1.tgz#c40b3da0222c500ff1e51c5d7014b60b79697c7a" - integrity sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ== - -postgres-date@~1.0.4: - version "1.0.7" - resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8" - integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q== - -postgres-interval@^1.1.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695" - integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ== - dependencies: - xtend "^4.0.0" - posthog-node@^5.34.2: version "5.34.2" resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-5.34.2.tgz#cf1022a0e5b8885e1d0c404a85522801fa3c5ef5" @@ -6931,6 +6420,24 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +protobufjs@^7.2.4: + version "7.6.2" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.6.2.tgz#2659f77bd8d54778814c274dc0df808f54c88918" + integrity sha512-N9EiLovGEQOJSPF26Ij7qUGvahfEnq0eeYZ02aigIedkmz1qZSwjnP9SBITHJuF/6MYbIW4HDN8zdYjsjqJKXQ== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.5" + "@protobufjs/eventemitter" "^1.1.1" + "@protobufjs/fetch" "^1.1.1" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.2" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.1" + "@types/node" ">=13.7.0" + long "^5.3.2" + proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" @@ -7067,14 +6574,6 @@ require-from-string@^2.0.2: resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== -require-in-the-middle@^8.0.0: - version "8.0.1" - resolved "https://registry.yarnpkg.com/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz#dbde2587f669398626d56b20c868ab87bf01cce4" - integrity sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ== - dependencies: - debug "^4.3.5" - module-details-from-path "^1.0.3" - require-relative@^0.8.7: version "0.8.7" resolved "https://registry.yarnpkg.com/require-relative/-/require-relative-0.8.7.tgz#7999539fc9e047a37928fa196f8e1563dabd36de" @@ -7138,6 +6637,18 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" +roarr@^2.15.3: + version "2.15.4" + resolved "https://registry.yarnpkg.com/roarr/-/roarr-2.15.4.tgz#f5fe795b7b838ccfe35dc608e0282b9eba2e7afd" + integrity sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A== + dependencies: + boolean "^3.0.1" + detect-node "^2.0.4" + globalthis "^1.0.1" + json-stringify-safe "^5.0.1" + semver-compare "^1.0.0" + sprintf-js "^1.1.2" + run-parallel@^1.1.9: version "1.2.0" resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" @@ -7221,6 +6732,11 @@ schema-utils@^4.3.0, schema-utils@^4.3.3: ajv-formats "^2.1.1" ajv-keywords "^5.1.0" +semver-compare@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" + integrity sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow== + semver-regex@^4.0.5: version "4.0.5" resolved "https://registry.yarnpkg.com/semver-regex/-/semver-regex-4.0.5.tgz#fbfa36c7ba70461311f5debcb3928821eb4f9180" @@ -7238,6 +6754,11 @@ semver@^6.3.0, semver@^6.3.1: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== +semver@^7.3.2: + version "7.8.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.8.1.tgz#bf4970b5e70fda0686363cc18bfe8805d5ed957e" + integrity sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg== + semver@^7.3.4, semver@^7.3.8: version "7.7.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.3.tgz#4b5f4143d007633a8dc671cd0a6ef9147b8bb946" @@ -7267,6 +6788,13 @@ send@~0.19.0, send@~0.19.1: range-parser "~1.2.1" statuses "~2.0.2" +serialize-error@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/serialize-error/-/serialize-error-7.0.1.tgz#f1360b0447f61ffb483ec4157c737fab7d778e18" + integrity sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw== + dependencies: + type-fest "^0.13.1" + serve-static@~1.16.2: version "1.16.3" resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.16.3.tgz#a97b74d955778583f3862a4f0b841eb4d5d78cf9" @@ -7313,6 +6841,40 @@ setprototypeof@1.2.0, setprototypeof@~1.2.0: resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== +sharp@^0.34.5: + version "0.34.5" + resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.34.5.tgz#b6f148e4b8c61f1797bde11a9d1cfebbae2c57b0" + integrity sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg== + dependencies: + "@img/colour" "^1.0.0" + detect-libc "^2.1.2" + semver "^7.7.3" + optionalDependencies: + "@img/sharp-darwin-arm64" "0.34.5" + "@img/sharp-darwin-x64" "0.34.5" + "@img/sharp-libvips-darwin-arm64" "1.2.4" + "@img/sharp-libvips-darwin-x64" "1.2.4" + "@img/sharp-libvips-linux-arm" "1.2.4" + "@img/sharp-libvips-linux-arm64" "1.2.4" + "@img/sharp-libvips-linux-ppc64" "1.2.4" + "@img/sharp-libvips-linux-riscv64" "1.2.4" + "@img/sharp-libvips-linux-s390x" "1.2.4" + "@img/sharp-libvips-linux-x64" "1.2.4" + "@img/sharp-libvips-linuxmusl-arm64" "1.2.4" + "@img/sharp-libvips-linuxmusl-x64" "1.2.4" + "@img/sharp-linux-arm" "0.34.5" + "@img/sharp-linux-arm64" "0.34.5" + "@img/sharp-linux-ppc64" "0.34.5" + "@img/sharp-linux-riscv64" "0.34.5" + "@img/sharp-linux-s390x" "0.34.5" + "@img/sharp-linux-x64" "0.34.5" + "@img/sharp-linuxmusl-arm64" "0.34.5" + "@img/sharp-linuxmusl-x64" "0.34.5" + "@img/sharp-wasm32" "0.34.5" + "@img/sharp-win32-arm64" "0.34.5" + "@img/sharp-win32-ia32" "0.34.5" + "@img/sharp-win32-x64" "0.34.5" + shebang-command@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" @@ -7430,6 +6992,11 @@ source-map@^0.7.3, source-map@^0.7.4: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.6.tgz#a3658ab87e5b6429c8a1f3ba0083d4c61ca3ef02" integrity sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ== +sprintf-js@^1.1.2: + version "1.1.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a" + integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA== + stack-utils@^2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.6.tgz#aaf0748169c02fc33c8232abccf933f54a1cc34f" @@ -7875,6 +7442,11 @@ type-detect@4.0.8: resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== +type-fest@^0.13.1: + version "0.13.1" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934" + integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg== + type-fest@^0.20.2: version "0.20.2" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" @@ -7995,6 +7567,11 @@ unbox-primitive@^1.1.0: has-symbols "^1.1.0" which-boxed-primitive "^1.1.1" +"undici-types@>=7.24.0 <7.24.7": + version "7.24.6" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.24.6.tgz#61275b485d7fd4e9d269c7cf04ec2873c9cc0f91" + integrity sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg== + undici-types@~6.19.8: version "6.19.8" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" @@ -8005,6 +7582,11 @@ undici-types@~7.18.0: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.18.2.tgz#29357a89e7b7ca4aef3bf0fd3fd0cd73884229e9" integrity sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w== +undici@^6.24.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.26.0.tgz#333a35b7f519c48d2dc6aeb38e4e91d9274e0652" + integrity sha512-4yqz8a3n5HmGTlsbADNtr/dJlhkh/55Rq798G6ibiULcXbDtaLpTl1pvdqcbFfeoj3iSi52lePFM7h9H21cw/A== + undici@^7.19.0: version "7.25.0" resolved "https://registry.yarnpkg.com/undici/-/undici-7.25.0.tgz#7d72fc429a0421769ca2966fd07cac875c85b781" @@ -8093,6 +7675,11 @@ uuid@11.0.3: resolved "https://registry.yarnpkg.com/uuid/-/uuid-11.0.3.tgz#248451cac9d1a4a4128033e765d137e2b2c49a3d" integrity sha512-d0z310fCWv5dJwnX1Y/MncBAqGMKEzlBb1AOf7z9K8ALnd0utBX/msg/fA0+sbyN1ihbMsLhrBlnl1ak7Wa0rg== +uuid@^14.0.0: + version "14.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-14.0.0.tgz#0af883220163d264ffe0c084f6b8a89b9666966d" + integrity sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg== + v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" @@ -8342,11 +7929,6 @@ write-file-atomic@^5.0.1: imurmurhash "^0.1.4" signal-exit "^4.0.1" -xtend@^4.0.0: - version "4.0.2" - resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" - integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== - y18n@^5.0.5: version "5.0.8" resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" From a81c92ceed6c70cba08be44d7043c377c650ff60 Mon Sep 17 00:00:00 2001 From: Mohamed Hafdi Idrissi <70617264+mhd-hi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:13:32 -0400 Subject: [PATCH 02/11] feat: update Node.js version to 22.22.3, enhance Dockerfile, and add embedding count endpoint --- .nvmrc | 2 +- Dockerfile | 43 +++++++++---------- docker-compose.yml | 3 ++ docs/onboarding.md | 11 ++++- package.json | 6 +-- src/embedding/dtos/embedding-count.dto.ts | 6 +++ .../embedding-course-indexer.service.ts | 20 +++++++-- src/embedding/embedding-worker.client.ts | 40 ++++++++++++++--- src/embedding/embedding.controller.ts | 8 ++++ src/embedding/embedding.service.ts | 8 ++++ src/embedding/qdrant-course-index.service.ts | 26 +++++++---- src/jobs/dtos/run-workers.dto.ts | 3 ++ src/jobs/jobs.constants.ts | 7 ++- src/jobs/jobs.controller.ts | 2 + src/jobs/jobs.module.ts | 2 + src/jobs/jobs.service.ts | 4 ++ 16 files changed, 143 insertions(+), 48 deletions(-) create mode 100644 src/embedding/dtos/embedding-count.dto.ts diff --git a/.nvmrc b/.nvmrc index 2bd5a0a..941d7c0 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -22 +22.22.3 diff --git a/Dockerfile b/Dockerfile index e9f7cbb..2de0587 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,71 +1,70 @@ # syntax=docker/dockerfile:1 -FROM node:22-bookworm-slim AS base +FROM node:22.22.3-bullseye-slim AS deps WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ - openssl \ - ca-certificates \ - tzdata \ python3 \ make \ g++ \ + ca-certificates \ && rm -rf /var/lib/apt/lists/* COPY package.json yarn.lock ./ RUN yarn install --frozen-lockfile --ignore-scripts -FROM base AS build +FROM node:22.22.3-bullseye-slim AS build WORKDIR /app +COPY --from=deps /app/node_modules ./node_modules COPY . ./ ENV NODE_ENV=production - RUN yarn build -FROM base AS dev +FROM node:22.22.3-bullseye-slim AS dev WORKDIR /app +COPY --from=deps /app/node_modules ./node_modules +COPY package.json yarn.lock ./ COPY prisma ./prisma +RUN apt-get update && apt-get install -y --no-install-recommends \ + openssl \ + ca-certificates \ + tzdata \ + && rm -rf /var/lib/apt/lists/* + +RUN yarn prisma:generate + ENV NODE_ENV=development ENV APP_ENV=development ENV TZ=America/Toronto EXPOSE 3001 +CMD ["yarn", "start:dev"] -CMD ["sh", "-c", "yarn prisma:generate && yarn start:dev"] - -FROM node:22-bookworm-slim AS production +FROM node:22.22.3-bullseye-slim AS production ARG APP_GIT_SHORT_SHA - ENV APP_GIT_SHORT_SHA=${APP_GIT_SHORT_SHA} ENV APP_ENV=production ENV TZ=America/Toronto WORKDIR /app +COPY --from=build /app/dist ./dist +COPY --from=deps /app/node_modules ./node_modules +COPY prisma ./prisma + RUN apt-get update && apt-get install -y --no-install-recommends \ openssl \ ca-certificates \ tzdata \ - python3 \ - make \ - g++ \ && rm -rf /var/lib/apt/lists/* -COPY package.json yarn.lock ./ -COPY prisma ./prisma - -RUN yarn install --production --frozen-lockfile --ignore-scripts - -COPY --from=build /app/dist ./dist - RUN yarn prisma:generate EXPOSE 3001 - CMD ["yarn", "start:prod"] diff --git a/docker-compose.yml b/docker-compose.yml index 0a1389e..e745d4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,6 +54,9 @@ services: EMBEDDING_DTYPE: ${EMBEDDING_DTYPE:-q4} EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-50} TRANSFORMERS_CACHE_DIR: ${TRANSFORMERS_CACHE_DIR:-/app/.cache/transformers} + CHOKIDAR_USEPOLLING: "true" + CHOKIDAR_INTERVAL: "100" + FORCE_POLLING: "true" depends_on: db: condition: service_healthy diff --git a/docs/onboarding.md b/docs/onboarding.md index fa77870..5dc5e62 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -2,10 +2,11 @@ ## Requirements -- Node.js 22 +- Node.js 22.22.0 - Yarn - PostgreSQL 16+ - VS Code +- nvm (recommended for Node.js version management) ## Optional @@ -75,6 +76,14 @@ Default body: ## Option B - Local setup +### 0. Set up Node.js with nvm + +```bash +nvm install 22.22.0 +nvm use 22.22.0 +node --version # Verify: should show v22.22.0+ +``` + ### 1. Clone the project ```bash diff --git a/package.json b/package.json index c4f5709..da554b6 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,9 @@ "scripts": { "build": "nest build", "start": "nest start", - "dev": "nest start --watch", - "start:dev": "nest start --watch", - "start:debug": "nest start --debug --watch", + "dev": "nest start --watch --preserveWatchOutput --no-shell", + "start:dev": "nest start --watch --preserveWatchOutput --no-shell", + "start:debug": "nest start --debug --watch --preserveWatchOutput --no-shell", "start:prod": "yarn prisma:migrate-prod && yarn prisma:seed && node dist/main", "docker:prod": "docker-compose -f docker-compose.yml up -d --build --force-recreate --remove-orphans", "typecheck": "tsc --noEmit", diff --git a/src/embedding/dtos/embedding-count.dto.ts b/src/embedding/dtos/embedding-count.dto.ts new file mode 100644 index 0000000..828a0f8 --- /dev/null +++ b/src/embedding/dtos/embedding-count.dto.ts @@ -0,0 +1,6 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class EmbeddingCountDto { + @ApiProperty({ example: 42 }) + public count!: number; +} diff --git a/src/embedding/embedding-course-indexer.service.ts b/src/embedding/embedding-course-indexer.service.ts index 607047c..7de992a 100644 --- a/src/embedding/embedding-course-indexer.service.ts +++ b/src/embedding/embedding-course-indexer.service.ts @@ -50,14 +50,18 @@ export class CourseEmbeddingIndexerService { for (let offset = 0; offset < rows.length; offset += batchSize) { const rowsBatch = rows.slice(offset, offset + batchSize); + const batchNumber = Math.floor(offset / batchSize) + 1; + const totalBatches = Math.ceil(rows.length / batchSize); - await this.processRowsBatch(rowsBatch, embeddingModel, counters, offset); + this.logger.debug(`Processing batch ${batchNumber}/${totalBatches} (offset ${offset}, size ${rowsBatch.length})`); + + await this.processRowsBatch(rowsBatch, embeddingModel, counters, offset, batchNumber, totalBatches); } const durationSeconds = ((Date.now() - startedAt) / 1000).toFixed(2); this.logger.log( - `${counters.indexedCount} cours indexés en ${durationSeconds} secondes, ${counters.errorCount} erreurs.`, + `COMPLETED: ${counters.indexedCount}/${rows.length} courses indexed in ${durationSeconds}s. Errors: ${counters.errorCount}. Missing: ${rows.length - counters.indexedCount - counters.errorCount}.`, ); } @@ -66,8 +70,11 @@ export class CourseEmbeddingIndexerService { embeddingModel: string, counters: IndexingCounters, offset: number, + batchNumber?: number, + totalBatches?: number, ): Promise { const preparedBatch: PreparedCourseEmbedding[] = []; + const batchInfo = batchNumber && totalBatches ? ` (batch ${batchNumber}/${totalBatches})` : ''; for (const row of rowsBatch) { try { @@ -81,7 +88,10 @@ export class CourseEmbeddingIndexerService { } } + this.logger.debug(`Prepared ${preparedBatch.length}/${rowsBatch.length} courses${batchInfo}. Errors: ${counters.errorCount}.`); + if (preparedBatch.length === 0) { + this.logger.warn(`Batch ${batchNumber || 'unknown'} had 0 prepared items, skipping.`); return; } @@ -89,9 +99,10 @@ export class CourseEmbeddingIndexerService { try { points = await this.embedPreparedBatch(preparedBatch); + this.logger.debug(`Embedded ${points.length} courses${batchInfo}.`); } catch (error) { this.logger.warn( - `Embedding batch failed at offset ${offset}. Falling back to item-by-item. Error: ${formatError(error)}`, + `Embedding batch failed at offset ${offset}${batchInfo}. Falling back to item-by-item. Error: ${formatError(error)}`, ); await this.processPreparedBatchOneByOne(preparedBatch, counters); @@ -101,9 +112,10 @@ export class CourseEmbeddingIndexerService { try { await this.qdrantCourseIndexService.upsertPoints(points); counters.indexedCount += points.length; + this.logger.log(`✓ Upserted ${points.length} courses to Qdrant${batchInfo}. Total indexed: ${counters.indexedCount}.`); } catch (error) { this.logger.error( - `Qdrant upsert failed at offset ${offset}. Stopping job. Error: ${formatError(error)}`, + `Qdrant upsert failed at offset ${offset}${batchInfo}. Stopping job. Error: ${formatError(error)}`, ); throw error; diff --git a/src/embedding/embedding-worker.client.ts b/src/embedding/embedding-worker.client.ts index b02fae2..02c0a7c 100644 --- a/src/embedding/embedding-worker.client.ts +++ b/src/embedding/embedding-worker.client.ts @@ -1,3 +1,4 @@ +import * as fs from 'node:fs'; import * as path from 'node:path'; import { Worker } from 'node:worker_threads'; @@ -25,33 +26,54 @@ type EmbedWorkerMessage = EmbedWorkerSuccessMessage | EmbedWorkerFailureMessage; @Injectable() export class EmbeddingWorkerClient implements OnModuleDestroy { private readonly logger = new Logger(EmbeddingWorkerClient.name); - private readonly worker: Worker; + private worker: Worker | null = null; private readonly pending = new Map(); private nextRequestId = 1; - constructor() { + private getWorkerPath(): string { const workerPath = path.join(__dirname, 'workers', 'bge-m3.worker.js'); - this.worker = new Worker(workerPath, { + if (!fs.existsSync(workerPath)) { + throw new Error(`Embedding worker not found at ${workerPath}`); + } + + return workerPath; + } + + private createWorker(): Worker { + const workerPath = this.getWorkerPath(); + this.logger.log(`Starting embedding worker from: ${workerPath}`); + + const worker = new Worker(workerPath, { env: process.env, }); - this.worker.on('message', (message: unknown) => { + worker.on('message', (message: unknown) => { this.handleWorkerMessage(message); }); - this.worker.on('error', (error: Error) => { + worker.on('error', (error: Error) => { this.logger.error(`Embedding worker error: ${error.message}`, error.stack); this.rejectAll(error); }); - this.worker.on('exit', (code: number) => { + worker.on('exit', (code: number) => { if (code !== 0) { const error = new Error(`Embedding worker exited with code ${code}`); this.logger.error(error.message); this.rejectAll(error); } }); + + return worker; + } + + private getWorker(): Worker { + if (!this.worker) { + this.worker = this.createWorker(); + } + + return this.worker; } public embed(texts: string[]): Promise { @@ -69,7 +91,7 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { reject, }); - this.worker.postMessage({ + this.getWorker().postMessage({ id, texts, model, @@ -79,6 +101,10 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { } public async onModuleDestroy(): Promise { + if (!this.worker) { + return; + } + await this.worker.terminate(); } diff --git a/src/embedding/embedding.controller.ts b/src/embedding/embedding.controller.ts index f3d2b7b..0e1680a 100644 --- a/src/embedding/embedding.controller.ts +++ b/src/embedding/embedding.controller.ts @@ -1,6 +1,7 @@ import { Controller, Get, Param, ParseIntPipe } from '@nestjs/common'; import { ApiOkResponse, ApiOperation, ApiParam, ApiTags } from '@nestjs/swagger'; +import { EmbeddingCountDto } from './dtos/embedding-count.dto'; import { EmbeddingViewDto } from './dtos/embedding-view.dto'; import { EmbeddingService } from './embedding.service'; @@ -16,6 +17,13 @@ export class EmbeddingController { return this.embeddingService.findAll(); } + @Get('count') + @ApiOperation({ summary: 'Get the total number of distinct courses in the embedding view' }) + @ApiOkResponse({ type: EmbeddingCountDto }) + public countCourses(): Promise { + return this.embeddingService.countCourses(); + } + @Get(':courseId') @ApiOperation({ summary: 'Return all rows for a given course (one per program it belongs to)' }) @ApiParam({ name: 'courseId', type: Number, example: 352413 }) diff --git a/src/embedding/embedding.service.ts b/src/embedding/embedding.service.ts index c732c2c..b2fcc1f 100644 --- a/src/embedding/embedding.service.ts +++ b/src/embedding/embedding.service.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; +import { EmbeddingCountDto } from './dtos/embedding-count.dto'; import { EmbeddingViewDto } from './dtos/embedding-view.dto'; @Injectable() @@ -20,4 +21,11 @@ export class EmbeddingService { ORDER BY program_id `; } + + public async countCourses(): Promise { + const result = await this.prisma.$queryRaw<[{ count: bigint }]>` + SELECT COUNT(DISTINCT course_id) as count FROM "v_courses_for_embedding" + `; + return { count: Number(result[0].count) }; + } } diff --git a/src/embedding/qdrant-course-index.service.ts b/src/embedding/qdrant-course-index.service.ts index 13278dc..084c018 100644 --- a/src/embedding/qdrant-course-index.service.ts +++ b/src/embedding/qdrant-course-index.service.ts @@ -70,15 +70,23 @@ export class QdrantCourseIndexService { const qdrantPoints = points.map(toQdrantUpsertPoint); - await retryTransient( - () => - this.client.upsert(this.collectionName, { - wait: true, - points: qdrantPoints, - }), - 3, - 1000, - ); + this.logger.debug(`Upserting ${qdrantPoints.length} points to collection ${this.collectionName}`); + + try { + await retryTransient( + () => + this.client.upsert(this.collectionName, { + wait: true, + points: qdrantPoints, + }), + 3, + 1000, + ); + this.logger.debug(`Successfully upserted ${qdrantPoints.length} points`); + } catch (error) { + this.logger.error(`Failed to upsert ${qdrantPoints.length} points: ${error}`); + throw error; + } } private validateCollection(info: unknown): void { diff --git a/src/jobs/dtos/run-workers.dto.ts b/src/jobs/dtos/run-workers.dto.ts index 1976bc3..5abb495 100644 --- a/src/jobs/dtos/run-workers.dto.ts +++ b/src/jobs/dtos/run-workers.dto.ts @@ -21,4 +21,7 @@ export class RunWorkersDto { @ApiProperty({ default: false }) public processSessions: boolean = false; + + @ApiProperty({ default: false, description: 'Run the course embedding index job (job:index).' }) + public processCourseEmbeddings: boolean = false; } diff --git a/src/jobs/jobs.constants.ts b/src/jobs/jobs.constants.ts index d1aba34..85e0ba5 100644 --- a/src/jobs/jobs.constants.ts +++ b/src/jobs/jobs.constants.ts @@ -1,6 +1,7 @@ import { Provider } from '@nestjs/common'; import { CourseCodeValidationPipe } from '../common/pipes/models/course/course-code-validation-pipe'; +import { CourseEmbeddingIndexerService } from '../embedding/embedding-course-indexer.service'; import { CourseInstancesJobService } from './workers/course-instances.worker'; import { CoursesJobService } from './workers/courses.worker'; import { ProgramsJobService } from './workers/programs.worker'; @@ -11,10 +12,14 @@ export const jobWorkerServiceMap = { CoursesJobService, CourseInstancesJobService, SessionsJobService, + CourseEmbeddingIndexerService, } as const; export const jobWorkerProviders: Provider[] = [ - ...Object.values(jobWorkerServiceMap), + ProgramsJobService, + CoursesJobService, + CourseInstancesJobService, + SessionsJobService, CourseCodeValidationPipe, ]; diff --git a/src/jobs/jobs.controller.ts b/src/jobs/jobs.controller.ts index 7428c90..3b9d964 100644 --- a/src/jobs/jobs.controller.ts +++ b/src/jobs/jobs.controller.ts @@ -21,6 +21,7 @@ export class JobsController { processCourseInstances = false, processProgramCourses = false, processSessions = false, + processCourseEmbeddings = false, } = body || {}; if (processAllJobs) { @@ -36,6 +37,7 @@ export class JobsController { if (processCourseInstances) jobs.push({ service: 'CourseInstancesJobService', method: 'processCourseInstances' }); if (processProgramCourses) jobs.push({ service: 'CoursesJobService', method: 'syncCourseDetailsWithCheminotData' }); if (processSessions) jobs.push({ service: 'SessionsJobService', method: 'processSessions' }); + if (processCourseEmbeddings) jobs.push({ service: 'CourseEmbeddingIndexerService', method: 'run' }); if (jobs.length === 0) { return { status: 'No jobs triggered (no flags set)' }; diff --git a/src/jobs/jobs.module.ts b/src/jobs/jobs.module.ts index c39a5ad..28a36d7 100644 --- a/src/jobs/jobs.module.ts +++ b/src/jobs/jobs.module.ts @@ -5,6 +5,7 @@ import { EtsModule } from '../common/api-helper/ets/ets.module'; import { PdfModule } from '../common/website-helper/pdf/pdf.module'; import { CourseModule } from '../course/course.module'; import { CourseInstanceModule } from '../course-instance/course-instance.module'; +import { EmbeddingModule } from '../embedding/embedding.module'; import { MonitoringModule } from '../monitoring/monitoring.module'; import { PrerequisiteModule } from '../prerequisite/prerequisite.module'; import { ProgramModule } from '../program/program.module'; @@ -23,6 +24,7 @@ import { JobsService } from './jobs.service'; ProgramModule, ProgramCourseModule, SessionModule, + EmbeddingModule, CheminotModule, EtsModule, diff --git a/src/jobs/jobs.service.ts b/src/jobs/jobs.service.ts index 9187253..29fe0f1 100644 --- a/src/jobs/jobs.service.ts +++ b/src/jobs/jobs.service.ts @@ -87,6 +87,10 @@ export class JobsService { // Create current Session and Prerequisite entities. // Data source: Horaire-cours PDF { service: 'SessionsJobService', method: 'processSessions' }, + + // Index course embeddings for RAG. + // Data source: Course data + { service: 'CourseEmbeddingIndexerService', method: 'run' }, ]; for (const [index, job] of jobs.entries()) { From 20eff829da1f52dd827f15fc6ee6a2ea4d952f54 Mon Sep 17 00:00:00 2001 From: Mohamed Hafdi Idrissi <70617264+mhd-hi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:56:02 -0400 Subject: [PATCH 03/11] cleanup --- .../embedding-course-indexer.service.ts | 29 +++- src/embedding/embedding-course.mapper.ts | 15 +- src/embedding/embedding.constants.ts | 1 + src/embedding/qdrant-course-index.service.ts | 136 +++++------------- src/embedding/qdrant-error.util.ts | 99 +++++++++++++ 5 files changed, 168 insertions(+), 112 deletions(-) create mode 100644 src/embedding/embedding.constants.ts create mode 100644 src/embedding/qdrant-error.util.ts diff --git a/src/embedding/embedding-course-indexer.service.ts b/src/embedding/embedding-course-indexer.service.ts index 7de992a..413470d 100644 --- a/src/embedding/embedding-course-indexer.service.ts +++ b/src/embedding/embedding-course-indexer.service.ts @@ -1,9 +1,10 @@ import { Injectable, Logger } from '@nestjs/common'; import { EmbeddingViewDto } from './dtos/embedding-view.dto'; +import { BGE_M3_VECTOR_SIZE } from './embedding.constants'; import { EmbeddingService } from './embedding.service'; import { - BGE_M3_VECTOR_SIZE, + computeCourseChangeKey, prepareCourseEmbedding, PreparedCourseEmbedding, } from './embedding-course.mapper'; @@ -15,6 +16,7 @@ import { type IndexingCounters = { indexedCount: number; + skippedCount: number; errorCount: number; }; @@ -35,6 +37,7 @@ export class CourseEmbeddingIndexerService { const counters: IndexingCounters = { indexedCount: 0, + skippedCount: 0, errorCount: 0, }; @@ -44,14 +47,26 @@ export class CourseEmbeddingIndexerService { await this.qdrantCourseIndexService.ensureCollection(); - const rows = await this.embeddingService.findAll(); + const [rows, existingHashes] = await Promise.all([ + this.embeddingService.findAll(), + this.qdrantCourseIndexService.getExistingTextHashes(), + ]); - this.logger.log(`Loaded ${rows.length} rows from v_courses_for_embedding.`); + this.logger.log(`Loaded ${rows.length} rows from v_courses_for_embedding. ${existingHashes.size} points already in Qdrant.`); - for (let offset = 0; offset < rows.length; offset += batchSize) { - const rowsBatch = rows.slice(offset, offset + batchSize); + const rowsToIndex = rows.filter((row) => { + const { id, hash } = computeCourseChangeKey(row); + return existingHashes.get(id) !== hash; + }); + + counters.skippedCount = rows.length - rowsToIndex.length; + + this.logger.log(`Skipping ${counters.skippedCount} unchanged courses. Indexing ${rowsToIndex.length} new/changed courses.`); + + for (let offset = 0; offset < rowsToIndex.length; offset += batchSize) { + const rowsBatch = rowsToIndex.slice(offset, offset + batchSize); const batchNumber = Math.floor(offset / batchSize) + 1; - const totalBatches = Math.ceil(rows.length / batchSize); + const totalBatches = Math.ceil(rowsToIndex.length / batchSize); this.logger.debug(`Processing batch ${batchNumber}/${totalBatches} (offset ${offset}, size ${rowsBatch.length})`); @@ -61,7 +76,7 @@ export class CourseEmbeddingIndexerService { const durationSeconds = ((Date.now() - startedAt) / 1000).toFixed(2); this.logger.log( - `COMPLETED: ${counters.indexedCount}/${rows.length} courses indexed in ${durationSeconds}s. Errors: ${counters.errorCount}. Missing: ${rows.length - counters.indexedCount - counters.errorCount}.`, + `COMPLETED: ${counters.indexedCount}/${rowsToIndex.length} courses indexed in ${durationSeconds}s. Skipped: ${counters.skippedCount}. Errors: ${counters.errorCount}.`, ); } diff --git a/src/embedding/embedding-course.mapper.ts b/src/embedding/embedding-course.mapper.ts index fc35c5a..a0bd3fc 100644 --- a/src/embedding/embedding-course.mapper.ts +++ b/src/embedding/embedding-course.mapper.ts @@ -1,9 +1,9 @@ +import { createHash } from 'node:crypto'; + import { v5 as uuidv5 } from 'uuid'; import { EmbeddingViewDto } from './dtos/embedding-view.dto'; -export const BGE_M3_VECTOR_SIZE = 1024; - const QDRANT_ID_NAMESPACE = process.env.QDRANT_ID_NAMESPACE ?? '5e7f1c4d-3d8a-45f1-87a4-9cf4de6f6b29'; @@ -31,6 +31,7 @@ export interface CourseEmbeddingPayload { availability: string[]; sessions: string[]; text: string; + text_hash: string; embedding_model: string; indexed_at: string; } @@ -45,6 +46,10 @@ export function toQdrantPointId(embeddingId: string): string { return uuidv5(embeddingId, QDRANT_ID_NAMESPACE); } +export function hashEmbeddingText(text: string): string { + return createHash('sha256').update(text).digest('hex'); +} + export function getCourseTypeLabel(type: string | null | undefined): string | undefined { if (!type) return undefined; return TYPE_LABELS[type] ?? type; @@ -123,6 +128,7 @@ export function buildCourseEmbeddingPayload( availability, sessions, text, + text_hash: hashEmbeddingText(text), embedding_model: embeddingModel, indexed_at: indexedAt, }; @@ -164,6 +170,11 @@ export function prepareCourseEmbedding( }; } +export function computeCourseChangeKey(row: EmbeddingViewDto): { id: string; hash: string } { + const text = buildCourseEmbeddingText(row); + return { id: toQdrantPointId(row.embedding_id), hash: hashEmbeddingText(text) }; +} + function clean(value: string | null | undefined): string { return normalizeWhitespace(value ?? '').trim(); } diff --git a/src/embedding/embedding.constants.ts b/src/embedding/embedding.constants.ts new file mode 100644 index 0000000..4c11c5d --- /dev/null +++ b/src/embedding/embedding.constants.ts @@ -0,0 +1 @@ +export const BGE_M3_VECTOR_SIZE = 1024; diff --git a/src/embedding/qdrant-course-index.service.ts b/src/embedding/qdrant-course-index.service.ts index 084c018..278905b 100644 --- a/src/embedding/qdrant-course-index.service.ts +++ b/src/embedding/qdrant-course-index.service.ts @@ -1,10 +1,14 @@ import { Injectable, Logger } from '@nestjs/common'; import { QdrantClient } from '@qdrant/js-client-rest'; +import { BGE_M3_VECTOR_SIZE } from './embedding.constants'; +import { CourseEmbeddingPayload } from './embedding-course.mapper'; import { - BGE_M3_VECTOR_SIZE, - CourseEmbeddingPayload, -} from './embedding-course.mapper'; + getNestedProperty, + isNotFoundError, + isRecord, + retryTransient, +} from './qdrant-error.util'; export interface CourseQdrantPoint { id: string; @@ -63,6 +67,32 @@ export class QdrantCourseIndexService { } } + public async getExistingTextHashes(): Promise> { + const hashes = new Map(); + let nextOffset: string | number | undefined = undefined; + + do { + const response = await this.client.scroll(this.collectionName, { + offset: nextOffset, + limit: 250, + with_payload: ['text_hash'], + with_vector: false, + }); + + for (const point of response.points) { + const hash = (point.payload as Record)?.text_hash; + if (typeof point.id === 'string' && typeof hash === 'string') { + hashes.set(point.id, hash); + } + } + + const raw = response.next_page_offset; + nextOffset = typeof raw === 'string' || typeof raw === 'number' ? raw : undefined; + } while (nextOffset !== undefined && nextOffset !== null); + + return hashes; + } + public async upsertPoints(points: CourseQdrantPoint[]): Promise { if (points.length === 0) { return; @@ -151,103 +181,3 @@ function parseVectorConfig(value: unknown): VectorConfig | undefined { distance, }; } - -async function retryTransient( - operation: () => Promise, - maxAttempts: number, - delayMs: number, -): Promise { - let lastError: unknown; - - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - return await operation(); - } catch (error) { - lastError = error; - - if (!isTransientError(error) || attempt === maxAttempts) { - throw error; - } - - await sleep(delayMs * attempt); - } - } - - throw lastError; -} - -function isNotFoundError(error: unknown): boolean { - return getStatusCode(error) === 404; -} - -function isTransientError(error: unknown): boolean { - const status = getStatusCode(error); - const code = getErrorCode(error); - - return ( - status === 408 || - status === 429 || - isServerErrorStatus(status) || - code === 'ECONNRESET' || - code === 'ECONNREFUSED' || - code === 'ETIMEDOUT' - ); -} - -function isServerErrorStatus(status: number | undefined): boolean { - return typeof status === 'number' && status >= 500; -} - -function getStatusCode(error: unknown): number | undefined { - if (!isRecord(error)) { - return undefined; - } - - const status = error.status; - - if (typeof status === 'number') { - return status; - } - - const statusCode = error.statusCode; - - if (typeof statusCode === 'number') { - return statusCode; - } - - const response = error.response; - - if (isRecord(response) && typeof response.status === 'number') { - return response.status; - } - - return undefined; -} - -function getErrorCode(error: unknown): string | undefined { - if (!isRecord(error)) { - return undefined; - } - - return typeof error.code === 'string' ? error.code : undefined; -} - -function getNestedProperty(value: unknown, path: string[]): unknown { - return path.reduce((currentValue, key) => { - if (!isRecord(currentValue)) { - return undefined; - } - - return currentValue[key]; - }, value); -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null; -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} diff --git a/src/embedding/qdrant-error.util.ts b/src/embedding/qdrant-error.util.ts new file mode 100644 index 0000000..b8fae03 --- /dev/null +++ b/src/embedding/qdrant-error.util.ts @@ -0,0 +1,99 @@ +export async function retryTransient( + operation: () => Promise, + maxAttempts: number, + delayMs: number, +): Promise { + let lastError: unknown; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await operation(); + } catch (error) { + lastError = error; + + if (!isTransientError(error) || attempt === maxAttempts) { + throw error; + } + + await sleep(delayMs * attempt); + } + } + + throw lastError; +} + +export function isNotFoundError(error: unknown): boolean { + return getStatusCode(error) === 404; +} + +export function isTransientError(error: unknown): boolean { + const status = getStatusCode(error); + const code = getErrorCode(error); + + return ( + status === 408 || + status === 429 || + isServerErrorStatus(status) || + code === 'ECONNRESET' || + code === 'ECONNREFUSED' || + code === 'ETIMEDOUT' + ); +} + +function isServerErrorStatus(status: number | undefined): boolean { + return typeof status === 'number' && status >= 500; +} + +function getStatusCode(error: unknown): number | undefined { + if (!isRecord(error)) { + return undefined; + } + + const status = error.status; + + if (typeof status === 'number') { + return status; + } + + const statusCode = error.statusCode; + + if (typeof statusCode === 'number') { + return statusCode; + } + + const response = error.response; + + if (isRecord(response) && typeof response.status === 'number') { + return response.status; + } + + return undefined; +} + +function getErrorCode(error: unknown): string | undefined { + if (!isRecord(error)) { + return undefined; + } + + return typeof error.code === 'string' ? error.code : undefined; +} + +export function getNestedProperty(value: unknown, path: string[]): unknown { + return path.reduce((currentValue, key) => { + if (!isRecord(currentValue)) { + return undefined; + } + + return currentValue[key]; + }, value); +} + +export function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} From 5b33241c58f890e18dc2845192b38cc81702079d Mon Sep 17 00:00:00 2001 From: Mohamed Hafdi Idrissi <70617264+mhd-hi@users.noreply.github.com> Date: Thu, 4 Jun 2026 03:49:13 -0400 Subject: [PATCH 04/11] fix transformers.js silencly failing --- docker-compose.yml | 6 ++-- src/embedding/embedding-worker.client.ts | 39 ++++++++++++++++-------- src/embedding/workers/bge-m3.worker.ts | 32 +++++++++++++------ 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e745d4a..950c0b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: context: . target: production ports: - - "${HOST_PORT:-3501}:${PORT:-3001}" + - "${PORT:-3501}:${PORT:-3001}" environment: DATABASE_URL: ${DATABASE_URL} APP_ENV: production @@ -35,13 +35,11 @@ services: context: . target: dev ports: - - "${HOST_PORT:-3501}:${PORT:-3001}" + - "${PORT:-3501}:${PORT:-3001}" volumes: - .:/app - /app/node_modules - transformers-cache:/app/.cache/transformers - mem_limit: 4g - memswap_limit: 4g environment: DATABASE_URL: ${DATABASE_URL} APP_ENV: development diff --git a/src/embedding/embedding-worker.client.ts b/src/embedding/embedding-worker.client.ts index 02c0a7c..e7376ec 100644 --- a/src/embedding/embedding-worker.client.ts +++ b/src/embedding/embedding-worker.client.ts @@ -7,6 +7,7 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; type PendingRequest = { resolve: (vectors: number[][]) => void; reject: (error: Error) => void; + timer: ReturnType; }; type EmbedWorkerSuccessMessage = { @@ -27,6 +28,7 @@ type EmbedWorkerMessage = EmbedWorkerSuccessMessage | EmbedWorkerFailureMessage; export class EmbeddingWorkerClient implements OnModuleDestroy { private readonly logger = new Logger(EmbeddingWorkerClient.name); private worker: Worker | null = null; + private workerTerminating = false; private readonly pending = new Map(); private nextRequestId = 1; @@ -58,11 +60,12 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { }); worker.on('exit', (code: number) => { - if (code !== 0) { + if (code !== 0 && !this.workerTerminating) { const error = new Error(`Embedding worker exited with code ${code}`); this.logger.error(error.message); this.rejectAll(error); } + this.workerTerminating = false; }); return worker; @@ -84,19 +87,28 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { const id = this.nextRequestId++; const model = process.env.EMBEDDING_MODEL ?? 'Xenova/bge-m3'; const dtype = process.env.EMBEDDING_DTYPE ?? 'q4'; + const timeoutMs = parseInt(process.env.EMBEDDING_WORKER_TIMEOUT_MS ?? '300000', 10); return new Promise((resolve, reject) => { - this.pending.set(id, { - resolve, - reject, - }); - - this.getWorker().postMessage({ - id, - texts, - model, - dtype, - }); + const timer = setTimeout(() => { + if (!this.pending.has(id)) { + return; + } + this.pending.delete(id); + + const error = new Error(`Embedding worker timed out after ${timeoutMs}ms (request ${id}).`); + this.logger.error(error.message + ' Terminating worker.'); + void this.worker?.terminate().then(() => { + this.worker = null; + }); + + reject(error); + }, timeoutMs); + + this.pending.set(id, { resolve, reject, timer }); + + this.logger.debug(`Posting embed request ${id}: ${texts.length} texts, model=${model}, dtype=${dtype}, timeout=${timeoutMs}ms`); + this.getWorker().postMessage({ id, texts, model, dtype }); }); } @@ -105,6 +117,7 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { return; } + this.workerTerminating = true; await this.worker.terminate(); } @@ -125,6 +138,7 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { return; } + clearTimeout(pendingRequest.timer); this.pending.delete(parsedMessage.id); if (parsedMessage.ok) { @@ -137,6 +151,7 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { private rejectAll(error: Error): void { for (const request of this.pending.values()) { + clearTimeout(request.timer); request.reject(error); } diff --git a/src/embedding/workers/bge-m3.worker.ts b/src/embedding/workers/bge-m3.worker.ts index 7c22817..f47734c 100644 --- a/src/embedding/workers/bge-m3.worker.ts +++ b/src/embedding/workers/bge-m3.worker.ts @@ -9,15 +9,15 @@ type EmbedRequest = { type EmbedResponse = | { - id: number; - ok: true; - vectors: number[][]; - } + id: number; + ok: true; + vectors: number[][]; + } | { - id: number; - ok: false; - error: string; - }; + id: number; + ok: false; + error: string; + }; type TensorLike = { data: ArrayLike | Iterable; @@ -88,16 +88,22 @@ async function handleMessage(message: unknown): Promise { return; } + console.log(`[bge-m3.worker] Request ${request.id}: loading extractor for ${request.texts.length} texts`); const extractor = await getExtractor(request.model, request.dtype); + console.log(`[bge-m3.worker] Request ${request.id}: running inference on ${request.texts.length} texts (max_length=1024)`); + const inferenceStart = Date.now(); const output = await extractor(request.texts, { pooling: 'mean', normalize: true, truncation: true, - max_length: 8192, + max_length: 1024, }); + console.log(`[bge-m3.worker] Request ${request.id}: inference done in ${Date.now() - inferenceStart}ms`); + console.log(`[bge-m3.worker] Request ${request.id}: converting tensor to vectors`); const vectors = tensorToVectors(output, request.texts.length); + console.log(`[bge-m3.worker] Request ${request.id}: done, returning ${vectors.length} vectors`); const response: EmbedResponse = { id: request.id, @@ -137,6 +143,8 @@ async function createExtractor( model: string, dtype?: string, ): Promise { + console.log(`[bge-m3.worker] Loading model: ${model} (dtype=${dtype ?? 'default'})`); + const transformersModule = (await import( '@huggingface/transformers' )) as unknown as TransformersModule; @@ -151,7 +159,11 @@ async function createExtractor( options.dtype = dtype; } - return transformersModule.pipeline('feature-extraction', model, options); + const extractor = await transformersModule.pipeline('feature-extraction', model, options); + + console.log(`[bge-m3.worker] Model ready: ${model}`); + + return extractor; } function parseEmbedRequest(message: unknown): EmbedRequest { From 31a282e06a9be66a669d7f02514356000dd3b74b Mon Sep 17 00:00:00 2001 From: Mohamed Hafdi Idrissi <70617264+mhd-hi@users.noreply.github.com> Date: Thu, 4 Jun 2026 04:25:08 -0400 Subject: [PATCH 05/11] fix devDependencies leaking in prod & transformers error handling --- Dockerfile | 22 +++++++++++++++++++--- package.json | 2 +- src/embedding/embedding-worker.client.ts | 3 +++ src/embedding/workers/bge-m3.worker.ts | 5 +++++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2de0587..6df57b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY package.json yarn.lock ./ RUN yarn install --frozen-lockfile --ignore-scripts +FROM node:22.22.3-bullseye-slim AS prod-deps + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + make \ + g++ \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY package.json yarn.lock ./ +RUN yarn install --frozen-lockfile --ignore-scripts --production + FROM node:22.22.3-bullseye-slim AS build WORKDIR /app @@ -21,6 +35,7 @@ COPY --from=deps /app/node_modules ./node_modules COPY . ./ ENV NODE_ENV=production +RUN yarn prisma:generate RUN yarn build FROM node:22.22.3-bullseye-slim AS dev @@ -49,13 +64,16 @@ FROM node:22.22.3-bullseye-slim AS production ARG APP_GIT_SHORT_SHA ENV APP_GIT_SHORT_SHA=${APP_GIT_SHORT_SHA} +ENV NODE_ENV=production ENV APP_ENV=production ENV TZ=America/Toronto WORKDIR /app +COPY package.json ./ COPY --from=build /app/dist ./dist -COPY --from=deps /app/node_modules ./node_modules +COPY --from=prod-deps /app/node_modules ./node_modules +COPY --from=build /app/node_modules/.prisma ./node_modules/.prisma COPY prisma ./prisma RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -64,7 +82,5 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ tzdata \ && rm -rf /var/lib/apt/lists/* -RUN yarn prisma:generate - EXPOSE 3001 CMD ["yarn", "start:prod"] diff --git a/package.json b/package.json index da554b6..4fbd1f0 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@nestjs/schedule": "^4.1.1", "@nestjs/swagger": "^11.2.6", "@prisma/client": "^5.22.0", + "prisma": "^5.22.0", "@qdrant/js-client-rest": "^1.18.0", "@types/unzipper": "^0.10.10", "axios": "^1.13.5", @@ -87,7 +88,6 @@ "knip": "^5.86.0", "prettier": "^3.3.3", "prettier-eslint": "^16.3.0", - "prisma": "^5.22.0", "source-map-support": "^0.5.21", "supertest": "^7.0.0", "ts-jest": "^29.4.6", diff --git a/src/embedding/embedding-worker.client.ts b/src/embedding/embedding-worker.client.ts index e7376ec..ecdea49 100644 --- a/src/embedding/embedding-worker.client.ts +++ b/src/embedding/embedding-worker.client.ts @@ -57,6 +57,7 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { worker.on('error', (error: Error) => { this.logger.error(`Embedding worker error: ${error.message}`, error.stack); this.rejectAll(error); + this.worker = null; }); worker.on('exit', (code: number) => { @@ -66,6 +67,7 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { this.rejectAll(error); } this.workerTerminating = false; + this.worker = null; }); return worker; @@ -119,6 +121,7 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { this.workerTerminating = true; await this.worker.terminate(); + this.worker = null; } private handleWorkerMessage(message: unknown): void { diff --git a/src/embedding/workers/bge-m3.worker.ts b/src/embedding/workers/bge-m3.worker.ts index f47734c..ec1c3a3 100644 --- a/src/embedding/workers/bge-m3.worker.ts +++ b/src/embedding/workers/bge-m3.worker.ts @@ -113,6 +113,11 @@ async function handleMessage(message: unknown): Promise { port.postMessage(response); } catch (error) { + if (requestId === -1) { + console.error('[bge-m3.worker] Cannot reply: message had no valid id, dropping error response'); + return; + } + const response: EmbedResponse = { id: requestId, ok: false, From 03f21a184429ee8b09001517f8d7c686819bb649 Mon Sep 17 00:00:00 2001 From: Mohamed Hafdi Idrissi <70617264+mhd-hi@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:08:58 -0400 Subject: [PATCH 06/11] (docker) include prisma generate in dev environnement --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6df57b9..6d28c84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ ENV APP_ENV=development ENV TZ=America/Toronto EXPOSE 3001 -CMD ["yarn", "start:dev"] +CMD ["sh", "-c", "yarn prisma:generate && yarn start:dev"] FROM node:22.22.3-bullseye-slim AS production From 27eada2be039f197d5417d26239504bd4537144f Mon Sep 17 00:00:00 2001 From: Mohamed Hafdi Idrissi <70617264+mhd-hi@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:31:00 -0400 Subject: [PATCH 07/11] sonarr chown docker prod --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6d28c84..96d224a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,7 +80,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ openssl \ ca-certificates \ tzdata \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && groupadd --gid 10001 appgroup \ + && useradd --uid 10001 --gid 10001 --no-create-home appuser \ + && chown -R appuser:appgroup /app + +USER 10001:10001 EXPOSE 3001 CMD ["yarn", "start:prod"] From d998c2e22bb67f294455100d20dbdeb94c7136f4 Mon Sep 17 00:00:00 2001 From: Mohamed Hafdi Idrissi <70617264+mhd-hi@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:31:51 -0400 Subject: [PATCH 08/11] unit --- test/jobs/jobs.service.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/jobs/jobs.service.test.ts b/test/jobs/jobs.service.test.ts index af42214..70de6e1 100644 --- a/test/jobs/jobs.service.test.ts +++ b/test/jobs/jobs.service.test.ts @@ -20,16 +20,17 @@ describe('JobsService', () => { }); it('should process all jobs in order and call runWorker with correct service/method', async () => { - const results = ['A', 'B', 'C', 'D', 'E', 'F']; + const results = ['A', 'B', 'C', 'D', 'E', 'F', 'G']; runWorkerSpy.mockImplementation(() => Promise.resolve(results.shift())); await service.processJobs(); - expect(runWorkerSpy).toHaveBeenCalledTimes(6); + expect(runWorkerSpy).toHaveBeenCalledTimes(7); expect(runWorkerSpy).toHaveBeenNthCalledWith(1, 'ProgramsJobService', 'processPrograms'); expect(runWorkerSpy).toHaveBeenNthCalledWith(2, 'CoursesJobService', 'processCourses'); expect(runWorkerSpy).toHaveBeenNthCalledWith(3, 'CoursesJobService', 'syncCourseDescriptionsFromEtsWebsite'); expect(runWorkerSpy).toHaveBeenNthCalledWith(4, 'CourseInstancesJobService', 'processCourseInstances'); expect(runWorkerSpy).toHaveBeenNthCalledWith(5, 'CoursesJobService', 'syncCourseDetailsWithCheminotData'); expect(runWorkerSpy).toHaveBeenNthCalledWith(6, 'SessionsJobService', 'processSessions'); + expect(runWorkerSpy).toHaveBeenNthCalledWith(7, 'CourseEmbeddingIndexerService', 'run'); }); it('should continue processing jobs even if one fails', async () => { @@ -39,9 +40,10 @@ describe('JobsService', () => { .mockResolvedValueOnce('ok3') .mockResolvedValueOnce('ok4') .mockResolvedValueOnce('ok5') - .mockResolvedValueOnce('ok6'); + .mockResolvedValueOnce('ok6') + .mockResolvedValueOnce('ok7'); await service.processJobs(); - expect(runWorkerSpy).toHaveBeenCalledTimes(6); + expect(runWorkerSpy).toHaveBeenCalledTimes(7); expect(loggerErrorSpy).toHaveBeenCalledWith( expect.stringContaining('Job 2 (CoursesJobService.processCourses) failed: fail2'), expect.any(String) @@ -55,7 +57,8 @@ describe('JobsService', () => { .mockResolvedValueOnce('ok3') .mockResolvedValueOnce('ok4') .mockResolvedValueOnce('ok5') - .mockResolvedValueOnce('ok6'); + .mockResolvedValueOnce('ok6') + .mockResolvedValueOnce('ok7'); await service.processJobs(); expect(loggerLogSpy).toHaveBeenCalledWith('Starting sequential job processing...'); expect(loggerLogSpy).toHaveBeenCalledWith('Starting job 1: ProgramsJobService.processPrograms'); @@ -64,6 +67,7 @@ describe('JobsService', () => { expect(loggerLogSpy).toHaveBeenCalledWith('Starting job 4: CourseInstancesJobService.processCourseInstances'); expect(loggerLogSpy).toHaveBeenCalledWith('Starting job 5: CoursesJobService.syncCourseDetailsWithCheminotData'); expect(loggerLogSpy).toHaveBeenCalledWith('Starting job 6: SessionsJobService.processSessions'); + expect(loggerLogSpy).toHaveBeenCalledWith('Starting job 7: CourseEmbeddingIndexerService.run'); expect(loggerLogSpy).toHaveBeenCalledWith('Job processing completed.'); expect(loggerLogSpy).toHaveBeenCalledWith( expect.stringContaining('Job 1 (ProgramsJobService.processPrograms) completed :'), From 0e8768c9dc13f201850ecbfd81f1534266988e35 Mon Sep 17 00:00:00 2001 From: Mohamed Hafdi Idrissi <70617264+mhd-hi@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:44:56 -0400 Subject: [PATCH 09/11] remove uuid package and generate manually instead & fix e2e --- package.json | 5 ++-- src/common/utils/uuid/uuidUtil.ts | 22 +++++++++++++++++ src/embedding/embedding-course.mapper.ts | 4 +-- test/common/utils/uuid/uuidUtil.test.ts | 31 ++++++++++++++++++++++++ yarn.lock | 5 ---- 5 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 src/common/utils/uuid/uuidUtil.ts create mode 100644 test/common/utils/uuid/uuidUtil.test.ts diff --git a/package.json b/package.json index 4fbd1f0..0b69c1f 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,6 @@ "@nestjs/schedule": "^4.1.1", "@nestjs/swagger": "^11.2.6", "@prisma/client": "^5.22.0", - "prisma": "^5.22.0", "@qdrant/js-client-rest": "^1.18.0", "@types/unzipper": "^0.10.10", "axios": "^1.13.5", @@ -56,10 +55,10 @@ "express": "^4.21.1", "pdf2json": "^4.0.0", "posthog-node": "^5.34.2", + "prisma": "^5.22.0", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", - "unzipper": "^0.12.3", - "uuid": "^14.0.0" + "unzipper": "^0.12.3" }, "devDependencies": { "@nestjs/cli": "^11.0.17", diff --git a/src/common/utils/uuid/uuidUtil.ts b/src/common/utils/uuid/uuidUtil.ts new file mode 100644 index 0000000..9ee3768 --- /dev/null +++ b/src/common/utils/uuid/uuidUtil.ts @@ -0,0 +1,22 @@ +import { createHash } from 'node:crypto'; + +export function uuidV5(name: string, namespace: string): string { + const namespaceBytes = Buffer.from(namespace.replace(/-/g, ''), 'hex'); + const nameBytes = Buffer.from(name, 'utf8'); + + const hash = createHash('sha1') + .update(namespaceBytes) + .update(nameBytes) + .digest(); + + hash[6] = (hash[6] & 0x0f) | 0x50; + hash[8] = (hash[8] & 0x3f) | 0x80; + + return [ + hash.subarray(0, 4).toString('hex'), + hash.subarray(4, 6).toString('hex'), + hash.subarray(6, 8).toString('hex'), + hash.subarray(8, 10).toString('hex'), + hash.subarray(10, 16).toString('hex'), + ].join('-'); +} diff --git a/src/embedding/embedding-course.mapper.ts b/src/embedding/embedding-course.mapper.ts index a0bd3fc..4056a9f 100644 --- a/src/embedding/embedding-course.mapper.ts +++ b/src/embedding/embedding-course.mapper.ts @@ -1,6 +1,6 @@ import { createHash } from 'node:crypto'; -import { v5 as uuidv5 } from 'uuid'; +import { uuidV5 } from '@/common/utils/uuid/uuidUtil'; import { EmbeddingViewDto } from './dtos/embedding-view.dto'; @@ -43,7 +43,7 @@ export interface PreparedCourseEmbedding { } export function toQdrantPointId(embeddingId: string): string { - return uuidv5(embeddingId, QDRANT_ID_NAMESPACE); + return uuidV5(embeddingId, QDRANT_ID_NAMESPACE); } export function hashEmbeddingText(text: string): string { diff --git a/test/common/utils/uuid/uuidUtil.test.ts b/test/common/utils/uuid/uuidUtil.test.ts new file mode 100644 index 0000000..c894080 --- /dev/null +++ b/test/common/utils/uuid/uuidUtil.test.ts @@ -0,0 +1,31 @@ +import { uuidV5 } from '@/common/utils/uuid/uuidUtil'; + +const UUID_V5_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +// RFC 4122 DNS namespace — well-known test vector source +const DNS_NAMESPACE = '6ba7b810-9dad-11d1-80b4-00c04fd430c8'; + +describe('uuidV5', () => { + it('returns a lowercase hyphenated UUID v5 string', () => { + expect(uuidV5('hello', DNS_NAMESPACE)).toMatch(UUID_V5_REGEX); + }); + + it('is deterministic for the same name and namespace', () => { + expect(uuidV5('hello', DNS_NAMESPACE)).toBe(uuidV5('hello', DNS_NAMESPACE)); + }); + + it('produces different UUIDs for different names', () => { + expect(uuidV5('foo', DNS_NAMESPACE)).not.toBe(uuidV5('bar', DNS_NAMESPACE)); + }); + + it('produces different UUIDs for different namespaces', () => { + const otherNamespace = '5e7f1c4d-3d8a-45f1-87a4-9cf4de6f6b29'; + + expect(uuidV5('hello', DNS_NAMESPACE)).not.toBe(uuidV5('hello', otherNamespace)); + }); + + it('matches RFC 4122 known test vector', () => { + // python: import uuid; str(uuid.uuid5(uuid.NAMESPACE_DNS, 'python.org')) + expect(uuidV5('python.org', DNS_NAMESPACE)).toBe('886313e1-3b8a-5372-9b90-0c9aee199e5d'); + }); +}); diff --git a/yarn.lock b/yarn.lock index 1900943..7fefd8e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7675,11 +7675,6 @@ uuid@11.0.3: resolved "https://registry.yarnpkg.com/uuid/-/uuid-11.0.3.tgz#248451cac9d1a4a4128033e765d137e2b2c49a3d" integrity sha512-d0z310fCWv5dJwnX1Y/MncBAqGMKEzlBb1AOf7z9K8ALnd0utBX/msg/fA0+sbyN1ihbMsLhrBlnl1ak7Wa0rg== -uuid@^14.0.0: - version "14.0.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-14.0.0.tgz#0af883220163d264ffe0c084f6b8a89b9666966d" - integrity sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg== - v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" From d47e294e63890d936d173ecac11c7d32f31c6db6 Mon Sep 17 00:00:00 2001 From: Mohamed Hafdi Idrissi <70617264+mhd-hi@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:18:45 -0400 Subject: [PATCH 10/11] coverage --- src/common/utils/uuid/uuidUtil.ts | 1 + .../embedding-course-indexer.service.test.ts | 245 +++++++++++++ .../embedding/embedding-course.mapper.test.ts | 345 ++++++++++++++++++ .../embedding/embedding-worker.client.test.ts | 149 ++++++++ test/embedding/embedding.controller.test.ts | 47 +++ test/embedding/embedding.service.test.ts | 51 +++ .../qdrant-course-index.service.test.ts | 226 ++++++++++++ test/embedding/qdrant-error.util.test.ts | 151 ++++++++ 8 files changed, 1215 insertions(+) create mode 100644 test/embedding/embedding-course-indexer.service.test.ts create mode 100644 test/embedding/embedding-course.mapper.test.ts create mode 100644 test/embedding/embedding-worker.client.test.ts create mode 100644 test/embedding/embedding.controller.test.ts create mode 100644 test/embedding/embedding.service.test.ts create mode 100644 test/embedding/qdrant-course-index.service.test.ts create mode 100644 test/embedding/qdrant-error.util.test.ts diff --git a/src/common/utils/uuid/uuidUtil.ts b/src/common/utils/uuid/uuidUtil.ts index 9ee3768..019889b 100644 --- a/src/common/utils/uuid/uuidUtil.ts +++ b/src/common/utils/uuid/uuidUtil.ts @@ -1,5 +1,6 @@ import { createHash } from 'node:crypto'; +// produces stable reproducible identifier (name, namespace) export function uuidV5(name: string, namespace: string): string { const namespaceBytes = Buffer.from(namespace.replace(/-/g, ''), 'hex'); const nameBytes = Buffer.from(name, 'utf8'); diff --git a/test/embedding/embedding-course-indexer.service.test.ts b/test/embedding/embedding-course-indexer.service.test.ts new file mode 100644 index 0000000..66d50b7 --- /dev/null +++ b/test/embedding/embedding-course-indexer.service.test.ts @@ -0,0 +1,245 @@ +import { EmbeddingViewDto } from '../../src/embedding/dtos/embedding-view.dto'; +import { BGE_M3_VECTOR_SIZE } from '../../src/embedding/embedding.constants'; +import { EmbeddingService } from '../../src/embedding/embedding.service'; +import { computeCourseChangeKey } from '../../src/embedding/embedding-course.mapper'; +import { CourseEmbeddingIndexerService } from '../../src/embedding/embedding-course-indexer.service'; +import { EmbeddingWorkerClient } from '../../src/embedding/embedding-worker.client'; +import { QdrantCourseIndexService } from '../../src/embedding/qdrant-course-index.service'; + +const makeVector = (): number[] => Array.from({ length: BGE_M3_VECTOR_SIZE }, () => 0.1); + +const buildRow = (overrides: Partial = {}): EmbeddingViewDto => ({ + embedding_id: '352507_182848', + course_id: 352507, + program_id: 182848, + code: 'LOG635', + title: 'Systèmes intelligents', + description: 'Description.', + cycle: 1, + program_title: 'Génie logiciel', + type: 'TRONC', + typical_session_index: 5, + unstructured_prerequisite: null, + prerequisite_codes: [], + has_prerequisites: false, + availability: ['JOUR'], + sessions: ['Automne 2026'], + ...overrides, +}); + +describe('CourseEmbeddingIndexerService', () => { + let service: CourseEmbeddingIndexerService; + let embeddingServiceMock: { findAll: jest.Mock }; + let workerClientMock: { embed: jest.Mock }; + let qdrantMock: { + ensureCollection: jest.Mock; + getExistingTextHashes: jest.Mock; + upsertPoints: jest.Mock; + }; + + beforeEach(() => { + jest.clearAllMocks(); + + embeddingServiceMock = { findAll: jest.fn() }; + workerClientMock = { embed: jest.fn() }; + qdrantMock = { + ensureCollection: jest.fn().mockResolvedValue(undefined), + getExistingTextHashes: jest.fn().mockResolvedValue(new Map()), + upsertPoints: jest.fn().mockResolvedValue(undefined), + }; + + service = new CourseEmbeddingIndexerService( + embeddingServiceMock as unknown as EmbeddingService, + workerClientMock as unknown as EmbeddingWorkerClient, + qdrantMock as unknown as QdrantCourseIndexService, + ); + + jest.spyOn(service['logger'], 'log').mockImplementation(() => {}); + jest.spyOn(service['logger'], 'debug').mockImplementation(() => {}); + jest.spyOn(service['logger'], 'warn').mockImplementation(() => {}); + jest.spyOn(service['logger'], 'error').mockImplementation(() => {}); + }); + + describe('run — happy path', () => { + it('ensures the Qdrant collection before doing any work', async () => { + embeddingServiceMock.findAll.mockResolvedValue([]); + await service.run(); + expect(qdrantMock.ensureCollection).toHaveBeenCalledTimes(1); + }); + + it('fetches all rows and existing hashes in parallel', async () => { + embeddingServiceMock.findAll.mockResolvedValue([]); + await service.run(); + expect(embeddingServiceMock.findAll).toHaveBeenCalledTimes(1); + expect(qdrantMock.getExistingTextHashes).toHaveBeenCalledTimes(1); + }); + + it('does not embed or upsert when there are no rows', async () => { + embeddingServiceMock.findAll.mockResolvedValue([]); + await service.run(); + expect(workerClientMock.embed).not.toHaveBeenCalled(); + expect(qdrantMock.upsertPoints).not.toHaveBeenCalled(); + }); + + it('indexes a new row that is not yet in Qdrant', async () => { + const row = buildRow(); + embeddingServiceMock.findAll.mockResolvedValue([row]); + workerClientMock.embed.mockResolvedValue([makeVector()]); + + await service.run(); + + expect(qdrantMock.upsertPoints).toHaveBeenCalledTimes(1); + const [points] = qdrantMock.upsertPoints.mock.calls[0] as [{ vector: number[] }[]]; + expect(points).toHaveLength(1); + expect(points[0].vector).toHaveLength(BGE_M3_VECTOR_SIZE); + }); + }); + + describe('run — change detection', () => { + it('skips a row whose hash already matches what is stored in Qdrant', async () => { + const row = buildRow(); + const { id, hash } = computeCourseChangeKey(row); + embeddingServiceMock.findAll.mockResolvedValue([row]); + qdrantMock.getExistingTextHashes.mockResolvedValue(new Map([[id, hash]])); + + await service.run(); + + expect(workerClientMock.embed).not.toHaveBeenCalled(); + expect(qdrantMock.upsertPoints).not.toHaveBeenCalled(); + }); + + it('re-indexes a row when its stored hash is stale', async () => { + const row = buildRow(); + const { id } = computeCourseChangeKey(row); + embeddingServiceMock.findAll.mockResolvedValue([row]); + qdrantMock.getExistingTextHashes.mockResolvedValue(new Map([[id, 'old-hash']])); + workerClientMock.embed.mockResolvedValue([makeVector()]); + + await service.run(); + + expect(qdrantMock.upsertPoints).toHaveBeenCalledTimes(1); + }); + }); + + describe('run — batching', () => { + it('processes rows in batches according to EMBEDDING_BATCH_SIZE', async () => { + const rows = [ + buildRow({ embedding_id: '1_1', course_id: 1, code: 'LOG001' }), + buildRow({ embedding_id: '2_1', course_id: 2, code: 'LOG002' }), + buildRow({ embedding_id: '3_1', course_id: 3, code: 'LOG003' }), + ]; + embeddingServiceMock.findAll.mockResolvedValue(rows); + process.env.EMBEDDING_BATCH_SIZE = '2'; + workerClientMock.embed + .mockResolvedValueOnce([makeVector(), makeVector()]) + .mockResolvedValueOnce([makeVector()]); + + await service.run(); + + expect(workerClientMock.embed).toHaveBeenCalledTimes(2); + delete process.env.EMBEDDING_BATCH_SIZE; + }); + + it('uses the default batch size of 50 when EMBEDDING_BATCH_SIZE is unset', async () => { + delete process.env.EMBEDDING_BATCH_SIZE; + const rows = Array.from({ length: 3 }, (_, i) => + buildRow({ embedding_id: `${i}_1`, course_id: i, code: `LOG00${i}` }), + ); + embeddingServiceMock.findAll.mockResolvedValue(rows); + workerClientMock.embed.mockResolvedValue(rows.map(makeVector)); + + await service.run(); + + expect(workerClientMock.embed).toHaveBeenCalledTimes(1); + }); + + it('falls back to default batch size when EMBEDDING_BATCH_SIZE is not a positive integer', async () => { + process.env.EMBEDDING_BATCH_SIZE = 'invalid'; + const row = buildRow(); + embeddingServiceMock.findAll.mockResolvedValue([row]); + workerClientMock.embed.mockResolvedValue([makeVector()]); + + await service.run(); + + expect(workerClientMock.embed).toHaveBeenCalledTimes(1); + delete process.env.EMBEDDING_BATCH_SIZE; + }); + }); + + describe('run — batch embed failure falls back to one-by-one', () => { + it('retries each item individually when the batch embed call throws', async () => { + const row = buildRow(); + embeddingServiceMock.findAll.mockResolvedValue([row]); + workerClientMock.embed + .mockRejectedValueOnce(new Error('batch failed')) + .mockResolvedValueOnce([makeVector()]); + + await service.run(); + + expect(workerClientMock.embed).toHaveBeenCalledTimes(2); + expect(qdrantMock.upsertPoints).toHaveBeenCalledTimes(1); + }); + + it('skips an item and continues when its individual embed fails', async () => { + const rows = [ + buildRow({ embedding_id: '1_1', course_id: 1, code: 'LOG001' }), + buildRow({ embedding_id: '2_1', course_id: 2, code: 'LOG002' }), + ]; + embeddingServiceMock.findAll.mockResolvedValue(rows); + workerClientMock.embed + .mockRejectedValueOnce(new Error('batch failed')) + .mockRejectedValueOnce(new Error('item 1 embed failed')) + .mockResolvedValueOnce([makeVector()]); + + await service.run(); + + // batch fail + item1 fail + item2 success + expect(workerClientMock.embed).toHaveBeenCalledTimes(3); + expect(qdrantMock.upsertPoints).toHaveBeenCalledTimes(1); + }); + + it('throws and stops when upsert fails during one-by-one processing', async () => { + const row = buildRow(); + embeddingServiceMock.findAll.mockResolvedValue([row]); + workerClientMock.embed + .mockRejectedValueOnce(new Error('batch failed')) + .mockResolvedValueOnce([makeVector()]); + qdrantMock.upsertPoints.mockRejectedValue(new Error('upsert failed')); + + await expect(service.run()).rejects.toThrow('upsert failed'); + }); + }); + + describe('run — upsert failure', () => { + it('throws and stops the job when upsertPoints fails on a batch', async () => { + const row = buildRow(); + embeddingServiceMock.findAll.mockResolvedValue([row]); + workerClientMock.embed.mockResolvedValue([makeVector()]); + qdrantMock.upsertPoints.mockRejectedValue(new Error('qdrant down')); + + await expect(service.run()).rejects.toThrow('qdrant down'); + }); + }); + + describe('run — embed output validation', () => { + it('falls back to one-by-one and skips the row when embed count mismatches', async () => { + const row = buildRow(); + embeddingServiceMock.findAll.mockResolvedValue([row]); + // Always returns 2 vectors for 1 item — mismatch at both batch and item level + workerClientMock.embed.mockResolvedValue([makeVector(), makeVector()]); + + await expect(service.run()).resolves.toBeUndefined(); + expect(qdrantMock.upsertPoints).not.toHaveBeenCalled(); + }); + + it('falls back to one-by-one and skips the row when vector size is wrong', async () => { + const row = buildRow(); + embeddingServiceMock.findAll.mockResolvedValue([row]); + // Returns a 512-dim vector instead of the required 1024 + workerClientMock.embed.mockResolvedValue([Array.from({ length: 512 }, () => 0.1)]); + + await expect(service.run()).resolves.toBeUndefined(); + expect(qdrantMock.upsertPoints).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/test/embedding/embedding-course.mapper.test.ts b/test/embedding/embedding-course.mapper.test.ts new file mode 100644 index 0000000..177dece --- /dev/null +++ b/test/embedding/embedding-course.mapper.test.ts @@ -0,0 +1,345 @@ +import { createHash } from 'node:crypto'; + +import { EmbeddingViewDto } from '../../src/embedding/dtos/embedding-view.dto'; +import { + buildCourseEmbeddingPayload, + buildCourseEmbeddingText, + computeCourseChangeKey, + getCourseTypeLabel, + hashEmbeddingText, + prepareCourseEmbedding, + toQdrantPointId, +} from '../../src/embedding/embedding-course.mapper'; + +const UUID_V5_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +const buildRow = (overrides: Partial = {}): EmbeddingViewDto => ({ + embedding_id: '352507_182848', + course_id: 352507, + program_id: 182848, + code: 'LOG635', + title: 'Systèmes intelligents', + description: 'Ce cours vise la compréhension.', + cycle: 1, + program_title: 'Génie logiciel', + type: 'TRONC', + typical_session_index: 5, + unstructured_prerequisite: null, + prerequisite_codes: ['LOG320', 'MAT350'], + has_prerequisites: true, + availability: ['JOUR'], + sessions: ['Automne 2026'], + ...overrides, +}); + +describe('hashEmbeddingText', () => { + it('produces a 64-char hex string', () => { + const result = hashEmbeddingText('hello'); + expect(result).toHaveLength(64); + expect(result).toMatch(/^[0-9a-f]+$/); + }); + + it('is deterministic for the same input', () => { + expect(hashEmbeddingText('test')).toBe(hashEmbeddingText('test')); + }); + + it('differs for different inputs', () => { + expect(hashEmbeddingText('a')).not.toBe(hashEmbeddingText('b')); + }); + + it('matches Node crypto sha256', () => { + const expected = createHash('sha256').update('hello').digest('hex'); + expect(hashEmbeddingText('hello')).toBe(expected); + }); +}); + +describe('toQdrantPointId', () => { + it('returns a UUID v5 string', () => { + expect(toQdrantPointId('352507_182848')).toMatch(UUID_V5_REGEX); + }); + + it('is deterministic for the same input', () => { + expect(toQdrantPointId('352507_182848')).toBe(toQdrantPointId('352507_182848')); + }); + + it('produces different IDs for different embedding IDs', () => { + expect(toQdrantPointId('352507_182848')).not.toBe(toQdrantPointId('352508_182848')); + }); + + it('produces a different id when QDRANT_ID_NAMESPACE changes', async () => { + const defaultId = toQdrantPointId('352507_182848'); + + jest.resetModules(); + process.env.QDRANT_ID_NAMESPACE = '11111111-1111-1111-1111-111111111111'; + const { toQdrantPointId: altFn } = await import('../../src/embedding/embedding-course.mapper'); + delete process.env.QDRANT_ID_NAMESPACE; + jest.resetModules(); + + expect(altFn('352507_182848')).toMatch(UUID_V5_REGEX); + expect(altFn('352507_182848')).not.toBe(defaultId); + }); +}); + +describe('getCourseTypeLabel', () => { + it('returns French label for CONCE', () => { + expect(getCourseTypeLabel('CONCE')).toBe('cours optionnel'); + }); + + it('returns French label for TRONC', () => { + expect(getCourseTypeLabel('TRONC')).toBe('tronc commun'); + }); + + it('returns French label for PROFI', () => { + expect(getCourseTypeLabel('PROFI')).toBe('cours de profil'); + }); + + it('returns the raw value for unknown types', () => { + expect(getCourseTypeLabel('UNKNOWN')).toBe('UNKNOWN'); + }); + + it('returns undefined for null', () => { + expect(getCourseTypeLabel(null)).toBeUndefined(); + }); + + it('returns undefined for undefined', () => { + expect(getCourseTypeLabel(undefined)).toBeUndefined(); + }); +}); + +describe('buildCourseEmbeddingText', () => { + it('includes code and title', () => { + const text = buildCourseEmbeddingText(buildRow()); + expect(text).toContain('LOG635'); + expect(text).toContain('Systèmes intelligents'); + }); + + it('includes description when present', () => { + const text = buildCourseEmbeddingText(buildRow({ description: 'Une description.' })); + expect(text).toContain('Une description.'); + }); + + it('includes prerequisite codes sorted', () => { + const text = buildCourseEmbeddingText(buildRow({ prerequisite_codes: ['MAT350', 'LOG320'] })); + expect(text).toContain('Préalables : LOG320, MAT350.'); + }); + + it('deduplicates prerequisite codes', () => { + const text = buildCourseEmbeddingText(buildRow({ prerequisite_codes: ['LOG320', 'LOG320'] })); + const count = (text.match(/LOG320/g) ?? []).length; + expect(count).toBe(1); + }); + + it('omits prerequisite section when array is empty', () => { + const text = buildCourseEmbeddingText(buildRow({ prerequisite_codes: [] })); + expect(text).not.toContain('Préalables :'); + }); + + it('includes unstructured prerequisite when present', () => { + const text = buildCourseEmbeddingText(buildRow({ unstructured_prerequisite: 'Approbation du directeur.' })); + expect(text).toContain('Préalables non structurés : Approbation du directeur.'); + }); + + it('omits unstructured prerequisite when null', () => { + const text = buildCourseEmbeddingText(buildRow({ unstructured_prerequisite: null })); + expect(text).not.toContain('Préalables non structurés'); + }); + + it('includes type label for known types', () => { + const text = buildCourseEmbeddingText(buildRow({ type: 'TRONC' })); + expect(text).toContain('Type : tronc commun.'); + }); + + it('omits type when null', () => { + const text = buildCourseEmbeddingText(buildRow({ type: null })); + expect(text).not.toContain('Type :'); + }); + + it('includes program title when present', () => { + const text = buildCourseEmbeddingText(buildRow({ program_title: 'Génie logiciel' })); + expect(text).toContain('Programme : Génie logiciel.'); + }); + + it('includes cycle when present', () => { + const text = buildCourseEmbeddingText(buildRow({ cycle: 2 })); + expect(text).toContain('Cycle : 2.'); + }); + + it('omits cycle when null', () => { + const text = buildCourseEmbeddingText(buildRow({ cycle: null })); + expect(text).not.toContain('Cycle :'); + }); + + it('includes typical_session_index when present', () => { + const text = buildCourseEmbeddingText(buildRow({ typical_session_index: 5 })); + expect(text).toContain('Session typique : 5.'); + }); + + it('omits typical_session_index when null', () => { + const text = buildCourseEmbeddingText(buildRow({ typical_session_index: null })); + expect(text).not.toContain('Session typique :'); + }); + + it('includes availability when present', () => { + const text = buildCourseEmbeddingText(buildRow({ availability: ['JOUR', 'SOIR'] })); + expect(text).toContain('Disponibilité :'); + expect(text).toContain('JOUR'); + expect(text).toContain('SOIR'); + }); + + it('omits availability when empty', () => { + const text = buildCourseEmbeddingText(buildRow({ availability: [] })); + expect(text).not.toContain('Disponibilité :'); + }); + + it('includes sessions when present', () => { + const text = buildCourseEmbeddingText(buildRow({ sessions: ['Automne 2026'] })); + expect(text).toContain('Sessions : Automne 2026.'); + }); + + it('omits sessions when empty', () => { + const text = buildCourseEmbeddingText(buildRow({ sessions: [] })); + expect(text).not.toContain('Sessions :'); + }); + + it('collapses extra whitespace', () => { + const text = buildCourseEmbeddingText(buildRow({ description: 'a b' })); + expect(text).not.toMatch(/\s{2,}/); + }); + + it('filters non-string entries in array fields', () => { + const text = buildCourseEmbeddingText(buildRow({ prerequisite_codes: [null, 'LOG320'] as unknown as string[] })); + expect(text).toContain('LOG320'); + expect(text).not.toContain('null'); + }); +}); + +describe('buildCourseEmbeddingPayload', () => { + it('includes all required fields', () => { + const text = 'embedded text'; + const payload = buildCourseEmbeddingPayload(buildRow(), text, 'Xenova/bge-m3', '2025-01-01T00:00:00.000Z'); + + expect(payload.embedding_id).toBe('352507_182848'); + expect(payload.course_id).toBe(352507); + expect(payload.program_id).toBe(182848); + expect(payload.code).toBe('LOG635'); + expect(payload.title).toBe('Systèmes intelligents'); + expect(payload.text).toBe(text); + expect(payload.text_hash).toBe(hashEmbeddingText(text)); + expect(payload.embedding_model).toBe('Xenova/bge-m3'); + expect(payload.indexed_at).toBe('2025-01-01T00:00:00.000Z'); + }); + + it('includes cycle when present', () => { + const payload = buildCourseEmbeddingPayload(buildRow({ cycle: 1 }), 'text', 'model', 'now'); + expect(payload.cycle).toBe(1); + }); + + it('omits cycle when null', () => { + const payload = buildCourseEmbeddingPayload(buildRow({ cycle: null }), 'text', 'model', 'now'); + expect(payload).not.toHaveProperty('cycle'); + }); + + it('includes type and type_label for known types', () => { + const payload = buildCourseEmbeddingPayload(buildRow({ type: 'CONCE' }), 'text', 'model', 'now'); + expect(payload.type).toBe('CONCE'); + expect(payload.type_label).toBe('cours optionnel'); + }); + + it('omits type fields when null', () => { + const payload = buildCourseEmbeddingPayload(buildRow({ type: null }), 'text', 'model', 'now'); + expect(payload).not.toHaveProperty('type'); + expect(payload).not.toHaveProperty('type_label'); + }); + + it('includes typical_session_index when present', () => { + const payload = buildCourseEmbeddingPayload(buildRow({ typical_session_index: 3 }), 'text', 'model', 'now'); + expect(payload.typical_session_index).toBe(3); + }); + + it('omits typical_session_index when null', () => { + const payload = buildCourseEmbeddingPayload(buildRow({ typical_session_index: null }), 'text', 'model', 'now'); + expect(payload).not.toHaveProperty('typical_session_index'); + }); + + it('includes unstructured_prerequisite when present', () => { + const payload = buildCourseEmbeddingPayload( + buildRow({ unstructured_prerequisite: 'Approbation.' }), + 'text', 'model', 'now', + ); + expect(payload.unstructured_prerequisite).toBe('Approbation.'); + }); + + it('omits unstructured_prerequisite when null', () => { + const payload = buildCourseEmbeddingPayload(buildRow({ unstructured_prerequisite: null }), 'text', 'model', 'now'); + expect(payload).not.toHaveProperty('unstructured_prerequisite'); + }); + + it('defaults description to empty string when null/undefined', () => { + const payload = buildCourseEmbeddingPayload( + buildRow({ description: null as unknown as string }), + 'text', 'model', 'now', + ); + expect(payload.description).toBe(''); + }); + + it('uses current time as indexed_at when not provided', () => { + const before = Date.now(); + const payload = buildCourseEmbeddingPayload(buildRow(), 'text', 'model'); + const after = Date.now(); + const indexedAtMs = new Date(payload.indexed_at).getTime(); + expect(indexedAtMs).toBeGreaterThanOrEqual(before); + expect(indexedAtMs).toBeLessThanOrEqual(after); + }); + + it('has_prerequisites reflects the row value', () => { + const truePayload = buildCourseEmbeddingPayload(buildRow({ has_prerequisites: true }), 'text', 'model', 'now'); + const falsePayload = buildCourseEmbeddingPayload(buildRow({ has_prerequisites: false }), 'text', 'model', 'now'); + expect(truePayload.has_prerequisites).toBe(true); + expect(falsePayload.has_prerequisites).toBe(false); + }); +}); + +describe('prepareCourseEmbedding', () => { + it('returns id, text, and payload', () => { + const result = prepareCourseEmbedding(buildRow(), 'Xenova/bge-m3'); + expect(result.id).toMatch(UUID_V5_REGEX); + expect(typeof result.text).toBe('string'); + expect(result.text.length).toBeGreaterThan(0); + expect(result.payload.code).toBe('LOG635'); + }); + + it('payload.text matches result.text', () => { + const result = prepareCourseEmbedding(buildRow(), 'Xenova/bge-m3'); + expect(result.payload.text).toBe(result.text); + }); + + it('payload.text_hash is hash of result.text', () => { + const result = prepareCourseEmbedding(buildRow(), 'Xenova/bge-m3'); + expect(result.payload.text_hash).toBe(hashEmbeddingText(result.text)); + }); +}); + +describe('computeCourseChangeKey', () => { + it('returns a UUID id and 64-char hash', () => { + const { id, hash } = computeCourseChangeKey(buildRow()); + expect(id).toMatch(UUID_V5_REGEX); + expect(hash).toHaveLength(64); + }); + + it('is deterministic for the same row', () => { + const row = buildRow(); + expect(computeCourseChangeKey(row)).toStrictEqual(computeCourseChangeKey(row)); + }); + + it('produces a different hash when title changes', () => { + const r1 = computeCourseChangeKey(buildRow({ title: 'Titre A' })); + const r2 = computeCourseChangeKey(buildRow({ title: 'Titre B' })); + expect(r1.hash).not.toBe(r2.hash); + }); + + it('produces the same id for the same embedding_id regardless of other fields', () => { + const r1 = computeCourseChangeKey(buildRow({ title: 'A' })); + const r2 = computeCourseChangeKey(buildRow({ title: 'B' })); + expect(r1.id).toBe(r2.id); + }); +}); diff --git a/test/embedding/embedding-worker.client.test.ts b/test/embedding/embedding-worker.client.test.ts new file mode 100644 index 0000000..2847fc3 --- /dev/null +++ b/test/embedding/embedding-worker.client.test.ts @@ -0,0 +1,149 @@ +import * as fs from 'node:fs'; +import { Worker } from 'node:worker_threads'; + +import { EmbeddingWorkerClient } from '../../src/embedding/embedding-worker.client'; + +jest.mock('node:fs'); +jest.mock('node:worker_threads'); + +const mockFsExistsSync = jest.mocked(fs.existsSync); +const MockWorker = jest.mocked(Worker); + +type WorkerEventMap = { + message: (msg: unknown) => void; + error: (err: Error) => void; + exit: (code: number) => void; +}; + +const buildMockWorker = () => { + const listeners: Partial = {}; + const worker = { + postMessage: jest.fn(), + terminate: jest.fn().mockResolvedValue(undefined), + on: jest.fn().mockImplementation((event: keyof WorkerEventMap, cb: WorkerEventMap[typeof event]) => { + listeners[event] = cb as never; + }), + emit: (event: keyof WorkerEventMap, ...args: unknown[]) => { + (listeners[event] as ((...a: unknown[]) => void) | undefined)?.(...args); + }, + }; + return worker; +}; + +describe('EmbeddingWorkerClient', () => { + let client: EmbeddingWorkerClient; + + beforeEach(() => { + jest.clearAllMocks(); + jest.useFakeTimers(); + client = new EmbeddingWorkerClient(); + jest.spyOn(client['logger'], 'log').mockImplementation(() => {}); + jest.spyOn(client['logger'], 'debug').mockImplementation(() => {}); + jest.spyOn(client['logger'], 'warn').mockImplementation(() => {}); + jest.spyOn(client['logger'], 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + describe('embed — empty input', () => { + it('returns an empty array without creating a worker', async () => { + await expect(client.embed([])).resolves.toStrictEqual([]); + expect(MockWorker).not.toHaveBeenCalled(); + }); + }); + + describe('embed — worker not found', () => { + it('throws when the worker script does not exist on disk', async () => { + mockFsExistsSync.mockReturnValue(false); + await expect(client.embed(['hello'])).rejects.toThrow('Embedding worker not found'); + }); + }); + + describe('onModuleDestroy', () => { + it('resolves immediately when no worker has been created', async () => { + await expect(client.onModuleDestroy()).resolves.toBeUndefined(); + }); + + it('terminates the worker and clears the reference', async () => { + mockFsExistsSync.mockReturnValue(true); + const mockWorker = buildMockWorker(); + MockWorker.mockImplementation(() => mockWorker as unknown as Worker); + + void client.embed(['text']); + await client.onModuleDestroy(); + + expect(mockWorker.terminate).toHaveBeenCalledTimes(1); + }); + }); + + describe('embed — message handling', () => { + let mockWorker: ReturnType; + + beforeEach(() => { + mockFsExistsSync.mockReturnValue(true); + mockWorker = buildMockWorker(); + MockWorker.mockImplementation(() => mockWorker as unknown as Worker); + }); + + it('resolves with vectors when the worker sends a success message', async () => { + const vectors = [[0.1, 0.2, 0.3]]; + const embedPromise = client.embed(['hello']); + mockWorker.emit('message', { id: 1, ok: true, vectors }); + await expect(embedPromise).resolves.toStrictEqual(vectors); + }); + + it('rejects when the worker sends a failure message', async () => { + const embedPromise = client.embed(['hello']); + mockWorker.emit('message', { id: 1, ok: false, error: 'model load failed' }); + await expect(embedPromise).rejects.toThrow('model load failed'); + }); + + it('warns and ignores messages that do not match any pending request', async () => { + const warnSpy = jest.spyOn(client['logger'], 'warn').mockImplementation(() => {}); + // Start an embed so the worker is created and its message listener is registered + const embedPromise = client.embed(['hello']); + mockWorker.emit('message', { id: 999, ok: true, vectors: [[0.1]] }); + expect(warnSpy).toHaveBeenCalled(); + // Resolve the pending request to avoid open handles + mockWorker.emit('message', { id: 1, ok: true, vectors: [[0.1]] }); + await embedPromise; + }); + + it('warns and ignores malformed messages from the worker', async () => { + const warnSpy = jest.spyOn(client['logger'], 'warn').mockImplementation(() => {}); + const embedPromise = client.embed(['hello']); + + mockWorker.emit('message', { not: 'valid' }); + expect(warnSpy).toHaveBeenCalled(); + + // clean up pending promise + mockWorker.emit('message', { id: 1, ok: true, vectors: [[0.1]] }); + await embedPromise; + }); + + it('rejects all pending requests when the worker emits an error', async () => { + const p1 = client.embed(['text-1']); + // Force a second request by re-creating worker expectation; both share the same worker + const p2 = client.embed(['text-2']); + + mockWorker.emit('error', new Error('worker crashed')); + + await expect(p1).rejects.toThrow('worker crashed'); + await expect(p2).rejects.toThrow('worker crashed'); + }); + + it('rejects the request and terminates the worker when the timeout fires', async () => { + process.env.EMBEDDING_WORKER_TIMEOUT_MS = '5000'; + const embedPromise = client.embed(['slow text']); + + jest.advanceTimersByTime(5001); + + await expect(embedPromise).rejects.toThrow('timed out'); + expect(mockWorker.terminate).toHaveBeenCalled(); + delete process.env.EMBEDDING_WORKER_TIMEOUT_MS; + }); + }); +}); diff --git a/test/embedding/embedding.controller.test.ts b/test/embedding/embedding.controller.test.ts new file mode 100644 index 0000000..66f7166 --- /dev/null +++ b/test/embedding/embedding.controller.test.ts @@ -0,0 +1,47 @@ +import { EmbeddingViewDto } from '../../src/embedding/dtos/embedding-view.dto'; +import { EmbeddingController } from '../../src/embedding/embedding.controller'; +import { EmbeddingService } from '../../src/embedding/embedding.service'; + +describe('EmbeddingController', () => { + let controller: EmbeddingController; + let serviceMock: { + findAll: jest.Mock; + findByCourseId: jest.Mock; + countCourses: jest.Mock; + }; + + beforeEach(() => { + serviceMock = { + findAll: jest.fn(), + findByCourseId: jest.fn(), + countCourses: jest.fn(), + }; + controller = new EmbeddingController(serviceMock as unknown as EmbeddingService); + }); + + describe('findAll', () => { + it('delegates to EmbeddingService.findAll and returns the result', async () => { + const rows: EmbeddingViewDto[] = []; + serviceMock.findAll.mockResolvedValue(rows); + await expect(controller.findAll()).resolves.toBe(rows); + expect(serviceMock.findAll).toHaveBeenCalledTimes(1); + }); + }); + + describe('countCourses', () => { + it('delegates to EmbeddingService.countCourses and returns the count', async () => { + serviceMock.countCourses.mockResolvedValue({ count: 10 }); + await expect(controller.countCourses()).resolves.toStrictEqual({ count: 10 }); + expect(serviceMock.countCourses).toHaveBeenCalledTimes(1); + }); + }); + + describe('findByCourseId', () => { + it('delegates to EmbeddingService.findByCourseId with the parsed courseId', async () => { + const rows: EmbeddingViewDto[] = []; + serviceMock.findByCourseId.mockResolvedValue(rows); + await expect(controller.findByCourseId(352507)).resolves.toBe(rows); + expect(serviceMock.findByCourseId).toHaveBeenCalledWith(352507); + }); + }); +}); diff --git a/test/embedding/embedding.service.test.ts b/test/embedding/embedding.service.test.ts new file mode 100644 index 0000000..8de59da --- /dev/null +++ b/test/embedding/embedding.service.test.ts @@ -0,0 +1,51 @@ +import { EmbeddingViewDto } from '../../src/embedding/dtos/embedding-view.dto'; +import { EmbeddingService } from '../../src/embedding/embedding.service'; +import { PrismaService } from '../../src/prisma/prisma.service'; + +describe('EmbeddingService', () => { + let service: EmbeddingService; + let prismaMock: { $queryRaw: jest.Mock }; + + beforeEach(() => { + prismaMock = { $queryRaw: jest.fn() }; + service = new EmbeddingService(prismaMock as unknown as PrismaService); + }); + + describe('findAll', () => { + it('returns all rows from the embedding view', async () => { + const rows: EmbeddingViewDto[] = [{ embedding_id: '1_2', course_id: 1 } as EmbeddingViewDto]; + prismaMock.$queryRaw.mockResolvedValue(rows); + await expect(service.findAll()).resolves.toBe(rows); + }); + + it('returns empty array when view is empty', async () => { + prismaMock.$queryRaw.mockResolvedValue([]); + await expect(service.findAll()).resolves.toStrictEqual([]); + }); + }); + + describe('findByCourseId', () => { + it('returns rows for the given course ID', async () => { + const rows: EmbeddingViewDto[] = [{ embedding_id: '352507_182848', course_id: 352507 } as EmbeddingViewDto]; + prismaMock.$queryRaw.mockResolvedValue(rows); + await expect(service.findByCourseId(352507)).resolves.toBe(rows); + }); + + it('returns empty array when course is not found', async () => { + prismaMock.$queryRaw.mockResolvedValue([]); + await expect(service.findByCourseId(99999)).resolves.toStrictEqual([]); + }); + }); + + describe('countCourses', () => { + it('converts bigint result to a plain number', async () => { + prismaMock.$queryRaw.mockResolvedValue([{ count: BigInt(42) }]); + await expect(service.countCourses()).resolves.toStrictEqual({ count: 42 }); + }); + + it('returns zero when there are no courses', async () => { + prismaMock.$queryRaw.mockResolvedValue([{ count: BigInt(0) }]); + await expect(service.countCourses()).resolves.toStrictEqual({ count: 0 }); + }); + }); +}); diff --git a/test/embedding/qdrant-course-index.service.test.ts b/test/embedding/qdrant-course-index.service.test.ts new file mode 100644 index 0000000..1c5d343 --- /dev/null +++ b/test/embedding/qdrant-course-index.service.test.ts @@ -0,0 +1,226 @@ +import { QdrantClient } from '@qdrant/js-client-rest'; + +import { BGE_M3_VECTOR_SIZE } from '../../src/embedding/embedding.constants'; +import type { CourseQdrantPoint } from '../../src/embedding/qdrant-course-index.service'; +import { QdrantCourseIndexService } from '../../src/embedding/qdrant-course-index.service'; + +jest.mock('@qdrant/js-client-rest', () => ({ + QdrantClient: jest.fn(), +})); + +const makeValidCollectionInfo = () => ({ + config: { + params: { + vectors: { size: BGE_M3_VECTOR_SIZE, distance: 'Cosine' }, + }, + }, +}); + +const makePoint = (id = 'uuid-1'): CourseQdrantPoint => ({ + id, + vector: Array.from({ length: BGE_M3_VECTOR_SIZE }, () => 0.1), + payload: { + embedding_id: '352507_182848', + course_id: 352507, + program_id: 182848, + code: 'LOG635', + title: 'Systèmes intelligents', + description: 'Description.', + program_title: 'Génie logiciel', + prerequisite_codes: [], + has_prerequisites: false, + availability: [], + sessions: [], + text: 'some text', + text_hash: 'abc123', + embedding_model: 'Xenova/bge-m3', + indexed_at: '2025-01-01T00:00:00.000Z', + }, +}); + +describe('QdrantCourseIndexService', () => { + let service: QdrantCourseIndexService; + let mockClient: { + getCollection: jest.Mock; + createCollection: jest.Mock; + scroll: jest.Mock; + upsert: jest.Mock; + }; + + beforeEach(() => { + jest.clearAllMocks(); + + mockClient = { + getCollection: jest.fn(), + createCollection: jest.fn(), + scroll: jest.fn(), + upsert: jest.fn(), + }; + + (QdrantClient as jest.Mock).mockImplementation(() => mockClient); + service = new QdrantCourseIndexService(); + + jest.spyOn(service['logger'], 'log').mockImplementation(() => {}); + jest.spyOn(service['logger'], 'debug').mockImplementation(() => {}); + jest.spyOn(service['logger'], 'warn').mockImplementation(() => {}); + jest.spyOn(service['logger'], 'error').mockImplementation(() => {}); + }); + + describe('ensureCollection', () => { + it('logs and skips creation when the collection already exists with valid config', async () => { + mockClient.getCollection.mockResolvedValue(makeValidCollectionInfo()); + + await expect(service.ensureCollection()).resolves.toBeUndefined(); + expect(mockClient.createCollection).not.toHaveBeenCalled(); + }); + + it('creates the collection when getCollection throws a 404 error', async () => { + mockClient.getCollection.mockRejectedValue({ status: 404 }); + mockClient.createCollection.mockResolvedValue(undefined); + + await service.ensureCollection(); + + expect(mockClient.createCollection).toHaveBeenCalledWith( + expect.any(String), + { vectors: { size: BGE_M3_VECTOR_SIZE, distance: 'Cosine' } }, + ); + }); + + it('re-throws non-404 errors from getCollection', async () => { + const error = { status: 500 }; + mockClient.getCollection.mockRejectedValue(error); + + await expect(service.ensureCollection()).rejects.toBe(error); + expect(mockClient.createCollection).not.toHaveBeenCalled(); + }); + + it('throws when the existing collection has the wrong vector size', async () => { + mockClient.getCollection.mockResolvedValue({ + config: { params: { vectors: { size: 512, distance: 'Cosine' } } }, + }); + + await expect(service.ensureCollection()).rejects.toThrow('Invalid Qdrant vector size'); + }); + + it('throws when the existing collection has the wrong distance metric', async () => { + mockClient.getCollection.mockResolvedValue({ + config: { params: { vectors: { size: BGE_M3_VECTOR_SIZE, distance: 'Dot' } } }, + }); + + await expect(service.ensureCollection()).rejects.toThrow('Invalid Qdrant distance'); + }); + + it('throws when the vector configuration cannot be read', async () => { + mockClient.getCollection.mockResolvedValue({ config: {} }); + + await expect(service.ensureCollection()).rejects.toThrow('Cannot read vector configuration'); + }); + + it('also reads vector config from the result.config path', async () => { + mockClient.getCollection.mockResolvedValue({ + result: { + config: { + params: { + vectors: { size: BGE_M3_VECTOR_SIZE, distance: 'Cosine' }, + }, + }, + }, + }); + + await expect(service.ensureCollection()).resolves.toBeUndefined(); + }); + }); + + describe('getExistingTextHashes', () => { + it('returns an empty map when there are no points', async () => { + mockClient.scroll.mockResolvedValue({ points: [], next_page_offset: null }); + + const result = await service.getExistingTextHashes(); + + expect(result).toBeInstanceOf(Map); + expect(result.size).toBe(0); + }); + + it('maps point id to its text_hash payload', async () => { + mockClient.scroll.mockResolvedValue({ + points: [{ id: 'uuid-1', payload: { text_hash: 'hash1' } }], + next_page_offset: null, + }); + + const result = await service.getExistingTextHashes(); + + expect(result.get('uuid-1')).toBe('hash1'); + }); + + it('paginates until next_page_offset is null', async () => { + mockClient.scroll + .mockResolvedValueOnce({ + points: [{ id: 'uuid-1', payload: { text_hash: 'hash1' } }], + next_page_offset: 250, + }) + .mockResolvedValueOnce({ + points: [{ id: 'uuid-2', payload: { text_hash: 'hash2' } }], + next_page_offset: null, + }); + + const result = await service.getExistingTextHashes(); + + expect(mockClient.scroll).toHaveBeenCalledTimes(2); + expect(result.size).toBe(2); + expect(result.get('uuid-2')).toBe('hash2'); + }); + + it('stops pagination when next_page_offset is undefined', async () => { + mockClient.scroll.mockResolvedValue({ + points: [], + next_page_offset: undefined, + }); + + await service.getExistingTextHashes(); + + expect(mockClient.scroll).toHaveBeenCalledTimes(1); + }); + + it('ignores points whose id or text_hash is not a string', async () => { + mockClient.scroll.mockResolvedValue({ + points: [ + { id: 1, payload: { text_hash: 'hash1' } }, + { id: 'uuid-2', payload: { text_hash: 42 } }, + ], + next_page_offset: null, + }); + + const result = await service.getExistingTextHashes(); + + expect(result.size).toBe(0); + }); + }); + + describe('upsertPoints', () => { + it('returns immediately without calling the client for an empty array', async () => { + await service.upsertPoints([]); + expect(mockClient.upsert).not.toHaveBeenCalled(); + }); + + it('calls client.upsert with transformed points', async () => { + mockClient.upsert.mockResolvedValue(undefined); + const point = makePoint(); + + await service.upsertPoints([point]); + + expect(mockClient.upsert).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + wait: true, + points: [expect.objectContaining({ id: point.id, vector: point.vector })], + }), + ); + }); + + it('throws immediately when upsert fails with a non-transient error', async () => { + mockClient.upsert.mockRejectedValue(new Error('bad request')); + + await expect(service.upsertPoints([makePoint()])).rejects.toThrow('bad request'); + }); + }); +}); diff --git a/test/embedding/qdrant-error.util.test.ts b/test/embedding/qdrant-error.util.test.ts new file mode 100644 index 0000000..db9632e --- /dev/null +++ b/test/embedding/qdrant-error.util.test.ts @@ -0,0 +1,151 @@ +import { + getNestedProperty, + isNotFoundError, + isRecord, + isTransientError, + retryTransient, +} from '../../src/embedding/qdrant-error.util'; + +describe('isRecord', () => { + it('returns true for plain objects', () => { + expect(isRecord({})).toBe(true); + expect(isRecord({ a: 1 })).toBe(true); + }); + + it('returns false for null', () => { + expect(isRecord(null)).toBe(false); + }); + + it('returns false for primitives', () => { + expect(isRecord(42)).toBe(false); + expect(isRecord('str')).toBe(false); + expect(isRecord(true)).toBe(false); + expect(isRecord(undefined)).toBe(false); + }); + + it('returns true for arrays', () => { + expect(isRecord([])).toBe(true); + }); +}); + +describe('getNestedProperty', () => { + it('retrieves a value at a nested path', () => { + expect(getNestedProperty({ a: { b: { c: 42 } } }, ['a', 'b', 'c'])).toBe(42); + }); + + it('returns undefined when an intermediate key is missing', () => { + expect(getNestedProperty({ a: 1 }, ['a', 'b'])).toBeUndefined(); + }); + + it('returns undefined when an intermediate value is not a record', () => { + expect(getNestedProperty({ a: 42 }, ['a', 'b'])).toBeUndefined(); + }); + + it('returns the root value for an empty path', () => { + const obj = { a: 1 }; + expect(getNestedProperty(obj, [])).toBe(obj); + }); + + it('returns undefined when root value is not a record', () => { + expect(getNestedProperty(null, ['a'])).toBeUndefined(); + }); +}); + +describe('isNotFoundError', () => { + it('returns true for an error with status 404', () => { + expect(isNotFoundError({ status: 404 })).toBe(true); + }); + + it('returns false for other status codes', () => { + expect(isNotFoundError({ status: 500 })).toBe(false); + expect(isNotFoundError({ status: 200 })).toBe(false); + }); + + it('returns false for non-record errors', () => { + expect(isNotFoundError(null)).toBe(false); + expect(isNotFoundError('not found')).toBe(false); + }); +}); + +describe('isTransientError', () => { + it('returns true for status 408', () => { + expect(isTransientError({ status: 408 })).toBe(true); + }); + + it('returns true for status 429', () => { + expect(isTransientError({ status: 429 })).toBe(true); + }); + + it('returns true for 5xx status codes', () => { + expect(isTransientError({ status: 500 })).toBe(true); + expect(isTransientError({ status: 503 })).toBe(true); + expect(isTransientError({ status: 599 })).toBe(true); + }); + + it('returns false for 4xx codes that are not 408/429', () => { + expect(isTransientError({ status: 400 })).toBe(false); + expect(isTransientError({ status: 404 })).toBe(false); + }); + + it('returns true for ECONNRESET', () => { + expect(isTransientError({ code: 'ECONNRESET' })).toBe(true); + }); + + it('returns true for ECONNREFUSED', () => { + expect(isTransientError({ code: 'ECONNREFUSED' })).toBe(true); + }); + + it('returns true for ETIMEDOUT', () => { + expect(isTransientError({ code: 'ETIMEDOUT' })).toBe(true); + }); + + it('reads statusCode as a fallback when status is absent', () => { + expect(isTransientError({ statusCode: 500 })).toBe(true); + }); + + it('reads response.status as a fallback when both status and statusCode are absent', () => { + expect(isTransientError({ response: { status: 503 } })).toBe(true); + }); + + it('returns false for non-record errors', () => { + expect(isTransientError(null)).toBe(false); + expect(isTransientError('error')).toBe(false); + }); +}); + +describe('retryTransient', () => { + it('resolves immediately when the operation succeeds on the first attempt', async () => { + const operation = jest.fn().mockResolvedValue('ok'); + await expect(retryTransient(operation, 3, 0)).resolves.toBe('ok'); + expect(operation).toHaveBeenCalledTimes(1); + }); + + it('retries on a transient error and resolves on the next attempt', async () => { + const operation = jest.fn() + .mockRejectedValueOnce({ status: 503 }) + .mockResolvedValueOnce('ok'); + await expect(retryTransient(operation, 3, 0)).resolves.toBe('ok'); + expect(operation).toHaveBeenCalledTimes(2); + }); + + it('throws immediately on non-transient errors without retrying', async () => { + const error = new Error('bad request'); + const operation = jest.fn().mockRejectedValue(error); + await expect(retryTransient(operation, 3, 0)).rejects.toBe(error); + expect(operation).toHaveBeenCalledTimes(1); + }); + + it('throws after exhausting all attempts on persistent transient errors', async () => { + const error = { status: 503 }; + const operation = jest.fn().mockRejectedValue(error); + await expect(retryTransient(operation, 3, 0)).rejects.toBe(error); + expect(operation).toHaveBeenCalledTimes(3); + }); + + it('throws on the last attempt even if error is transient', async () => { + const transientError = { status: 429 }; + const operation = jest.fn().mockRejectedValue(transientError); + await expect(retryTransient(operation, 1, 0)).rejects.toBe(transientError); + expect(operation).toHaveBeenCalledTimes(1); + }); +}); From 3a3ea4dd6ad5abfc13f756e6aa92f144a94cd53f Mon Sep 17 00:00:00 2001 From: Mohamed Hafdi Idrissi <70617264+mhd-hi@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:21:03 -0400 Subject: [PATCH 11/11] sonarr --- src/common/utils/uuid/uuidUtil.ts | 2 +- src/embedding/embedding-worker.client.ts | 6 ++++-- src/embedding/workers/bge-m3.worker.ts | 8 ++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/common/utils/uuid/uuidUtil.ts b/src/common/utils/uuid/uuidUtil.ts index 019889b..d795664 100644 --- a/src/common/utils/uuid/uuidUtil.ts +++ b/src/common/utils/uuid/uuidUtil.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'; // produces stable reproducible identifier (name, namespace) export function uuidV5(name: string, namespace: string): string { - const namespaceBytes = Buffer.from(namespace.replace(/-/g, ''), 'hex'); + const namespaceBytes = Buffer.from(namespace.replaceAll('-', ''), 'hex'); const nameBytes = Buffer.from(name, 'utf8'); const hash = createHash('sha1') diff --git a/src/embedding/embedding-worker.client.ts b/src/embedding/embedding-worker.client.ts index ecdea49..b4196bf 100644 --- a/src/embedding/embedding-worker.client.ts +++ b/src/embedding/embedding-worker.client.ts @@ -89,7 +89,7 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { const id = this.nextRequestId++; const model = process.env.EMBEDDING_MODEL ?? 'Xenova/bge-m3'; const dtype = process.env.EMBEDDING_DTYPE ?? 'q4'; - const timeoutMs = parseInt(process.env.EMBEDDING_WORKER_TIMEOUT_MS ?? '300000', 10); + const timeoutMs = Number.parseInt(process.env.EMBEDDING_WORKER_TIMEOUT_MS ?? '300000', 10); return new Promise((resolve, reject) => { const timer = setTimeout(() => { @@ -100,7 +100,9 @@ export class EmbeddingWorkerClient implements OnModuleDestroy { const error = new Error(`Embedding worker timed out after ${timeoutMs}ms (request ${id}).`); this.logger.error(error.message + ' Terminating worker.'); - void this.worker?.terminate().then(() => { + this.worker?.terminate().then(() => { + this.worker = null; + }).catch(() => { this.worker = null; }); diff --git a/src/embedding/workers/bge-m3.worker.ts b/src/embedding/workers/bge-m3.worker.ts index ec1c3a3..25157b2 100644 --- a/src/embedding/workers/bge-m3.worker.ts +++ b/src/embedding/workers/bge-m3.worker.ts @@ -134,7 +134,7 @@ async function getExtractor( ): Promise { const key = `${model}:${dtype ?? 'default'}`; - if (!extractorState || extractorState.key !== key) { + if (extractorState?.key !== key) { extractorState = { key, promise: createExtractor(model, dtype), @@ -182,7 +182,7 @@ function parseEmbedRequest(message: unknown): EmbedRequest { const dtype = message.dtype; if (typeof id !== 'number' || !Number.isInteger(id)) { - throw new Error('Invalid worker message: "id" must be an integer.'); + throw new TypeError('Invalid worker message: "id" must be an integer.'); } if (!Array.isArray(texts) || !texts.every((text) => typeof text === 'string')) { @@ -220,11 +220,11 @@ function tensorToVectors( throw new Error('Embedding output is empty.'); } - if (output.dims && output.dims.length === 2) { + if (output.dims?.length === 2) { const [batchSize, vectorSize] = output.dims; if (!Number.isInteger(batchSize) || !Number.isInteger(vectorSize)) { - throw new Error(`Invalid embedding tensor dims: ${output.dims.join(', ')}.`); + throw new TypeError(`Invalid embedding tensor dims: ${output.dims.join(', ')}.`); } if (batchSize !== expectedBatchSize) {