From 6d2e4d5519f73d2513c9133a3b04cc616a2c46e6 Mon Sep 17 00:00:00 2001 From: Alexander Kireev Date: Fri, 22 May 2026 03:57:23 +0700 Subject: [PATCH] =?UTF-8?q?feat:=20CoachAnalyzer=20+=20extractUserAssistan?= =?UTF-8?q?tPairs=20=E2=80=94=20post-hoc=20skill=20grading?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавляет `CoachAnalyzer` class — post-hoc анализ closed-lead conversation'ов. Для каждой пары (user-question, bot-reply) дёргает `gradeSkills` (@chatman-media/rag) и пишет результаты в skill-outcomes repo. Используется: - admin-UI: win-rate per skill для inspect'а - coach-proposals: данные для proposeStyleEdits - shadow-evaluations: pair-wise сравнения styles Извлечён из чужого монорепо (chatman-media/lead-engine — SaaS-платформа). Чтобы класс мог работать в любой DAL-shape (lead-engine использует Drizzle multi-tenant repos; standalone tg-chatbot — что-то другое), все упоминания конкретных типов БД заменены на in-file interfaces: - CoachAnalyzerLead — { id, tenantId } - CoachAnalyzerMessage — { id, role, text } - CoachAnalyzerMessages — recent(conversationId, limit) - CoachAnalyzerSkillOutcomes — record(...): Promise ChatClient + gradeSkills остаются из @chatman-media/rag (existing peer dep). Также exported `extractUserAssistantPairs` для standalone использования — свернуть flat-массив messages в alternating user→assistant pairs. 8 unit tests покрывают: - extractUserAssistantPairs (alternating / human-role / два user подряд / system skip / пустой text) - CoachAnalyzer (анализ пар + write outcomes, idempotency, no-pairs case) Bump: 0.1.1 → 0.2.0 (minor — только additions, никаких breaking changes). Co-Authored-By: Claude Opus 4.7 --- package.json | 2 +- src/__tests__/coach-analyzer.test.ts | 215 +++++++++++++++++++++++++++ src/coach-analyzer.ts | 177 ++++++++++++++++++++++ src/index.ts | 16 ++ 4 files changed, 409 insertions(+), 1 deletion(-) create mode 100644 src/__tests__/coach-analyzer.test.ts create mode 100644 src/coach-analyzer.ts diff --git a/package.json b/package.json index 23ff47b..7227f42 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@chatman-media/sales", - "version": "0.1.1", + "version": "0.2.0", "description": "LLM-powered sales funnel engine — persona composition, A/B testing, ELO rating, self-play evaluation for conversational bots.", "type": "module", "main": "./dist/index.js", diff --git a/src/__tests__/coach-analyzer.test.ts b/src/__tests__/coach-analyzer.test.ts new file mode 100644 index 0000000..c2a99c1 --- /dev/null +++ b/src/__tests__/coach-analyzer.test.ts @@ -0,0 +1,215 @@ +import type { ChatClient, ChatMessage } from "@chatman-media/rag"; +import { describe, expect, it } from "bun:test"; +import { + CoachAnalyzer, + type CoachAnalyzerLead, + type CoachAnalyzerMessage, + extractUserAssistantPairs, +} from "../coach-analyzer.ts"; + +function msg( + opts: { id: number; role: CoachAnalyzerMessage["role"]; text: string }, +): CoachAnalyzerMessage { + return { id: opts.id, role: opts.role, text: opts.text }; +} + +describe("extractUserAssistantPairs", () => { + it("Собирает alternating user→assistant пары", () => { + const out = extractUserAssistantPairs([ + msg({ id: 1, role: "user", text: "привет" }), + msg({ id: 2, role: "assistant", text: "здравствуй" }), + msg({ id: 3, role: "user", text: "сколько" }), + msg({ id: 4, role: "assistant", text: "уточню" }), + ]); + expect(out).toEqual([ + { userText: "привет", assistantText: "здравствуй", assistantMessageId: 2 }, + { userText: "сколько", assistantText: "уточню", assistantMessageId: 4 }, + ]); + }); + + it("human-роль считается assistant'ом (operator reply)", () => { + const out = extractUserAssistantPairs([ + msg({ id: 1, role: "user", text: "?" }), + msg({ id: 2, role: "human", text: "manual reply" }), + ]); + expect(out).toEqual([{ userText: "?", assistantText: "manual reply", assistantMessageId: 2 }]); + }); + + it("два user подряд (бот не ответил) — пропускает первый", () => { + const out = extractUserAssistantPairs([ + msg({ id: 1, role: "user", text: "first" }), + msg({ id: 2, role: "user", text: "second" }), + msg({ id: 3, role: "assistant", text: "reply" }), + ]); + expect(out).toEqual([{ userText: "second", assistantText: "reply", assistantMessageId: 3 }]); + }); + + it("system-роль игнорируется", () => { + const out = extractUserAssistantPairs([ + msg({ id: 1, role: "system", text: "init" }), + msg({ id: 2, role: "user", text: "hi" }), + msg({ id: 3, role: "assistant", text: "hi back" }), + ]); + expect(out).toEqual([{ userText: "hi", assistantText: "hi back", assistantMessageId: 3 }]); + }); + + it("пустой text → skip pair", () => { + const out = extractUserAssistantPairs([ + msg({ id: 1, role: "user", text: "" }), + msg({ id: 2, role: "assistant", text: "reply" }), + ]); + expect(out).toEqual([]); + }); +}); + +class FakeChat implements ChatClient { + public calls = 0; + public lastMessages: ChatMessage[] | undefined; + constructor(private readonly outputs: string[]) {} + async complete(messages: ChatMessage[]): Promise { + this.lastMessages = messages; + const out = this.outputs[this.calls] ?? "[]"; + this.calls += 1; + return out; + } +} + +class FakeSkillOutcomes { + public recorded: Array<{ + leadId: number; + skillSlug: string; + outcome: string; + messageId: number; + }> = []; + private seen = new Set(); + async record(opts: { + leadId: number; + skillSlug: string; + outcome: "won" | "lost" | "draw"; + source: string; + conversationId: number; + messageId: number; + styleSlug: string | null; + nowEpoch: number; + }): Promise { + const key = `${opts.leadId}:${opts.skillSlug}:${opts.source}`; + if (this.seen.has(key)) return false; + this.seen.add(key); + this.recorded.push({ + leadId: opts.leadId, + skillSlug: opts.skillSlug, + outcome: opts.outcome, + messageId: opts.messageId, + }); + return true; + } +} + +class FakeMessages { + constructor(private readonly rows: readonly CoachAnalyzerMessage[]) {} + async recent(_conversationId: number, _limit: number): Promise { + return [...this.rows]; + } +} + +describe("CoachAnalyzer", () => { + const lead: CoachAnalyzerLead = { id: 42, tenantId: 1 }; + + it("анализирует пары и пишет skill_outcomes", async () => { + const chat = new FakeChat(['["social-proof-stat","tactical-empathy"]', '["mirroring"]']); + const analyzer = new CoachAnalyzer({ + availableSlugs: ["social-proof-stat", "tactical-empathy", "mirroring"], + resolveChat: () => chat, + }); + const messages = new FakeMessages([ + msg({ id: 1, role: "user", text: "сколько платят" }), + msg({ id: 2, role: "assistant", text: "70% наших закрывают за месяц" }), + msg({ id: 3, role: "user", text: "не уверена" }), + msg({ id: 4, role: "assistant", text: "не уверена?" }), + ]); + const outcomes = new FakeSkillOutcomes(); + const result = await analyzer.analyzeLead( + { messages, skillOutcomes: outcomes }, + { + lead, + conversationId: 100, + styleSlug: "alina-infinity-v1", + outcome: "won", + source: "lead_submitted", + nowEpoch: 1700000000, + }, + ); + expect(result.pairsAnalyzed).toBe(2); + expect(result.outcomesRecorded).toBe(3); + expect(result.outcomesDuplicate).toBe(0); + expect(chat.calls).toBe(2); + expect(outcomes.recorded.map((r) => r.skillSlug).sort()).toEqual([ + "mirroring", + "social-proof-stat", + "tactical-empathy", + ]); + }); + + it("idempotency: повторный analyze того же lead → 0 newly recorded", async () => { + const chat = new FakeChat(['["mirroring"]', '["mirroring"]']); + const analyzer = new CoachAnalyzer({ + availableSlugs: ["mirroring"], + resolveChat: () => chat, + }); + const messages = new FakeMessages([ + msg({ id: 1, role: "user", text: "x" }), + msg({ id: 2, role: "assistant", text: "x?" }), + ]); + const outcomes = new FakeSkillOutcomes(); + await analyzer.analyzeLead( + { messages, skillOutcomes: outcomes }, + { + lead, + conversationId: 100, + styleSlug: null, + outcome: "lost", + source: "lead_rejected", + nowEpoch: 0, + }, + ); + const r2 = await analyzer.analyzeLead( + { messages, skillOutcomes: outcomes }, + { + lead, + conversationId: 100, + styleSlug: null, + outcome: "lost", + source: "lead_rejected", + nowEpoch: 0, + }, + ); + expect(r2.outcomesRecorded).toBe(0); + expect(r2.outcomesDuplicate).toBeGreaterThan(0); + }); + + it("нет user→assistant пар → 0 анализов, 0 LLM call'ов", async () => { + const chat = new FakeChat([]); + const analyzer = new CoachAnalyzer({ + availableSlugs: ["mirroring"], + resolveChat: () => chat, + }); + const messages = new FakeMessages([ + msg({ id: 1, role: "system", text: "init" }), + msg({ id: 2, role: "user", text: "lone user" }), + ]); + const outcomes = new FakeSkillOutcomes(); + const result = await analyzer.analyzeLead( + { messages, skillOutcomes: outcomes }, + { + lead, + conversationId: 100, + styleSlug: null, + outcome: "draw", + source: "lead_ghosted", + nowEpoch: 0, + }, + ); + expect(result.pairsAnalyzed).toBe(0); + expect(chat.calls).toBe(0); + }); +}); diff --git a/src/coach-analyzer.ts b/src/coach-analyzer.ts new file mode 100644 index 0000000..4ba97e6 --- /dev/null +++ b/src/coach-analyzer.ts @@ -0,0 +1,177 @@ +import { type ChatClient, gradeSkills } from "@chatman-media/rag"; + +/** + * Coach analyzer — post-hoc разбор completed conversation'ов. После того как + * lead закрылся (won/lost/ghosted), для каждой пары (user-question, bot-reply) + * в transcript'е дёргает gradeSkills из @chatman-media/rag — возвращённые + * skill-slugs пишутся в skill-outcomes таблицу с одним общим outcome'ом + * лида. + * + * Результат используется: + * - admin-UI: win-rate per skill для inspect'а + * - coach-proposals: LLM-предложения по улучшению style'а на основе + * skill-aggregates + * - shadow-evaluations: pair-wise сравнения старого vs нового style'а + * + * Не блокирует основной reply pipeline — запускается отдельным admin-cron'ом. + * Idempotent: uniq на (lead_id, skill_slug, source) защищает от дублей. + * + * NB: package-agnostic. Caller передаёт минимальные репозитории-интерфейсы + * (CoachAnalyzerMessages, CoachAnalyzerSkillOutcomes), сам по себе модуль + * не зависит от схемы БД. Это позволяет использовать его и из multi-tenant + * SaaS-платформы (lead-engine), и из standalone tg-chatbot. + */ +export interface CoachAnalyzerOpts { + /** Available skill slugs из styles[*].skills (или global skills table). */ + availableSlugs: readonly string[]; + /** Chat-LLM для gradeSkills. Тот же что reply-strategy. */ + resolveChat: (tenantId: number) => ChatClient; + /** Опционально: lightweight model для grading (cheaper than main chat). */ + gradingModel?: string; +} + +/** + * Минимальная shape сообщения для extractUserAssistantPairs. Caller'ский + * MessageRow обычно богаче (id, conversationId, createdAt, ...), но + * CoachAnalyzer'у достаточно того что ниже. + */ +export interface CoachAnalyzerMessage { + id: number; + role: "user" | "assistant" | "human" | "system" | string; + text: string; +} + +/** + * Минимальный lead-snapshot — то что CoachAnalyzer использует. tenantId нужен + * для resolveChat (multi-tenant). id нужен для skill_outcomes.lead_id. + */ +export interface CoachAnalyzerLead { + id: number; + tenantId: number; +} + +/** Минимальный messages-repo contract. */ +export interface CoachAnalyzerMessages { + recent(conversationId: number, limit: number): Promise; +} + +/** + * Минимальный skill-outcomes-repo contract. `record` должен быть idempotent: + * возвращает true если строка реально вставлена, false если уже была + * (ON CONFLICT DO NOTHING / uniq-violation handled внутри). + */ +export interface CoachAnalyzerSkillOutcomes { + record(opts: { + leadId: number; + skillSlug: string; + outcome: "won" | "lost" | "draw"; + source: string; + conversationId: number; + messageId: number; + styleSlug: string | null; + nowEpoch: number; + }): Promise; +} + +export interface AnalyzeLeadOpts { + lead: CoachAnalyzerLead; + conversationId: number; + styleSlug: string | null; + /** + * Outcome для всех skill_outcomes этого лида. Извлекается caller'ом из + * lead.state (например, ready_to_work → 'won', rejected → 'lost'). + */ + outcome: "won" | "lost" | "draw"; + /** Источник outcome'а: lead_submitted/lead_rejected/lead_ghosted/manual/self_play. */ + source: string; + nowEpoch: number; +} + +export interface AnalysisResult { + /** Сколько message-пар (user+assistant) проанализировано. */ + pairsAnalyzed: number; + /** Сколько skill_outcomes реально вставлено (после дедупа). */ + outcomesRecorded: number; + /** Сколько skill_outcomes уже было (conflict) — для idempotency-tracking. */ + outcomesDuplicate: number; +} + +export class CoachAnalyzer { + constructor(private readonly opts: CoachAnalyzerOpts) {} + + /** + * Анализирует все user→assistant пары в conversation, записывает skill_outcomes. + * Pair = последовательное user-сообщение и сразу следующее assistant-сообщение + * (skip system/human/самостоятельные). + */ + async analyzeLead( + deps: { + messages: CoachAnalyzerMessages; + skillOutcomes: CoachAnalyzerSkillOutcomes; + }, + input: AnalyzeLeadOpts, + ): Promise { + // Грузим всю историю в порядке от старого к новому. Используем большой + // limit — реалистично one full lead это ~100 сообщений; >1000 будет + // chunk'аться в отдельных итерациях. + const all = await deps.messages.recent(input.conversationId, 1000); + + const pairs = extractUserAssistantPairs(all); + if (pairs.length === 0) { + return { pairsAnalyzed: 0, outcomesRecorded: 0, outcomesDuplicate: 0 }; + } + + const chat = this.opts.resolveChat(input.lead.tenantId); + let recorded = 0; + let duplicate = 0; + for (const pair of pairs) { + // gradeSkills сам catches LLM exceptions → возвращает [] на failure. + const skills = await gradeSkills({ + question: pair.userText, + reply: pair.assistantText, + availableSlugs: this.opts.availableSlugs, + chat, + ...(this.opts.gradingModel ? { model: this.opts.gradingModel } : {}), + }); + for (const slug of skills) { + const inserted = await deps.skillOutcomes.record({ + leadId: input.lead.id, + skillSlug: slug, + outcome: input.outcome, + source: input.source, + conversationId: input.conversationId, + messageId: pair.assistantMessageId, + styleSlug: input.styleSlug, + nowEpoch: input.nowEpoch, + }); + if (inserted) recorded += 1; + else duplicate += 1; + } + } + return { pairsAnalyzed: pairs.length, outcomesRecorded: recorded, outcomesDuplicate: duplicate }; + } +} + +/** + * Из flat-массива messages извлекает пары (user → следующий assistant/human). + * Стандартный alternating-pattern: user, assistant, user, assistant. Если + * после user идёт ещё user (бот не отвечал) — pair пропускается. + */ +export function extractUserAssistantPairs( + messages: readonly CoachAnalyzerMessage[], +): Array<{ + userText: string; + assistantText: string; + assistantMessageId: number; +}> { + const pairs: Array<{ userText: string; assistantText: string; assistantMessageId: number }> = []; + for (let i = 0; i < messages.length - 1; i++) { + const a = messages[i]!; + const b = messages[i + 1]!; + if (a.role !== "user") continue; + if (b.role !== "assistant" && b.role !== "human") continue; + if (!a.text || !b.text) continue; + pairs.push({ userText: a.text, assistantText: b.text, assistantMessageId: b.id }); + } + return pairs; +} diff --git a/src/index.ts b/src/index.ts index 3114da1..5284cc8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,22 @@ export { pickVariant, } from "./ab-router.ts"; // ─── Coach ─────────────────────────────────────────────────────────────────── +// `proposeStyleEdits` (coach.ts) — LLM coach: читает worst self-play +// transcripts и предлагает edits для Style spec'а. Out-of-band, admin-trigger. +// `CoachAnalyzer` (coach-analyzer.ts) — post-hoc grader: для closed-lead'ов +// проходит transcript и записывает skill_outcomes через gradeSkills. +// Package-agnostic (принимает interfaces) — работает в любой DAL-shape. +export { + type AnalyzeLeadOpts, + type AnalysisResult, + CoachAnalyzer, + type CoachAnalyzerLead, + type CoachAnalyzerMessage, + type CoachAnalyzerMessages, + type CoachAnalyzerOpts, + type CoachAnalyzerSkillOutcomes, + extractUserAssistantPairs, +} from "./coach-analyzer.ts"; export { applyEditsToStyle, type CoachInput,