From f3e92d54fac101d852f4f99ac8828f8711d34ae9 Mon Sep 17 00:00:00 2001 From: bbsngg Date: Fri, 17 Apr 2026 00:51:02 -0400 Subject: [PATCH] feat(telemetry): add autoresearch usage tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds dedicated autoresearch_* telemetry events covering the full usage funnel: hub discovery, pack/workflow selection, configuration, message submission, and backend run lifecycle (started / task_completed / finished / cancel_requested). Also backfills data-telemetry-id on key DOM elements so existing ui_click events gain semantic identifiers. Privacy: user input content is never sent — only input_length, command name (from scenario_id), and attachment counts. Project paths are SHA256-hashed. Existing server-side sanitizer continues to strip any content/prompt/output fields as a second line of defense. Co-Authored-By: Claude Opus 4.7 --- server/routes/auto-research.js | 64 ++++++++++- src/components/AutoResearchHub.tsx | 102 ++++++++++++++++-- .../chat/hooks/useChatComposerState.ts | 19 +++- .../subcomponents/AutoResearchDropdown.tsx | 41 ++++++- .../subcomponents/GuidedPromptStarter.tsx | 26 ++++- .../view/subcomponents/SidebarHeader.tsx | 12 ++- src/utils/autoresearchTelemetry.ts | 81 ++++++++++++++ 7 files changed, 326 insertions(+), 19 deletions(-) create mode 100644 src/utils/autoresearchTelemetry.ts diff --git a/server/routes/auto-research.js b/server/routes/auto-research.js index b0fb4e5c..2999f1df 100644 --- a/server/routes/auto-research.js +++ b/server/routes/auto-research.js @@ -12,6 +12,7 @@ import { spawnGemini, abortGeminiSession, isGeminiSessionActive } from '../gemin import { queryOpenRouter, abortOpenRouterSession, isOpenRouterSessionActive } from '../openrouter.js'; import { sendAutoResearchCompletionEmail } from '../utils/auto-research-mailer.js'; import { getGeminiApiKeyForUser, withGeminiApiKeyEnv } from '../utils/geminiApiKey.js'; +import { enqueueTelemetryEvent } from '../telemetry.js'; const router = express.Router(); @@ -49,6 +50,34 @@ function normalizePermissionMode(permissionMode) { return AUTO_RESEARCH_DEFAULT_PERMISSION_MODE; } +function hashProjectPath(projectPath) { + if (!projectPath) return null; + return crypto.createHash('sha256').update(String(projectPath)).digest('hex').slice(0, 16); +} + +function emitRunTelemetry(name, run, extra = {}) { + if (!run) return; + try { + enqueueTelemetryEvent({ + name, + source: 'auto-research-backend', + data: { + run_id: run.id, + user_id: run.user_id ?? null, + project_name: run.project_name ?? null, + project_path_hash: hashProjectPath(run.project_path), + provider: run.provider ?? null, + status: run.status ?? null, + total_tasks: run.total_tasks ?? null, + completed_tasks: run.completed_tasks ?? null, + ...extra, + }, + }); + } catch (error) { + console.error('[AutoResearch] telemetry emit failed:', error?.message || error); + } +} + function abortActiveSession(provider, sessionId) { if (provider === 'codex') { return abortCodexSession(sessionId); @@ -380,28 +409,44 @@ async function runAutoResearch(runId, userId, projectName, projectPath) { throw new Error(`Task ${task.id} did not transition to done after execution`); } - autoResearchDb.updateRun(runId, { + const afterTaskRun = autoResearchDb.updateRun(runId, { completedTasks: pipelineState.completedTaskCount, totalTasks: pipelineState.tasks.length, currentTaskId: null, }); + + emitRunTelemetry('autoresearch_run_task_completed', afterTaskRun, { + task_id_hash: task.id ? hashProjectPath(String(task.id)) : null, + task_stage: task.stage ?? null, + }); } - autoResearchDb.updateRun(runId, { + const completedRun = autoResearchDb.updateRun(runId, { status: 'completed', currentTaskId: null, completedTasks: pipelineState.completedTaskCount, totalTasks: pipelineState.tasks.length, finishedAt: new Date().toISOString(), }); + + emitRunTelemetry('autoresearch_run_finished', completedRun, { + outcome: 'completed', + duration_ms: runState.startedAt ? Date.now() - runState.startedAt : null, + }); } catch (error) { const isCancelled = runState.cancelRequested || /cancelled by user/i.test(String(error?.message || '')); - autoResearchDb.updateRun(runId, { + const finishedRun = autoResearchDb.updateRun(runId, { status: isCancelled ? 'cancelled' : 'failed', error: error.message, currentTaskId: null, finishedAt: new Date().toISOString(), }); + + emitRunTelemetry('autoresearch_run_finished', finishedRun, { + outcome: isCancelled ? 'cancelled' : 'failed', + duration_ms: runState.startedAt ? Date.now() - runState.startedAt : null, + error_type: error?.name || 'Error', + }); } finally { activeRuns.delete(runId); await deliverCompletionEmail(runId, userId, projectName); @@ -498,6 +543,13 @@ router.post('/:projectName/start', async (req, res) => { provider, model, permissionMode, + startedAt: Date.now(), + }); + + emitRunTelemetry('autoresearch_run_started', run, { + model, + permission_mode: permissionMode, + actionable_task_count: pipelineState.actionableTaskCount, }); void runAutoResearch(runId, userId, projectName, projectPath); @@ -524,6 +576,12 @@ router.post('/:projectName/cancel', async (req, res) => { const runtime = activeRuns.get(activeRun.id); const sessionStillActive = isRunSessionStillActive(activeRun); + + emitRunTelemetry('autoresearch_run_cancel_requested', activeRun, { + had_active_runtime: Boolean(runtime), + session_still_active: sessionStillActive, + }); + if (runtime) { runtime.cancelRequested = true; if (runtime.sessionId || activeRun.session_id) { diff --git a/src/components/AutoResearchHub.tsx b/src/components/AutoResearchHub.tsx index fef38219..a12d9793 100644 --- a/src/components/AutoResearchHub.tsx +++ b/src/components/AutoResearchHub.tsx @@ -18,6 +18,7 @@ import { } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import { api } from '../utils/api'; +import { emitAutoresearchEvent } from '../utils/autoresearchTelemetry'; import { AUTO_RESEARCH_PACKS, type LocaleKey, type PackDef } from '../constants/autoResearchPacks'; import useLocalStorage from '../hooks/useLocalStorage'; @@ -170,6 +171,13 @@ export default function AutoResearchHub() { const [expandedWorkflows, setExpandedWorkflows] = useState>(new Set()); + useEffect(() => { + emitAutoresearchEvent('autoresearch_hub_viewed', { + pack_count: PACKS.length, + workflow_count: PACKS.reduce((sum, p) => sum + p.workflows.length, 0), + }); + }, []); + const copyToClipboard = useCallback((text: string) => { navigator.clipboard.writeText(text).then(() => { setCopiedCommand(text); @@ -182,28 +190,59 @@ export default function AutoResearchHub() { const mcpOpt = pack.mcp.find(m => m.key === mcpKey) || pack.mcp[0]; const apiKeys: Record = {}; + let envVarsFilled = 0; if (mcpOpt) { for (const ev of mcpOpt.envVars) { - if (apiKeyInputs[ev.name]) apiKeys[ev.name] = apiKeyInputs[ev.name]; + if (apiKeyInputs[ev.name]) { + apiKeys[ev.name] = apiKeyInputs[ev.name]; + envVarsFilled += 1; + } } } + emitAutoresearchEvent('autoresearch_configure_clicked', { + pack: pack.name, + mcp_key: mcpKey || null, + env_vars_required: mcpOpt?.envVars.length ?? 0, + env_vars_filled: envVarsFilled, + }); + setConfiguring(true); setConfigResult(null); try { const resp = await api.communityTools.configure(null, mcpKey, apiKeys, null); if (!resp.ok) { const text = await resp.text(); + emitAutoresearchEvent('autoresearch_configure_result', { + pack: pack.name, + mcp_key: mcpKey || null, + success: false, + error_type: 'http_error', + http_status: resp.status, + }); setConfigResult({ success: false, message: `Server error (${resp.status}): ${text}` }); return; } const data = await resp.json(); + emitAutoresearchEvent('autoresearch_configure_result', { + pack: pack.name, + mcp_key: mcpKey || null, + success: Boolean(data.success), + error_type: data.success ? null : 'server_reported_error', + error_count: Array.isArray(data.errors) ? data.errors.length : 0, + }); setConfigResult({ success: data.success, message: data.success ? t.configSuccess : (data.errors || []).map((e: { error: string }) => e.error).join('; '), }); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); + emitAutoresearchEvent('autoresearch_configure_result', { + pack: pack.name, + mcp_key: mcpKey || null, + success: false, + error_type: 'exception', + }); setConfigResult({ success: false, message: msg }); } finally { setConfiguring(false); @@ -285,7 +324,11 @@ export default function AutoResearchHub() { ))} @@ -460,8 +539,13 @@ export default function AutoResearchHub() { )} {/* Configure button */} - {configResult && ( diff --git a/src/components/chat/hooks/useChatComposerState.ts b/src/components/chat/hooks/useChatComposerState.ts index 4bef1391..bd87da54 100644 --- a/src/components/chat/hooks/useChatComposerState.ts +++ b/src/components/chat/hooks/useChatComposerState.ts @@ -43,6 +43,7 @@ import { type SlashCommand, useSlashCommands } from './useSlashCommands'; import type { Project, ProjectSession, SessionProvider } from '../../../types/app'; import { escapeRegExp } from '../utils/chatFormatting'; import { isAutoResearchScenario } from '../utils/autoResearch'; +import { emitAutoresearchEvent } from '../../../utils/autoresearchTelemetry'; import type { SessionMode } from '../../../types/app'; import type { BtwOverlayState } from '../view/subcomponents/BtwOverlay'; @@ -1132,10 +1133,26 @@ export function useChatComposerState({ } // Auto-bypass permissions for autoresearch workflows - const effectivePermissionMode = isAutoResearchScenario(attachedPrompt?.scenarioId) + const isAutoresearchMessage = isAutoResearchScenario(attachedPrompt?.scenarioId); + const effectivePermissionMode = isAutoresearchMessage ? 'bypassPermissions' : permissionMode; + if (isAutoresearchMessage) { + const scenarioId = attachedPrompt?.scenarioId ?? null; + const commandFromScenario = scenarioId?.startsWith('autoresearch-') + ? scenarioId.slice('autoresearch-'.length) + : null; + emitAutoresearchEvent('autoresearch_message_sent', { + scenario_id: scenarioId, + command: commandFromScenario, + input_length: currentInput.trim().length, + has_attachments: currentAttachedFiles.length > 0, + attachment_count: currentAttachedFiles.length, + provider, + }); + } + const selectedThinkingMode = thinkingModes.find((mode: { id: string; prefix?: string }) => mode.id === thinkingMode); if (selectedThinkingMode && selectedThinkingMode.prefix) { messageContent = `${selectedThinkingMode.prefix}: ${messageContent}`; diff --git a/src/components/chat/view/subcomponents/AutoResearchDropdown.tsx b/src/components/chat/view/subcomponents/AutoResearchDropdown.tsx index 5df99b48..698ccfc7 100644 --- a/src/components/chat/view/subcomponents/AutoResearchDropdown.tsx +++ b/src/components/chat/view/subcomponents/AutoResearchDropdown.tsx @@ -2,6 +2,7 @@ import React, { useState, useEffect, useRef, useMemo } from 'react'; import { ChevronDown, ChevronRight, FlaskConical, Settings } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import { AUTO_RESEARCH_PACKS, type LocaleKey } from '../../../../constants/autoResearchPacks'; +import { emitAutoresearchEvent } from '../../../../utils/autoresearchTelemetry'; import type { AttachedPrompt } from '../../types/types'; function resolveLocaleKey(lang: string): LocaleKey { @@ -49,6 +50,12 @@ export default function AutoResearchDropdown({ }, [open]); const select = (command: string, packName: string, wfName: string) => { + emitAutoresearchEvent('autoresearch_workflow_selected', { + pack: packName, + workflow_name: wfName, + command, + source: 'chat-dropdown', + }); if (setAttachedPrompt) { setAttachedPrompt({ scenarioId: `autoresearch-${command}`, @@ -64,11 +71,31 @@ export default function AutoResearchDropdown({ setExpandedPack(null); }; + const togglePackExpanded = (packName: string, isExpanded: boolean) => { + emitAutoresearchEvent('autoresearch_pack_expanded', { + pack: packName, + expanded: !isExpanded, + source: 'chat-dropdown', + }); + setExpandedPack(isExpanded ? null : packName); + }; + + const toggleDropdown = () => { + const willOpen = !open; + emitAutoresearchEvent('autoresearch_dropdown_toggled', { + open: willOpen, + source: 'chat-dropdown', + }); + setOpen(willOpen); + if (!willOpen) setExpandedPack(null); + }; + return (