Skip to content

feat(telemetry): add autoresearch usage tracking #193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bbsngg wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(telemetry): add autoresearch usage tracking #193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 61 additions & 3 deletions server/routes/auto-research.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { spawnGemini, abortGeminiSession, isGeminiSessionActive } from '../gemin
import { queryOpenRouter, abortOpenRouterSession, isOpenRouterSessionActive } from '../openrouter.js';
import { sendAutoResearchCompletionEmail } from '../utils/auto-research-mailer.js';
import { getGeminiApiKeyForUser, withGeminiApiKeyEnv } from '../utils/geminiApiKey.js';
import { enqueueTelemetryEvent } from '../telemetry.js';

const router = express.Router();

Expand Down Expand Up @@ -49,6 +50,34 @@ function normalizePermissionMode(permissionMode) {
return AUTO_RESEARCH_DEFAULT_PERMISSION_MODE;
}

function hashProjectPath(projectPath) {
if (!projectPath) return null;
return crypto.createHash('sha256').update(String(projectPath)).digest('hex').slice(0, 16);
}

function emitRunTelemetry(name, run, extra = {}) {
if (!run) return;
try {
enqueueTelemetryEvent({
name,
source: 'auto-research-backend',
data: {
run_id: run.id,
user_id: run.user_id ?? null,
project_name: run.project_name ?? null,
project_path_hash: hashProjectPath(run.project_path),
provider: run.provider ?? null,
status: run.status ?? null,
total_tasks: run.total_tasks ?? null,
completed_tasks: run.completed_tasks ?? null,
...extra,
},
});
} catch (error) {
console.error('[AutoResearch] telemetry emit failed:', error?.message || error);
}
}

function abortActiveSession(provider, sessionId) {
if (provider === 'codex') {
return abortCodexSession(sessionId);
Expand Down Expand Up @@ -380,28 +409,44 @@ async function runAutoResearch(runId, userId, projectName, projectPath) {
throw new Error(`Task ${task.id} did not transition to done after execution`);
}

autoResearchDb.updateRun(runId, {
const afterTaskRun = autoResearchDb.updateRun(runId, {
completedTasks: pipelineState.completedTaskCount,
totalTasks: pipelineState.tasks.length,
currentTaskId: null,
});

emitRunTelemetry('autoresearch_run_task_completed', afterTaskRun, {
task_id_hash: task.id ? hashProjectPath(String(task.id)) : null,
task_stage: task.stage ?? null,
});
}

autoResearchDb.updateRun(runId, {
const completedRun = autoResearchDb.updateRun(runId, {
status: 'completed',
currentTaskId: null,
completedTasks: pipelineState.completedTaskCount,
totalTasks: pipelineState.tasks.length,
finishedAt: new Date().toISOString(),
});

emitRunTelemetry('autoresearch_run_finished', completedRun, {
outcome: 'completed',
duration_ms: runState.startedAt ? Date.now() - runState.startedAt : null,
});
} catch (error) {
const isCancelled = runState.cancelRequested || /cancelled by user/i.test(String(error?.message || ''));
autoResearchDb.updateRun(runId, {
const finishedRun = autoResearchDb.updateRun(runId, {
status: isCancelled ? 'cancelled' : 'failed',
error: error.message,
currentTaskId: null,
finishedAt: new Date().toISOString(),
});

emitRunTelemetry('autoresearch_run_finished', finishedRun, {
outcome: isCancelled ? 'cancelled' : 'failed',
duration_ms: runState.startedAt ? Date.now() - runState.startedAt : null,
error_type: error?.name || 'Error',
});
} finally {
activeRuns.delete(runId);
await deliverCompletionEmail(runId, userId, projectName);
Expand Down Expand Up @@ -498,6 +543,13 @@ router.post('/:projectName/start', async (req, res) => {
provider,
model,
permissionMode,
startedAt: Date.now(),
});

emitRunTelemetry('autoresearch_run_started', run, {
model,
permission_mode: permissionMode,
actionable_task_count: pipelineState.actionableTaskCount,
});

void runAutoResearch(runId, userId, projectName, projectPath);
Expand All @@ -524,6 +576,12 @@ router.post('/:projectName/cancel', async (req, res) => {

const runtime = activeRuns.get(activeRun.id);
const sessionStillActive = isRunSessionStillActive(activeRun);

emitRunTelemetry('autoresearch_run_cancel_requested', activeRun, {
had_active_runtime: Boolean(runtime),
session_still_active: sessionStillActive,
});

if (runtime) {
runtime.cancelRequested = true;
if (runtime.sessionId || activeRun.session_id) {
Expand Down
102 changes: 93 additions & 9 deletions src/components/AutoResearchHub.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
} from 'lucide-react';
import { useTranslation } from 'react-i18next';
import { api } from '../utils/api';
import { emitAutoresearchEvent } from '../utils/autoresearchTelemetry';
import { AUTO_RESEARCH_PACKS, type LocaleKey, type PackDef } from '../constants/autoResearchPacks';
import useLocalStorage from '../hooks/useLocalStorage';

Expand Down Expand Up @@ -170,6 +171,13 @@ export default function AutoResearchHub() {

const [expandedWorkflows, setExpandedWorkflows] = useState<Set<string>>(new Set());

useEffect(() => {
emitAutoresearchEvent('autoresearch_hub_viewed', {
pack_count: PACKS.length,
workflow_count: PACKS.reduce((sum, p) => sum + p.workflows.length, 0),
});
}, []);

const copyToClipboard = useCallback((text: string) => {
navigator.clipboard.writeText(text).then(() => {
setCopiedCommand(text);
Expand All @@ -182,28 +190,59 @@ export default function AutoResearchHub() {
const mcpOpt = pack.mcp.find(m => m.key === mcpKey) || pack.mcp[0];

const apiKeys: Record<string, string> = {};
let envVarsFilled = 0;
if (mcpOpt) {
for (const ev of mcpOpt.envVars) {
if (apiKeyInputs[ev.name]) apiKeys[ev.name] = apiKeyInputs[ev.name];
if (apiKeyInputs[ev.name]) {
apiKeys[ev.name] = apiKeyInputs[ev.name];
envVarsFilled += 1;
}
}
}

emitAutoresearchEvent('autoresearch_configure_clicked', {
pack: pack.name,
mcp_key: mcpKey || null,
env_vars_required: mcpOpt?.envVars.length ?? 0,
env_vars_filled: envVarsFilled,
});

setConfiguring(true);
setConfigResult(null);
try {
const resp = await api.communityTools.configure(null, mcpKey, apiKeys, null);
if (!resp.ok) {
const text = await resp.text();
emitAutoresearchEvent('autoresearch_configure_result', {
pack: pack.name,
mcp_key: mcpKey || null,
success: false,
error_type: 'http_error',
http_status: resp.status,
});
setConfigResult({ success: false, message: `Server error (${resp.status}): ${text}` });
return;
}
const data = await resp.json();
emitAutoresearchEvent('autoresearch_configure_result', {
pack: pack.name,
mcp_key: mcpKey || null,
success: Boolean(data.success),
error_type: data.success ? null : 'server_reported_error',
error_count: Array.isArray(data.errors) ? data.errors.length : 0,
});
setConfigResult({
success: data.success,
message: data.success ? t.configSuccess : (data.errors || []).map((e: { error: string }) => e.error).join('; '),
});
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
emitAutoresearchEvent('autoresearch_configure_result', {
pack: pack.name,
mcp_key: mcpKey || null,
success: false,
error_type: 'exception',
});
setConfigResult({ success: false, message: msg });
} finally {
setConfiguring(false);
Expand Down Expand Up @@ -285,15 +324,23 @@ export default function AutoResearchHub() {
<button
type="button"
className="inline-flex h-8 items-center gap-1.5 rounded-full px-3 text-xs font-medium text-slate-700 hover:bg-sky-100/60 hover:text-slate-900 dark:text-sky-100/80 dark:hover:bg-sky-900/30 dark:hover:text-sky-100"
onClick={() => setGuideCollapsed(!guideCollapsed)}
onClick={() => {
emitAutoresearchEvent('autoresearch_guide_toggled', { collapsed: !guideCollapsed });
setGuideCollapsed(!guideCollapsed);
}}
data-telemetry-id="autoresearch-hub-guide-toggle"
>
{guideCollapsed ? <ChevronDown className="h-4 w-4" /> : <ChevronUp className="h-4 w-4" />}
{guideCollapsed ? t.guideExpand : t.guideCollapse}
</button>
<button
type="button"
className="inline-flex h-8 items-center gap-1.5 rounded-full px-3 text-xs font-medium text-slate-700 hover:bg-sky-100/60 hover:text-slate-900 dark:text-sky-100/80 dark:hover:bg-sky-900/30 dark:hover:text-sky-100"
onClick={() => setGuideDismissed(true)}
onClick={() => {
emitAutoresearchEvent('autoresearch_guide_dismissed', {});
setGuideDismissed(true);
}}
data-telemetry-id="autoresearch-hub-guide-dismiss"
>
<X className="h-4 w-4" />
{t.guideDismiss}
Expand Down Expand Up @@ -352,7 +399,18 @@ export default function AutoResearchHub() {
{/* Collapsible workflows list */}
<button
type="button"
onClick={() => setExpandedWorkflows(prev => { const n = new Set(prev); n.has(pack.name) ? n.delete(pack.name) : n.add(pack.name); return n; })}
onClick={() => setExpandedWorkflows(prev => {
const n = new Set(prev);
const willExpand = !n.has(pack.name);
emitAutoresearchEvent('autoresearch_pack_expanded', {
pack: pack.name,
expanded: willExpand,
source: 'hub',
});
if (n.has(pack.name)) n.delete(pack.name); else n.add(pack.name);
return n;
})}
data-telemetry-id={`autoresearch-hub-pack-workflows-${pack.name}`}
className="mt-3 flex w-full items-center justify-between rounded-lg border border-border/40 bg-muted/30 px-3 py-2 text-left transition-colors hover:bg-muted/60"
>
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
Expand Down Expand Up @@ -381,7 +439,17 @@ export default function AutoResearchHub() {
<div className="border-t border-border/40">
<button
type="button"
onClick={() => setExpandedConfig(prev => { const n = new Set(prev); n.has(pack.name) ? n.delete(pack.name) : n.add(pack.name); return n; })}
onClick={() => setExpandedConfig(prev => {
const n = new Set(prev);
const willExpand = !n.has(pack.name);
emitAutoresearchEvent('autoresearch_config_section_opened', {
pack: pack.name,
expanded: willExpand,
});
if (n.has(pack.name)) n.delete(pack.name); else n.add(pack.name);
return n;
})}
data-telemetry-id={`autoresearch-hub-config-${pack.name}`}
className="flex w-full items-center justify-between px-5 py-3 transition-colors hover:bg-muted/30"
>
<span className="flex items-center gap-2 text-sm font-semibold text-foreground">
Expand All @@ -400,8 +468,19 @@ export default function AutoResearchHub() {
{pack.mcp.length > 1 && (
<div className="mb-2 flex flex-wrap gap-1.5">
{pack.mcp.map(opt => (
<button key={opt.key} type="button" onClick={() => setSelectedMcp(p => ({ ...p, [pack.name]: opt.key }))}
className={`rounded-lg border px-3 py-1.5 text-xs font-medium transition-all ${mcpKey === opt.key ? 'border-sky-400 bg-sky-100 text-sky-700 shadow-sm dark:border-sky-600 dark:bg-sky-900/40 dark:text-sky-200' : 'border-border/60 bg-background text-muted-foreground hover:bg-muted/60'}`}>
<button
key={opt.key}
type="button"
onClick={() => {
emitAutoresearchEvent('autoresearch_mcp_selected', {
pack: pack.name,
mcp_key: opt.key,
});
setSelectedMcp(p => ({ ...p, [pack.name]: opt.key }));
}}
data-telemetry-id={`autoresearch-hub-mcp-${pack.name}-${opt.key}`}
className={`rounded-lg border px-3 py-1.5 text-xs font-medium transition-all ${mcpKey === opt.key ? 'border-sky-400 bg-sky-100 text-sky-700 shadow-sm dark:border-sky-600 dark:bg-sky-900/40 dark:text-sky-200' : 'border-border/60 bg-background text-muted-foreground hover:bg-muted/60'}`}
>
{opt.label}
</button>
))}
Expand Down Expand Up @@ -460,8 +539,13 @@ export default function AutoResearchHub() {
)}

{/* Configure button */}
<button type="button" disabled={configuring} onClick={() => handleConfigure(pack)}
className="inline-flex w-full items-center justify-center gap-2 rounded-xl bg-sky-600 px-5 py-2.5 text-sm font-semibold text-white shadow-sm transition-all hover:bg-sky-700 active:scale-[0.98] disabled:opacity-60 dark:bg-sky-500 dark:hover:bg-sky-600">
<button
type="button"
disabled={configuring}
onClick={() => handleConfigure(pack)}
data-telemetry-id={`autoresearch-hub-configure-${pack.name}`}
className="inline-flex w-full items-center justify-center gap-2 rounded-xl bg-sky-600 px-5 py-2.5 text-sm font-semibold text-white shadow-sm transition-all hover:bg-sky-700 active:scale-[0.98] disabled:opacity-60 dark:bg-sky-500 dark:hover:bg-sky-600"
>
{configuring ? <><Loader2 className="h-4 w-4 animate-spin" />{t.configApplying}</> : <><Settings className="h-4 w-4" />{t.configApply}</>}
</button>
{configResult && (
Expand Down
19 changes: 18 additions & 1 deletion src/components/chat/hooks/useChatComposerState.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import { type SlashCommand, useSlashCommands } from './useSlashCommands';
import type { Project, ProjectSession, SessionProvider } from '../../../types/app';
import { escapeRegExp } from '../utils/chatFormatting';
import { isAutoResearchScenario } from '../utils/autoResearch';
import { emitAutoresearchEvent } from '../../../utils/autoresearchTelemetry';
import type { SessionMode } from '../../../types/app';
import type { BtwOverlayState } from '../view/subcomponents/BtwOverlay';

Expand Down Expand Up @@ -1132,10 +1133,26 @@ export function useChatComposerState({
}

// Auto-bypass permissions for autoresearch workflows
const effectivePermissionMode = isAutoResearchScenario(attachedPrompt?.scenarioId)
const isAutoresearchMessage = isAutoResearchScenario(attachedPrompt?.scenarioId);
const effectivePermissionMode = isAutoresearchMessage
? 'bypassPermissions'
: permissionMode;

if (isAutoresearchMessage) {
const scenarioId = attachedPrompt?.scenarioId ?? null;
const commandFromScenario = scenarioId?.startsWith('autoresearch-')
? scenarioId.slice('autoresearch-'.length)
: null;
emitAutoresearchEvent('autoresearch_message_sent', {
scenario_id: scenarioId,
command: commandFromScenario,
input_length: currentInput.trim().length,
has_attachments: currentAttachedFiles.length > 0,
attachment_count: currentAttachedFiles.length,
provider,
});
}

const selectedThinkingMode = thinkingModes.find((mode: { id: string; prefix?: string }) => mode.id === thinkingMode);
if (selectedThinkingMode && selectedThinkingMode.prefix) {
messageContent = `${selectedThinkingMode.prefix}: ${messageContent}`;
Expand Down
Loading
Loading