From d2ad754391fcf2d7210687caf750170e8d0e8b84 Mon Sep 17 00:00:00 2001 From: "jianhua.wang" Date: Sun, 5 Jul 2026 19:30:07 +0800 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20add=20state-manager=20service=20?= =?UTF-8?q?=E2=80=94=20agent=20memory,=20distillation,=20task=20tracking?= =?UTF-8?q?=20&=20event=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New OctoBus service providing persistent state management for AI agents: Core modules: - Memory Engine: Remember/Recall/Forget with TTL and typed storage - Distillation Engine: Automatic memory compression to prevent unbounded growth - Task State Machine: Structured task lifecycle (pending→running→done/failed) - Event Log: Append-only event recording with time-range queries - Type Registry: Extensible memory type system (cache, log, correction, promise, summary) - TTL Manager: Automatic expiration for time-sensitive entries - Storage Adapter: JSON file-based persistence with atomic writes RPC interface (proto/state.proto): - 28 RPC methods covering project context, meetings, tasks, events, memory, and user config - User personal config via single JSON file (identity, kanban fields, instance mapping) Configuration: - config.schema.json: service-level config (port, data root, user config path) - config/user-config.example.json: template for per-user identity & integration settings - secret.schema.json: no secrets required (state-manager is internal) This service enables agents to maintain persistent context across sessions, automatically compress historical data, track task progress, and integrate with external systems (kanban, calendar, CRM) through a unified state layer. Signed-off-by: jianhua.wang --- services/state-manager/bin/state-service.js | 844 ++++++++++++++++++ services/state-manager/config.schema.json | 20 + .../config/user-config.example.json | 58 ++ .../state-manager/lib/distillation-engine.js | 415 +++++++++ services/state-manager/lib/event-log.js | 255 ++++++ services/state-manager/lib/memory-engine.js | 367 ++++++++ services/state-manager/lib/memory-index.js | 347 +++++++ services/state-manager/lib/storage-adapter.js | 111 +++ services/state-manager/lib/task-machine.js | 416 +++++++++ services/state-manager/lib/ttl-manager.js | 130 +++ services/state-manager/lib/type-registry.js | 113 +++ services/state-manager/package.json | 15 + services/state-manager/proto/state.proto | 582 ++++++++++++ services/state-manager/secret.schema.json | 4 + services/state-manager/service.json | 12 + 15 files changed, 3689 insertions(+) create mode 100644 services/state-manager/bin/state-service.js create mode 100644 services/state-manager/config.schema.json create mode 100644 services/state-manager/config/user-config.example.json create mode 100644 services/state-manager/lib/distillation-engine.js create mode 100644 services/state-manager/lib/event-log.js create mode 100644 services/state-manager/lib/memory-engine.js create mode 100644 services/state-manager/lib/memory-index.js create mode 100644 services/state-manager/lib/storage-adapter.js create mode 100644 services/state-manager/lib/task-machine.js create mode 100644 services/state-manager/lib/ttl-manager.js create mode 100644 services/state-manager/lib/type-registry.js create mode 100644 services/state-manager/package.json create mode 100644 services/state-manager/proto/state.proto create mode 100644 services/state-manager/secret.schema.json create mode 100644 services/state-manager/service.json diff --git a/services/state-manager/bin/state-service.js b/services/state-manager/bin/state-service.js new file mode 100644 index 00000000..cfc9f577 --- /dev/null +++ b/services/state-manager/bin/state-service.js @@ -0,0 +1,844 @@ +#!/usr/bin/env node +/** + * Agent State Manager - OctoBus Service Entry Point + * + * Capability modules (28 RPCs): + * 1. Project Config — read projects.yaml, provide query/match (3 RPCs) + * 2. Weekly Buffer — accumulate summaries, clear after sync (3 RPCs, delegated to memory engine commitment type) + * 3. Meeting Tracking — register meetings, track state, query pending (3 RPCs, delegated to memory engine commitment type) + * 4. User Config — read user-config.json personal config (1 RPC) + * 5. Memory Engine — Remember/Recall/Forget + stats (4 RPCs) + * 6. Distillation Engine — action_log compression + session summary + correction management (2 RPCs) + * 7. Task State Machine — task lifecycle + step tracking + timeout detection (6 RPCs) + * 8. Event Log — append-only event records + range queries (3 RPCs) + * 9. General Storage — simple key-value store (backward-compatible API) (2 RPCs) + * + * OctoBus service display name mapping: + * state-manager → Agent State Manager (this package) + * dingtalk-aitable → DingTalk AITable + * dingtalk-calendar → DingTalk Calendar + * dingtalk-doc → DingTalk Doc + * dingtalk-message → DingTalk Message + * dingtalk-todo → DingTalk Todo + */ + +import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { StorageAdapter } from '../lib/storage-adapter.js'; +import { MemoryIndex } from '../lib/memory-index.js'; +import { TTLManager } from '../lib/ttl-manager.js'; +import { MemoryEngine } from '../lib/memory-engine.js'; +import { DistillationEngine } from '../lib/distillation-engine.js'; +import { TaskMachine } from '../lib/task-machine.js'; +import { EventLog } from '../lib/event-log.js'; + +// ====== Config ====== +const config = { + dataDir: process.env.DATA_DIR || './data', + projectsConfigPath: process.env.PROJECTS_CONFIG_PATH || process.env.PROJECTS_CONFIG || './config/projects.yaml', + userConfigPath: process.env.USER_CONFIG_PATH || process.env.USER_CONFIG || './config/user-config.json', +}; + +// ====== Project Config Cache ====== +let projectsCache = null; +let projectsCacheTime = 0; +const CACHE_TTL = 60000; // 1 minute + +async function loadProjects() { + const now = Date.now(); + if (projectsCache && (now - projectsCacheTime) < CACHE_TTL) { + return projectsCache; + } + + const yaml = await import('js-yaml'); + const raw = await readFile(config.projectsConfigPath, 'utf-8'); + const parsed = yaml.load(raw); + + projectsCache = parsed; + projectsCacheTime = now; + return parsed; +} + +// ====== JSON Persistence Helpers ====== +async function loadJSON(filePath, defaultValue = {}) { + try { + const raw = await readFile(filePath, 'utf-8'); + return JSON.parse(raw); + } catch { + return defaultValue; + } +} + +async function saveJSON(filePath, data) { + await mkdir(dirname(filePath), { recursive: true }); + await writeFile(filePath, JSON.stringify(data, null, 2), 'utf-8'); +} + +// ====== Meeting State Machine (migrated to memory engine, this constant is for internal filtering only) ====== +const VALID_MEETING_STATES = ['registered', 'notes_submitted', 'doc_created', 'reminded']; + +// ====== Memory Engine (global singleton) ====== +const storageAdapter = new StorageAdapter(config.dataDir); +const memoryIndex = new MemoryIndex(); +const ttlManager = new TTLManager(memoryIndex, storageAdapter); +const memoryEngine = new MemoryEngine(memoryIndex, storageAdapter, ttlManager); + +// ====== Distillation Engine ====== +const distillationEngine = new DistillationEngine(memoryEngine); + +// ====== Task State Machine ====== +const taskMachine = new TaskMachine(config.dataDir); + +// ====== Event Log ====== +const eventLog = new EventLog(config.dataDir); + +// ====== Startup Initialization ====== +async function initialize() { + // 1. Load tenant/userId from user-config + await memoryEngine.loadIdentity(config.userConfigPath); + + // 2. Load existing memories from files into index + const allEntries = await storageAdapter.loadAll(); + memoryIndex.rebuild(allEntries); + + // 3. Start TTL background scan + ttlManager.start(); + + // 4. Initialize task state machine + await taskMachine.initialize(); + + // 5. Initialize event log + await eventLog.initialize(); + + console.log(`[MemoryEngine] initialized: tenant=${memoryEngine.tenant}, user=${memoryEngine.userId}, entries=${memoryIndex.entries.size}`); + console.log(`[TaskMachine] ${taskMachine.tasks.size} tasks, [EventLog] ${eventLog.events.length} events`); +} + +// Initialization (runs immediately after service start) +const initPromise = initialize(); + +// ====== Service Definition ====== +const service = defineService({ + handlers: { + + // ====== Project Config ====== + + /** + * Get project config by key + */ + 'state.manager.v1.StateManagerService/GetProject': async (ctx) => { + const { key } = ctx.request; + try { + const parsed = await loadProjects(); + const projects = parsed.projects || {}; + const proj = projects[key]; + if (!proj) { + return { success: false, error: `Project "${key}" not found` }; + } + return { + success: true, + project: { + key, + name: proj.name || '', + nodeId: proj.node_id || '', + customer: proj.customer || '', + keywords: proj.keywords || [], + detailFolder: proj.detail_folder || '', + background: proj.background || '', + }, + error: '', + }; + } catch (err) { + return { success: false, error: err.message }; + } + }, + + /** + * List all projects + */ + 'state.manager.v1.StateManagerService/ListProjects': async () => { + try { + const parsed = await loadProjects(); + const projects = parsed.projects || {}; + const list = Object.entries(projects).map(([key, proj]) => ({ + key, + name: proj.name || '', + nodeId: proj.node_id || '', + customer: proj.customer || '', + keywords: proj.keywords || [], + detailFolder: proj.detail_folder || '', + background: proj.background || '', + })); + return { success: true, projects: list, error: '' }; + } catch (err) { + return { success: false, projects: [], error: err.message }; + } + }, + + /** + * Match project by keywords + * Ported from Python doc_creator.py match_customer_to_project() + */ + 'state.manager.v1.StateManagerService/MatchProject': async (ctx) => { + const { text } = ctx.request; + try { + const parsed = await loadProjects(); + const projects = parsed.projects || {}; + + let bestMatch = null; + let bestScore = 0; + + for (const [key, proj] of Object.entries(projects)) { + const keywords = proj.keywords || []; + for (const kw of keywords) { + if (text.includes(kw)) { + const score = kw.length / Math.max(text.length, 1); + if (score > bestScore) { + bestScore = score; + bestMatch = { + key, + name: proj.name || '', + nodeId: proj.node_id || '', + customer: proj.customer || '', + keywords, + detailFolder: proj.detail_folder || '', + background: proj.background || '', + }; + } + } + } + } + + return { + success: true, + project: bestMatch || { key: '', name: '', nodeId: '', customer: '', keywords: [], detailFolder: '', background: '' }, + confidence: bestMatch ? Math.min(bestScore * 5, 1.0) : 0, + error: '', + }; + } catch (err) { + return { success: false, confidence: 0, error: err.message }; + } + }, + + // ====== Weekly Pending Buffer (migrated to memory engine commitment type) ====== + + /** + * Add a pending weekly summary item + * Internally delegates to Remember(type=commitment, key=weekly_item:{date}:{hash}) + */ + 'state.manager.v1.StateManagerService/AddPendingItem': async (ctx) => { + await initPromise; + const { summary, category, date } = ctx.request; + const itemDate = date || new Date().toISOString().slice(0, 10); + const itemCategory = category || 'Communication'; + const key = `weekly_item:${itemDate}:${(summary || '').slice(0, 20)}`; + const value = JSON.stringify({ + summary: summary || '', + category: itemCategory, + date: itemDate, + }); + + try { + const result = await memoryEngine.remember({ + type: 'commitment', + key, + value, + ttl_seconds: 30 * 86400, // 30 days + }); + + // Count current pending total + const all = await memoryEngine.recall({ prefix: 'weekly_item:', type: 'commitment', limit: 100 }); + + return { success: result.success, pendingCount: all.entries.length, error: result.error || '' }; + } catch (err) { + return { success: false, pendingCount: 0, error: err.message }; + } + }, + + /** + * Get all pending sync items + * Internally delegates to Recall(prefix=weekly_item:, type=commitment) + */ + 'state.manager.v1.StateManagerService/GetPendingItems': async () => { + await initPromise; + try { + const result = await memoryEngine.recall({ prefix: 'weekly_item:', type: 'commitment', limit: 100 }); + const items = (result.entries || []).map((entry) => { + try { + const parsed = JSON.parse(entry.value); + return { + summary: parsed.summary || '', + category: parsed.category || '', + date: parsed.date || '', + }; + } catch { + return { summary: entry.value || '', category: '', date: '' }; + } + }); + return { success: true, items, error: '' }; + } catch (err) { + return { success: false, items: [], error: err.message }; + } + }, + + /** + * Clear pending buffer (called after successful sync) + * Internally delegates to Forget(prefix=weekly_item:, type=commitment) + records last_synced + */ + 'state.manager.v1.StateManagerService/ClearPending': async () => { + await initPromise; + try { + // Get current entry count (for returning clearedCount) + const all = await memoryEngine.recall({ prefix: 'weekly_item:', type: 'commitment', limit: 100 }); + const clearedCount = all.entries.length; + + // Batch delete all weekly_item entries + await memoryEngine.forget({ prefix: 'weekly_item:', type: 'commitment' }); + + // Record last_synced time + await memoryEngine.remember({ + type: 'commitment', + key: 'weekly:last_synced', + value: JSON.stringify({ last_synced: new Date().toISOString() }), + ttl_seconds: 30 * 86400, + }); + + return { success: true, clearedCount, error: '' }; + } catch (err) { + return { success: false, clearedCount: 0, error: err.message }; + } + }, + + // ====== Meeting Tracking (migrated to memory engine commitment type) ====== + + /** + * Register a meeting + * Internally delegates to Remember(type=commitment, key=meeting:{eventId}) + */ + 'state.manager.v1.StateManagerService/RegisterMeeting': async (ctx) => { + await initPromise; + const { eventId, summary, startTime, endTime, attendees } = ctx.request; + + try { + // Check if already exists + const existing = await memoryEngine.recall({ key: `meeting:${eventId}`, type: 'commitment' }); + + if (existing.entries && existing.entries.length > 0) { + // Already exists, skip but update last_check + return { success: true, isNew: false, error: '' }; + } + + // New meeting, write to memory engine + const value = JSON.stringify({ + eventId: eventId || '', + summary: summary || '', + startTime: startTime || '', + endTime: endTime || '', + attendees: attendees || [], + state: 'registered', + registeredAt: new Date().toISOString(), + }); + + await memoryEngine.remember({ + type: 'commitment', + key: `meeting:${eventId}`, + value, + ttl_seconds: 30 * 86400, // 30 days + }); + + return { success: true, isNew: true, error: '' }; + } catch (err) { + return { success: false, isNew: false, error: err.message }; + } + }, + + /** + * Get pending meetings (ended but no minutes submitted yet) + * Internally delegates to Recall(prefix=meeting:, type=commitment) + filtering + */ + 'state.manager.v1.StateManagerService/GetPendingMeetings': async (ctx) => { + await initPromise; + const graceHours = ctx.request.graceHours || 24; + + try { + const result = await memoryEngine.recall({ prefix: 'meeting:', type: 'commitment', limit: 200 }); + const now = new Date(); + const graceMs = graceHours * 3600 * 1000; + + const meetings = (result.entries || []) + .map((entry) => { + try { + return JSON.parse(entry.value); + } catch { + return null; + } + }) + .filter((m) => { + if (!m || m.state !== 'registered') return false; + if (!m.endTime && !m.end_time) return false; + const endTime = new Date(m.endTime || m.end_time); + return (now - endTime) > graceMs; + }) + .map((m) => ({ + eventId: m.eventId || m.event_id || '', + summary: m.summary || '', + startTime: m.startTime || m.start_time || '', + endTime: m.endTime || m.end_time || '', + attendees: m.attendees || [], + state: m.state || 'registered', + })); + + return { success: true, meetings, error: '' }; + } catch (err) { + return { success: false, meetings: [], error: err.message }; + } + }, + + /** + * Update meeting state + * Internally delegates to Recall + Remember to update memory entry + */ + 'state.manager.v1.StateManagerService/UpdateMeetingState': async (ctx) => { + await initPromise; + const { eventId, newState } = ctx.request; + + if (!VALID_MEETING_STATES.includes(newState)) { + return { success: false, error: `Invalid state "${newState}", valid values: ${VALID_MEETING_STATES.join(', ')}` }; + } + + try { + const existing = await memoryEngine.recall({ key: `meeting:${eventId}`, type: 'commitment' }); + + if (!existing.entries || existing.entries.length === 0) { + return { success: false, error: `Meeting "${eventId}" not found` }; + } + + const entry = existing.entries[0]; + let meeting; + try { + meeting = JSON.parse(entry.value); + } catch { + return { success: false, error: `Meeting "${eventId}" data corrupted` }; + } + + meeting.state = newState; + meeting.updatedAt = new Date().toISOString(); + + await memoryEngine.remember({ + type: 'commitment', + key: `meeting:${eventId}`, + value: JSON.stringify(meeting), + ttl_seconds: 30 * 86400, + }); + + return { success: true, error: '' }; + } catch (err) { + return { success: false, error: err.message }; + } + }, + + // ====== User Config ====== + + /** + * Read user personal config + * Config file: config/user-config.json (filled by deployer per actual environment) + */ + 'state.manager.v1.StateManagerService/GetUserConfig': async () => { + try { + const raw = await readFile(config.userConfigPath, 'utf-8'); + const userConfig = JSON.parse(raw); + return { success: true, config: userConfig, error: '' }; + } catch (err) { + if (err.code === 'ENOENT') { + return { + success: false, + config: null, + error: `User config file not found: ${config.userConfigPath}, please create and fill in personal config`, + }; + } + return { success: false, config: null, error: `Failed to read config: ${err.message}` }; + } + }, + + // ====== Agent Memory ====== + + /** + * Write a memory entry + */ + 'state.manager.v1.StateManagerService/Remember': async (ctx) => { + await initPromise; // Ensure initialization is complete + const result = await memoryEngine.remember(ctx.request); + + // Log to event log (non-blocking) + eventLog.logEvent({ + event_type: 'memory.remember', + actor: 'agent', + description: `Remember: type=${ctx.request.type}, key=${ctx.request.key}, unchanged=${result.unchanged}`, + level: 'info', + }).catch(() => {}); + + return { + success: result.success, + unchanged: result.unchanged || false, + evictedCount: result.evictedCount || 0, + error: result.error || '', + }; + }, + + /** + * Read memory (exact match or prefix match) + */ + 'state.manager.v1.StateManagerService/Recall': async (ctx) => { + await initPromise; + const result = await memoryEngine.recall(ctx.request); + return { + success: result.success, + entries: result.entries || [], + error: result.error || '', + }; + }, + + /** + * Delete memory + */ + 'state.manager.v1.StateManagerService/Forget': async (ctx) => { + await initPromise; + const result = await memoryEngine.forget(ctx.request); + + // Log to event log + eventLog.logEvent({ + event_type: 'memory.forget', + actor: 'agent', + description: `Forget: key=${ctx.request.key || ctx.request.prefix}, deleted=${result.deletedCount}`, + level: 'info', + }); + + return { + success: result.success, + deletedCount: result.deletedCount || 0, + error: result.error || '', + }; + }, + + /** + * Get memory engine runtime statistics + */ + 'state.manager.v1.StateManagerService/GetMemoryStats': async () => { + await initPromise; + const stats = memoryEngine.getStats(); + return { + success: true, + totalEntries: stats.totalEntries || 0, + typeStats: stats.types || {}, + rememberCalls: stats.rememberCalls || 0, + recallCalls: stats.recallCalls || 0, + forgetCalls: stats.forgetCalls || 0, + totalEvictions: stats.totalEvictions || 0, + totalDedups: stats.totalDedups || 0, + tenant: stats.tenant || '', + userId: stats.userId || '', + error: '', + }; + }, + + // ====== Distillation Engine ====== + + /** + * Manually trigger a full distillation + */ + 'state.manager.v1.StateManagerService/Distill': async () => { + await initPromise; + const result = await distillationEngine.distill(); + + eventLog.logEvent({ + event_type: 'distillation.run', + actor: 'system', + description: `Distill: actionLog=${JSON.stringify(result.summary?.actionLog)}, session=${JSON.stringify(result.summary?.sessionSummary)}`, + level: result.success ? 'info' : 'error', + }); + + const s = result.summary || {}; + return { + success: result.success, + actionLog: s.actionLog ? { compressed: s.actionLog.compressed || 0, created: s.actionLog.created || 0, deleted: s.actionLog.deleted || 0, reason: s.actionLog.reason || '' } : null, + sessionSummary: s.sessionSummary ? { compressed: s.sessionSummary.compressed || 0, created: s.sessionSummary.created || 0, deleted: s.sessionSummary.deleted || 0, reason: s.sessionSummary.reason || '' } : null, + correctionsUpgraded: s.userCorrection?.upgraded || 0, + correctionsDemoted: s.userCorrection?.demoted || 0, + error: result.error || '', + }; + }, + + /** + * Get distillation statistics + */ + 'state.manager.v1.StateManagerService/GetDistillStats': async () => { + await initPromise; + const stats = distillationEngine.getStats(); + return { + success: true, + totalRuns: stats.totalRuns || 0, + lastRunAt: stats.lastRunAt || '', + actionLogCompressed: stats.actionLogCompressed || 0, + sessionSummaryCompressed: stats.sessionSummaryCompressed || 0, + correctionsUpgraded: stats.correctionsUpgraded || 0, + error: '', + }; + }, + + // ====== Task State Machine ====== + + /** + * Create a task + */ + 'state.manager.v1.StateManagerService/CreateTask': async (ctx) => { + await initPromise; + const r = ctx.request; + const result = await taskMachine.createTask({ + task_id: r.taskId || r.task_id, + name: r.name, + description: r.description, + assignee: r.assignee, + due_date: r.dueDate || r.due_date, + steps: r.steps, + priority: r.priority, + }); + + eventLog.logEvent({ + event_type: 'task.created', + actor: ctx.request.assignee || 'agent', + description: `CreateTask: ${ctx.request.name} (${result.task_id})`, + level: 'info', + }); + + return { success: result.success, taskId: result.task_id || '', error: result.error || '' }; + }, + + /** + * Update task state + */ + 'state.manager.v1.StateManagerService/UpdateTask': async (ctx) => { + await initPromise; + const r = ctx.request; + const result = await taskMachine.updateTask({ + task_id: r.taskId || r.task_id, + new_state: r.newState || r.new_state, + reason: r.reason, + }); + + eventLog.logEvent({ + event_type: 'task.state_changed', + actor: 'agent', + description: `UpdateTask: ${ctx.request.taskId || ctx.request.task_id} ${result.old_state} → ${result.new_state}`, + level: result.new_state === 'failed' ? 'warning' : 'info', + }); + + return { + success: result.success, + oldState: result.old_state || '', + newState: result.new_state || '', + error: result.error || '', + }; + }, + + /** + * Add/update a step + */ + 'state.manager.v1.StateManagerService/UpdateStep': async (ctx) => { + await initPromise; + const r = ctx.request; + const result = await taskMachine.updateStep({ + task_id: r.taskId || r.task_id, + step_id: r.stepId || r.step_id, + name: r.name, + state: r.state, + notes: r.notes, + }); + return { success: result.success, stepId: result.step_id || '', error: result.error || '' }; + }, + + /** + * Get a single task + */ + 'state.manager.v1.StateManagerService/GetTask': async (ctx) => { + await initPromise; + const taskId = ctx.request.taskId || ctx.request.task_id; + const result = taskMachine.getTask(taskId); + return { + success: result.success, + task: result.task || null, + error: result.error || '', + }; + }, + + /** + * List tasks (with filtering) + */ + 'state.manager.v1.StateManagerService/ListTasks': async (ctx) => { + await initPromise; + const r = ctx.request; + const result = taskMachine.listTasks({ + state: r.state, + assignee: r.assignee, + priority: r.priority, + due_before: r.dueBefore || r.due_before, + due_after: r.dueAfter || r.due_after, + limit: r.limit, + }); + return { + success: result.success, + tasks: result.tasks || [], + total: result.total || 0, + error: result.error || '', + }; + }, + + /** + * Get task statistics + */ + 'state.manager.v1.StateManagerService/GetTaskStats': async () => { + await initPromise; + const stats = taskMachine.getStats(); + return { + success: true, + totalTasks: stats.totalTasks || 0, + stateCounts: stats.stateCounts || {}, + totalCreated: stats.totalCreated || 0, + totalCompleted: stats.totalCompleted || 0, + totalFailed: stats.totalFailed || 0, + totalTimedOut: stats.totalTimedOut || 0, + error: '', + }; + }, + + // ====== Event Log ====== + + /** + * Log an event + */ + 'state.manager.v1.StateManagerService/LogEvent': async (ctx) => { + await initPromise; + let data = ctx.request.data || '{}'; + try { JSON.parse(data); } catch { data = JSON.stringify({ raw: data }); } + const result = await eventLog.logEvent({ + event_type: ctx.request.eventType || ctx.request.event_type || 'unknown', + actor: ctx.request.actor || 'unknown', + description: ctx.request.description || '', + data: JSON.parse(data), + level: ctx.request.level || 'info', + }); + return { success: result.success, eventId: result.event_id || '', error: result.error || '' }; + }, + + /** + * Query events + */ + 'state.manager.v1.StateManagerService/QueryEvents': async (ctx) => { + await initPromise; + const r = ctx.request; + const result = await eventLog.queryEvents({ + event_type: r.eventType || r.event_type, + actor: r.actor, + level: r.level, + since: r.since, + until: r.until, + search: r.search, + limit: r.limit, + }); + return { + success: result.success, + events: (result.events || []).map(e => ({ + eventId: e.event_id, + eventType: e.event_type, + actor: e.actor, + description: e.description, + data: JSON.stringify(e.data), + level: e.level, + timestamp: e.timestamp, + })), + total: result.total || 0, + error: result.error || '', + }; + }, + + /** + * Get event log statistics + */ + 'state.manager.v1.StateManagerService/GetEventStats': async () => { + await initPromise; + const stats = eventLog.getStats(); + return { + success: true, + totalEvents: stats.totalEvents || 0, + totalLogged: stats.totalLogged || 0, + totalRotated: stats.totalRotated || 0, + oldestEvent: stats.oldestEvent || '', + newestEvent: stats.newestEvent || '', + error: '', + }; + }, + + // ====== General KV Storage ====== + + /** + * Read state + */ + 'state.manager.v1.StateManagerService/GetState': async (ctx) => { + const { key } = ctx.request; + const filePath = join(config.dataDir, 'kv_state.json'); + + try { + const data = await loadJSON(filePath, {}); + return { success: true, value: JSON.stringify(data[key] ?? null), error: '' }; + } catch (err) { + return { success: false, value: '', error: err.message }; + } + }, + + /** + * Write state + */ + 'state.manager.v1.StateManagerService/SetState': async (ctx) => { + const { key, value } = ctx.request; + const filePath = join(config.dataDir, 'kv_state.json'); + + try { + const data = await loadJSON(filePath, {}); + try { + data[key] = JSON.parse(value); + } catch { + data[key] = value; + } + await saveJSON(filePath, data); + + return { success: true, error: '' }; + } catch (err) { + return { success: false, error: err.message }; + } + }, + }, +}); + +runServiceMain(service); + +// ====== Graceful Shutdown ====== +async function gracefulShutdown(signal) { + console.log(`[StateService] received ${signal}, shutting down gracefully...`); + + // 1. Stop TTL scan + ttlManager.stop(); + + // 2. Stop task timeout detection + taskMachine.stop(); + + // 3. Final persistence: wait for all in-flight writes + await storageAdapter.waitForPendingWrites(); + await taskMachine.shutdown(); + await eventLog.shutdown(); + + console.log('[StateService] shutdown complete'); + process.exit(0); +} + +process.on('SIGTERM', () => gracefulShutdown('SIGTERM')); +process.on('SIGINT', () => gracefulShutdown('SIGINT')); diff --git a/services/state-manager/config.schema.json b/services/state-manager/config.schema.json new file mode 100644 index 00000000..61a910b0 --- /dev/null +++ b/services/state-manager/config.schema.json @@ -0,0 +1,20 @@ +{ + "type": "object", + "properties": { + "dataDir": { + "type": "string", + "description": "Data directory path, can be overridden via DATA_DIR env var", + "default": "./data" + }, + "projectsConfigPath": { + "type": "string", + "description": "Project config file path, can be overridden via PROJECTS_CONFIG env var", + "default": "./config/projects.yaml" + }, + "userConfigPath": { + "type": "string", + "description": "User personal config file path (identity/kanban/CRM/instance mapping), can be overridden via USER_CONFIG env var. New users only need to edit this one file during deployment.", + "default": "./config/user-config.json" + } + } +} diff --git a/services/state-manager/config/user-config.example.json b/services/state-manager/config/user-config.example.json new file mode 100644 index 00000000..b5dbb047 --- /dev/null +++ b/services/state-manager/config/user-config.example.json @@ -0,0 +1,58 @@ +{ + "_README": "Personal config file — change this one file when switching users/environments. All agents read via GetUserConfig.", + "identity": { + "user_id": "YOUR_LOGIN_ID", + "display_name": "Your Name", + "role": "Your role (pre-sales/sales/tech-support)", + "crm_user_id": "CRM user ID (look up via ListUsers on first use, can leave empty for agent auto-lookup)", + "default_sales_user_id": "Default sales userId", + "default_sales_name": "Default sales name", + "team": "Your team name" + }, + "kanban": { + "base_id": "AITable base ID (from alidocs URL nodeId)", + "table_id": "AITable table ID", + "instance": "OctoBus instance name (e.g. aitable-instance)", + "fields": { + "start_date": "Start date field ID", + "end_date": "Due date field ID", + "task_type": "Task type field ID", + "task_desc": "Task description field ID", + "sales": "Sales field ID", + "owner": "Owner field ID", + "notes": "Description field ID", + "customer_l1": "Customer tier field ID", + "customer_l2": "Sub-department field ID", + "status": "Status field ID", + "presale": "Pre-sales field ID", + "crm_link": "CRM project link field ID" + }, + "task_types": { + "communication": [{"id": "option ID", "name": "Communication"}], + "documentation": [{"id": "option ID", "name": "Documentation"}], + "bidding": [{"id": "option ID", "name": "Bidding"}], + "poc": [{"id": "option ID", "name": "POC & Others"}] + }, + "statuses": { + "completed": [{"id": "option ID", "name": "Done"}], + "in_progress": [{"id": "option ID", "name": "In Progress"}] + }, + "customers": { + "customers": [ + {"id": "option ID", "name": "Customer Name"} + ] + } + }, + "instances": { + "aitable": "aitable-instance", + "doc": "doc-instance", + "calendar": "calendar-instance", + "todo": "todo-instance", + "message": "message-instance", + "crm": "crm-instance", + "state": "state-instance" + }, + "knowledge_file": "/opt/knowledge/company-products.md", + "company_name": "Your Company", + "company_english": "CompanyNameEn" +} diff --git a/services/state-manager/lib/distillation-engine.js b/services/state-manager/lib/distillation-engine.js new file mode 100644 index 00000000..fc71a907 --- /dev/null +++ b/services/state-manager/lib/distillation-engine.js @@ -0,0 +1,415 @@ +/** + * DistillationEngine — Memory Distillation Engine + * + * Core value: makes memory more than just "store and retrieve" — + * it continuously compresses and refines. + * Three levels of distillation: + * 1. action_log → session_summary (weekly summary, ~10:1 compression) + * 2. session_summary → monthly_summary (monthly rollup, ~10:1 compression) + * 3. user_correction pattern extraction (multiple corrections for same key → upgrade to permanent preference) + * + * The distillation engine is driven by deterministic rules, not LLM. + * It is a "clever tool", not an agent. + */ + +export class DistillationEngine { + /** + * @param {import('./memory-engine.js').MemoryEngine} memoryEngine + * @param {object} opts + * @param {number} opts.weeklyWindowDays - action_log weekly summary window (default 7 days) + * @param {number} opts.monthlyWindowDays - session_summary monthly rollup window (default 30 days) + * @param {number} opts.correctionThreshold - Number of corrections for same key to trigger upgrade (default 3) + */ + constructor(memoryEngine, opts = {}) { + this.engine = memoryEngine; + this.weeklyWindowDays = opts.weeklyWindowDays || 7; + this.monthlyWindowDays = opts.monthlyWindowDays || 30; + this.correctionThreshold = opts.correctionThreshold || 3; + + // Distillation statistics + this._stats = { + totalRuns: 0, + lastRunAt: null, + actionLogCompressed: 0, + sessionSummaryCompressed: 0, + correctionsUpgraded: 0, + errors: 0, + }; + } + + /** + * Run a full distillation pass (all types) + * @returns {object} { success, summary, error } + */ + async distill() { + this._stats.totalRuns++; + this._stats.lastRunAt = new Date().toISOString(); + + const summary = { + actionLog: null, + sessionSummary: null, + userCorrection: null, + }; + + try { + // 1. action_log weekly summary + summary.actionLog = await this._distillActionLogs(); + + // 2. session_summary monthly rollup + summary.sessionSummary = await this._distillSessionSummaries(); + + // 3. user_correction pattern extraction + summary.userCorrection = await this._distillUserCorrections(); + + return { success: true, summary, error: '' }; + } catch (err) { + this._stats.errors++; + return { success: false, summary, error: err.message }; + } + } + + /** + * Get distillation statistics + */ + getStats() { + return { ...this._stats }; + } + + // ─── 1. Action Log Weekly Summary ─── + + /** + * Compress action_log entries within 7 days into 1 session_summary + * @returns {object} { compressed, created, deleted } + */ + async _distillActionLogs() { + const now = new Date(); + const windowMs = this.weeklyWindowDays * 86400 * 1000; + const cutoff = new Date(now.getTime() - windowMs); + + // Collect action_log entries within the window + const typeEntries = this.engine.index.getEntriesByType('action_log'); + const candidates = []; + for (const [fullKey, entry] of typeEntries) { + const created = new Date(entry.createdAt); + if (created >= cutoff && created < now) { + candidates.push({ fullKey, entry }); + } + } + + if (candidates.length < 3) { + // Too few, not worth compressing + return { compressed: 0, created: 0, deleted: 0, reason: 'too_few_entries' }; + } + + // Analyze action_logs: extract patterns + const analysis = this._analyzeActionLogs(candidates); + + // Build summary + const summaryValue = JSON.stringify({ + source: 'distillation:action_log', + period: { + from: cutoff.toISOString(), + to: now.toISOString(), + }, + totalActions: candidates.length, + topActions: analysis.topActions, + topEntities: analysis.topEntities, + frequency: analysis.frequency, + patterns: analysis.patterns, + }); + + // Write session_summary + const weekLabel = `${cutoff.toISOString().slice(0, 10)}_to_${now.toISOString().slice(0, 10)}`; + const writeResult = await this.engine.remember({ + key: `distilled:action_log:${weekLabel}`, + value: summaryValue, + type: 'session_summary', + ttl_seconds: 30 * 86400, // 30 days + confidence: 1.0, + }); + + let deletedCount = 0; + if (writeResult.success) { + // Delete compressed action_log entries + for (const { fullKey } of candidates) { + const entry = this.engine.index.delete(fullKey); + if (entry) deletedCount++; + } + // Persist action_log + await this.engine._persistType('action_log'); + // Persist session_summary (since a new entry was just written) + await this.engine._persistType('session_summary'); + } + + this._stats.actionLogCompressed += deletedCount; + + return { + compressed: candidates.length, + created: writeResult.success ? 1 : 0, + deleted: deletedCount, + }; + } + + /** + * Analyze action_log entries and extract patterns + */ + _analyzeActionLogs(candidates) { + const actionCounts = new Map(); // action → count + const entityCounts = new Map(); // entity → count + const dailyCounts = new Map(); // date → count + const patterns = []; + + for (const { entry } of candidates) { + // Extract action and entity from key + // Key format: "action:::" or similar structure + const originalKey = entry._originalKey || ''; + const parts = originalKey.split(':'); + const action = parts[0] || 'unknown'; + const entity = parts[1] || 'unknown'; + + // Count action frequency + actionCounts.set(action, (actionCounts.get(action) || 0) + 1); + + // Count entity frequency + entityCounts.set(entity, (entityCounts.get(entity) || 0) + 1); + + // Count daily frequency + const date = entry.createdAt.slice(0, 10); + dailyCounts.set(date, (dailyCounts.get(date) || 0) + 1); + + // Parse value to extract extra info + try { + const val = typeof entry.value === 'string' ? JSON.parse(entry.value) : entry.value; + if (val && typeof val === 'object') { + // Record success/failure + if (val.success === false) { + patterns.push(`Failed action: ${action}(${entity})`); + } + } + } catch { + // Value is not JSON, skip + } + } + + // Top 5 actions + const topActions = [...actionCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) + .map(([action, count]) => ({ action, count })); + + // Top 5 entities + const topEntities = [...entityCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) + .map(([entity, count]) => ({ entity, count })); + + // Frequency distribution + const dailyArr = [...dailyCounts.entries()].sort((a, b) => a[0].localeCompare(b[0])); + const avgPerDay = candidates.length / Math.max(dailyArr.length, 1); + const frequency = { + avgPerDay: Math.round(avgPerDay * 10) / 10, + activeDays: dailyArr.length, + totalDays: this.weeklyWindowDays, + }; + + // Pattern detection + const uniquePatterns = [...new Set(patterns)].slice(0, 5); + + return { topActions, topEntities, frequency, patterns: uniquePatterns }; + } + + // ─── 2. Session Summary Monthly Rollup ─── + + /** + * Compress session_summary entries within 30 days into 1 monthly rollup + * @returns {object} { compressed, created, deleted } + */ + async _distillSessionSummaries() { + const now = new Date(); + const windowMs = this.monthlyWindowDays * 86400 * 1000; + const cutoff = new Date(now.getTime() - windowMs); + + const typeEntries = this.engine.index.getEntriesByType('session_summary'); + const candidates = []; + for (const [fullKey, entry] of typeEntries) { + const created = new Date(entry.createdAt); + // Only compress non-distilled session_summary (avoid recursive compression) + if (created >= cutoff && created < now) { + const val = typeof entry.value === 'string' ? entry.value : JSON.stringify(entry.value); + if (!val.includes('distillation:monthly_rollup')) { + candidates.push({ fullKey, entry }); + } + } + } + + if (candidates.length < 2) { + return { compressed: 0, created: 0, deleted: 0, reason: 'too_few_entries' }; + } + + // Merge all session_summary entries + const analysis = this._mergeSessionSummaries(candidates); + + const summaryValue = JSON.stringify({ + source: 'distillation:monthly_rollup', + period: { + from: cutoff.toISOString(), + to: now.toISOString(), + }, + sessionCount: candidates.length, + allTopics: analysis.topTopics, + keyDecisions: analysis.keyDecisions, + pendingItems: analysis.pendingItems, + entitiesTouched: analysis.topEntities, + }); + + const monthLabel = cutoff.toISOString().slice(0, 7); // YYYY-MM + const writeResult = await this.engine.remember({ + key: `distilled:monthly:${monthLabel}`, + value: summaryValue, + type: 'session_summary', + ttl_seconds: 90 * 86400, // Monthly rollup retained for 90 days + confidence: 1.0, + }); + + let deletedCount = 0; + if (writeResult.success) { + for (const { fullKey } of candidates) { + const entry = this.engine.index.delete(fullKey); + if (entry) deletedCount++; + } + await this.engine._persistType('session_summary'); + } + + this._stats.sessionSummaryCompressed += deletedCount; + + return { + compressed: candidates.length, + created: writeResult.success ? 1 : 0, + deleted: deletedCount, + }; + } + + /** + * Merge multiple session_summary entries + */ + _mergeSessionSummaries(candidates) { + const topicCounts = new Map(); + const decisions = []; + const pendingItems = []; + const entityCounts = new Map(); + + for (const { entry } of candidates) { + try { + const val = typeof entry.value === 'string' ? JSON.parse(entry.value) : entry.value; + if (!val || typeof val !== 'object') continue; + + // Merge topics + if (Array.isArray(val.topics)) { + for (const t of val.topics) { + topicCounts.set(t, (topicCounts.get(t) || 0) + 1); + } + } + + // Merge decisions (keep all, deduplicate) + if (Array.isArray(val.decisions)) { + for (const d of val.decisions) { + if (!decisions.includes(d)) decisions.push(d); + } + } + + // Merge pending (keep only uncompleted) + if (Array.isArray(val.pending)) { + for (const p of val.pending) { + if (!pendingItems.includes(p)) pendingItems.push(p); + } + } + + // Merge entities_touched + if (Array.isArray(val.entities_touched)) { + for (const e of val.entities_touched) { + entityCounts.set(e, (entityCounts.get(e) || 0) + 1); + } + } + } catch { + // Parse failed, skip + } + } + + const topTopics = [...topicCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([topic, count]) => ({ topic, count })); + + const topEntities = [...entityCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([entity, count]) => ({ entity, count })); + + return { + topTopics, + keyDecisions: decisions.slice(0, 20), + pendingItems: pendingItems.slice(0, 10), + topEntities, + }; + } + + // ─── 3. User Correction Capacity Management ─── + + /** + * user_correction capacity management: + * When user_correction entries approach the capacity limit, demote the oldest + * entries to action_log. This keeps the latest correction records and frees + * space for new corrections. + * + * Note: user_correction is designed for same-key overwrite (keeps latest), + * so there will never be "multiple entries for the same key". + * Distillation focuses on overall capacity control. + * @returns {object} { demoted } + */ + async _distillUserCorrections() { + const typeEntries = this.engine.index.getEntriesByType('user_correction'); + const maxEntries = 100; // user_correction capacity limit + const threshold = Math.floor(maxEntries * 0.7); // Trigger cleanup at 70% + + if (typeEntries.size < threshold) { + return { upgraded: 0, demoted: 0, reason: 'below_threshold' }; + } + + // Sort by creation time, oldest first + const sorted = [...typeEntries.entries()] + .sort((a, b) => new Date(a[1].createdAt) - new Date(b[1].createdAt)); + + // Demote oldest entries (keep latest 50%) + const toDemote = Math.floor(sorted.length * 0.5); + let demoted = 0; + + for (let i = 0; i < toDemote; i++) { + const [fullKey, entry] = sorted[i]; + + // Skip protected entries + if (entry.protected) continue; + + // Demote to action_log + await this.engine.remember({ + key: `correction_archived:${entry._originalKey || fullKey}`, + value: typeof entry.value === 'string' ? entry.value : JSON.stringify(entry.value), + type: 'action_log', + ttl_seconds: 7 * 86400, + confidence: 0.5, + }); + + // Delete from user_correction + this.engine.index.delete(fullKey); + demoted++; + } + + if (demoted > 0) { + await this.engine._persistType('user_correction'); + await this.engine._persistType('action_log'); + } + + return { upgraded: 0, demoted }; + } +} + +export default DistillationEngine; diff --git a/services/state-manager/lib/event-log.js b/services/state-manager/lib/event-log.js new file mode 100644 index 00000000..f3eeb71d --- /dev/null +++ b/services/state-manager/lib/event-log.js @@ -0,0 +1,255 @@ +/** + * 事件日志（EventLog）— Append-only 事件日志 + * + * 记录智能体平台中"发生了什么"，用于审计/调试/回溯。 + * 与 Loader 的 Event Bus 互补： + * - Loader Event Bus：实时触发（pub/sub），"什么时候跑" + * - EventLog：历史记录（append-only），"发生了什么" + * + * 设计原则： + * - Append-only，不删除不修改 + * - 按时间排序，支持范围查询 + * - 文件按大小轮转（maxEvents 条后新建文件） + * - 不引入 pub/sub，与 Loader 零耦合 + */ + +import { readFile, writeFile, mkdir, readdir, rename } from 'node:fs/promises'; +import { join, dirname } from 'node:path'; + +const DEFAULT_MAX_EVENTS = 5000; // 单文件最大事件数 +const DEFAULT_MAX_FILES = 5; // 最多保留的轮转文件数 + +export class EventLog { + /** + * @param {string} dataDir + * @param {object} opts + * @param {number} opts.maxEvents - 单文件最大事件数 + * @param {number} opts.maxFiles - 轮转文件数上限 + */ + constructor(dataDir, opts = {}) { + this.dataDir = dataDir; + this.filePath = join(dataDir, 'event_log.json'); + this.maxEvents = opts.maxEvents || DEFAULT_MAX_EVENTS; + this.maxFiles = opts.maxFiles || DEFAULT_MAX_FILES; + + // 内存中的事件缓冲（最近的事件） + this.events = []; + this._dirty = false; + + // 统计 + this._stats = { + totalLogged: 0, + totalRotated: 0, + }; + } + + // ─── 生命周期 ─── + + async initialize() { + await this._load(); + console.log(`[EventLog] initialized: ${this.events.length} events loaded`); + } + + async shutdown() { + if (this._dirty) await this._save(); + } + + // ─── 写入事件 ─── + + /** + * 记录一个事件 + * @param {object} request + * @param {string} request.event_type - 事件类型（如 agent.remember, loader.triggered, task.state_changed） + * @param {string} request.actor - 触发者（智能体名称/系统/用户） + * @param {string} request.description - 事件描述 + * @param {object} request.data - 附加数据 + * @param {string} request.level - info / warning / error + * @returns {object} { success, event_id, error } + */ + async logEvent(request) { + try { + const eventId = `evt_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + const event = { + event_id: eventId, + event_type: request.event_type || 'unknown', + actor: request.actor || 'system', + description: request.description || '', + data: request.data || {}, + level: request.level || 'info', + timestamp: new Date().toISOString(), + }; + + this.events.push(event); + this._stats.totalLogged++; + this._dirty = true; + + // 检查是否需要轮转 + if (this.events.length >= this.maxEvents) { + await this._rotate(); + } + + // 立即持久化（事件日志要求可靠写入） + await this._save(); + + return { success: true, event_id: eventId, error: '' }; + } catch (err) { + return { success: false, event_id: '', error: err.message }; + } + } + + // ─── 查询事件 ─── + + /** + * 查询事件 + * @param {object} filter + * @param {string} filter.event_type - 按事件类型过滤 + * @param {string} filter.actor - 按触发者过滤 + * @param {string} filter.level - 按级别过滤 + * @param {string} filter.since - 起始时间（ISO 8601） + * @param {string} filter.until - 结束时间（ISO 8601） + * @param {string} filter.search - 关键词搜索（在 description 中匹配） + * @param {number} filter.limit - 最大返回条数（默认 50） + * @returns {object} { success, events, total, error } + */ + async queryEvents(filter = {}) { + let results = [...this.events]; + + // 按事件类型过滤 + if (filter.event_type) { + const types = filter.event_type.split(','); + results = results.filter(e => types.includes(e.event_type)); + } + + // 按触发者过滤 + if (filter.actor) { + results = results.filter(e => e.actor === filter.actor); + } + + // 按级别过滤 + if (filter.level) { + results = results.filter(e => e.level === filter.level); + } + + // 按时间范围过滤 + if (filter.since) { + const since = new Date(filter.since); + results = results.filter(e => new Date(e.timestamp) >= since); + } + if (filter.until) { + const until = new Date(filter.until); + results = results.filter(e => new Date(e.timestamp) <= until); + } + + // 关键词搜索 + if (filter.search) { + const keyword = filter.search.toLowerCase(); + results = results.filter(e => + (e.description || '').toLowerCase().includes(keyword) || + (e.event_type || '').toLowerCase().includes(keyword) + ); + } + + // 按时间倒序（最新的在前） + results.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp)); + + const total = results.length; + const limit = filter.limit || 50; + results = results.slice(0, limit); + + return { success: true, events: results, total, error: '' }; + } + + /** + * 获取事件日志统计 + */ + getStats() { + const typeCounts = {}; + const actorCounts = {}; + const levelCounts = { info: 0, warning: 0, error: 0 }; + + for (const event of this.events) { + typeCounts[event.event_type] = (typeCounts[event.event_type] || 0) + 1; + actorCounts[event.actor] = (actorCounts[event.actor] || 0) + 1; + if (levelCounts[event.level] !== undefined) { + levelCounts[event.level]++; + } + } + + return { + totalEvents: this.events.length, + ...this._stats, + typeCounts, + actorCounts, + levelCounts, + oldestEvent: this.events.length > 0 ? this.events[0].timestamp : null, + newestEvent: this.events.length > 0 ? this.events[this.events.length - 1].timestamp : null, + }; + } + + // ─── 内部方法 ─── + + /** + * 文件轮转：当前文件重命名为 .1，旧文件递增 + */ + async _rotate() { + try { + // 递增轮转文件编号 + for (let i = this.maxFiles - 1; i >= 1; i--) { + const from = join(this.dataDir, `event_log.${i}.json`); + const to = join(this.dataDir, `event_log.${i + 1}.json`); + try { + await rename(from, to); + } catch { + // 文件不存在，跳过 + } + } + + // 当前文件 → .1 + try { + await rename(this.filePath, join(this.dataDir, 'event_log.1.json')); + } catch { + // 当前文件不存在，忽略 + } + + // 清空内存 + this.events = []; + this._stats.totalRotated++; + this._dirty = false; + } catch (err) { + console.error('[EventLog] rotation error:', err.message); + } + } + + async _load() { + try { + await mkdir(this.dataDir, { recursive: true }); + const raw = await readFile(this.filePath, 'utf-8'); + const data = JSON.parse(raw); + if (Array.isArray(data.events)) { + this.events = data.events; + } + } catch { + // 文件不存在，从空开始 + } + } + + async _save() { + try { + await mkdir(this.dataDir, { recursive: true }); + const data = { + version: 1, + updatedAt: new Date().toISOString(), + eventCount: this.events.length, + events: this.events, + }; + const tmpPath = this.filePath + '.tmp'; + await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); + await rename(tmpPath, this.filePath); + this._dirty = false; + } catch (err) { + console.error('[EventLog] save error:', err.message); + } + } +} + +export default EventLog; diff --git a/services/state-manager/lib/memory-engine.js b/services/state-manager/lib/memory-engine.js new file mode 100644 index 00000000..296a84ce --- /dev/null +++ b/services/state-manager/lib/memory-engine.js @@ -0,0 +1,367 @@ +/** + * 记忆引擎（MemoryEngine）— Remember/Recall/Forget 核心逻辑 + * + * 职责： + * - 前缀拼接（tenant:user:type:key） + * - 去重检查（dedup_window 内同 key 同值跳过） + * - 容量管理（超限触发 LRU/oldest 淘汰） + * - 置信度保护（high confidence + 永不过期 → 不被淘汰） + */ + +import { resolve as resolveType, listTypes } from './type-registry.js'; + +/** + * 操作日志（审计/调试用），保留最近 MAX_LOG_SIZE 条 + */ +const MAX_LOG_SIZE = 500; + +export class MemoryEngine { + /** + * @param {import('./memory-index.js').MemoryIndex} index + * @param {import('./storage-adapter.js').StorageAdapter} storage + * @param {import('./ttl-manager.js').TTLManager} ttlManager + * @param {object} config - { tenant, userId } + */ + constructor(index, storage, ttlManager, config = {}) { + this.index = index; + this.storage = storage; + this.ttlManager = ttlManager; + this.tenant = config.tenant || 'default'; + this.userId = config.userId || 'shared'; + + // 操作日志（ring buffer） + this._opLog = []; + // 累计统计 + this._stats = { + rememberCalls: 0, + recallCalls: 0, + forgetCalls: 0, + rememberErrors: 0, + recallErrors: 0, + forgetErrors: 0, + totalEvictions: 0, + totalDedups: 0, + }; + } + + /** + * 记录操作日志 + */ + _log(op, detail) { + this._opLog.push({ op, detail, ts: new Date().toISOString() }); + if (this._opLog.length > MAX_LOG_SIZE) { + this._opLog.splice(0, this._opLog.length - MAX_LOG_SIZE); + } + } + + /** + * 获取引擎运行统计 + */ + getStats() { + const indexStats = this.index.getStats(); + return { + ...this._stats, + ...indexStats, + tenant: this.tenant, + userId: this.userId, + recentOps: this._opLog.slice(-20), + }; + } + + /** + * 拼接完整 key：{tenant}:{user}:{type}:{key} + */ + _fullKey(type, key) { + return `${this.tenant}:${this.userId}:${type}:${key}`; + } + + /** + * 从 user-config 加载 tenant 和 userId + */ + async loadIdentity(userConfigPath) { + try { + const { readFile } = await import('node:fs/promises'); + const raw = await readFile(userConfigPath, 'utf-8'); + const config = JSON.parse(raw); + this.tenant = config.company_english || config.companyName || 'default'; + this.userId = config.identity?.user_id || 'shared'; + } catch { + // 配置文件不存在时用默认值 + } + } + + // ─── Remember ─── + + /** + * 写入一条记忆 + * @param {object} request - { key, value, type, ttl_seconds, dedup_window_sec, confidence } + * @returns {object} { success, unchanged, evicted_count, error } + */ + async remember(request) { + try { + this._stats.rememberCalls++; + const resolved = resolveType(request); + const fullKey = this._fullKey(resolved.type, resolved.key); + const now = new Date(); + + // 去重检查 + const existing = this.index.get(fullKey); + if (existing && resolved.dedupWindowSec > 0) { + const createdMs = new Date(existing.createdAt).getTime(); + const windowMs = resolved.dedupWindowSec * 1000; + if (now.getTime() - createdMs < windowMs) { + // 去重窗口内：同 key 同值 → 跳过 + if (existing.value === resolved.value) { + this._stats.totalDedups++; + this._log('remember', { key: resolved.key, type: resolved.type, result: 'dedup' }); + return { success: true, unchanged: true, evictedCount: 0, error: '' }; + } + // 同 key 异值 → 更新（不跳过） + } + } + + // 计算过期时间 + const expiresAt = resolved.ttlSeconds > 0 + ? new Date(now.getTime() + resolved.ttlSeconds * 1000).toISOString() + : null; + + // 构建记忆条目 + const entry = { + key: fullKey, + _originalKey: resolved.key, + value: resolved.value, + type: resolved.type, + confidence: resolved.confidence, + createdAt: existing ? existing.createdAt : now.toISOString(), + updatedAt: now.toISOString(), + expiresAt, + lastAccessAt: now.toISOString(), + accessCount: (existing?.accessCount || 0) + 1, + protected: resolved.protected, + }; + + // 容量检查 + 淘汰 + let evictedCount = 0; + const currentCount = this.index.getTypeCount(resolved.type); + const maxEntries = resolved.maxEntries; + if (!existing && currentCount >= maxEntries) { + evictedCount = await this._evict(resolved.type, resolved.evictPolicy, currentCount - maxEntries + 1); + this._stats.totalEvictions += evictedCount; + } + + // 写入索引 + this.index.set(fullKey, entry); + + // 持久化 + await this._persistType(resolved.type); + + this._log('remember', { key: resolved.key, type: resolved.type, result: existing ? 'updated' : 'created', evicted: evictedCount }); + return { success: true, unchanged: false, evictedCount, error: '' }; + } catch (err) { + this._stats.rememberErrors++; + this._log('remember', { key: request.key, type: request.type, result: 'error', error: err.message }); + return { success: false, unchanged: false, evictedCount: 0, error: err.message }; + } + } + + // ─── Recall ─── + + /** + * 读取记忆 + * @param {object} request - { key, prefix, type, limit } + * @returns {object} { success, entries, error } + */ + async recall(request) { + try { + this._stats.recallCalls++; + const limit = request.limit || 10; + let results = []; + + if (request.key) { + // 精确匹配 + const type = request.type || 'entity_cache'; + const fullKey = this._fullKey(type, request.key); + + let entry = this.index.get(fullKey); + + // 如果精确 key 找不到，遍历所有已注册类型查找 + if (!entry) { + for (const t of listTypes()) { + const tryKey = this._fullKey(t.type, request.key); + entry = this.index.get(tryKey); + if (entry) break; + } + } + + if (entry) { + // TTL lazy check + if (this.ttlManager.checkAndExpire(entry.key)) { + await this._persistType(entry.type); + results = []; + } else { + // 更新访问信息 + entry.lastAccessAt = new Date().toISOString(); + entry.accessCount = (entry.accessCount || 0) + 1; + this.index.set(entry.key, entry); + + results = [{ + key: entry._originalKey || request.key, + value: entry.value, + type: entry.type, + confidence: entry.confidence, + createdAt: entry.createdAt, + expiresAt: entry.expiresAt || '', + }]; + } + } + } else if (request.prefix) { + // 前缀匹配 + const type = request.type || ''; + const prefix = type + ? this._fullKey(type, request.prefix) + : `${this.tenant}:${this.userId}:${request.prefix}`; + + const matches = this.index.getByPrefix(prefix.endsWith(':') ? prefix : prefix + ':', { + type: request.type || null, + limit, + }); + + // 也尝试不含尾冒号的前缀 + if (matches.length === 0) { + const altMatches = this.index.getByPrefix(prefix, { + type: request.type || null, + limit, + }); + results = altMatches.map(e => ({ + key: e._originalKey || e.key, + value: e.value, + type: e.type, + confidence: e.confidence, + createdAt: e.createdAt, + expiresAt: e.expiresAt || '', + })); + } else { + results = matches.map(e => ({ + key: e._originalKey || e.key, + value: e.value, + type: e.type, + confidence: e.confidence, + createdAt: e.createdAt, + expiresAt: e.expiresAt || '', + })); + } + + // 更新访问信息 + for (const r of results) { + for (const [k, v] of this.index.entries) { + if (v._originalKey === r.key && v.type === r.type) { + v.lastAccessAt = new Date().toISOString(); + v.accessCount = (v.accessCount || 0) + 1; + break; + } + } + } + } + + this._log('recall', { key: request.key || request.prefix, type: request.type, results: results.length }); + return { success: true, entries: results, error: '' }; + } catch (err) { + this._stats.recallErrors++; + this._log('recall', { key: request.key || request.prefix, result: 'error', error: err.message }); + return { success: false, entries: [], error: err.message }; + } + } + + // ─── Forget ─── + + /** + * 删除记忆 + * @param {object} request - { key, prefix, type } + * @returns {object} { success, deleted_count, error } + */ + async forget(request) { + try { + this._stats.forgetCalls++; + let deletedCount = 0; + const dirtyTypes = new Set(); + + if (request.key) { + // 精确删除 + const type = request.type || 'entity_cache'; + const fullKey = this._fullKey(type, request.key); + const entry = this.index.delete(fullKey); + if (entry) { + deletedCount = 1; + dirtyTypes.add(entry.type); + } + } else if (request.prefix) { + // 前缀批量删除 + const type = request.type || ''; + const prefix = type + ? this._fullKey(type, request.prefix) + : `${this.tenant}:${this.userId}:${request.prefix}`; + + const matches = this.index.getByPrefix(prefix.endsWith(':') ? prefix : prefix + ':', { + type: request.type || null, + limit: 100000, + }); + + for (const match of matches) { + const fullKey = match._fullKey || match.key; + const entry = this.index.delete(fullKey); + if (entry) { + deletedCount++; + dirtyTypes.add(entry.type); + } + } + } + + // 持久化受影响的 type + for (const type of dirtyTypes) { + await this._persistType(type); + } + + this._log('forget', { key: request.key || request.prefix, type: request.type, deleted: deletedCount }); + return { success: true, deletedCount, error: '' }; + } catch (err) { + this._stats.forgetErrors++; + this._log('forget', { key: request.key || request.prefix, result: 'error', error: err.message }); + return { success: false, deletedCount: 0, error: err.message }; + } + } + + // ─── 内部方法 ─── + + /** + * 淘汰指定类型的条目 + * @param {string} type + * @param {string} policy - 'lru' | 'oldest' | 'compress' + * @param {number} count - 要淘汰的条目数 + * @returns {number} 实际淘汰的条目数 + */ + async _evict(type, policy, count) { + const candidates = this.index.getCandidatesForEviction(type, policy, count); + let evicted = 0; + + for (const key of candidates) { + const entry = this.index.delete(key); + if (entry) evicted++; + } + + if (evicted > 0) { + await this._persistType(type); + } + + return evicted; + } + + /** + * 持久化指定类型到文件（O(1) 通过 per-type 索引） + */ + async _persistType(type) { + const entries = this.index.getEntriesByType(type); + await this.storage.saveType(type, entries); + } +} + +export default MemoryEngine; diff --git a/services/state-manager/lib/memory-index.js b/services/state-manager/lib/memory-index.js new file mode 100644 index 00000000..96660ce1 --- /dev/null +++ b/services/state-manager/lib/memory-index.js @@ -0,0 +1,347 @@ +/** + * 记忆索引（MemoryIndex）— 内存前缀索引 + TTL 过期表 + 容量计数器 + * + * 前缀索引：按 key 的 `:` 分割段构建嵌套 Map，支持 O(k) 前缀查询 + * TTL 过期表：按过期时间排序的数组，Active Sweep 只扫描即将过期的条目 + * 容量计数器：per-type 计数，超限触发 LRU 淘汰 + */ + +export class MemoryIndex { + constructor() { + // 前缀索引：嵌套 Map + // "acme:user1:entity_cache:crm_user:self" → + // trie.acme.user1.entity_cache.crm_user.self = entry + this.trie = new Map(); + + // 全量 key → entry 的扁平 Map（用于精确查找） + this.entries = new Map(); + + // per-type entries Map（避免 _persistType 全量遍历） + // type → Map + this.entriesByType = new Map(); + + // TTL 过期表：{ expiresAt, key } 按时间排序 + this.ttlIndex = []; + + // per-type 容量计数 + this.typeCounts = new Map(); + + // 是否需要重排 TTL 过期表 + this._ttlSorted = true; + } + + /** + * 插入或更新一条记忆 + */ + set(fullKey, entry) { + const old = this.entries.get(fullKey); + + // 更新扁平 Map + this.entries.set(fullKey, entry); + + // 更新 per-type Map + const type = entry.type; + if (!this.entriesByType.has(type)) { + this.entriesByType.set(type, new Map()); + } + this.entriesByType.get(type).set(fullKey, entry); + + // 更新前缀索引 + this._insertTrie(fullKey, entry); + + // 更新 TTL 过期表 + if (old && old.expiresAt) { + // 移除旧的 TTL 记录 + this.ttlIndex = this.ttlIndex.filter(t => t.key !== fullKey); + } + if (entry.expiresAt) { + this.ttlIndex.push({ expiresAt: entry.expiresAt, key: fullKey }); + this._ttlSorted = false; + } + + // 更新容量计数 + if (!old) { + this.typeCounts.set(type, (this.typeCounts.get(type) || 0) + 1); + } + } + + /** + * 精确查找 + */ + get(fullKey) { + return this.entries.get(fullKey) || null; + } + + /** + * 前缀查找 + * @param {string} prefix - 如 "acme:user1:entity_cache:" + * @param {object} opts - { type, limit } + * @returns {object[]} 匹配的 entry 列表，按 createdAt 倒序 + */ + getByPrefix(prefix, opts = {}) { + const segments = prefix.split(':').filter(Boolean); + let node = this.trie; + + // 沿前缀段逐层深入 + for (const seg of segments) { + if (!node.has(seg)) return []; + node = node.get(seg); + } + + // 收集该节点下所有叶子 + const results = []; + this._collectLeaves(node, results); + + // 按 type 过滤 + let filtered = opts.type + ? results.filter(e => e.type === opts.type) + : results; + + // 按 confidence 过滤（< 0.3 的不返回） + filtered = filtered.filter(e => e.confidence >= 0.3); + + // TTL lazy check：过期的不返回 + const now = Date.now(); + filtered = filtered.filter(e => { + if (!e.expiresAt) return true; + return new Date(e.expiresAt).getTime() > now; + }); + + // 按 createdAt 倒序 + filtered.sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt)); + + // 限制条数 + if (opts.limit && opts.limit > 0) { + filtered = filtered.slice(0, opts.limit); + } + + return filtered; + } + + /** + * 删除一条记忆 + */ + delete(fullKey) { + const entry = this.entries.get(fullKey); + if (!entry) return null; + + // 移除扁平 Map + this.entries.delete(fullKey); + + // 移除 per-type Map + const typeMap = this.entriesByType.get(entry.type); + if (typeMap) typeMap.delete(fullKey); + + // 移除前缀索引 + this._deleteTrie(fullKey); + + // 移除 TTL 记录 + this.ttlIndex = this.ttlIndex.filter(t => t.key !== fullKey); + + // 更新容量计数 + this.typeCounts.set(entry.type, Math.max(0, (this.typeCounts.get(entry.type) || 1) - 1)); + + return entry; + } + + /** + * 批量删除（按前缀 + type） + * @returns {number} 删除条数 + */ + deleteByPrefix(prefix, type = null) { + const matches = this.getByPrefix(prefix, { type, limit: 100000 }); + let count = 0; + for (const entry of matches) { + this.delete(entry._fullKey || this._findFullKey(entry)); + count++; + } + return count; + } + + /** + * 获取指定类型的条目数 + */ + getTypeCount(type) { + return this.typeCounts.get(type) || 0; + } + + /** + * 获取已过期的条目 key 列表 + * @param {Date} now - 当前时间 + * @returns {string[]} 已过期但未被清理的 key + */ + getExpiredKeys(now = new Date()) { + this._ensureTtlSorted(); + + const nowMs = now.getTime(); + const expired = []; + + // TTL 过期表按时间排序，二分查找过期分界点 + for (const item of this.ttlIndex) { + if (new Date(item.expiresAt).getTime() <= nowMs) { + const entry = this.entries.get(item.key); + // 跳过 protected 条目 + if (entry && !entry.protected) { + expired.push(item.key); + } + } else { + break; // 排序了，后面的都不会过期 + } + } + + return expired; + } + + /** + * 获取指定类型中最旧/最久未访问的条目（LRU/oldest 淘汰用） + * @param {string} type + * @param {string} policy - 'lru' | 'oldest' | 'compress' + * @param {number} count - 要淘汰的条目数 + * @returns {string[]} 要淘汰的 key 列表 + */ + getCandidatesForEviction(type, policy, count) { + const entries = []; + for (const [key, entry] of this.entries) { + if (entry.type === type && !entry.protected) { + entries.push({ key, entry }); + } + } + + if (policy === 'oldest') { + entries.sort((a, b) => new Date(a.entry.createdAt) - new Date(b.entry.createdAt)); + } else if (policy === 'lru') { + entries.sort((a, b) => new Date(a.entry.lastAccessAt || a.entry.createdAt) - new Date(b.entry.lastAccessAt || b.entry.createdAt)); + } else if (policy === 'compress') { + // compress：同 key 前缀的只保留最新的 + // 返回非最新的同 key 条目 + const seen = new Map(); + const toEvict = []; + entries.sort((a, b) => new Date(b.entry.createdAt) - new Date(a.entry.createdAt)); + for (const e of entries) { + const shortKey = e.entry._originalKey || e.key; + if (seen.has(shortKey)) { + toEvict.push(e.key); + } else { + seen.set(shortKey, e.key); + } + } + return toEvict.slice(0, count); + } + + return entries.slice(0, count).map(e => e.key); + } + + /** + * 获取指定类型的所有条目 Map（O(1)，避免全量遍历） + * @param {string} type + * @returns {Map} + */ + getEntriesByType(type) { + return this.entriesByType.get(type) || new Map(); + } + + /** + * 获取引擎统计信息 + */ + getStats() { + const typeStats = {}; + for (const [type, count] of this.typeCounts) { + typeStats[type] = { + count, + protected: [...(this.entriesByType.get(type) || [])].filter(([, e]) => e.protected).length, + }; + } + return { + totalEntries: this.entries.size, + ttlIndexSize: this.ttlIndex.length, + types: typeStats, + }; + } + + /** + * 批量重建索引（从 StorageAdapter 加载后调用） + */ + rebuild(allEntries) { + this.trie.clear(); + this.entries.clear(); + this.entriesByType.clear(); + this.ttlIndex = []; + this.typeCounts.clear(); + this._ttlSorted = true; + + for (const [type, typeEntries] of allEntries) { + for (const [fullKey, entry] of typeEntries) { + this.set(fullKey, entry); + } + } + } + + // ─── 内部方法 ─── + + _insertTrie(fullKey, entry) { + const segments = fullKey.split(':'); + let node = this.trie; + for (const seg of segments) { + if (!node.has(seg)) { + node.set(seg, new Map()); + } + node = node.get(seg); + } + // 最深层的 Map 存 entry（用特殊 key） + node.set('__entry__', entry); + } + + _deleteTrie(fullKey) { + const segments = fullKey.split(':'); + let node = this.trie; + const path = [this.trie]; + + for (const seg of segments) { + if (!node.has(seg)) return; + node = node.get(seg); + path.push(node); + } + + // 删除叶子 entry + node.delete('__entry__'); + + // 回溯清理空分支（可选优化，MVP 不做） + } + + _collectLeaves(node, results) { + if (node.has('__entry__')) { + const entry = node.get('__entry__'); + results.push({ ...entry, _fullKey: this._reconstructKey(entry) }); + } + for (const [key, child] of node) { + if (key !== '__entry__' && child instanceof Map) { + this._collectLeaves(child, results); + } + } + } + + _reconstructKey(entry) { + // 从 entry 的 metadata 重建 fullKey + // 这是一个简化实现：遍历 entries Map 反查 + for (const [k, v] of this.entries) { + if (v === entry) return k; + } + return ''; + } + + _findFullKey(entry) { + for (const [k, v] of this.entries) { + if (v.createdAt === entry.createdAt && v.type === entry.type) return k; + } + return ''; + } + + _ensureTtlSorted() { + if (!this._ttlSorted) { + this.ttlIndex.sort((a, b) => new Date(a.expiresAt) - new Date(b.expiresAt)); + this._ttlSorted = true; + } + } +} + +export default MemoryIndex; diff --git a/services/state-manager/lib/storage-adapter.js b/services/state-manager/lib/storage-adapter.js new file mode 100644 index 00000000..fb456368 --- /dev/null +++ b/services/state-manager/lib/storage-adapter.js @@ -0,0 +1,111 @@ +/** + * 存储适配器（StorageAdapter）— JSON file per type 持久化存储 + * + * 每种记忆类型一个 JSON 文件，原子写入（write tmp → rename）。 + * 文件格式版本化，便于未来迁移。 + */ + +import { readFile, writeFile, mkdir, rename, unlink } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { dirname, join } from 'node:path'; + +const FILE_VERSION = 1; +const MEMORY_TYPES = ['entity_cache', 'action_log', 'user_correction', 'commitment', 'session_summary']; + +export class StorageAdapter { + constructor(dataDir) { + this.dataDir = dataDir; + // Promise 队列锁：每个 type 一条串行链，避免并发写同一文件 + this._writeChains = new Map(); + } + + /** + * 获取指定类型的文件路径 + */ + _filePath(type) { + return join(this.dataDir, `memory_${type}.json`); + } + + /** + * 读取指定类型的所有条目 + * @returns {Map} key → entry + */ + async loadType(type) { + const filePath = this._filePath(type); + try { + const raw = await readFile(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (parsed.version !== FILE_VERSION) { + // 未来版本迁移点 + } + return new Map(Object.entries(parsed.entries || {})); + } catch { + return new Map(); + } + } + + /** + * 写入指定类型的所有条目（原子写） + * 使用 Promise 链保证同一 type 的写操作串行执行 + * @param {string} type + * @param {Map} entries + */ + async saveType(type, entries) { + // Promise 链：新写入排在上一次写入之后，天然串行 + const prev = this._writeChains.get(type) || Promise.resolve(); + const next = prev.then(async () => { + const filePath = this._filePath(type); + await mkdir(dirname(filePath), { recursive: true }); + + const data = { + version: FILE_VERSION, + updatedAt: new Date().toISOString(), + entryCount: entries.size, + entries: Object.fromEntries(entries), + }; + + // 原子写：先写临时文件，再 rename + const tmpPath = filePath + '.tmp'; + await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); + await rename(tmpPath, filePath); + }).catch(err => { + console.error(`[StorageAdapter] saveType(${type}) error:`, err.message); + }); + this._writeChains.set(type, next); + return next; + } + + /** + * 加载所有记忆类型的条目 + * @returns {Map>} type → (key → entry) + */ + async loadAll() { + const result = new Map(); + for (const type of MEMORY_TYPES) { + result.set(type, await this.loadType(type)); + } + return result; + } + + /** + * 等待所有进行中的写操作完成（优雅停机用） + */ + async waitForPendingWrites() { + const pending = [...this._writeChains.values()]; + await Promise.allSettled(pending); + } + + /** + * 删除指定类型的存储文件（用于测试或重置） + */ + async deleteType(type) { + const filePath = this._filePath(type); + try { + await unlink(filePath); + } catch { + // 文件不存在，忽略 + } + } +} + +export default StorageAdapter; diff --git a/services/state-manager/lib/task-machine.js b/services/state-manager/lib/task-machine.js new file mode 100644 index 00000000..29b88552 --- /dev/null +++ b/services/state-manager/lib/task-machine.js @@ -0,0 +1,416 @@ +/** + * TaskMachine — Task State Machine + * + * Manages the lifecycle of multi-step tasks: + * pending → running → done / failed / cancelled / timed_out + * + * Core value: lets agents track "which step a long task has reached", + * instead of relying on LLM memory to chain multi-step flows. + * + * Persistence: task_state.json (atomic write) + */ + +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { dirname } from 'node:path'; + +// Valid state transitions +const VALID_TRANSITIONS = { + pending: ['running', 'cancelled'], + running: ['done', 'failed', 'cancelled', 'timed_out', 'pending'], + done: [], // terminal state + failed: ['pending'], // can retry + cancelled: ['pending'], // can restore + timed_out: ['pending', 'running'], // can retry or continue +}; + +const VALID_STATES = Object.keys(VALID_TRANSITIONS); + +export class TaskMachine { + /** + * @param {string} dataDir - Data directory + * @param {object} opts + * @param {number} opts.checkIntervalMs - Timeout check interval (default 60000ms) + */ + constructor(dataDir, opts = {}) { + this.filePath = `${dataDir}/task_state.json`; + this.tasks = new Map(); // taskId → task + this.checkIntervalMs = opts.checkIntervalMs || 60000; + this._timer = null; + this._dirty = false; + + // Statistics + this._stats = { + totalCreated: 0, + totalCompleted: 0, + totalFailed: 0, + totalTimedOut: 0, + }; + } + + // ─── Lifecycle ─── + + async initialize() { + await this._load(); + this.startTimeoutChecker(); + console.log(`[TaskMachine] initialized: ${this.tasks.size} tasks loaded`); + } + + startTimeoutChecker() { + if (this._timer) return; + this._timer = setInterval(() => this._checkTimeouts(), this.checkIntervalMs); + if (this._timer.unref) this._timer.unref(); + } + + stop() { + if (this._timer) { + clearInterval(this._timer); + this._timer = null; + } + } + + async shutdown() { + this.stop(); + if (this._dirty) await this._save(); + } + + // ─── Create Task ─── + + /** + * @param {object} request + * @param {string} request.task_id - Task ID (optional, auto-generated) + * @param {string} request.name - Task name + * @param {string} request.description - Task description + * @param {string} request.assignee - Assignee + * @param {string} request.due_date - Due date ISO 8601 + * @param {string[]} request.steps - Initial step list + * @param {string} request.priority - high / medium / low + * @param {object} request.metadata - Additional data + * @returns {object} { success, task_id, error } + */ + async createTask(request) { + try { + const taskId = request.task_id || `task_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + + if (this.tasks.has(taskId)) { + return { success: false, task_id: taskId, error: `Task "${taskId}" already exists` }; + } + + const now = new Date().toISOString(); + const task = { + task_id: taskId, + name: request.name || '', + description: request.description || '', + assignee: request.assignee || '', + state: 'pending', + priority: request.priority || 'medium', + due_date: request.due_date || null, + steps: (request.steps || []).map((s, i) => ({ + step_id: `step_${i}`, + name: typeof s === 'string' ? s : s.name, + state: 'pending', + completed_at: null, + notes: typeof s === 'string' ? '' : (s.notes || ''), + })), + current_step: null, + metadata: request.metadata || {}, + created_at: now, + updated_at: now, + state_history: [{ state: 'pending', at: now, reason: 'created' }], + }; + + this.tasks.set(taskId, task); + this._stats.totalCreated++; + this._dirty = true; + await this._save(); + + return { success: true, task_id: taskId, error: '' }; + } catch (err) { + return { success: false, task_id: '', error: err.message }; + } + } + + // ─── Update Task State ─── + + /** + * @param {object} request + * @param {string} request.task_id + * @param {string} request.new_state + * @param {string} request.reason - Reason for state change + * @returns {object} { success, old_state, new_state, error } + */ + async updateTask(request) { + try { + const task = this.tasks.get(request.task_id); + if (!task) { + return { success: false, old_state: '', new_state: '', error: `Task "${request.task_id}" not found` }; + } + + const oldState = task.state; + const newState = request.new_state; + + // Validate state transition + if (!VALID_STATES.includes(newState)) { + return { success: false, old_state: oldState, new_state: '', error: `Invalid state "${newState}"` }; + } + + const allowed = VALID_TRANSITIONS[oldState] || []; + if (!allowed.includes(newState)) { + return { + success: false, + old_state: oldState, + new_state: '', + error: `Transition from "${oldState}" to "${newState}" not allowed, allowed: ${allowed.join(', ')}`, + }; + } + + const now = new Date().toISOString(); + task.state = newState; + task.updated_at = now; + task.state_history.push({ state: newState, at: now, reason: request.reason || '' }); + + // Terminal state statistics + if (newState === 'done') this._stats.totalCompleted++; + if (newState === 'failed') this._stats.totalFailed++; + if (newState === 'timed_out') this._stats.totalTimedOut++; + + this._dirty = true; + await this._save(); + + return { success: true, old_state: oldState, new_state: newState, error: '' }; + } catch (err) { + return { success: false, old_state: '', new_state: '', error: err.message }; + } + } + + // ─── Add/Update Step ─── + + /** + * @param {object} request + * @param {string} request.task_id + * @param {string} request.step_id - Step ID (optional when adding new) + * @param {string} request.name - Step name + * @param {string} request.state - pending / running / done / skipped + * @param {string} request.notes - Notes + * @returns {object} { success, step_id, error } + */ + async updateStep(request) { + try { + const task = this.tasks.get(request.task_id); + if (!task) { + return { success: false, step_id: '', error: `Task "${request.task_id}" not found` }; + } + + let step = null; + if (request.step_id) { + step = task.steps.find(s => s.step_id === request.step_id); + } + + const now = new Date().toISOString(); + + if (step) { + // Update existing step + if (request.state) step.state = request.state; + if (request.notes) step.notes = request.notes; + if (request.name) step.name = request.name; + if (request.state === 'done') step.completed_at = now; + + // Auto-advance current_step + if (request.state === 'done') { + const nextPending = task.steps.find(s => s.state === 'pending'); + task.current_step = nextPending ? nextPending.step_id : null; + } + } else { + // Add new step + const stepId = request.step_id || `step_${task.steps.length}`; + step = { + step_id: stepId, + name: request.name || '', + state: request.state || 'pending', + completed_at: request.state === 'done' ? now : null, + notes: request.notes || '', + }; + task.steps.push(step); + } + + task.updated_at = now; + this._dirty = true; + await this._save(); + + return { success: true, step_id: step.step_id, error: '' }; + } catch (err) { + return { success: false, step_id: '', error: err.message }; + } + } + + // ─── Query Tasks ─── + + /** + * @param {string} taskId + * @returns {object} { success, task, error } + */ + getTask(taskId) { + const task = this.tasks.get(taskId); + if (!task) { + return { success: false, task: null, error: `Task "${taskId}" not found` }; + } + return { success: true, task: this._formatTask(task), error: '' }; + } + + /** + * @param {object} filter - { state, assignee, priority, due_before, due_after, limit } + * @returns {object} { success, tasks, total, error } + */ + listTasks(filter = {}) { + let results = [...this.tasks.values()]; + + if (filter.state) { + const states = filter.state.split(','); + results = results.filter(t => states.includes(t.state)); + } + if (filter.assignee) { + results = results.filter(t => t.assignee === filter.assignee); + } + if (filter.priority) { + results = results.filter(t => t.priority === filter.priority); + } + if (filter.due_before) { + const before = new Date(filter.due_before); + results = results.filter(t => t.due_date && new Date(t.due_date) <= before); + } + if (filter.due_after) { + const after = new Date(filter.due_after); + results = results.filter(t => t.due_date && new Date(t.due_date) >= after); + } + + // Sort by priority + creation time + const priorityOrder = { high: 0, medium: 1, low: 2 }; + results.sort((a, b) => { + const pa = priorityOrder[a.priority] ?? 1; + const pb = priorityOrder[b.priority] ?? 1; + if (pa !== pb) return pa - pb; + return new Date(b.created_at) - new Date(a.created_at); + }); + + const total = results.length; + const limit = filter.limit || 50; + results = results.slice(0, limit); + + return { + success: true, + tasks: results.map(t => this._formatTask(t)), + total, + error: '', + }; + } + + /** + * Get task state machine statistics + */ + getStats() { + const stateCounts = {}; + for (const state of VALID_STATES) { + stateCounts[state] = 0; + } + for (const task of this.tasks.values()) { + stateCounts[task.state] = (stateCounts[task.state] || 0) + 1; + } + + return { + totalTasks: this.tasks.size, + stateCounts, + ...this._stats, + }; + } + + // ─── Timeout Detection ─── + + async _checkTimeouts() { + const now = new Date(); + let changed = false; + + for (const [taskId, task] of this.tasks) { + // Only check pending and running tasks + if (task.state !== 'pending' && task.state !== 'running') continue; + if (!task.due_date) continue; + + const due = new Date(task.due_date); + if (now > due) { + // Timed out + task.state = 'timed_out'; + task.updated_at = now.toISOString(); + task.state_history.push({ + state: 'timed_out', + at: now.toISOString(), + reason: `Past due date ${task.due_date}`, + }); + this._stats.totalTimedOut++; + changed = true; + + console.log(`[TaskMachine] task "${taskId}" timed out (due: ${task.due_date})`); + } + } + + if (changed) { + this._dirty = true; + await this._save(); + } + } + + // ─── Internal Methods ─── + + _formatTask(task) { + const totalSteps = task.steps.length; + const doneSteps = task.steps.filter(s => s.state === 'done').length; + return { + task_id: task.task_id, + name: task.name, + description: task.description, + assignee: task.assignee, + state: task.state, + priority: task.priority, + due_date: task.due_date || '', + current_step: task.current_step || '', + steps: task.steps, + progress: totalSteps > 0 ? Math.round((doneSteps / totalSteps) * 100) : 0, + metadata: task.metadata, + created_at: task.created_at, + updated_at: task.updated_at, + state_history: task.state_history, + }; + } + + async _load() { + try { + const raw = await readFile(this.filePath, 'utf-8'); + const data = JSON.parse(raw); + if (data.tasks && typeof data.tasks === 'object') { + for (const [id, task] of Object.entries(data.tasks)) { + this.tasks.set(id, task); + } + } + } catch { + // File does not exist, start from empty + } + } + + async _save() { + try { + await mkdir(dirname(this.filePath), { recursive: true }); + const data = { + version: 1, + updatedAt: new Date().toISOString(), + taskCount: this.tasks.size, + tasks: Object.fromEntries(this.tasks), + }; + const tmpPath = this.filePath + '.tmp'; + await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); + const { rename } = await import('node:fs/promises'); + await rename(tmpPath, this.filePath); + this._dirty = false; + } catch (err) { + console.error('[TaskMachine] save error:', err.message); + } + } +} + +export default TaskMachine; diff --git a/services/state-manager/lib/ttl-manager.js b/services/state-manager/lib/ttl-manager.js new file mode 100644 index 00000000..8e540e88 --- /dev/null +++ b/services/state-manager/lib/ttl-manager.js @@ -0,0 +1,130 @@ +/** + * 过期管理器（TTLManager）— 记忆过期管理 + * + * 双策略（Redis 同款）： + * 1. Lazy check：Recall 时检查过期，过期则删除 + * 2. Active sweep：每 60 秒扫描 TTL 过期表，批量清理 + * + * 参考：https://stackharbor.com/en/knowledge-base/redis-key-expiration-patterns/ + */ + +export class TTLManager { + /** + * @param {import('./memory-index.js').MemoryIndex} index + * @param {import('./storage-adapter.js').StorageAdapter} storage + * @param {object} opts + * @param {number} opts.sweepIntervalMs - 扫描间隔（默认 60000ms） + */ + constructor(index, storage, opts = {}) { + this.index = index; + this.storage = storage; + this.sweepIntervalMs = opts.sweepIntervalMs || 60000; + this._timer = null; + this._sweeping = false; + } + + /** + * 启动定时扫描 + */ + start() { + if (this._timer) return; + this._timer = setInterval(() => this._sweep(), this.sweepIntervalMs); + // 不阻止进程退出 + if (this._timer.unref) this._timer.unref(); + } + + /** + * 停止定时扫描 + */ + stop() { + if (this._timer) { + clearInterval(this._timer); + this._timer = null; + } + } + + /** + * 检查单条记忆是否过期（Lazy check，Recall 时调用） + * @returns {boolean} true=已过期并已删除，false=未过期 + */ + checkAndExpire(fullKey) { + const entry = this.index.get(fullKey); + if (!entry) return true; // 不存在视为"过期" + + if (!entry.expiresAt) return false; // 永不过期 + + const now = Date.now(); + const expiresMs = new Date(entry.expiresAt).getTime(); + + if (expiresMs <= now) { + // 过期了，删除（protected 不删） + if (!entry.protected) { + this.index.delete(fullKey); + // 标记需要持久化 + this._markDirty(entry.type); + } + return true; + } + + return false; + } + + /** + * 主动扫描过期条目（Active sweep） + */ + async _sweep() { + if (this._sweeping) return; + this._sweeping = true; + + try { + const now = new Date(); + const expiredKeys = this.index.getExpiredKeys(now); + + if (expiredKeys.length === 0) return; + + // 按 type 分组，批量删除后批量持久化 + const dirtyTypes = new Set(); + + for (const key of expiredKeys) { + const entry = this.index.get(key); + if (entry) { + dirtyTypes.add(entry.type); + } + this.index.delete(key); + } + + // 持久化受影响的 type 文件 + for (const type of dirtyTypes) { + await this._persistType(type); + } + + // 自适应：如果过期条目占比 > 25%，立即再扫一轮（Redis 同款逻辑） + const totalWithTTL = this.index.ttlIndex.length; + if (totalWithTTL > 0 && expiredKeys.length / totalWithTTL > 0.25) { + setImmediate(() => this._sweep()); + } + } catch (err) { + // 扫描失败不影响服务 + console.error('[TTLManager] sweep error:', err.message); + } finally { + this._sweeping = false; + } + } + + /** + * 持久化指定类型的所有条目到文件（O(1) 通过 per-type 索引） + */ + async _persistType(type) { + const entries = this.index.getEntriesByType(type); + await this.storage.saveType(type, entries); + } + + _markDirty(type) { + // 简易脏标记：下次 sweep 时持久化 + // 对于 MVP，Recall 触发的删除会在下次 sweep 时持久化 + // 这意味着最多 60 秒的窗口内，进程崩溃可能导致已删除条目复活 + // 可接受：下次 Recall 仍会 lazy check + } +} + +export default TTLManager; diff --git a/services/state-manager/lib/type-registry.js b/services/state-manager/lib/type-registry.js new file mode 100644 index 00000000..595b9b5b --- /dev/null +++ b/services/state-manager/lib/type-registry.js @@ -0,0 +1,113 @@ +/** + * TypeRegistry — Default parameter configuration for five memory types + * + * Each memory type has different TTL, dedup window, confidence, capacity limit, + * and eviction policy. Remember calls use defaults from this registry when + * parameters are unspecified. To add a new memory type, simply add an entry + * here — no code changes needed. + */ + +const SECONDS = { + MINUTE: 60, + HOUR: 3600, + DAY: 86400, + WEEK: 7 * 86400, + MONTH: 30 * 86400, +}; + +const TYPE_REGISTRY = { + entity_cache: { + defaultTTL: SECONDS.WEEK, + dedupWindow: 0, + defaultConfidence: 1.0, + maxEntries: 200, + evictPolicy: 'lru', + description: 'Query result cache (CRM IDs, project details, etc. — expensive query results)', + label: 'Entity Cache', + }, + action_log: { + defaultTTL: SECONDS.WEEK, + dedupWindow: SECONDS.DAY, + defaultConfidence: 1.0, + maxEntries: 1000, + evictPolicy: 'oldest', + description: 'Action log (todo creation, kanban writes, etc. — write operation records for idempotent dedup)', + label: 'Action Log', + }, + user_correction: { + defaultTTL: 0, + dedupWindow: 0, + defaultConfidence: 0.9, + maxEntries: 100, + evictPolicy: 'compress', + description: 'User correction (user-initiated fixes to agent behavior — highest-value memory)', + label: 'User Correction', + }, + commitment: { + defaultTTL: SECONDS.MONTH, + dedupWindow: 0, + defaultConfidence: 1.0, + maxEntries: 100, + evictPolicy: 'lru', + description: 'Commitment (future tasks the user mentioned they will do)', + label: 'Commitment', + }, + session_summary: { + defaultTTL: SECONDS.MONTH, + dedupWindow: 0, + defaultConfidence: 1.0, + maxEntries: 50, + evictPolicy: 'oldest', + description: 'Session summary (key decisions and pending items at the end of each session)', + label: 'Session Summary', + }, +}; + +/** + * Resolve Remember request parameters, filling in defaults + * @param {object} request - { key, value, type, ttl_seconds, dedup_window_sec, confidence } + * @returns {object} resolved - complete parameters with defaults filled in + */ +export function resolve(request) { + const type = request.type || 'entity_cache'; + const registry = TYPE_REGISTRY[type]; + + if (!registry) { + throw new Error(`Unknown memory type "${type}", available types: ${Object.keys(TYPE_REGISTRY).join(', ')}`); + } + + const ttlSeconds = request.ttl_seconds === -1 + ? 0 // -1 means never expire + : (request.ttl_seconds || registry.defaultTTL); + + return { + key: request.key, + value: request.value, + type, + ttlSeconds, + dedupWindowSec: request.dedup_window_sec || registry.dedupWindow, + confidence: request.confidence || registry.defaultConfidence, + maxEntries: registry.maxEntries, + evictPolicy: registry.evictPolicy, + protected: (request.confidence || registry.defaultConfidence) >= 0.9 && ttlSeconds === 0, + }; +} + +/** + * Get the registry config for a specific type + */ +export function getTypeConfig(type) { + return TYPE_REGISTRY[type] || null; +} + +/** + * List all available types + */ +export function listTypes() { + return Object.entries(TYPE_REGISTRY).map(([type, config]) => ({ + type, + ...config, + })); +} + +export default TYPE_REGISTRY; diff --git a/services/state-manager/package.json b/services/state-manager/package.json new file mode 100644 index 00000000..28c15d52 --- /dev/null +++ b/services/state-manager/package.json @@ -0,0 +1,15 @@ +{ + "name": "octobus-state-manager", + "displayName": "State Manager", + "version": "0.2.0", + "description": "Agent State Manager — Memory Engine + Distillation + Task State Machine + Event Log", + "private": true, + "type": "module", + "bin": { + "state-manager": "bin/state-service.js" + }, + "dependencies": { + "@chaitin-ai/octobus-sdk": "^0.5.0", + "js-yaml": "^4.1.0" + } +} diff --git a/services/state-manager/proto/state.proto b/services/state-manager/proto/state.proto new file mode 100644 index 00000000..d52dd46b --- /dev/null +++ b/services/state-manager/proto/state.proto @@ -0,0 +1,582 @@ +syntax = "proto3"; +package state.manager.v1; + +// Agent State Manager +// OctoBus service package: state-manager -> Agent State Manager +// Provides: project config, weekly buffer, meeting tracking, user config, memory engine, distillation engine, task state machine, event log, general storage +service StateManagerService { + // ====== Project Config ====== + // Get project config by key + rpc GetProject(GetProjectRequest) returns (GetProjectResponse); + // List all projects + rpc ListProjects(ListProjectsRequest) returns (ListProjectsResponse); + // Match project by keywords + rpc MatchProject(MatchProjectRequest) returns (MatchProjectResponse); + + // ====== Weekly Pending Buffer (migrated to memory engine, internally delegates to Remember/Recall/Forget) ====== + // Add a pending weekly summary item -> Remember(type=commitment, key=weekly_item:...) + rpc AddPendingItem(AddPendingItemRequest) returns (AddPendingItemResponse); + // Get all pending sync items -> Recall(prefix=weekly_item:, type=commitment) + rpc GetPendingItems(GetPendingItemsRequest) returns (GetPendingItemsResponse); + // Clear pending buffer -> Forget(prefix=weekly_item:, type=commitment) + rpc ClearPending(ClearPendingRequest) returns (ClearPendingResponse); + + // ====== Meeting Tracking (migrated to memory engine, internally delegates to Remember/Recall/Forget) ====== + // Register a meeting -> Remember(type=commitment, key=meeting:{eventId}) + rpc RegisterMeeting(RegisterMeetingRequest) returns (RegisterMeetingResponse); + // Get pending meetings -> Recall(prefix=meeting:, type=commitment) + filtering + rpc GetPendingMeetings(GetPendingMeetingsRequest) returns (GetPendingMeetingsResponse); + // Update meeting state -> Recall + Remember update + rpc UpdateMeetingState(UpdateMeetingStateRequest) returns (UpdateMeetingStateResponse); + + // ====== User Config ====== + // Read user personal config (identity/kanban/CRM/defaults) + rpc GetUserConfig(GetUserConfigRequest) returns (GetUserConfigResponse); + + // ====== Agent Memory ====== + // Write a memory entry + rpc Remember(RememberRequest) returns (RememberResponse); + // Read memory (exact match or prefix match) + rpc Recall(RecallRequest) returns (RecallResponse); + // Delete memory + rpc Forget(ForgetRequest) returns (ForgetResponse); + // Get memory engine runtime statistics + rpc GetMemoryStats(GetMemoryStatsRequest) returns (GetMemoryStatsResponse); + + // ====== Distillation Engine ====== + // Manually trigger a full distillation pass + rpc Distill(DistillRequest) returns (DistillResponse); + // Get distillation statistics + rpc GetDistillStats(GetDistillStatsRequest) returns (GetDistillStatsResponse); + + // ====== Task State Machine ====== + // Create a task + rpc CreateTask(CreateTaskRequest) returns (CreateTaskResponse); + // Update task state + rpc UpdateTask(UpdateTaskRequest) returns (UpdateTaskResponse); + // Add/update a step + rpc UpdateStep(UpdateStepRequest) returns (UpdateStepResponse); + // Get a single task + rpc GetTask(GetTaskRequest) returns (GetTaskResponse); + // List tasks (with filtering) + rpc ListTasks(ListTasksRequest) returns (ListTasksResponse); + // Get task statistics + rpc GetTaskStats(GetTaskStatsRequest) returns (GetTaskStatsResponse); + + // ====== Event Log ====== + // Log an event + rpc LogEvent(LogEventRequest) returns (LogEventResponse); + // Query events + rpc QueryEvents(QueryEventsRequest) returns (QueryEventsResponse); + // Get event log statistics + rpc GetEventStats(GetEventStatsRequest) returns (GetEventStatsResponse); + + // ====== General KV ====== + rpc GetState(GetStateRequest) returns (GetStateResponse); + rpc SetState(SetStateRequest) returns (SetStateResponse); +} + +// ====== Project Config ====== + +message GetProjectRequest { + string key = 1; // Project key (e.g. yidong_ai_plus) +} + +message ProjectConfig { + string key = 1; + string name = 2; + string node_id = 3; + string customer = 4; + repeated string keywords = 5; + string detail_folder = 6; + string background = 7; // Customer background (pinned info) +} + +message GetProjectResponse { + bool success = 1; + ProjectConfig project = 2; + string error = 3; +} + +message ListProjectsRequest {} + +message ListProjectsResponse { + bool success = 1; + repeated ProjectConfig projects = 2; + string error = 3; +} + +message MatchProjectRequest { + string text = 1; // Text to match (e.g. calendar title or note content) +} + +message MatchProjectResponse { + bool success = 1; + ProjectConfig project = 2; // Matched project (may be empty) + float confidence = 3; // Match confidence 0-1 + string error = 4; +} + +// ====== Weekly Pending Buffer ====== + +message AddPendingItemRequest { + string summary = 1; // One-line summary (max 30 chars) + string category = 2; // Category: Communication / Documentation / Bidding / POC & Others + string date = 3; // Date YYYY-MM-DD +} + +message AddPendingItemResponse { + bool success = 1; + int32 pending_count = 2; // Current pending total + string error = 3; +} + +message GetPendingItemsRequest {} + +message PendingItem { + string summary = 1; + string category = 2; + string date = 3; +} + +message GetPendingItemsResponse { + bool success = 1; + repeated PendingItem items = 2; + string error = 3; +} + +message ClearPendingRequest {} + +message ClearPendingResponse { + bool success = 1; + int32 cleared_count = 2; // Number of entries cleared + string error = 3; +} + +// ====== Meeting Tracking ====== + +message RegisterMeetingRequest { + string event_id = 1; // Calendar event ID + string summary = 2; // Meeting title + string start_time = 3; // ISO time + string end_time = 4; // ISO time + repeated string attendees = 5; +} + +message RegisterMeetingResponse { + bool success = 1; + bool is_new = 2; // Whether newly registered (false = already exists) + string error = 3; +} + +message GetPendingMeetingsRequest { + int32 grace_hours = 1; // Hours past end time to be considered pending (default 24) +} + +message MeetingInfo { + string event_id = 1; + string summary = 2; + string start_time = 3; + string end_time = 4; + repeated string attendees = 5; + string state = 6; // registered / notes_submitted / doc_created / reminded +} + +message GetPendingMeetingsResponse { + bool success = 1; + repeated MeetingInfo meetings = 2; + string error = 3; +} + +message UpdateMeetingStateRequest { + string event_id = 1; + string new_state = 2; // notes_submitted / doc_created / reminded +} + +message UpdateMeetingStateResponse { + bool success = 1; + string error = 2; +} + +// ====== User Config ====== + +message GetUserConfigRequest {} + +message KanbanFieldConfig { + string start_date = 1; // field ID for Start Date + string end_date = 2; // field ID for Due Date + string task_type = 3; // field ID for Task Type + string task_desc = 4; // field ID for Task Description + string sales = 5; // field ID for Sales + string owner = 6; // field ID for Owner + string notes = 7; // field ID for Description + string customer_l1 = 8; // field ID for Customer Tier + string customer_l2 = 9; // field ID for Sub-department + string status = 10; // field ID for Status + string presale = 11; // field ID for Pre-sales + string crm_link = 12; // field ID for CRM Project Link +} + +message KanbanOptionConfig { + string id = 1; + string name = 2; +} + +message KanbanTaskTypeConfig { + repeated KanbanOptionConfig communication = 1; // Communication + repeated KanbanOptionConfig documentation = 2; // Documentation + repeated KanbanOptionConfig bidding = 3; // Bidding + repeated KanbanOptionConfig poc = 4; // POC & Others +} + +message KanbanStatusConfig { + repeated KanbanOptionConfig completed = 1; // Done + repeated KanbanOptionConfig in_progress = 2; // In Progress +} + +message KanbanCustomerConfig { + repeated KanbanOptionConfig customers = 1; +} + +message KanbanConfig { + string base_id = 1; + string table_id = 2; + string instance = 3; // OctoBus instance name (e.g. "aitable-instance") + KanbanFieldConfig fields = 4; + KanbanTaskTypeConfig task_types = 5; + KanbanStatusConfig statuses = 6; + KanbanCustomerConfig customers = 7; +} + +message IdentityConfig { + string user_id = 1; // Login name (e.g. "user01") + string display_name = 2; // Display name (e.g. "Alice") + string role = 3; // Role (e.g. "Pre-sales") + string crm_user_id = 4; // CRM system user ID (look up via ListUsers on first use) + string default_sales_user_id = 5;// Default sales userId (e.g. "sales01") + string default_sales_name = 6; // Default sales name (e.g. "Bob") + string team = 7; // Team name (e.g. "Enterprise Pre-sales") +} + +message OctoBusInstanceConfig { + string aitable = 1; // e.g. "aitable-instance" + string doc = 2; // e.g. "doc-instance" + string calendar = 3; // e.g. "calendar-instance" + string todo = 4; // e.g. "todo-instance" + string message = 5; // e.g. "message-instance" + string crm = 6; // e.g. "crm-instance" + string state = 7; // e.g. "state-instance" +} + +message UserConfig { + IdentityConfig identity = 1; + KanbanConfig kanban = 2; + OctoBusInstanceConfig instances = 3; + string knowledge_file = 4; // Product knowledge file path (e.g. "/opt/knowledge/company-products.md") + string company_name = 5; // Company name (e.g. "Acme Corp") + string company_english = 6; // Company English name (e.g. "Acme") +} + +message GetUserConfigResponse { + bool success = 1; + UserConfig config = 2; + string error = 3; +} + +// ====== Agent Memory ====== + +message RememberRequest { + string key = 1; // Memory key (e.g. "crm_user:self") + string value = 2; // JSON string + string type = 3; // entity_cache | action_log | user_correction | commitment | session_summary + int32 ttl_seconds = 4; // 0=use type default, -1=never expire + int32 dedup_window_sec = 5; // 0=use type default + double confidence = 6; // 0=use type default +} + +message RememberResponse { + bool success = 1; + bool unchanged = 2; // Dedup hit, not actually written + int32 evicted_count = 3; // Entries evicted due to capacity + string error = 4; +} + +message RecallRequest { + string key = 1; // Exact match (mutually exclusive with prefix) + string prefix = 2; // Prefix match + string type = 3; // Filter by type + int32 limit = 4; // Max entries to return, default 10 +} + +message MemoryEntry { + string key = 1; + string value = 2; // JSON string + string type = 3; + double confidence = 4; + string created_at = 5; // ISO 8601 + string expires_at = 6; // ISO 8601, empty=never expire +} + +message RecallResponse { + bool success = 1; + repeated MemoryEntry entries = 2; + string error = 3; +} + +message ForgetRequest { + string key = 1; // Exact delete (mutually exclusive with prefix) + string prefix = 2; // Batch prefix delete + string type = 3; // Filter by type +} + +message ForgetResponse { + bool success = 1; + int32 deleted_count = 2; + string error = 3; +} + +// ====== General KV ====== + +message GetStateRequest { + string key = 1; +} + +message GetStateResponse { + bool success = 1; + string value = 2; // JSON string + string error = 3; +} + +message SetStateRequest { + string key = 1; + string value = 2; // JSON string +} + +message SetStateResponse { + bool success = 1; + string error = 2; +} + +// ====== Memory Engine Statistics ====== + +message GetMemoryStatsRequest {} + +message TypeStats { + int32 count = 1; + int32 protected_count = 2; +} + +message GetMemoryStatsResponse { + bool success = 1; + int32 total_entries = 2; + map type_stats = 3; + int64 remember_calls = 4; + int64 recall_calls = 5; + int64 forget_calls = 6; + int64 total_evictions = 7; + int64 total_dedups = 8; + string tenant = 9; + string user_id = 10; + string error = 11; +} + +// ====== Distillation Engine ====== + +message DistillRequest {} + +message DistillSummary { + int32 compressed = 1; + int32 created = 2; + int32 deleted = 3; + string reason = 4; +} + +message DistillResponse { + bool success = 1; + DistillSummary action_log = 2; + DistillSummary session_summary = 3; + int32 corrections_upgraded = 4; + int32 corrections_demoted = 5; + string error = 6; +} + +message GetDistillStatsRequest {} + +message GetDistillStatsResponse { + bool success = 1; + int32 total_runs = 2; + string last_run_at = 3; + int64 action_log_compressed = 4; + int64 session_summary_compressed = 5; + int64 corrections_upgraded = 6; + string error = 7; +} + +// ====== Task State Machine ====== + +message TaskStep { + string step_id = 1; + string name = 2; + string state = 3; // pending / running / done / skipped + string completed_at = 4; + string notes = 5; +} + +message StateHistoryEntry { + string state = 1; + string at = 2; + string reason = 3; +} + +message TaskInfo { + string task_id = 1; + string name = 2; + string description = 3; + string assignee = 4; + string state = 5; + string priority = 6; + string due_date = 7; + string current_step = 8; + repeated TaskStep steps = 9; + int32 progress = 10; // 0-100 completion percentage + string created_at = 11; + string updated_at = 12; + repeated StateHistoryEntry state_history = 13; +} + +message CreateTaskRequest { + string task_id = 1; // Optional, auto-generated + string name = 2; + string description = 3; + string assignee = 4; + string due_date = 5; // ISO 8601 + repeated string steps = 6; + string priority = 7; // high / medium / low +} + +message CreateTaskResponse { + bool success = 1; + string task_id = 2; + string error = 3; +} + +message UpdateTaskRequest { + string task_id = 1; + string new_state = 2; // pending / running / done / failed / cancelled / timed_out + string reason = 3; +} + +message UpdateTaskResponse { + bool success = 1; + string old_state = 2; + string new_state = 3; + string error = 4; +} + +message UpdateStepRequest { + string task_id = 1; + string step_id = 2; // Optional, adds new step if omitted + string name = 3; + string state = 4; // pending / running / done / skipped + string notes = 5; +} + +message UpdateStepResponse { + bool success = 1; + string step_id = 2; + string error = 3; +} + +message GetTaskRequest { + string task_id = 1; +} + +message GetTaskResponse { + bool success = 1; + TaskInfo task = 2; + string error = 3; +} + +message ListTasksRequest { + string state = 1; // Comma-separated, e.g. "pending,running" + string assignee = 2; + string priority = 3; + string due_before = 4; // ISO 8601 + string due_after = 5; + int32 limit = 6; +} + +message ListTasksResponse { + bool success = 1; + repeated TaskInfo tasks = 2; + int32 total = 3; + string error = 4; +} + +message GetTaskStatsRequest {} + +message GetTaskStatsResponse { + bool success = 1; + int32 total_tasks = 2; + map state_counts = 3; + int64 total_created = 4; + int64 total_completed = 5; + int64 total_failed = 6; + int64 total_timed_out = 7; + string error = 8; +} + +// ====== Event Log ====== + +message LogEventRequest { + string event_type = 1; // e.g. agent.remember, loader.triggered + string actor = 2; // Triggered by + string description = 3; + string data = 4; // JSON string + string level = 5; // info / warning / error +} + +message LogEventResponse { + bool success = 1; + string event_id = 2; + string error = 3; +} + +message EventInfo { + string event_id = 1; + string event_type = 2; + string actor = 3; + string description = 4; + string data = 5; // JSON string + string level = 6; + string timestamp = 7; +} + +message QueryEventsRequest { + string event_type = 1; // Comma-separated + string actor = 2; + string level = 3; + string since = 4; // ISO 8601 + string until = 5; + string search = 6; // Keyword search + int32 limit = 7; +} + +message QueryEventsResponse { + bool success = 1; + repeated EventInfo events = 2; + int32 total = 3; + string error = 4; +} + +message GetEventStatsRequest {} + +message GetEventStatsResponse { + bool success = 1; + int32 total_events = 2; + int64 total_logged = 3; + int64 total_rotated = 4; + string oldest_event = 5; + string newest_event = 6; + string error = 7; +} diff --git a/services/state-manager/secret.schema.json b/services/state-manager/secret.schema.json new file mode 100644 index 00000000..e1a8346a --- /dev/null +++ b/services/state-manager/secret.schema.json @@ -0,0 +1,4 @@ +{ + "type": "object", + "properties": {} +} diff --git a/services/state-manager/service.json b/services/state-manager/service.json new file mode 100644 index 00000000..26369aa8 --- /dev/null +++ b/services/state-manager/service.json @@ -0,0 +1,12 @@ +{ + "schema": "chaitin.octobus.service.v1", + "name": "state-manager", + "displayName": "Agent State Manager", + "description": "Persistent memory, cognitive distillation, task tracking, and audit logging for stateless AI agents — Memory Engine, Distillation Engine, Task State Machine, and Event Log.", + "proto": { + "roots": ["proto"], + "files": ["proto/state.proto"] + }, + "configSchema": "config.schema.json", + "secretSchema": "secret.schema.json" +} From 1fd34f6c1598fae78319be64b2f3bf413f008f44 Mon Sep 17 00:00:00 2001 From: "jianhua.wang" Date: Sun, 5 Jul 2026 19:34:08 +0800 Subject: [PATCH 2/9] =?UTF-8?q?feat:=20add=20DingTalk=20service=20packages?= =?UTF-8?q?=20=E2=80=94=20AITable,=20Calendar,=20Doc,=20Drive,=20Message,?= =?UTF-8?q?=20Todo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 6 OctoBus service packages for DingTalk (钉钉) API integration, enabling AI agents to interact with DingTalk workspace tools. Services included: - dingtalk__aitable: AITable CRUD, kanban operations, record management with configurable field mapping via kanban.json - dingtalk__calendar: Calendar event queries, meeting scheduling, free/busy time detection - dingtalk__doc: Document read/write, folder management, weekly report sync, knowledge base operations - dingtalk__drive: File upload/download, folder management, doc space ops - dingtalk__message: Group messaging, robot push, Ding notification, email - dingtalk__todo: Todo creation, completion, querying Each service follows the standard OctoBus service pattern: - bin/: Service entry point with RPC handlers - proto/: gRPC service definition - config.schema.json / secret.schema.json: Configuration schemas - package.json / service.json: Service metadata Configuration for aitable kanban field mapping is externalized to config/kanban.json with generic key names, allowing deployments to customize without modifying source code. Signed-off-by: jianhua.wang --- services/dingtalk__aitable/bin/service.js | 217 +++++++ services/dingtalk__aitable/config.schema.json | 15 + services/dingtalk__aitable/config/kanban.json | 28 + services/dingtalk__aitable/package.json | 15 + .../dingtalk__aitable/proto/aitable.proto | 56 ++ services/dingtalk__aitable/secret.schema.json | 4 + services/dingtalk__aitable/service.json | 12 + services/dingtalk__calendar/bin/service.js | 61 ++ .../dingtalk__calendar/config.schema.json | 10 + services/dingtalk__calendar/package.json | 14 + .../dingtalk__calendar/proto/calendar.proto | 43 ++ .../dingtalk__calendar/secret.schema.json | 4 + services/dingtalk__calendar/service.json | 12 + services/dingtalk__doc/bin/doc-service.js | 608 ++++++++++++++++++ services/dingtalk__doc/config.schema.json | 30 + services/dingtalk__doc/lib/doc-memory.js | 89 +++ services/dingtalk__doc/lib/dws-runner.js | 128 ++++ services/dingtalk__doc/lib/safety-check.js | 60 ++ services/dingtalk__doc/package.json | 14 + services/dingtalk__doc/proto/doc.proto | 225 +++++++ services/dingtalk__doc/secret.schema.json | 4 + services/dingtalk__doc/service.json | 12 + services/dingtalk__drive/bin/drive-service.js | 252 ++++++++ services/dingtalk__drive/config.schema.json | 20 + services/dingtalk__drive/package.json | 15 + services/dingtalk__drive/proto/drive.proto | 155 +++++ services/dingtalk__drive/secret.schema.json | 4 + services/dingtalk__drive/service.json | 12 + .../proto/dingtalk_group_robot.proto | 2 +- services/dingtalk__message/bin/service.js | 50 ++ services/dingtalk__message/config.schema.json | 10 + services/dingtalk__message/package.json | 14 + .../dingtalk__message/proto/message.proto | 40 ++ services/dingtalk__message/secret.schema.json | 4 + services/dingtalk__message/service.json | 12 + services/dingtalk__todo/bin/service.js | 96 +++ services/dingtalk__todo/config.schema.json | 10 + services/dingtalk__todo/package.json | 14 + services/dingtalk__todo/proto/todo.proto | 51 ++ services/dingtalk__todo/secret.schema.json | 4 + services/dingtalk__todo/service.json | 12 + 41 files changed, 2437 insertions(+), 1 deletion(-) create mode 100644 services/dingtalk__aitable/bin/service.js create mode 100644 services/dingtalk__aitable/config.schema.json create mode 100644 services/dingtalk__aitable/config/kanban.json create mode 100644 services/dingtalk__aitable/package.json create mode 100644 services/dingtalk__aitable/proto/aitable.proto create mode 100644 services/dingtalk__aitable/secret.schema.json create mode 100644 services/dingtalk__aitable/service.json create mode 100644 services/dingtalk__calendar/bin/service.js create mode 100644 services/dingtalk__calendar/config.schema.json create mode 100644 services/dingtalk__calendar/package.json create mode 100644 services/dingtalk__calendar/proto/calendar.proto create mode 100644 services/dingtalk__calendar/secret.schema.json create mode 100644 services/dingtalk__calendar/service.json create mode 100644 services/dingtalk__doc/bin/doc-service.js create mode 100644 services/dingtalk__doc/config.schema.json create mode 100644 services/dingtalk__doc/lib/doc-memory.js create mode 100644 services/dingtalk__doc/lib/dws-runner.js create mode 100644 services/dingtalk__doc/lib/safety-check.js create mode 100644 services/dingtalk__doc/package.json create mode 100644 services/dingtalk__doc/proto/doc.proto create mode 100644 services/dingtalk__doc/secret.schema.json create mode 100644 services/dingtalk__doc/service.json create mode 100644 services/dingtalk__drive/bin/drive-service.js create mode 100644 services/dingtalk__drive/config.schema.json create mode 100644 services/dingtalk__drive/package.json create mode 100644 services/dingtalk__drive/proto/drive.proto create mode 100644 services/dingtalk__drive/secret.schema.json create mode 100644 services/dingtalk__drive/service.json create mode 100644 services/dingtalk__message/bin/service.js create mode 100644 services/dingtalk__message/config.schema.json create mode 100644 services/dingtalk__message/package.json create mode 100644 services/dingtalk__message/proto/message.proto create mode 100644 services/dingtalk__message/secret.schema.json create mode 100644 services/dingtalk__message/service.json create mode 100644 services/dingtalk__todo/bin/service.js create mode 100644 services/dingtalk__todo/config.schema.json create mode 100644 services/dingtalk__todo/package.json create mode 100644 services/dingtalk__todo/proto/todo.proto create mode 100644 services/dingtalk__todo/secret.schema.json create mode 100644 services/dingtalk__todo/service.json diff --git a/services/dingtalk__aitable/bin/service.js b/services/dingtalk__aitable/bin/service.js new file mode 100644 index 00000000..3eccee6f --- /dev/null +++ b/services/dingtalk__aitable/bin/service.js @@ -0,0 +1,217 @@ +#!/usr/bin/env node +/** + * DingTalk AI Table Service — OctoBus 服务包 + */ +import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; +import { execFile } from 'child_process'; +import { readFileSync } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// 加载看板配置（从 kanban.json 读取，避免 js-yaml 依赖） +let kanbanConfig = null; +function loadKanbanConfig(configPath) { + if (kanbanConfig) return kanbanConfig; + try { + const resolved = configPath || join(__dirname, '..', 'config', 'kanban.json'); + const raw = readFileSync(resolved, 'utf-8'); + kanbanConfig = JSON.parse(raw); + } catch (err) { + console.error(`[aitable] Failed to load kanban.json: ${err.message}`); + kanbanConfig = {}; + } + return kanbanConfig; +} + +function runDws(command, timeout = 60000) { + return new Promise((resolve) => { + execFile('sh', ['-c', `dws ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + (error, stdout) => { + const raw = stdout.trim(); + let data = null; + try { data = JSON.parse(raw); } catch { data = raw; } + // dws 退出码 0 不代表业务成功，要看 data.status + const status = data && typeof data === 'object' ? String(data.status || '').toLowerCase() : ''; + const businessFailed = status === 'error' || status === 'failed'; + if (error && error.code !== 0) { + resolve({ success: false, data, error: (data && data.summary) || error.message }); + } else if (businessFailed) { + const errMsg = (data && data.error && (data.error.message || data.error)) || (data && data.summary) || 'dws business error'; + resolve({ success: false, data, error: typeof errMsg === 'object' ? JSON.stringify(errMsg) : String(errMsg) }); + } else { + resolve({ success: true, data, error: '' }); + } + }); + }); +} + +const shellEscape = (s) => "'" + String(s).replace(/'/g, "'\\''") + "'"; + +const service = defineService({ + handlers: { + 'dingtalk.aitable.v1.AITableService/CreateRecord': async (ctx) => { + const { baseId, tableId, fields } = ctx.request; + // dws CLI expects --fields as array: [{"cells": {"fieldId": value, ...}}] + // Values can be strings, or JSON-stringified objects/arrays for complex fields + const cells = {}; + for (const [key, val] of Object.entries(fields || {})) { + // Try to parse JSON strings back to objects/arrays for complex fields + if (typeof val === 'string' && (val.startsWith('{') || val.startsWith('['))) { + try { cells[key] = JSON.parse(val); } catch { cells[key] = val; } + } else { + cells[key] = val; + } + } + const recordsArray = JSON.stringify([{ cells }]); + const cmd = `aitable record create --base-id ${shellEscape(baseId)} --table-id ${shellEscape(tableId)} --fields ${shellEscape(recordsArray)}`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, recordId: '', error: res.error }; + + // dws 把结果包在 data.data 里；newRecordIds 是新建记录 ID 列表 + const r = res.data?.data || res.data?.result || res.data; + const recordId = (Array.isArray(r?.newRecordIds) && r.newRecordIds[0]) + || (Array.isArray(r) && r[0]?.recordId) + || r?.recordId || ''; + return { success: true, recordId, error: '' }; + }, + + 'dingtalk.aitable.v1.AITableService/QueryRecords': async (ctx) => { + const { baseId, tableId, limit } = ctx.request; + const cmd = `aitable record list --base-id ${shellEscape(baseId)} --table-id ${shellEscape(tableId)} --limit ${limit || 10}`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, records: [], error: res.error }; + + // dws 把结果包在 data.data.records；每条记录字段在 cells（不是 fields） + const rawRecords = res.data?.data?.records || res.data?.result?.records || res.data?.records || []; + const list = Array.isArray(rawRecords) ? rawRecords : []; + const records = list.map((r) => ({ + recordId: r.recordId || r.id || '', + fields: r.cells || r.fields || {}, + })); + return { success: true, records, error: '' }; + }, + + /** + * 看板记录创建（业务封装） + * 读 projects.yaml 的 kanban 配置，自动映射字段 ID → 构造 fields → 调用 CreateRecord + */ + 'dingtalk.aitable.v1.AITableService/CreateKanbanRecord': async (ctx) => { + const { summary, category, customer, date, taskType, description, salesman, owner } = ctx.request; + + // 1. 加载配置 + const kanban = loadKanbanConfig(process.env.KANBAN_CONFIG_PATH); + if (!kanban) { + return { success: false, recordId: '', error: 'projects.yaml 中缺少 kanban 配置' }; + } + + // 2. 映射任务类型 + const taskTypes = kanban.task_types || {}; + let taskTypeOption = null; + // 先精确匹配 taskType，再回退到 category + const lookupKey = taskType || category; + if (taskTypes[lookupKey]) { + taskTypeOption = taskTypes[lookupKey]; + } else { + // 模糊匹配：key 包含 category 的关键词 + for (const [key, val] of Object.entries(taskTypes)) { + if (key.includes(lookupKey) || lookupKey.includes(key)) { + taskTypeOption = val; + break; + } + } + } + if (!taskTypeOption) { + // 最终回退：用 category 本身作为名称 + taskTypeOption = { id: '', name: lookupKey }; + } + + // 3. 映射客户分类 + const customers = kanban.customers || {}; + let customerOption = null; + if (customers[customer]) { + customerOption = customers[customer]; + } else { + // 模糊匹配：客户名中是否包含已知的 key + for (const [key, val] of Object.entries(customers)) { + if (customer.includes(key) || key.includes(customer)) { + customerOption = val; + break; + } + } + } + if (!customerOption && customer) { + customerOption = { id: '', name: customer }; + } + + // 4. 提取二级部门（客户名中"-"后的部分） + let department2 = ''; + if (customer && customer.includes('-')) { + department2 = customer.split('-').slice(1).join('-'); + } + + // 5. 构造 fields + const fields = {}; + const kf = kanban.fields || {}; + // 日期字段（两个日期列） + fields[kf.date_start || '5wlytmi'] = date || ''; + fields[kf.date_end || '5sxy4yq'] = date || ''; + // 任务类型 + fields[kf.task_type || 'wiisqll'] = taskTypeOption; + // 任务描述 + fields[kf.description || 'q3aj1jn'] = summary || ''; + // Owner — 从 user-config 读取，无配置时不设置默认值 + const ownerUser = owner || ''; + if (ownerUser) fields[kf.owner || 'kkcfjj6'] = [{ userId: ownerUser }]; + // 销售 — 从 user-config 读取，无配置时不设置默认值 + const salesUser = salesman || ''; + if (salesUser) fields[kf.salesman || '6olqudb'] = [{ userId: salesUser }]; + // 周报摘要/说明 + fields[kf.weekly_summary || 'x2ysb28'] = description || summary || ''; + // 客户分类 + if (customerOption) { + fields[kf.customer_tier || 'dacnewp'] = customerOption; + } + // 二级部门 + if (department2) { + fields[kf.sub_department || 'cgrfhw7'] = department2; + } + // 状态（固定：已完成） + if (kanban.status_done) { + fields[kf.status || 'v5j63kh'] = kanban.status_done; + } + + // 6. 调用 CreateRecord + const cells = {}; + for (const [key, val] of Object.entries(fields)) { + if (typeof val === 'object') { + cells[key] = val; + } else { + // Try to parse JSON strings back to objects/arrays for complex fields + if (typeof val === 'string' && (val.startsWith('{') || val.startsWith('['))) { + try { cells[key] = JSON.parse(val); } catch { cells[key] = val; } + } else { + cells[key] = val; + } + } + } + const recordsArray = JSON.stringify([{ cells }]); + const cmd = `aitable record create --base-id ${shellEscape(kanban.base_id)} --table-id ${shellEscape(kanban.table_id)} --fields ${shellEscape(recordsArray)}`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, recordId: '', error: res.error }; + + const r = res.data?.data || res.data?.result || res.data; + const recordId = (Array.isArray(r?.newRecordIds) && r.newRecordIds[0]) + || (Array.isArray(r) && r[0]?.recordId) + || r?.recordId || ''; + return { success: true, recordId, error: '' }; + }, + }, +}); + +runServiceMain(service); diff --git a/services/dingtalk__aitable/config.schema.json b/services/dingtalk__aitable/config.schema.json new file mode 100644 index 00000000..7c64861e --- /dev/null +++ b/services/dingtalk__aitable/config.schema.json @@ -0,0 +1,15 @@ +{ + "type": "object", + "properties": { + "dwsPath": { + "type": "string", + "description": "dws CLI path", + "default": "dws" + }, + "kanbanConfigPath": { + "type": "string", + "description": "看板配置文件路径（kanban.json），用于字段映射", + "default": "./config/kanban.json" + } + } +} diff --git a/services/dingtalk__aitable/config/kanban.json b/services/dingtalk__aitable/config/kanban.json new file mode 100644 index 00000000..33cff27f --- /dev/null +++ b/services/dingtalk__aitable/config/kanban.json @@ -0,0 +1,28 @@ +{ + "_README": "Kanban config template — replace with your own AITable base/table ID and field option IDs", + "base_id": "YOUR_AITABLE_BASE_ID", + "table_id": "YOUR_AITABLE_TABLE_ID", + "fields": { + "date_start": "5wlytmi", + "date_end": "5sxy4yq", + "task_type": "wiisqll", + "description": "q3aj1jn", + "owner": "kkcfjj6", + "salesman": "6olqudb", + "weekly_summary": "x2ysb28", + "customer_tier": "dacnewp", + "sub_department": "cgrfhw7", + "status": "v5j63kh" + }, + "task_types": { + "communication": { "id": "OPTION_ID", "name": "Communication" }, + "documentation": { "id": "OPTION_ID", "name": "Documentation" }, + "bidding": { "id": "OPTION_ID", "name": "Bidding" }, + "poc": { "id": "OPTION_ID", "name": "POC" } + }, + "customers": { + "customer_a": { "id": "OPTION_ID", "name": "Customer A Name" }, + "customer_b": { "id": "OPTION_ID", "name": "Customer B Name" } + }, + "status_done": { "id": "OPTION_ID", "name": "Done" } +} diff --git a/services/dingtalk__aitable/package.json b/services/dingtalk__aitable/package.json new file mode 100644 index 00000000..cfab4beb --- /dev/null +++ b/services/dingtalk__aitable/package.json @@ -0,0 +1,15 @@ +{ + "name": "@office-agent/dingtalk-aitable", + "displayName": "DingTalk AITable", + "version": "0.1.0", + "description": "DingTalk AITable — record CRUD, kanban operations, field management", + "type": "module", + "main": "bin/service.js", + "bin": "bin/service.js", + "scripts": { + "start": "node bin/service.js" + }, + "dependencies": { + "@chaitin-ai/octobus-sdk": "^0.1.0" + } +} diff --git a/services/dingtalk__aitable/proto/aitable.proto b/services/dingtalk__aitable/proto/aitable.proto new file mode 100644 index 00000000..8eb34ea2 --- /dev/null +++ b/services/dingtalk__aitable/proto/aitable.proto @@ -0,0 +1,56 @@ +syntax = "proto3"; +package dingtalk.aitable.v1; + +service AITableService { + rpc CreateRecord(CreateRecordRequest) returns (CreateRecordResponse); + rpc QueryRecords(QueryRecordsRequest) returns (QueryRecordsResponse); + rpc CreateKanbanRecord(CreateKanbanRecordRequest) returns (CreateKanbanRecordResponse); +} + +message CreateRecordRequest { + string base_id = 1; // AI Table base ID + string table_id = 2; // Table ID + map fields = 3; // 字段名 → 值 +} + +message CreateRecordResponse { + bool success = 1; + string record_id = 2; + string error = 3; +} + +message QueryRecordsRequest { + string base_id = 1; + string table_id = 2; + string filter = 3; // 筛选条件（可选） + int32 limit = 4; +} + +message AITableRecord { + string record_id = 1; + map fields = 2; +} + +message QueryRecordsResponse { + bool success = 1; + repeated AITableRecord records = 2; + string error = 3; +} + +// 看板记录创建（业务封装，自动处理字段映射） +message CreateKanbanRecordRequest { + string summary = 1; // 任务描述，如"【交流】与XX沟通XX项目" + string category = 2; // 任务类型：沟通交流/文档材料/投标谈判/POC及其他 + string customer = 3; // 一级客户分类 + string date = 4; // 日期 YYYY-MM-DD + string task_type = 5; // 具体任务类型名（对应 projects.yaml 中 task_types 的 key） + string description = 6; // 必要说明 + string salesman = 7; // 销售 userId（从 user-config 读取，不设硬编码默认值） + string owner = 8; // 任务 Owner userId +} + +message CreateKanbanRecordResponse { + bool success = 1; + string record_id = 2; + string error = 3; +} diff --git a/services/dingtalk__aitable/secret.schema.json b/services/dingtalk__aitable/secret.schema.json new file mode 100644 index 00000000..e1a8346a --- /dev/null +++ b/services/dingtalk__aitable/secret.schema.json @@ -0,0 +1,4 @@ +{ + "type": "object", + "properties": {} +} diff --git a/services/dingtalk__aitable/service.json b/services/dingtalk__aitable/service.json new file mode 100644 index 00000000..20186190 --- /dev/null +++ b/services/dingtalk__aitable/service.json @@ -0,0 +1,12 @@ +{ + "schema": "chaitin.octobus.service.v1", + "name": "dingtalk__aitable", + "displayName": "DingTalk AITable Service", + "description": "DingTalk AITable integration — record creation, querying, and kanban board writing via dws CLI.", + "proto": { + "roots": ["proto"], + "files": ["proto/aitable.proto"] + }, + "configSchema": "config.schema.json", + "secretSchema": "secret.schema.json" +} diff --git a/services/dingtalk__calendar/bin/service.js b/services/dingtalk__calendar/bin/service.js new file mode 100644 index 00000000..41f604cb --- /dev/null +++ b/services/dingtalk__calendar/bin/service.js @@ -0,0 +1,61 @@ +#!/usr/bin/env node +/** + * DingTalk Calendar Service — OctoBus 服务包 + */ +import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; +import { execFile } from 'child_process'; + +function runDws(command, timeout = 60000) { + return new Promise((resolve) => { + execFile('sh', ['-c', `dws ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + (error, stdout) => { + const raw = stdout.trim(); + let data = null; + try { data = JSON.parse(raw); } catch { data = raw; } + if (error && error.code !== 0) resolve({ success: false, data, error: error.message }); + else resolve({ success: true, data, error: '' }); + }); + }); +} + +const shellEscape = (s) => "'" + String(s).replace(/'/g, "'\\''") + "'"; + +const service = defineService({ + handlers: { + 'dingtalk.calendar.v1.CalendarService/ListEvents': async (ctx) => { + const { start, end } = ctx.request; + const res = await runDws(`calendar event list --start ${shellEscape(start)} --end ${shellEscape(end)}`); + if (!res.success) return { success: false, events: [], error: res.error }; + + const rawEvents = res.data?.result?.events || res.data?.events || []; + const list = Array.isArray(rawEvents) ? rawEvents : []; + const events = list.map((e) => ({ + id: e.id || '', + summary: e.summary || '', + start: e.start?.dateTime || e.start || '', + end: e.end?.dateTime || e.end || '', + isAllDay: e.isAllDay || false, + description: e.description || '', + location: e.location?.displayName || e.location || '', + attendees: (e.attendees || []).map((a) => a.displayName || a.name || ''), + })); + return { success: true, events, error: '' }; + }, + + 'dingtalk.calendar.v1.CalendarService/CreateEvent': async (ctx) => { + const { title, start, end, location, attendees } = ctx.request; + let cmd = `calendar event create --title ${shellEscape(title || 'Untitled')} --start ${shellEscape(start)} --end ${shellEscape(end)}`; + if (location) cmd += ` --location ${shellEscape(location)}`; + if (attendees && attendees.length > 0) cmd += ` --attendees ${shellEscape(attendees.join(','))}`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, eventId: '', error: res.error }; + + const r = res.data?.result || res.data; + const eventId = r?.id || r?.eventId || (Array.isArray(r) && r[0]?.id) || ''; + return { success: true, eventId, error: '' }; + }, + }, +}); + +runServiceMain(service); diff --git a/services/dingtalk__calendar/config.schema.json b/services/dingtalk__calendar/config.schema.json new file mode 100644 index 00000000..8ca24efa --- /dev/null +++ b/services/dingtalk__calendar/config.schema.json @@ -0,0 +1,10 @@ +{ + "type": "object", + "properties": { + "dwsPath": { + "type": "string", + "description": "dws CLI path", + "default": "dws" + } + } +} diff --git a/services/dingtalk__calendar/package.json b/services/dingtalk__calendar/package.json new file mode 100644 index 00000000..f6850e26 --- /dev/null +++ b/services/dingtalk__calendar/package.json @@ -0,0 +1,14 @@ +{ + "name": "@office-agent/dingtalk-calendar", + "displayName": "DingTalk Calendar", + "version": "0.1.0", + "description": "DingTalk Calendar — schedule queries, meeting management, free-busy detection", + "type": "module", + "main": "bin/service.js", + "scripts": { + "start": "node bin/service.js" + }, + "dependencies": { + "@chaitin-ai/octobus-sdk": "^0.1.0" + } +} diff --git a/services/dingtalk__calendar/proto/calendar.proto b/services/dingtalk__calendar/proto/calendar.proto new file mode 100644 index 00000000..d315dad5 --- /dev/null +++ b/services/dingtalk__calendar/proto/calendar.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; +package dingtalk.calendar.v1; + +service CalendarService { + rpc ListEvents(ListEventsRequest) returns (ListEventsResponse); + rpc CreateEvent(CreateEventRequest) returns (CreateEventResponse); +} + +message ListEventsRequest { + string start = 1; // ISO 时间（起始） + string end = 2; // ISO 时间（结束） +} + +message CalendarEvent { + string id = 1; + string summary = 2; + string start = 3; + string end = 4; + bool is_all_day = 5; + string description = 6; + string location = 7; + repeated string attendees = 8; +} + +message ListEventsResponse { + bool success = 1; + repeated CalendarEvent events = 2; + string error = 3; +} + +message CreateEventRequest { + string title = 1; + string start = 2; // ISO 时间 + string end = 3; // ISO 时间 + string location = 4; // 地点（可选） + repeated string attendees = 5; // 参与者 userId 列表（可选） +} + +message CreateEventResponse { + bool success = 1; + string event_id = 2; + string error = 3; +} diff --git a/services/dingtalk__calendar/secret.schema.json b/services/dingtalk__calendar/secret.schema.json new file mode 100644 index 00000000..e1a8346a --- /dev/null +++ b/services/dingtalk__calendar/secret.schema.json @@ -0,0 +1,4 @@ +{ + "type": "object", + "properties": {} +} diff --git a/services/dingtalk__calendar/service.json b/services/dingtalk__calendar/service.json new file mode 100644 index 00000000..6e1f710b --- /dev/null +++ b/services/dingtalk__calendar/service.json @@ -0,0 +1,12 @@ +{ + "schema": "chaitin.octobus.service.v1", + "name": "dingtalk__calendar", + "displayName": "DingTalk Calendar Service", + "description": "DingTalk Calendar integration — schedule querying and meeting creation via dws CLI.", + "proto": { + "roots": ["proto"], + "files": ["proto/calendar.proto"] + }, + "configSchema": "config.schema.json", + "secretSchema": "secret.schema.json" +} diff --git a/services/dingtalk__doc/bin/doc-service.js b/services/dingtalk__doc/bin/doc-service.js new file mode 100644 index 00000000..ac363580 --- /dev/null +++ b/services/dingtalk__doc/bin/doc-service.js @@ -0,0 +1,608 @@ +#!/usr/bin/env node +/** + * DingTalk Document Service - OctoBus 服务包入口 + * + * 封装钉钉文档的所有操作： + * - 文档读写（ReadDoc / UpdateDoc / CreateDoc） + * - 倒序插入简报（InsertBrief - 复合方法） + * - 块级操作（ListBlocks / UpdateBlock / InsertBlockAfter） + * - 搜索（SearchDocs） + * - 上下文记忆（GetLastContext） + * + * 关键设计决策： + * - InsertBrief 是复合方法，内部封装了记忆读取、备份、安全校验、插入全流程 + * - Agent 只需调用一次 InsertBrief，不需要关心文档操作细节 + * - 所有 dws 操作经过 dws-runner.js 封装，统一处理错误和超时 + */ + +import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; +import { runDws, writeTempFile, cleanupTempFile, shellEscape } from '../lib/dws-runner.js'; +import { DocMemory } from '../lib/doc-memory.js'; +import { safetyCheck, cleanHtmlTags, backupDocument } from '../lib/safety-check.js'; + +// 服务配置（从 OctoBus instance config 注入） +const config = { + dwsPath: process.env.DWS_PATH || 'dws', + stateDataDir: process.env.STATE_DATA_DIR || './data', + backupDir: process.env.BACKUP_DIR || './backups', + maxDocSize: parseInt(process.env.MAX_DOC_SIZE || '10000'), + dwsTimeout: parseInt(process.env.DWS_TIMEOUT || '60000'), +}; + +const docMemory = new DocMemory(config.stateDataDir); + +const service = defineService({ + handlers: { + /** + * 读取文档全文 + */ + 'dingtalk.doc.v1.DocService/ReadDoc': async (ctx) => { + const { nodeId } = ctx.request; + const result = await runDws( + `doc read --node-id ${nodeId}`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + // dws doc read 返回 JSON 对象: {markdown, nodeId, docUrl, ...} + const raw = result.data; + const markdown = result.success + ? (typeof raw === 'object' && raw !== null ? (raw.markdown || raw.content || '') : String(raw || '')) + : ''; + return { + success: result.success, + content: markdown, + error: result.error, + }; + }, + + /** + * 全文覆盖更新（<10000字） + */ + 'dingtalk.doc.v1.DocService/UpdateDoc': async (ctx) => { + const { nodeId, content } = ctx.request; + + // 清理 HTML 标签 + const cleanContent = cleanHtmlTags(content); + + // 写入临时文件 + const tmpFile = await writeTempFile(cleanContent); + try { + const result = await runDws( + `doc update --node-id ${nodeId} --mode overwrite --content-file ${shellEscape(tmpFile)}`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + return { success: result.success, error: result.error }; + } finally { + await cleanupTempFile(tmpFile); + } + }, + + /** + * 创建新文档 + */ + 'dingtalk.doc.v1.DocService/CreateDoc': async (ctx) => { + const { title, content, parentFolder } = ctx.request; + + const tmpFile = await writeTempFile(content || ''); + try { + let cmd = `doc create --title ${shellEscape(title)} --content-file ${shellEscape(tmpFile)}`; + if (parentFolder) { + cmd += ` --folder-id ${parentFolder}`; + } + const result = await runDws(cmd, { dwsPath: config.dwsPath, timeout: config.dwsTimeout }); + + // 从返回中提取 nodeId 和 url + const nodeId = result.data?.nodeId || result.data?.node_id || ''; + const url = result.data?.url || ''; + + return { success: result.success, nodeId, url, error: result.error }; + } finally { + await cleanupTempFile(tmpFile); + } + }, + + /** + * 倒序插入简报（核心复合方法） + * + * 流程： + * 1. 读取 doc_memory → 获取 header 位置 + * 2. 读取当前文档全文 + * 3. 备份原文档 + * 4. 安全校验（新内容不能太短） + * 5. 在 header 后倒序插入新简报 + * 6. 写回文档 + * 7. 更新 doc_memory + */ + 'dingtalk.doc.v1.DocService/InsertBrief': async (ctx) => { + const { nodeId, briefContent, title } = ctx.request; + + try { + // 1. 读取记忆 + const memory = await docMemory.get(nodeId); + const headerLength = memory?.header_length; + + // 2. 读取文档 + const readResult = await runDws( + `doc read --node-id ${nodeId}`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + if (!readResult.success) { + return { success: false, error: `读取文档失败: ${readResult.error}` }; + } + + const originalContent = (() => { + const raw = readResult.data; + if (typeof raw === 'object' && raw !== null) { + return raw.markdown || raw.content || JSON.stringify(raw); + } + return String(raw || ''); + })(); + + // 3. 备份 + const backupPath = await backupDocument(nodeId, originalContent, config.backupDir); + + // 4. 安全校验 + const headerEnd = headerLength ?? DocMemory.findHeaderEnd(originalContent); + const header = originalContent.substring(0, headerEnd); + const body = originalContent.substring(headerEnd); + const newContent = header + '\n' + briefContent + '\n\n' + body; + + const check = safetyCheck(originalContent, newContent); + if (!check.safe) { + return { success: false, error: `安全校验失败: ${check.reason}` }; + } + + // 5. 写回文档 + const cleanContent = cleanHtmlTags(newContent); + const tmpFile = await writeTempFile(cleanContent); + try { + const writeResult = await runDws( + `doc update --node-id ${nodeId} --mode overwrite --content-file ${shellEscape(tmpFile)}`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + if (!writeResult.success) { + return { success: false, backupPath, error: `写入文档失败: ${writeResult.error}` }; + } + } finally { + await cleanupTempFile(tmpFile); + } + + // 6. 更新记忆 + await docMemory.set(nodeId, { + headerLength: headerEnd, + lastBriefTitle: title, + }); + + return { success: true, backupPath, error: '' }; + } catch (err) { + return { success: false, error: `InsertBrief 异常: ${err.message}` }; + } + }, + + /** + * 列出文档块 + */ + 'dingtalk.doc.v1.DocService/ListBlocks': async (ctx) => { + const { nodeId, startIndex, endIndex } = ctx.request; + const result = await runDws( + `doc block list --node ${nodeId} --start-index ${startIndex} --end-index ${endIndex}`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + + if (!result.success) { + return { success: false, blocks: [], error: result.error }; + } + + // dws 返回 {blocks: [{blockType, element: {id, index, heading/paragraph}}], totalCount, success} + const rawBlocks = (result.data && typeof result.data === 'object' && Array.isArray(result.data.blocks)) + ? result.data.blocks : []; + const blocks = rawBlocks.map((b) => { + const el = b.element || b; + const heading = el.heading || {}; + const paragraph = el.paragraph || {}; + const orderedList = el.orderedList || {}; + const content = heading.text || paragraph.text || orderedList.text || el.text || ''; + const levelStr = heading.level || ''; + const level = typeof levelStr === 'string' ? parseInt(levelStr.replace('heading-', '') || '0') : (levelStr || 0); + return { + blockId: el.id || '', + index: el.index ?? b.index ?? 0, + blockType: b.blockType || el.blockType || '', + content, + level, + }; + }); + + return { success: true, blocks, error: '' }; + }, + + /** + * 更新块内容 + */ + 'dingtalk.doc.v1.DocService/UpdateBlock': async (ctx) => { + const { nodeId, blockId, content } = ctx.request; + const tmpFile = await writeTempFile(content); + try { + const result = await runDws( + `doc block update --node ${nodeId} --block-id ${blockId} --content-file ${shellEscape(tmpFile)}`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + return { success: result.success, error: result.error }; + } finally { + await cleanupTempFile(tmpFile); + } + }, + + /** + * 在指定块之后插入新块 + */ + 'dingtalk.doc.v1.DocService/InsertBlockAfter': async (ctx) => { + const { nodeId, afterBlockId, blockType, content, level } = ctx.request; + const tmpFile = await writeTempFile(content); + try { + let cmd = `doc block insert --node ${nodeId} --after-block-id ${afterBlockId} --type ${blockType} --content-file ${shellEscape(tmpFile)}`; + if (level && blockType === 'heading') { + cmd += ` --level ${level}`; + } + const result = await runDws(cmd, { dwsPath: config.dwsPath, timeout: config.dwsTimeout }); + const newBlockId = result.data?.blockId || result.data?.id || ''; + return { success: result.success, newBlockId, error: result.error }; + } finally { + await cleanupTempFile(tmpFile); + } + }, + + /** + * 搜索文档 + */ + 'dingtalk.doc.v1.DocService/SearchDocs': async (ctx) => { + const { keyword, maxResults } = ctx.request; + const limit = maxResults || 10; + const result = await runDws( + `doc search --keyword ${shellEscape(keyword)} --limit ${limit}`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + + if (!result.success) { + return { success: false, results: [], error: result.error }; + } + + // dws 返回 {documents: [{name, nodeId, docUrl, contentType}], hasMore, success} + const rawDocs = (result.data && typeof result.data === 'object' && Array.isArray(result.data.documents)) + ? result.data.documents : []; + const results = rawDocs.map((d) => ({ + nodeId: d.nodeId || '', + title: d.name || d.title || '', + url: d.docUrl || d.url || '', + snippet: d.description || '', + })); + + return { success: true, results, error: '' }; + }, + + /** + * 获取上次交流上下文 + */ + 'dingtalk.doc.v1.DocService/GetLastContext': async (ctx) => { + const { nodeId, maxLength } = ctx.request; + const maxLen = maxLength || 800; + + try { + const memory = await docMemory.get(nodeId); + if (!memory?.last_brief_title) { + return { success: true, context: '', lastTitle: '', error: '' }; + } + + // 读取文档，找到上次简报 + const readResult = await runDws( + `doc read --node-id ${nodeId}`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + if (!readResult.success) { + return { success: false, context: '', lastTitle: memory.last_brief_title, error: readResult.error }; + } + + const content = (() => { + const raw = readResult.data; + if (typeof raw === 'object' && raw !== null) { + return raw.markdown || raw.content || JSON.stringify(raw); + } + return String(raw || ''); + })(); + const titleIdx = content.indexOf(memory.last_brief_title); + if (titleIdx === -1) { + return { success: true, context: '', lastTitle: memory.last_brief_title, error: '上次简报标题未找到' }; + } + + // 提取上次简报内容（到下一个同级标题或末尾） + const afterTitle = content.substring(titleIdx); + const nextHeading = afterTitle.indexOf('\n# ', 1); + const context = nextHeading > 0 + ? afterTitle.substring(0, Math.min(nextHeading, maxLen)) + : afterTitle.substring(0, maxLen); + + return { success: true, context, lastTitle: memory.last_brief_title, error: '' }; + } catch (err) { + return { success: false, context: '', lastTitle: '', error: err.message }; + } + }, + + /** + * 周会记录同步（移植自 Python weekly_sync.py） + * + * 流程： + * 1. 分批获取文档块（每批200个） + * 2. 从本周一开始，逐周查找目标周标题（heading-2 含日期） + * 3. 在周标题下查找 userName（orderedList.text 匹配） + * 4. 找到后收集分类块，更新计数并插入条目 + * 5. 若所有周标题下都没找到 userName → savedForLater=true + */ + 'dingtalk.doc.v1.DocService/SyncWeekly': async (ctx) => { + const { nodeId, userName, entries } = ctx.request; + + if (!entries || entries.length === 0) { + return { success: true, weekId: '', syncedCount: 0, savedForLater: false, error: '' }; + } + + // ── 辅助函数 ── + const listBlocks = async (start, end) => { + const result = await runDws( + `doc block list --node ${nodeId} --start-index ${start} --end-index ${end}`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + if (!result.success) return []; + const rawBlocks = (result.data && typeof result.data === 'object' && Array.isArray(result.data.blocks)) + ? result.data.blocks : []; + return rawBlocks.map((b) => { + const el = b.element || b; + const heading = el.heading || {}; + const paragraph = el.paragraph || {}; + const orderedList = el.orderedList || {}; + return { + blockId: el.id || '', + index: el.index ?? b.index ?? 0, + blockType: b.blockType || el.blockType || '', + content: heading.text || paragraph.text || orderedList.text || el.text || '', + level: (() => { + const lv = heading.level || ''; + return typeof lv === 'string' ? parseInt(lv.replace('heading-', '') || '0') : (lv || 0); + })(), + }; + }); + }; + + const updateBlock = async (blockId, text) => { + const safeText = shellEscape(text); + const result = await runDws( + `doc block update --node ${nodeId} --block-id ${blockId} --type orderedList --text ${safeText} --fix-jsonml`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + return result.success; + }; + + const insertBlockAfter = async (refBlockId, text) => { + const safeText = shellEscape(text); + const result = await runDws( + `doc block insert --node ${nodeId} --ref-block ${refBlockId} --type orderedList --text ${safeText} --fix-jsonml`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + return result.success; + }; + + // ── 1. 分批获取所有块 ── + const allBlocks = []; + const BATCH_SIZE = 200; + for (let start = 0; start < 2000; start += BATCH_SIZE) { + const batch = await listBlocks(start, start + BATCH_SIZE); + if (batch.length === 0) break; + allBlocks.push(...batch); + if (batch.length < BATCH_SIZE) break; + } + if (allBlocks.length === 0) { + return { success: false, weekId: '', syncedCount: 0, savedForLater: false, error: '无法获取文档块' }; + } + + // ── 2. 计算目标周日期（本周一） ── + const now = new Date(); + const cstOffset = 8 * 60 * 60 * 1000; + const cstNow = new Date(now.getTime() + cstOffset); + const dayOfWeek = cstNow.getUTCDay(); // 0=Sun, 1=Mon + const mondayOffset = dayOfWeek === 0 ? -6 : 1 - dayOfWeek; + const monday = new Date(cstNow.getTime() + mondayOffset * 24 * 60 * 60 * 1000); + const targetWeekId = monday.toISOString().slice(0, 10); // YYYY-MM-DD + + // ── 3. 找到所有周标题 ── + const weekHeaders = []; // {index, weekId, blockId} + const dateRegex = /^(\d{4}-\d{2}-\d{2})/; + for (let i = 0; i < allBlocks.length; i++) { + const b = allBlocks[i]; + if (b.blockType === 'heading' && b.level === 2) { + const m = b.content.match(dateRegex); + if (m) { + weekHeaders.push({ index: i, weekId: m[1], blockId: b.blockId }); + } + } + } + + // 按日期排序，优先找 >= targetWeekId 的周标题 + weekHeaders.sort((a, b) => a.weekId.localeCompare(b.weekId)); + const candidateWeeks = weekHeaders.filter(w => w.weekId >= targetWeekId); + const weeksToSearch = candidateWeeks.length > 0 ? candidateWeeks : weekHeaders; + + if (weeksToSearch.length === 0) { + return { success: false, weekId: '', syncedCount: 0, savedForLater: true, error: '文档中无周标题' }; + } + + // ── 4. 逐周查找 userName ── + const CATEGORY_KEYWORDS = { + 'Communication': 'Communication', + 'Documentation': 'Documentation', + 'Bidding': 'Bidding', + 'POC & Others': 'POC & Others', + }; + + for (const week of weeksToSearch) { + // 确定本周标题的范围（到下一个周标题或文档末尾） + const weekEnd = weekHeaders.find(w => w.weekId > week.weekId)?.index ?? allBlocks.length; + + // 在周范围内找 userName + let userNameIdx = null; + for (let i = week.index + 1; i < weekEnd; i++) { + const b = allBlocks[i]; + if (b.blockType === 'orderedList' && b.content.trim() === userName) { + userNameIdx = i; + break; + } + } + if (userNameIdx === null) { continue; } + + // ── 5. 从 userName 开始收集分类块 ── + const categories = {}; // { categoryName: { headerBlockId, entryBlockIds: [] } } + let currentCat = null; + + for (let i = userNameIdx + 1; i < weekEnd; i++) { + const b = allBlocks[i]; + if (b.blockType === 'heading') break; // 遇到下一个周标题 + + if (b.blockType === 'orderedList') { + const text = b.content.trim(); + + // 遇到下一个人名（2-4个汉字，不是分类关键词，不是"近1周"/"本周计划"） + if (/^[一-鿿]{2,4}$/.test(text) && + text !== userName && text !== '近1周' && text !== '本周计划' && + !text.startsWith('【')) { + break; + } + + // 遇到"本周计划"则停止 + if (text.startsWith('本周计划')) break; + + // 检查是否是分类标题 + let foundCat = false; + for (const [catName, keyword] of Object.entries(CATEGORY_KEYWORDS)) { + if (text.includes(keyword)) { + categories[catName] = { headerBlockId: b.blockId, entryBlockIds: [] }; + currentCat = catName; + foundCat = true; + break; + } + } + + // 非分类标题 → 属于当前分类的条目 + if (!foundCat && currentCat && categories[currentCat]) { + categories[currentCat].entryBlockIds.push(b.blockId); + } + } + } + + if (Object.keys(categories).length === 0) { continue; } + + // ── 6. 按分类分组 entries ── + const grouped = {}; + for (const entry of entries) { + let cat = 'Communication'; + for (const catName of Object.keys(CATEGORY_KEYWORDS)) { + if (entry.category && entry.category.includes(catName)) { + cat = catName; + break; + } + } + if (!grouped[cat]) grouped[cat] = []; + grouped[cat].push(entry.summary); + } + + // ── 7. 更新每个分类 ── + let syncedCount = 0; + + for (const [catName, summaries] of Object.entries(grouped)) { + if (!categories[catName]) { continue; } + const catInfo = categories[catName]; + + // 更新分类标题计数 + const newHeader = `【${catName}*${summaries.length}】`; + + // 插入条目 + for (const summary of summaries) { + const refId = catInfo.entryBlockIds.length > 0 + ? catInfo.entryBlockIds[catInfo.entryBlockIds.length - 1] + : catInfo.headerBlockId; + const ok = await insertBlockAfter(refId, summary); + if (ok) { + syncedCount++; + catInfo.entryBlockIds.push('newly-inserted'); // track for subsequent inserts + } + } + } + + return { success: syncedCount > 0, weekId: week.weekId, syncedCount, savedForLater: false, error: '' }; + } + + // ── 所有周标题下都没找到 userName → 留存 ── + return { success: false, weekId: '', syncedCount: 0, savedForLater: true, error: `在所有周标题下均未找到「${userName}」` }; + }, + + /** + * 移动文档到指定文件夹（整理归档用） + * 封装 dws doc move --node --folder + */ + 'dingtalk.doc.v1.DocService/MoveDoc': async (ctx) => { + const { nodeId, targetFolderId } = ctx.request; + if (!nodeId || !targetFolderId) { + return { success: false, error: 'node_id 和 target_folder_id 都必填' }; + } + const result = await runDws( + `doc move --node ${shellEscape(nodeId)} --folder ${shellEscape(targetFolderId)}`, + { dwsPath: config.dwsPath, timeout: config.dwsTimeout } + ); + return { success: result.success, error: result.error }; + }, + + /** + * 创建文件夹 + * 封装 dws doc folder create --name [--folder ] + */ + 'dingtalk.doc.v1.DocService/CreateFolder': async (ctx) => { + const { name, parentFolderId } = ctx.request; + if (!name) { + return { success: false, folderId: '', error: 'name 必填' }; + } + let cmd = `doc folder create --name ${shellEscape(name)}`; + if (parentFolderId) { + cmd += ` --folder ${shellEscape(parentFolderId)}`; + } + const result = await runDws(cmd, { dwsPath: config.dwsPath, timeout: config.dwsTimeout }); + const folderId = result.data?.nodeId || result.data?.node_id || result.data?.id || ''; + return { success: result.success, folderId, error: result.error }; + }, + + /** + * 列出文件夹下的文档节点（folder_id 空=根目录），用于查找散落文档做整理归档 + * 封装 dws doc list [--folder ] [--page-size N] + */ + 'dingtalk.doc.v1.DocService/ListDocs': async (ctx) => { + const { folderId, pageSize } = ctx.request; + let cmd = `doc list`; + if (folderId) { + cmd += ` --folder ${shellEscape(folderId)}`; + } + if (pageSize && pageSize > 0) { + cmd += ` --page-size ${pageSize}`; + } + const result = await runDws(cmd, { dwsPath: config.dwsPath, timeout: config.dwsTimeout }); + const rawItems = result.data?.nodes || result.data?.items || (Array.isArray(result.data) ? result.data : []); + const docs = (Array.isArray(rawItems) ? rawItems : []).map((it) => ({ + nodeId: it.nodeId || it.node_id || it.id || '', + title: it.name || it.title || '', + type: (it.nodeType || it.node_type || '').includes('folder') ? 'folder' : 'doc', + url: it.docUrl || it.url || '', + })); + return { + success: result.success, + docs, + nextPageToken: result.data?.nextPageToken || result.data?.next_page_token || '', + error: result.error, + }; + }, + }, +}); + +runServiceMain(service); diff --git a/services/dingtalk__doc/config.schema.json b/services/dingtalk__doc/config.schema.json new file mode 100644 index 00000000..06964523 --- /dev/null +++ b/services/dingtalk__doc/config.schema.json @@ -0,0 +1,30 @@ +{ + "type": "object", + "properties": { + "dwsPath": { + "type": "string", + "description": "dws CLI 可执行文件路径", + "default": "dws" + }, + "stateDataDir": { + "type": "string", + "description": "state-manager 统一数据目录（doc_memory.json 等持久化文件存放于此），可通过 STATE_DATA_DIR 环境变量覆盖", + "default": "./data" + }, + "backupDir": { + "type": "string", + "description": "文档备份目录，可通过 BACKUP_DIR 环境变量覆盖", + "default": "./backups" + }, + "maxDocSize": { + "type": "integer", + "description": "全文更新的最大文档字符数，超过此值需使用块编辑", + "default": 10000 + }, + "dwsTimeout": { + "type": "integer", + "description": "dws 命令超时时间（毫秒）", + "default": 60000 + } + } +} diff --git a/services/dingtalk__doc/lib/doc-memory.js b/services/dingtalk__doc/lib/doc-memory.js new file mode 100644 index 00000000..87ba1a05 --- /dev/null +++ b/services/dingtalk__doc/lib/doc-memory.js @@ -0,0 +1,89 @@ +/** + * 文档记忆管理 + * 移植自 Python 版本 src/brief/document_ops.py 的记忆机制 + * + * 记忆文件：doc_memory.json + * 结构：{ [nodeId]: { header_length, last_brief_title, last_updated } } + * + * 作用： + * - 避免每次全量扫描文档来定位 header 结束位置 + * - 记住上次简报标题，用于提取"上次交流上下文" + * + * 2026-07 重构：dataDir 改为统一使用 state-manager 的数据目录， + * 通过 STATE_DATA_DIR 环境变量注入，消除独立 dataDir 配置。 + */ + +import { readFile, writeFile, mkdir } from 'fs/promises'; +import { dirname, join } from 'path'; + +export class DocMemory { + constructor(stateDataDir) { + this.dataDir = stateDataDir; + this.filePath = join(stateDataDir, 'doc_memory.json'); + this.cache = null; + } + + /** + * 加载记忆（懒加载 + 缓存） + */ + async load() { + if (this.cache) return this.cache; + + try { + const raw = await readFile(this.filePath, 'utf-8'); + this.cache = JSON.parse(raw); + } catch { + this.cache = {}; + } + return this.cache; + } + + /** + * 保存记忆到文件 + */ + async save() { + if (!this.cache) return; + await mkdir(dirname(this.filePath), { recursive: true }); + await writeFile(this.filePath, JSON.stringify(this.cache, null, 2), 'utf-8'); + } + + /** + * 获取指定文档的记忆 + */ + async get(nodeId) { + const memory = await this.load(); + return memory[nodeId] || null; + } + + /** + * 更新文档记忆 + */ + async set(nodeId, { headerLength, lastBriefTitle }) { + const memory = await this.load(); + memory[nodeId] = { + header_length: headerLength, + last_brief_title: lastBriefTitle, + last_updated: new Date().toISOString(), + }; + await this.save(); + } + + /** + * 找到文档 header 结束位置（第一个 # 标题的行号） + * 移植自 Python _find_header_end() + */ + static findHeaderEnd(content) { + const lines = content.split('\n'); + for (let i = 0; i < lines.length; i++) { + // 跳过空行和文档头部（通常以 # 开头的项目标题行之前） + if (lines[i].startsWith('# ') && i > 0) { + return i; + } + } + // 如果没有找到，返回第一个非空行之后 + for (let i = 0; i < lines.length; i++) { + if (lines[i].trim() !== '') return i + 1; + } + return 0; + } +} diff --git a/services/dingtalk__doc/lib/dws-runner.js b/services/dingtalk__doc/lib/dws-runner.js new file mode 100644 index 00000000..3b853339 --- /dev/null +++ b/services/dingtalk__doc/lib/dws-runner.js @@ -0,0 +1,128 @@ +/** + * dws CLI 子进程封装 + * 移植自 Python 版本 src/core/dws.py + * + * 注意事项（踩坑记录）： + * - 所有命令自动附加 --yes 跳过交互确认 + * - 临时文件必须用 UTF-8 无 BOM 写入 + * - doc update 需要 --mode overwrite + --content-file（不是 --text） + * - 大文档（>10000字）不能用 overwrite，要用块级编辑 + */ + +import { execFile } from 'child_process'; +import { writeFile, unlink } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { randomBytes } from 'crypto'; + +/** + * 把字符串清洗为合法 UTF-8：lone surrogate 和非法序列替换为 U+FFFD。 + * 钉钉文档内容偶含 lone surrogate，若不清洗，agent 把内容转给 LLM 时 + * gRPC/protobuf 序列化会报 "string field contains invalid UTF-8" 导致整轮运行失败。 + * TextEncoder 会把 lone surrogate 编成 U+FFFD 字节，TextDecoder 再解回干净字符串。 + */ +const sanitizeUtf8 = (s) => { + if (typeof s !== 'string' || s === '') return s; + try { + return new TextDecoder('utf-8', { fatal: false }).decode(new TextEncoder().encode(s)); + } catch { + return s.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + if (typeof value === 'string') return sanitizeUtf8(value); + if (Array.isArray(value)) return value.map(sanitizeDeep); + if (value && typeof value === 'object') { + const out = {}; + for (const k of Object.keys(value)) out[k] = sanitizeDeep(value[k]); + return out; + } + return value; +}; + + +/** + * 执行 dws CLI 命令 + * @param {string} command - dws 子命令（如 "doc read --node-id xxx"） + * @param {object} options - 选项 + * @param {string} options.dwsPath - dws 可执行文件路径 + * @param {number} options.timeout - 超时毫秒数 + * @returns {Promise<{success: boolean, data: any, rawOutput: string, error: string}>} + */ +export async function runDws(command, options = {}) { + const dwsPath = options.dwsPath || process.env.DWS_PATH || 'dws'; + const timeout = options.timeout || 60000; + + // 始终附加 --yes 跳过交互确认 + const fullCommand = `${command} --yes`; + + return new Promise((resolve) => { + execFile('sh', ['-c', `${dwsPath} ${fullCommand}`], { + timeout, + maxBuffer: 10 * 1024 * 1024, // 10MB + }, (error, stdout, stderr) => { + if (error && error.killed) { + resolve({ success: false, data: null, rawOutput: '', error: `dws command timed out after ${timeout}ms` }); + return; + } + + const rawOutput = sanitizeUtf8(stdout.trim()); + + // 尝试解析 JSON 输出 + let data = null; + try { + data = JSON.parse(rawOutput); + // 检查业务层错误 + if (data && data.success === false) { + resolve({ success: false, data: sanitizeDeep(data), rawOutput, error: sanitizeUtf8(data.message || 'Business error') }); + return; + } + } catch { + // 非 JSON 输出，返回原始文本 + data = rawOutput; + } + + // 统一清洗 data 中的字符串，剔除 lone surrogate / 非法 UTF-8 + data = sanitizeDeep(data); + + if (error && error.code !== 0) { + resolve({ success: false, data, rawOutput, error: sanitizeUtf8(stderr || error.message) }); + return; + } + + resolve({ success: true, data, rawOutput, error: '' }); + }); + }); +} + +/** + * 写入临时文件（UTF-8 无 BOM） + * 移植自 Python 版本的 mode="wb" + .encode("utf-8") + */ +export async function writeTempFile(content) { + const suffix = randomBytes(4).toString('hex'); + const filePath = join(tmpdir(), `dws-temp-${suffix}.md`); + // Node.js Buffer.from(string, 'utf8') 不带 BOM；先清洗 lone surrogate 保证写入合法 UTF-8 + await writeFile(filePath, Buffer.from(sanitizeUtf8(content), 'utf8')); + return filePath; +} + +/** + * 清理临时文件 + */ +export async function cleanupTempFile(filePath) { + try { + await unlink(filePath); + } catch { + // 忽略清理失败 + } +} + +/** + * Shell 转义 + */ +export function shellEscape(str) { + return `'${str.replace(/'/g, "'\\''")}'`; +} diff --git a/services/dingtalk__doc/lib/safety-check.js b/services/dingtalk__doc/lib/safety-check.js new file mode 100644 index 00000000..093a1c53 --- /dev/null +++ b/services/dingtalk__doc/lib/safety-check.js @@ -0,0 +1,60 @@ +/** + * 安全检查 + 备份 + * 移植自 Python 版本 src/brief/document_ops.py + * + * 安全策略： + * - 新内容长度不能小于原文档的 50%（防止数据丢失） + * - 每次更新前自动备份原文档 + * + * 踩坑记录： + * - 文档读取的内容含 HTML span 标签，原样写回会被 API 拒绝 "invalid control characters" + * - 需要在写入前清理这些标签 + */ + +import { writeFile, mkdir, copyFile } from 'fs/promises'; +import { join } from 'path'; + +/** + * 安全检查：新内容不能太短 + * @param {string} originalContent - 原文档内容 + * @param {string} newContent - 新文档内容 + * @returns {{safe: boolean, reason: string}} + */ +export function safetyCheck(originalContent, newContent) { + if (!originalContent || originalContent.length === 0) { + return { safe: true, reason: '' }; + } + + const ratio = newContent.length / originalContent.length; + if (ratio < 0.5) { + return { + safe: false, + reason: `新内容长度 (${newContent.length}) 仅为原文档 (${originalContent.length}) 的 ${(ratio * 100).toFixed(1)}%，可能丢失数据`, + }; + } + + return { safe: true, reason: '' }; +} + +/** + * 清理 HTML span 标签 + * 钉钉文档 API 返回的内容可能含 标签，写回时会被拒绝 + */ +export function cleanHtmlTags(content) { + // 移除 和 标签，保留内容 + return content.replace(/<\/?span[^>]*>/g, ''); +} + +/** + * 备份文档到指定目录 + */ +export async function backupDocument(nodeId, content, backupDir) { + await mkdir(backupDir, { recursive: true }); + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `${nodeId}_${timestamp}.md`; + const filepath = join(backupDir, filename); + + await writeFile(filepath, content, 'utf-8'); + return filepath; +} diff --git a/services/dingtalk__doc/package.json b/services/dingtalk__doc/package.json new file mode 100644 index 00000000..b183edbb --- /dev/null +++ b/services/dingtalk__doc/package.json @@ -0,0 +1,14 @@ +{ + "name": "@office-agent/dingtalk-doc", + "displayName": "DingTalk Doc", + "version": "0.1.0", + "description": "DingTalk Doc — document read/write, folder management, knowledge base operations", + "type": "module", + "main": "bin/doc-service.js", + "scripts": { + "start": "node bin/doc-service.js" + }, + "dependencies": { + "@chaitin-ai/octobus-sdk": "^0.1.0" + } +} diff --git a/services/dingtalk__doc/proto/doc.proto b/services/dingtalk__doc/proto/doc.proto new file mode 100644 index 00000000..9b7e0eb5 --- /dev/null +++ b/services/dingtalk__doc/proto/doc.proto @@ -0,0 +1,225 @@ +syntax = "proto3"; +package dingtalk.doc.v1; + +// 钉钉文档操作服务 +// 封装 dws CLI 的文档读写、块编辑、备份等操作 +service DocService { + // 读取文档全文 + rpc ReadDoc(ReadDocRequest) returns (ReadDocResponse); + // 全文覆盖更新（<10000字的文档） + rpc UpdateDoc(UpdateDocRequest) returns (UpdateDocResponse); + // 创建新文档 + rpc CreateDoc(CreateDocRequest) returns (CreateDocResponse); + // 倒序插入简报（内部自动处理记忆+安全校验+备份） + // 这是一个复合方法，Agent 只需调用一次 + rpc InsertBrief(InsertBriefRequest) returns (InsertBriefResponse); + // 块级操作（用于周记录等大文档编辑） + rpc ListBlocks(ListBlocksRequest) returns (ListBlocksResponse); + rpc UpdateBlock(UpdateBlockRequest) returns (UpdateBlockResponse); + rpc InsertBlockAfter(InsertBlockAfterRequest) returns (InsertBlockAfterResponse); + // 文档搜索 + rpc SearchDocs(SearchDocsRequest) returns (SearchDocsResponse); + // 获取上次交流上下文（从记忆读取） + rpc GetLastContext(GetLastContextRequest) returns (GetLastContextResponse); + // 周会记录同步（块级编辑，移植自 Python weekly_sync） + rpc SyncWeekly(SyncWeeklyRequest) returns (SyncWeeklyResponse); + // 移动文档到指定文件夹（封装 dws doc move，用于整理归档） + rpc MoveDoc(MoveDocRequest) returns (MoveDocResponse); + // 创建文件夹（封装 dws doc folder create） + rpc CreateFolder(CreateFolderRequest) returns (CreateFolderResponse); + // 列出文件夹下的文档节点（封装 dws doc list，用于整理归档时查找散落文档） + rpc ListDocs(ListDocsRequest) returns (ListDocsResponse); +} + +// ====== ReadDoc ====== +message ReadDocRequest { + string node_id = 1; // 钉钉文档 nodeId +} + +message ReadDocResponse { + bool success = 1; + string content = 2; // 文档 Markdown 内容 + string error = 3; +} + +// ====== UpdateDoc ====== +message UpdateDocRequest { + string node_id = 1; + string content = 2; // 新的完整内容 +} + +message UpdateDocResponse { + bool success = 1; + string error = 2; +} + +// ====== CreateDoc ====== +message CreateDocRequest { + string title = 1; // 文档标题 + string content = 2; // 初始内容 + string parent_folder = 3; // 父文件夹 nodeId（可选） +} + +message CreateDocResponse { + bool success = 1; + string node_id = 2; // 新文档的 nodeId + string url = 3; // 文档访问链接 + string error = 4; +} + +// ====== InsertBrief ====== +message InsertBriefRequest { + string node_id = 1; // 目标文档 nodeId + string brief_content = 2; // 简报 Markdown 内容 + string title = 3; // 简报标题（用于记忆） +} + +message InsertBriefResponse { + bool success = 1; + string backup_path = 2; // 备份文件路径 + string error = 3; +} + +// ====== ListBlocks ====== +message ListBlocksRequest { + string node_id = 1; + int32 start_index = 2; // 起始索引（0-based） + int32 end_index = 3; // 结束索引（不含） +} + +message BlockInfo { + string block_id = 1; + int32 index = 2; + string block_type = 3; // heading, paragraph, list, etc. + string content = 4; // 块的文本内容 + int32 level = 5; // heading level (1-6) +} + +message ListBlocksResponse { + bool success = 1; + repeated BlockInfo blocks = 2; + string error = 3; +} + +// ====== UpdateBlock ====== +message UpdateBlockRequest { + string node_id = 1; + string block_id = 2; + string content = 3; // 新的块内容 +} + +message UpdateBlockResponse { + bool success = 1; + string error = 2; +} + +// ====== InsertBlockAfter ====== +message InsertBlockAfterRequest { + string node_id = 1; + string after_block_id = 2; // 在此块之后插入 + string block_type = 3; // heading / paragraph / list + string content = 4; + int32 level = 5; // heading level（仅 heading 有效） +} + +message InsertBlockAfterResponse { + bool success = 1; + string new_block_id = 2; + string error = 3; +} + +// ====== SearchDocs ====== +message SearchDocsRequest { + string keyword = 1; + int32 max_results = 2; // 默认 10 +} + +message DocSearchResult { + string node_id = 1; + string title = 2; + string url = 3; + string snippet = 4; // 匹配片段 +} + +message SearchDocsResponse { + bool success = 1; + repeated DocSearchResult results = 2; + string error = 3; +} + +// ====== GetLastContext ====== +message GetLastContextRequest { + string node_id = 1; // 文档 nodeId + int32 max_length = 2; // 最大返回字符数（默认 800） +} + +message GetLastContextResponse { + bool success = 1; + string context = 2; // 上次交流的简报内容 + string last_title = 3; // 上次简报标题 + string error = 4; +} + +// ====== SyncWeekly ====== +message SyncWeeklyRequest { + string node_id = 1; // 周会记录文档 nodeId + string user_name = 2; // Target user name (e.g. "Alice") + repeated WeeklyEntry entries = 3; // 待同步条目 +} + +message WeeklyEntry { + string summary = 1; // 一句话摘要 + string category = 2; // 沟通交流/文档材料/投标谈判/POC及其他 + string date = 3; // YYYY-MM-DD +} + +message SyncWeeklyResponse { + bool success = 1; + string week_id = 2; // 实际写入的周日期 + int32 synced_count = 3; // 成功写入的条目数 + bool saved_for_later = 4; // true = 未找到目标段落，条目已留存 + string error = 5; +} + +// ====== MoveDoc ====== +message MoveDocRequest { + string node_id = 1; // 要移动的文档 nodeId + string target_folder_id = 2; // 目标父文件夹 nodeId +} + +message MoveDocResponse { + bool success = 1; + string error = 2; +} + +// ====== CreateFolder ====== +message CreateFolderRequest { + string name = 1; // 文件夹名称（必填） + string parent_folder_id = 2; // 父文件夹 nodeId（可选，空=根目录） +} + +message CreateFolderResponse { + bool success = 1; + string folder_id = 2; // 新建文件夹的 nodeId + string error = 3; +} + +// ====== ListDocs ====== +message ListDocsRequest { + string folder_id = 1; // 父文件夹 nodeId（空=根目录） + int32 page_size = 2; // 每页数量（默认 50） +} + +message DocNodeInfo { + string node_id = 1; + string title = 2; + string type = 3; // doc / folder + string url = 4; +} + +message ListDocsResponse { + bool success = 1; + repeated DocNodeInfo docs = 2; + string next_page_token = 3; // 分页 token（空=无下一页） + string error = 4; +} diff --git a/services/dingtalk__doc/secret.schema.json b/services/dingtalk__doc/secret.schema.json new file mode 100644 index 00000000..e1a8346a --- /dev/null +++ b/services/dingtalk__doc/secret.schema.json @@ -0,0 +1,4 @@ +{ + "type": "object", + "properties": {} +} diff --git a/services/dingtalk__doc/service.json b/services/dingtalk__doc/service.json new file mode 100644 index 00000000..36196409 --- /dev/null +++ b/services/dingtalk__doc/service.json @@ -0,0 +1,12 @@ +{ + "schema": "chaitin.octobus.service.v1", + "name": "dingtalk__doc", + "displayName": "DingTalk Document Service", + "description": "DingTalk Document integration — document read/write, block editing, and briefing insertion via dws CLI.", + "proto": { + "roots": ["proto"], + "files": ["proto/doc.proto"] + }, + "configSchema": "config.schema.json", + "secretSchema": "secret.schema.json" +} diff --git a/services/dingtalk__drive/bin/drive-service.js b/services/dingtalk__drive/bin/drive-service.js new file mode 100644 index 00000000..b1e07530 --- /dev/null +++ b/services/dingtalk__drive/bin/drive-service.js @@ -0,0 +1,252 @@ +#!/usr/bin/env node +/** + * 钉钉云盘（DingTalk Drive）— OctoBus 服务包入口 + * + * 封装 dws drive CLI 的所有操作： + * - 文件上传（UploadFile） + * - 文件下载（DownloadFile） + * - 文件列表（ListFiles） + * - 空间列表（ListSpaces） + * - 文件元信息（GetFileInfo） + * - 文件夹管理（CreateFolder / DeleteFile） + * - 文档空间上传（UploadToDocSpace） + * + * 设计原则： + * - 每个 RPC 对应一次或多次 dws CLI 调用 + * - 统一错误处理和超时控制 + * - camelCase/snake_case 双兼容（OctoBus 可能传 camelCase） + */ + +import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; +import { execFile } from 'child_process'; + +// ─── dws CLI 封装 ─── + +function runDws(command, timeout = 120000) { + const dwsPath = process.env.DWS_PATH || 'dws'; + return new Promise((resolve) => { + execFile('sh', ['-c', `${dwsPath} ${command} --yes --format json`], { timeout, maxBuffer: 50 * 1024 * 1024 }, + (error, stdout) => { + const raw = stdout.trim(); + let data = null; + try { data = JSON.parse(raw); } catch { data = raw; } + if (error && error.code !== 0) resolve({ success: false, data, error: error.message }); + else resolve({ success: true, data, error: '' }); + }); + }); +} + +const shellEscape = (s) => "'" + String(s).replace(/'/g, "'\\''") + "'"; + +// ─── 兼容字段提取 ─── + +const get = (r, ...keys) => { + for (const k of keys) if (r[k] !== undefined && r[k] !== null && r[k] !== '') return r[k]; + return undefined; +}; + +// ─── 服务定义 ─── + +const service = defineService({ + handlers: { + + // ====== 上传文件到云盘 ====== + 'dingtalk.drive.v1.DriveService/UploadFile': async (ctx) => { + const r = ctx.request; + const filePath = get(r, 'filePath', 'file_path'); + const fileName = get(r, 'fileName', 'file_name'); + const spaceId = get(r, 'spaceId', 'space_id') || process.env.DEFAULT_SPACE_ID || ''; + const folderId = get(r, 'folderId', 'folder_id'); + const mimeType = get(r, 'mimeType', 'mime_type'); + + if (!filePath) return { success: false, error: 'filePath is required' }; + + let cmd = `drive upload --file ${shellEscape(filePath)}`; + if (fileName) cmd += ` --file-name ${shellEscape(fileName)}`; + if (spaceId) cmd += ` --space-id ${shellEscape(spaceId)}`; + if (folderId) cmd += ` --folder ${shellEscape(folderId)}`; + if (mimeType) cmd += ` --mime-type ${shellEscape(mimeType)}`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, error: res.error }; + + const d = res.data || {}; + return { + success: true, + dentryId: d.dentryId || d.dentry_id || d.nodeId || '', + fileUrl: d.url || d.fileUrl || d.file_url || '', + error: '', + }; + }, + + // ====== 下载文件 ====== + 'dingtalk.drive.v1.DriveService/DownloadFile': async (ctx) => { + const r = ctx.request; + const dentryId = get(r, 'dentryId', 'dentry_id'); + const outputPath = get(r, 'outputPath', 'output_path'); + + if (!dentryId) return { success: false, error: 'dentryId is required' }; + if (!outputPath) return { success: false, error: 'outputPath is required' }; + + const cmd = `drive download --node ${shellEscape(dentryId)} --output ${shellEscape(outputPath)}`; + const res = await runDws(cmd); + if (!res.success) return { success: false, error: res.error }; + + const d = res.data || {}; + return { + success: true, + localPath: d.localPath || d.local_path || outputPath, + fileSize: d.size || d.fileSize || d.file_size || 0, + error: '', + }; + }, + + // ====== 列出文件 ====== + 'dingtalk.drive.v1.DriveService/ListFiles': async (ctx) => { + const r = ctx.request; + const spaceId = get(r, 'spaceId', 'space_id') || process.env.DEFAULT_SPACE_ID || ''; + const folderId = get(r, 'folderId', 'folder_id'); + const maxResults = get(r, 'maxResults', 'max_results') || 20; + const nextToken = get(r, 'nextToken', 'next_token'); + + let cmd = `drive list`; + if (spaceId) cmd += ` --space-id ${shellEscape(spaceId)}`; + if (folderId) cmd += ` --folder ${shellEscape(folderId)}`; + if (maxResults) cmd += ` --max-results ${maxResults}`; + if (nextToken) cmd += ` --next-token ${shellEscape(nextToken)}`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, files: [], error: res.error }; + + const d = res.data || {}; + const items = (d.items || d.files || d.entries || []); + const files = items.map(f => ({ + dentryId: f.dentryId || f.dentry_id || f.nodeId || '', + name: f.name || '', + type: f.type || (f.isFolder ? 'folder' : 'file'), + size: f.size || 0, + updatedAt: f.updatedAt || f.updated_at || f.modifiedTime || '', + url: f.url || f.fileUrl || '', + })); + + return { + success: true, + files, + nextToken: d.nextToken || d.next_token || '', + error: '', + }; + }, + + // ====== 列出空间 ====== + 'dingtalk.drive.v1.DriveService/ListSpaces': async (ctx) => { + const r = ctx.request; + const spaceType = get(r, 'spaceType', 'space_type'); + + let cmd = `drive list-spaces`; + if (spaceType) cmd += ` --type ${shellEscape(spaceType)}`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, spaces: [], error: res.error }; + + const d = res.data || {}; + const items = (d.items || d.spaces || d.entries || []); + const spaces = items.map(s => ({ + spaceId: s.spaceId || s.space_id || s.id || '', + name: s.name || '', + type: s.type || s.spaceType || '', + })); + + return { success: true, spaces, error: '' }; + }, + + // ====== 获取文件信息 ====== + 'dingtalk.drive.v1.DriveService/GetFileInfo': async (ctx) => { + const r = ctx.request; + const dentryId = get(r, 'dentryId', 'dentry_id'); + + if (!dentryId) return { success: false, error: 'dentryId is required' }; + + const cmd = `drive info --node ${shellEscape(dentryId)}`; + const res = await runDws(cmd); + if (!res.success) return { success: false, error: res.error }; + + const d = res.data || {}; + return { + success: true, + name: d.name || '', + type: d.type || '', + size: d.size || 0, + url: d.url || d.fileUrl || '', + spaceId: d.spaceId || d.space_id || '', + createdAt: d.createdAt || d.created_at || d.createTime || '', + updatedAt: d.updatedAt || d.updated_at || d.modifiedTime || '', + error: '', + }; + }, + + // ====== 创建文件夹 ====== + 'dingtalk.drive.v1.DriveService/CreateFolder': async (ctx) => { + const r = ctx.request; + const name = get(r, 'name'); + const spaceId = get(r, 'spaceId', 'space_id') || process.env.DEFAULT_SPACE_ID || ''; + const parentFolderId = get(r, 'parentFolderId', 'parent_folder_id'); + + if (!name) return { success: false, error: 'name is required' }; + + let cmd = `drive mkdir --name ${shellEscape(name)}`; + if (spaceId) cmd += ` --space-id ${shellEscape(spaceId)}`; + if (parentFolderId) cmd += ` --parent ${shellEscape(parentFolderId)}`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, error: res.error }; + + const d = res.data || {}; + return { + success: true, + dentryId: d.dentryId || d.dentry_id || d.nodeId || '', + error: '', + }; + }, + + // ====== 删除文件 ====== + 'dingtalk.drive.v1.DriveService/DeleteFile': async (ctx) => { + const r = ctx.request; + const dentryId = get(r, 'dentryId', 'dentry_id'); + + if (!dentryId) return { success: false, error: 'dentryId is required' }; + + const cmd = `drive delete --node ${shellEscape(dentryId)}`; + const res = await runDws(cmd); + return { success: res.success, error: res.error || '' }; + }, + + // ====== 上传到文档空间 ====== + 'dingtalk.drive.v1.DriveService/UploadToDocSpace': async (ctx) => { + const r = ctx.request; + const filePath = get(r, 'filePath', 'file_path'); + const folderNodeId = get(r, 'folderNodeId', 'folder_node_id'); + const workspaceId = get(r, 'workspaceId', 'workspace_id'); + const convert = get(r, 'convert') || false; + + if (!filePath) return { success: false, error: 'filePath is required' }; + + let cmd = `doc upload --file ${shellEscape(filePath)}`; + if (folderNodeId) cmd += ` --folder ${shellEscape(folderNodeId)}`; + if (workspaceId) cmd += ` --workspace ${shellEscape(workspaceId)}`; + if (convert) cmd += ` --convert`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, error: res.error }; + + const d = res.data || {}; + return { + success: true, + nodeId: d.nodeId || d.node_id || d.dentryId || '', + docUrl: d.docUrl || d.doc_url || d.url || '', + error: '', + }; + }, + }, +}); + +runServiceMain(service); diff --git a/services/dingtalk__drive/config.schema.json b/services/dingtalk__drive/config.schema.json new file mode 100644 index 00000000..07cd1412 --- /dev/null +++ b/services/dingtalk__drive/config.schema.json @@ -0,0 +1,20 @@ +{ + "schema": "chaitin.octobus.config.v1", + "properties": { + "dwsPath": { + "type": "string", + "description": "dws CLI executable path", + "default": "dws" + }, + "defaultSpaceId": { + "type": "string", + "description": "Default drive space ID for uploads", + "default": "" + }, + "dwsTimeout": { + "type": "integer", + "description": "Default timeout for dws commands in milliseconds", + "default": 120000 + } + } +} diff --git a/services/dingtalk__drive/package.json b/services/dingtalk__drive/package.json new file mode 100644 index 00000000..676f0299 --- /dev/null +++ b/services/dingtalk__drive/package.json @@ -0,0 +1,15 @@ +{ + "name": "@office-agent/dingtalk-drive", + "displayName": "DingTalk Drive", + "version": "0.1.0", + "description": "DingTalk Drive — file upload/download, folder management", + "type": "module", + "main": "bin/drive-service.js", + "bin": "bin/drive-service.js", + "scripts": { + "start": "node bin/drive-service.js" + }, + "dependencies": { + "@chaitin-ai/octobus-sdk": "^0.5.0" + } +} diff --git a/services/dingtalk__drive/proto/drive.proto b/services/dingtalk__drive/proto/drive.proto new file mode 100644 index 00000000..8e95d221 --- /dev/null +++ b/services/dingtalk__drive/proto/drive.proto @@ -0,0 +1,155 @@ +syntax = "proto3"; + +package dingtalk.drive.v1; + +service DriveService { + // 上传文件到云盘 + rpc UploadFile(UploadFileRequest) returns (UploadFileResponse); + // 从云盘下载文件 + rpc DownloadFile(DownloadFileRequest) returns (DownloadFileResponse); + // 列出文件/文件夹 + rpc ListFiles(ListFilesRequest) returns (ListFilesResponse); + // 列出可用空间 + rpc ListSpaces(ListSpacesRequest) returns (ListSpacesResponse); + // 获取文件元信息 + rpc GetFileInfo(GetFileInfoRequest) returns (GetFileInfoResponse); + // 创建文件夹 + rpc CreateFolder(CreateFolderRequest) returns (CreateFolderResponse); + // 删除文件 + rpc DeleteFile(DeleteFileRequest) returns (DeleteFileResponse); + // 上传文件到文档空间（可作为附件插入文档） + rpc UploadToDocSpace(UploadToDocSpaceRequest) returns (UploadToDocSpaceResponse); +} + +// ====== 上传文件到云盘 ====== + +message UploadFileRequest { + string file_path = 1; // 本地文件路径（容器内路径） + string file_name = 2; // 可选，覆盖文件名 + string space_id = 3; // 目标空间 ID + string folder_id = 4; // 目标文件夹 dentryUuid（可选） + string mime_type = 5; // MIME 类型（可选，自动检测） +} + +message UploadFileResponse { + bool success = 1; + string dentry_id = 2; // 上传后的文件 ID + string file_url = 3; // 文件访问 URL + string error = 4; +} + +// ====== 下载文件 ====== + +message DownloadFileRequest { + string dentry_id = 1; // 文件 ID + string output_path = 2; // 保存路径（容器内） +} + +message DownloadFileResponse { + bool success = 1; + string local_path = 2; // 下载后的本地路径 + int64 file_size = 3; // 文件大小（字节） + string error = 4; +} + +// ====== 列出文件 ====== + +message ListFilesRequest { + string space_id = 1; + string folder_id = 2; // 可选，不传则列出根目录 + int32 max_results = 3; // 默认 20 + string next_token = 4; // 分页 token +} + +message ListFilesResponse { + bool success = 1; + repeated FileEntry files = 2; + string next_token = 3; + string error = 4; +} + +message FileEntry { + string dentry_id = 1; + string name = 2; + string type = 3; // file / folder + int64 size = 4; + string updated_at = 5; + string url = 6; +} + +// ====== 列出空间 ====== + +message ListSpacesRequest { + string space_type = 1; // org / personal（可选） +} + +message ListSpacesResponse { + bool success = 1; + repeated SpaceEntry spaces = 2; + string error = 3; +} + +message SpaceEntry { + string space_id = 1; + string name = 2; + string type = 3; // org / personal +} + +// ====== 获取文件信息 ====== + +message GetFileInfoRequest { + string dentry_id = 1; +} + +message GetFileInfoResponse { + bool success = 1; + string name = 2; + string type = 3; + int64 size = 4; + string url = 5; + string space_id = 6; + string created_at = 7; + string updated_at = 8; + string error = 9; +} + +// ====== 创建文件夹 ====== + +message CreateFolderRequest { + string name = 1; + string space_id = 2; + string parent_folder_id = 3; // 父文件夹 ID（可选） +} + +message CreateFolderResponse { + bool success = 1; + string dentry_id = 2; + string error = 3; +} + +// ====== 删除文件 ====== + +message DeleteFileRequest { + string dentry_id = 1; +} + +message DeleteFileResponse { + bool success = 1; + string error = 2; +} + +// ====== 上传到文档空间 ====== + +message UploadToDocSpaceRequest { + string file_path = 1; // 本地文件路径 + string folder_node_id = 2; // 文档空间文件夹节点 ID + string workspace_id = 3; // 知识库 ID（可选） + bool convert = 4; // 是否转换为在线文档 +} + +message UploadToDocSpaceResponse { + bool success = 1; + string node_id = 2; // 上传后的文档节点 ID + string doc_url = 3; // 文档 URL + string error = 4; +} diff --git a/services/dingtalk__drive/secret.schema.json b/services/dingtalk__drive/secret.schema.json new file mode 100644 index 00000000..eb6e7838 --- /dev/null +++ b/services/dingtalk__drive/secret.schema.json @@ -0,0 +1,4 @@ +{ + "schema": "chaitin.octobus.secret.v1", + "properties": {} +} diff --git a/services/dingtalk__drive/service.json b/services/dingtalk__drive/service.json new file mode 100644 index 00000000..79e3a0aa --- /dev/null +++ b/services/dingtalk__drive/service.json @@ -0,0 +1,12 @@ +{ + "schema": "chaitin.octobus.service.v1", + "name": "dingtalk__drive", + "displayName": "DingTalk Drive Service", + "description": "DingTalk Drive integration — file upload/download and folder management via dws CLI.", + "proto": { + "roots": ["proto"], + "files": ["proto/drive.proto"] + }, + "configSchema": "config.schema.json", + "secretSchema": "secret.schema.json" +} diff --git a/services/dingtalk__group-robot/proto/dingtalk_group_robot.proto b/services/dingtalk__group-robot/proto/dingtalk_group_robot.proto index 87c7aa5b..2a5e2fdb 100644 --- a/services/dingtalk__group-robot/proto/dingtalk_group_robot.proto +++ b/services/dingtalk__group-robot/proto/dingtalk_group_robot.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package DingDing_GroupRobot; -option go_package = "miner/grpc-service/DingDing_GroupRobot"; +option go_package = "octobus/dingtalk_group_robot"; // DingDing_GroupRobot 提供钉钉群自定义机器人消息推送能力，通过统一 Engine gRPC service 封装钉钉 Webhook 接口。 service DingDing_GroupRobot { diff --git a/services/dingtalk__message/bin/service.js b/services/dingtalk__message/bin/service.js new file mode 100644 index 00000000..48c836b3 --- /dev/null +++ b/services/dingtalk__message/bin/service.js @@ -0,0 +1,50 @@ +#!/usr/bin/env node +/** + * DingTalk Message Service — OctoBus 服务包 + */ +import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; +import { execFile } from 'child_process'; + +function runDws(command, timeout = 60000) { + return new Promise((resolve) => { + execFile('sh', ['-c', `dws ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + (error, stdout) => { + const raw = stdout.trim(); + let data = null; + try { data = JSON.parse(raw); } catch { data = raw; } + if (error && error.code !== 0) resolve({ success: false, data, error: error.message }); + else resolve({ success: true, data, error: '' }); + }); + }); +} + +const shellEscape = (s) => "'" + String(s).replace(/'/g, "'\\''") + "'"; + +const service = defineService({ + handlers: { + 'dingtalk.message.v1.MessageService/SendDing': async (ctx) => { + const { userId, content } = ctx.request; + const cmd = `ding message send --user-ids ${shellEscape(userId)} --content ${shellEscape(content || '')}`; + const res = await runDws(cmd); + return { success: res.success, error: res.error }; + }, + + 'dingtalk.message.v1.MessageService/SendGroupMessage': async (ctx) => { + const { groupId, chatId, content } = ctx.request; + const conversationId = chatId || groupId; + if (!conversationId) return { success: false, error: 'chatId or groupId required' }; + const cmd = `chat message send --conversation-id ${shellEscape(conversationId)} --content ${shellEscape(content || '')} --content-type text`; + const res = await runDws(cmd); + return { success: res.success, error: res.error }; + }, + + 'dingtalk.message.v1.MessageService/SendMail': async (ctx) => { + const { to, subject, body } = ctx.request; + const cmd = `mail message send --to ${shellEscape(to)} --subject ${shellEscape(subject || '')} --body ${shellEscape(body || '')}`; + const res = await runDws(cmd); + return { success: res.success, error: res.error }; + }, + }, +}); + +runServiceMain(service); diff --git a/services/dingtalk__message/config.schema.json b/services/dingtalk__message/config.schema.json new file mode 100644 index 00000000..8ca24efa --- /dev/null +++ b/services/dingtalk__message/config.schema.json @@ -0,0 +1,10 @@ +{ + "type": "object", + "properties": { + "dwsPath": { + "type": "string", + "description": "dws CLI path", + "default": "dws" + } + } +} diff --git a/services/dingtalk__message/package.json b/services/dingtalk__message/package.json new file mode 100644 index 00000000..d09eb4f8 --- /dev/null +++ b/services/dingtalk__message/package.json @@ -0,0 +1,14 @@ +{ + "name": "@office-agent/dingtalk-message", + "displayName": "DingTalk Message", + "version": "0.1.0", + "description": "DingTalk Message — group chat messaging, robot push, notification delivery", + "type": "module", + "main": "bin/service.js", + "scripts": { + "start": "node bin/service.js" + }, + "dependencies": { + "@chaitin-ai/octobus-sdk": "^0.1.0" + } +} diff --git a/services/dingtalk__message/proto/message.proto b/services/dingtalk__message/proto/message.proto new file mode 100644 index 00000000..fc39db01 --- /dev/null +++ b/services/dingtalk__message/proto/message.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; +package dingtalk.message.v1; + +service MessageService { + rpc SendDing(SendDingRequest) returns (SendDingResponse); + rpc SendGroupMessage(SendGroupMessageRequest) returns (SendGroupMessageResponse); + rpc SendMail(SendMailRequest) returns (SendMailResponse); +} + +message SendDingRequest { + string user_id = 1; // 接收者 userId + string content = 2; // 消息内容 +} + +message SendDingResponse { + bool success = 1; + string error = 2; +} + +message SendGroupMessageRequest { + string group_id = 1; // 群 ID（可选，与 chat_id 二选一） + string chat_id = 2; // 会话 ID（可选） + string content = 3; +} + +message SendGroupMessageResponse { + bool success = 1; + string error = 2; +} + +message SendMailRequest { + string to = 1; // 收件人邮箱 + string subject = 2; + string body = 3; +} + +message SendMailResponse { + bool success = 1; + string error = 2; +} diff --git a/services/dingtalk__message/secret.schema.json b/services/dingtalk__message/secret.schema.json new file mode 100644 index 00000000..e1a8346a --- /dev/null +++ b/services/dingtalk__message/secret.schema.json @@ -0,0 +1,4 @@ +{ + "type": "object", + "properties": {} +} diff --git a/services/dingtalk__message/service.json b/services/dingtalk__message/service.json new file mode 100644 index 00000000..dd4afaa7 --- /dev/null +++ b/services/dingtalk__message/service.json @@ -0,0 +1,12 @@ +{ + "schema": "chaitin.octobus.service.v1", + "name": "dingtalk__message", + "displayName": "DingTalk Message Service", + "description": "DingTalk Message integration — DING messages, group messages, and email via dws CLI.", + "proto": { + "roots": ["proto"], + "files": ["proto/message.proto"] + }, + "configSchema": "config.schema.json", + "secretSchema": "secret.schema.json" +} diff --git a/services/dingtalk__todo/bin/service.js b/services/dingtalk__todo/bin/service.js new file mode 100644 index 00000000..be4b8d58 --- /dev/null +++ b/services/dingtalk__todo/bin/service.js @@ -0,0 +1,96 @@ +#!/usr/bin/env node +/** + * DingTalk Todo Service — OctoBus 服务包 + * 封装 dws CLI 的待办操作 + */ +import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; +import { execFile } from 'child_process'; + +function runDws(command, timeout = 60000) { + const dwsPath = 'dws'; + return new Promise((resolve) => { + execFile('sh', ['-c', `${dwsPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + (error, stdout) => { + const raw = stdout.trim(); + let data = null; + try { data = JSON.parse(raw); } catch { data = raw; } + if (error && error.code !== 0) resolve({ success: false, data, error: error.message }); + else resolve({ success: true, data, error: '' }); + }); + }); +} + +const shellEscape = (s) => "'" + String(s).replace(/'/g, "'\\''") + "'"; + +const service = defineService({ + handlers: { + 'dingtalk.todo.v1.TodoService/CreateTodo': async (ctx) => { + const { title, description, dueDate, assigneeId } = ctx.request; + // dws CLI flags: --title (required), --due (ISO-8601), --executors (required, userId comma-separated), --priority + // dws does NOT have --description; embed it in the title if provided + const displayTitle = description ? `${title || 'Untitled'} (${description})` : (title || 'Untitled'); + let cmd = `todo task create --title ${shellEscape(displayTitle)}`; + if (dueDate) cmd += ` --due ${shellEscape(dueDate)}`; + // executors is required by dws; use assigneeId, env USER_ID, or fallback + const executor = assigneeId || process.env.USER_ID || 'default'; + if (!assigneeId && !process.env.USER_ID) { + console.warn('[dingtalk-todo] assigneeId not provided and USER_ID not set, using "default"'); + } + cmd += ` --executors ${shellEscape(executor)}`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, todoId: '', error: res.error }; + + const r = res.data?.result || res.data; + const todoId = Array.isArray(r) && r.length > 0 ? (r[0].id || r[0].taskId || '') : ''; + return { success: true, todoId, error: '' }; + }, + + 'dingtalk.todo.v1.TodoService/ListTodos': async (ctx) => { + const { isDone, limit } = ctx.request; + let cmd = `todo task list --size ${limit || 10}`; + if (isDone === true) cmd += ` --status true`; + else if (isDone === false) cmd += ` --status false`; + + const res = await runDws(cmd); + if (!res.success) return { success: false, todos: [], error: res.error }; + + const rawList = res.data?.result?.todoList || res.data?.result || res.data?.todoList || []; + const list = Array.isArray(rawList) ? rawList : []; + const todos = list.map((t) => ({ + todoId: t.id || t.taskId || '', + title: t.subject || t.title || '', + description: t.description || '', + isDone: t.done === true || t.isDone === true, + dueDate: t.dueTime || t.dueDate || '', + createdAt: t.createdTime || t.createdAt || '', + })); + return { success: true, todos, error: '' }; + }, + + 'dingtalk.todo.v1.TodoService/MarkDone': async (ctx) => { + const { keyword } = ctx.request; + // First list all pending todos + const listRes = await runDws('todo task list --status false --size 50'); + if (!listRes.success) return { success: false, matchedCount: 0, error: listRes.error }; + + const rawList = listRes.data?.result?.todoList || listRes.data?.result || []; + const list = Array.isArray(rawList) ? rawList : []; + + let matched = 0; + for (const t of list) { + const subject = t.subject || t.title || ''; + if (keyword && subject.includes(keyword)) { + const id = t.id || t.taskId; + if (id) { + await runDws(`todo task update --task-id ${shellEscape(id)} --done true`); + matched++; + } + } + } + return { success: true, matchedCount: matched, error: '' }; + }, + }, +}); + +runServiceMain(service); diff --git a/services/dingtalk__todo/config.schema.json b/services/dingtalk__todo/config.schema.json new file mode 100644 index 00000000..8ca24efa --- /dev/null +++ b/services/dingtalk__todo/config.schema.json @@ -0,0 +1,10 @@ +{ + "type": "object", + "properties": { + "dwsPath": { + "type": "string", + "description": "dws CLI path", + "default": "dws" + } + } +} diff --git a/services/dingtalk__todo/package.json b/services/dingtalk__todo/package.json new file mode 100644 index 00000000..d96a6796 --- /dev/null +++ b/services/dingtalk__todo/package.json @@ -0,0 +1,14 @@ +{ + "name": "@office-agent/dingtalk-todo", + "displayName": "DingTalk Todo", + "version": "0.1.0", + "description": "DingTalk Todo — task creation, completion, queries, and reminders", + "type": "module", + "main": "bin/service.js", + "scripts": { + "start": "node bin/service.js" + }, + "dependencies": { + "@chaitin-ai/octobus-sdk": "^0.1.0" + } +} diff --git a/services/dingtalk__todo/proto/todo.proto b/services/dingtalk__todo/proto/todo.proto new file mode 100644 index 00000000..d77566a9 --- /dev/null +++ b/services/dingtalk__todo/proto/todo.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; +package dingtalk.todo.v1; + +service TodoService { + rpc CreateTodo(CreateTodoRequest) returns (CreateTodoResponse); + rpc ListTodos(ListTodosRequest) returns (ListTodosResponse); + rpc MarkDone(MarkDoneRequest) returns (MarkDoneResponse); +} + +message CreateTodoRequest { + string title = 1; + string description = 2; + string due_date = 3; // ISO 时间（可选） + string assignee_id = 4; // 钉钉 userId（默认自己） +} + +message CreateTodoResponse { + bool success = 1; + string todo_id = 2; + string error = 3; +} + +message ListTodosRequest { + bool is_done = 1; // 筛选完成状态（可选） + int32 limit = 2; +} + +message TodoItem { + string todo_id = 1; + string title = 2; + string description = 3; + bool is_done = 4; + string due_date = 5; + string created_at = 6; +} + +message ListTodosResponse { + bool success = 1; + repeated TodoItem todos = 2; + string error = 3; +} + +message MarkDoneRequest { + string keyword = 1; // 按关键词匹配待办标题 +} + +message MarkDoneResponse { + bool success = 1; + int32 matched_count = 2; + string error = 3; +} diff --git a/services/dingtalk__todo/secret.schema.json b/services/dingtalk__todo/secret.schema.json new file mode 100644 index 00000000..e1a8346a --- /dev/null +++ b/services/dingtalk__todo/secret.schema.json @@ -0,0 +1,4 @@ +{ + "type": "object", + "properties": {} +} diff --git a/services/dingtalk__todo/service.json b/services/dingtalk__todo/service.json new file mode 100644 index 00000000..77628f99 --- /dev/null +++ b/services/dingtalk__todo/service.json @@ -0,0 +1,12 @@ +{ + "schema": "chaitin.octobus.service.v1", + "name": "dingtalk__todo", + "displayName": "DingTalk Todo Service", + "description": "DingTalk Todo integration — todo creation, listing, and completion marking via dws CLI.", + "proto": { + "roots": ["proto"], + "files": ["proto/todo.proto"] + }, + "configSchema": "config.schema.json", + "secretSchema": "secret.schema.json" +} From 5ddb33a2b468aa6fe198bebe043c8ba4327e1fdf Mon Sep 17 00:00:00 2001 From: "jianhua.wang" Date: Sun, 5 Jul 2026 20:06:20 +0800 Subject: [PATCH 3/9] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20state-manager?= =?UTF-8?q?=20v3=20=E2=80=94=20=E9=80=9A=E7=94=A8=E8=AE=B0=E5=BF=86?= =?UTF-8?q?=E4=B8=8E=E8=92=B8=E9=A6=8F=E5=BC=95=E6=93=8E=EF=BC=8817=20?= =?UTF-8?q?=E9=80=9A=E7=94=A8=20RPC=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v3 核心改造：从 6 个智能体专属状态层升级为社区可用的 AI 记忆基础设施 改造内容： - 剥离 11 个业务 RPC（GetProject/MatchProject/AddPendingItem/RegisterMeeting/ UpdateMeetingState/GetPendingMeetings/ClearPending/GetPendingItems/ GetUserConfig/ListProjects/UpdateStep 等） - 保留 17 个通用 RPC：记忆(4) + 蒸馏(2) + 任务(6) + 事件(3) + KV(2) - 删除 user-config.json（业务专属配置），tenant/userId 改为环境变量注入 - 移除 js-yaml 依赖（不再需要加载 projects.yaml） - memory-engine: loadIdentity() → setIdentity()，不再依赖配置文件 - distillation-engine: weekly → summary（通用化命名） - type-registry: 描述改为通用场景（不再引用 CRM/kanban） - 新增 README.md + docs/v3-design.md 设计决策：业务逻辑归属 agent 层，OctoBus 只做通用能力 - 旧 MatchProject → Recall(prefix='project:') + agent LLM 判断 - 旧 AddPendingItem → Remember(type=commitment) - 旧 RegisterMeeting → Remember(type=commitment, key='meeting:{id}') - 旧 GetUserConfig → GetState(key='user_config') 或环境变量 Signed-off-by: jianhua.wang --- services/state-manager/README.md | 121 +++++ services/state-manager/bin/state-service.js | 472 ++---------------- services/state-manager/config.schema.json | 14 +- .../config/user-config.example.json | 58 --- services/state-manager/docs/v3-design.md | 170 +++++++ .../state-manager/lib/distillation-engine.js | 14 +- services/state-manager/lib/memory-engine.js | 17 +- services/state-manager/lib/type-registry.js | 4 +- services/state-manager/package.json | 7 +- services/state-manager/proto/state.proto | 279 ++--------- 10 files changed, 386 insertions(+), 770 deletions(-) create mode 100644 services/state-manager/README.md delete mode 100644 services/state-manager/config/user-config.example.json create mode 100644 services/state-manager/docs/v3-design.md diff --git a/services/state-manager/README.md b/services/state-manager/README.md new file mode 100644 index 00000000..2e197900 --- /dev/null +++ b/services/state-manager/README.md @@ -0,0 +1,121 @@ +# state-manager — Agent Memory & Distillation Engine + +A generic state management service for AI agents, built on [OctoBus](https://github.com/chaitin/OctoBus). + +## Why This Exists + +AI agents today are stateless — every conversation starts from zero. This service gives agents: + +- **Persistent memory** that survives across sessions (Remember/Recall/Forget) +- **Automatic distillation** that compresses old memories to prevent unbounded growth +- **Task tracking** with structured state machines (not just "I'll do it" → forgotten) +- **Event logging** for full audit trails + +The key differentiator is **distillation** — not just storing data, but actively compressing, summarizing, and prioritizing it. This transforms the service from passive storage into an active cognitive layer. + +## Architecture + +``` +┌─────────────────────────────────────────────┐ +│ gRPC / Connect API (17 RPCs) │ +├──────────┬──────────────┬──────────┬────────┤ +│ Memory │ Distillation │ Task │ Event │ +│ Engine │ Engine │ Machine │ Log │ +├──────────┴──────────────┴──────────┴────────┤ +│ Type Registry │ TTL Manager │ Index │ +├─────────────────────────────────────────────┤ +│ Storage Adapter (pluggable) │ +│ JSON File → SQLite → etcd → Redis │ +└─────────────────────────────────────────────┘ +``` + +## Five Memory Types + +| Type | TTL | Dedup | Use Case | +|------|-----|-------|----------| +| `entity_cache` | 7 days | None | Cache expensive query results | +| `action_log` | 7 days | 24h | Idempotent dedup + audit trail | +| `user_correction` | Forever | None | Highest-value: user fixes to agent behavior | +| `commitment` | 30 days | None | Track future promises ("follow up next week") | +| `session_summary` | 30 days | None | Key decisions + pending items per session | + +Add custom types via the Type Registry — no code changes needed. + +## Quick Start + +### 1. Install as OctoBus service + +```bash +octobus service import /path/to/state-manager +``` + +### 2. Configure environment + +```bash +export DATA_DIR=/path/to/data # Persistent storage directory +export TENANT=my_company # Tenant ID for key namespacing +export USER_ID=my_agent # User ID for memory isolation +``` + +### 3. Use from agent + +```bash +# Remember a query result +grpcurl -plaintext \ + -H "x-octobus-capset: default" \ + -H "x-octobus-instance: state-manager" \ + -d '{"type":"entity_cache","key":"customer:acme","value":"{\"id\":\"123\",\"name\":\"Acme Corp\"}"}' \ + $CAP_GRPC_TARGET state.manager.v1.StateManagerService/Remember + +# Recall it later +grpcurl -plaintext \ + -d '{"key":"customer:acme"}' \ + $CAP_GRPC_TARGET state.manager.v1.StateManagerService/Recall + +# Trigger memory compression +grpcurl -plaintext \ + -d '{}' \ + $CAP_GRPC_TARGET state.manager.v1.StateManagerService/Distill +``` + +## RPC Reference (17 methods) + +### Memory (4) +- `Remember(key, value, type, ttl_seconds, dedup_window_sec, confidence)` — Write memory +- `Recall(key | prefix, type, limit)` — Read memory +- `Forget(key | prefix, type)` — Delete memory +- `GetMemoryStats()` — Memory statistics + +### Distillation (2) +- `Distill()` — Trigger compression pass +- `GetDistillStats()` — Compression statistics + +### Task (6) +- `CreateTask(name, description, assignee, due_date, steps, priority)` — Create task +- `UpdateTask(task_id, new_state, reason)` — Update state (valid transitions only) +- `UpdateStep(task_id, step_id, name, state, notes)` — Update step +- `GetTask(task_id)` — Get task details +- `ListTasks(state, assignee, priority, due_before, due_after, limit)` — List with filters +- `GetTaskStats()` — Task statistics + +### Event Log (3) +- `LogEvent(event_type, actor, description, data, level)` — Append event +- `QueryEvents(event_type, actor, level, since, until, search, limit)` — Query events +- `GetEventStats()` — Event statistics + +### KV Store (2) +- `GetState(key)` — Read value +- `SetState(key, value)` — Write value + +## Design Decisions + +See [docs/v3-design.md](docs/v3-design.md) for the full design document, including: +- Five memory types and their lifecycle +- Three-level distillation strategy +- Business logic separation (service = generic primitives, agent = business logic) +- Multi-tenant key namespacing +- Pluggable storage adapter design + +## License + +Same as OctoBus. diff --git a/services/state-manager/bin/state-service.js b/services/state-manager/bin/state-service.js index cfc9f577..2c11b96d 100644 --- a/services/state-manager/bin/state-service.js +++ b/services/state-manager/bin/state-service.js @@ -1,30 +1,20 @@ #!/usr/bin/env node /** - * Agent State Manager - OctoBus Service Entry Point + * Agent State Manager — Generic Memory & Distillation Service * - * Capability modules (28 RPCs): - * 1. Project Config — read projects.yaml, provide query/match (3 RPCs) - * 2. Weekly Buffer — accumulate summaries, clear after sync (3 RPCs, delegated to memory engine commitment type) - * 3. Meeting Tracking — register meetings, track state, query pending (3 RPCs, delegated to memory engine commitment type) - * 4. User Config — read user-config.json personal config (1 RPC) - * 5. Memory Engine — Remember/Recall/Forget + stats (4 RPCs) - * 6. Distillation Engine — action_log compression + session summary + correction management (2 RPCs) - * 7. Task State Machine — task lifecycle + step tracking + timeout detection (6 RPCs) - * 8. Event Log — append-only event records + range queries (3 RPCs) - * 9. General Storage — simple key-value store (backward-compatible API) (2 RPCs) + * A generic state management service for AI agents, providing: + * 1. Memory Engine — Remember/Recall/Forget with typed storage, TTL, and dedup (4 RPCs) + * 2. Distillation Engine — Automatic memory compression (2 RPCs) + * 3. Task State Machine — Structured task lifecycle tracking (6 RPCs) + * 4. Event Log — Append-only event recording (3 RPCs) + * 5. KV Store — Simple key-value persistence (2 RPCs) * - * OctoBus service display name mapping: - * state-manager → Agent State Manager (this package) - * dingtalk-aitable → DingTalk AITable - * dingtalk-calendar → DingTalk Calendar - * dingtalk-doc → DingTalk Doc - * dingtalk-message → DingTalk Message - * dingtalk-todo → DingTalk Todo + * Business-specific logic (project matching, weekly reports, meeting tracking, + * CRM integration, etc.) should be implemented at the agent/skill layer using + * the generic primitives provided here (Remember/Recall/Forget + Task + EventLog). */ import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; -import { readFile, writeFile, mkdir } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; import { StorageAdapter } from '../lib/storage-adapter.js'; import { MemoryIndex } from '../lib/memory-index.js'; import { TTLManager } from '../lib/ttl-manager.js'; @@ -36,48 +26,10 @@ import { EventLog } from '../lib/event-log.js'; // ====== Config ====== const config = { dataDir: process.env.DATA_DIR || './data', - projectsConfigPath: process.env.PROJECTS_CONFIG_PATH || process.env.PROJECTS_CONFIG || './config/projects.yaml', - userConfigPath: process.env.USER_CONFIG_PATH || process.env.USER_CONFIG || './config/user-config.json', + tenant: process.env.TENANT || process.env.COMPANY_NAME || 'default', + userId: process.env.USER_ID || 'default', }; -// ====== Project Config Cache ====== -let projectsCache = null; -let projectsCacheTime = 0; -const CACHE_TTL = 60000; // 1 minute - -async function loadProjects() { - const now = Date.now(); - if (projectsCache && (now - projectsCacheTime) < CACHE_TTL) { - return projectsCache; - } - - const yaml = await import('js-yaml'); - const raw = await readFile(config.projectsConfigPath, 'utf-8'); - const parsed = yaml.load(raw); - - projectsCache = parsed; - projectsCacheTime = now; - return parsed; -} - -// ====== JSON Persistence Helpers ====== -async function loadJSON(filePath, defaultValue = {}) { - try { - const raw = await readFile(filePath, 'utf-8'); - return JSON.parse(raw); - } catch { - return defaultValue; - } -} - -async function saveJSON(filePath, data) { - await mkdir(dirname(filePath), { recursive: true }); - await writeFile(filePath, JSON.stringify(data, null, 2), 'utf-8'); -} - -// ====== Meeting State Machine (migrated to memory engine, this constant is for internal filtering only) ====== -const VALID_MEETING_STATES = ['registered', 'notes_submitted', 'doc_created', 'reminded']; - // ====== Memory Engine (global singleton) ====== const storageAdapter = new StorageAdapter(config.dataDir); const memoryIndex = new MemoryIndex(); @@ -95,8 +47,8 @@ const eventLog = new EventLog(config.dataDir); // ====== Startup Initialization ====== async function initialize() { - // 1. Load tenant/userId from user-config - await memoryEngine.loadIdentity(config.userConfigPath); + // 1. Set tenant/userId from environment + memoryEngine.setIdentity(config.tenant, config.userId); // 2. Load existing memories from files into index const allEntries = await storageAdapter.loadAll(); @@ -122,352 +74,13 @@ const initPromise = initialize(); const service = defineService({ handlers: { - // ====== Project Config ====== - - /** - * Get project config by key - */ - 'state.manager.v1.StateManagerService/GetProject': async (ctx) => { - const { key } = ctx.request; - try { - const parsed = await loadProjects(); - const projects = parsed.projects || {}; - const proj = projects[key]; - if (!proj) { - return { success: false, error: `Project "${key}" not found` }; - } - return { - success: true, - project: { - key, - name: proj.name || '', - nodeId: proj.node_id || '', - customer: proj.customer || '', - keywords: proj.keywords || [], - detailFolder: proj.detail_folder || '', - background: proj.background || '', - }, - error: '', - }; - } catch (err) { - return { success: false, error: err.message }; - } - }, - - /** - * List all projects - */ - 'state.manager.v1.StateManagerService/ListProjects': async () => { - try { - const parsed = await loadProjects(); - const projects = parsed.projects || {}; - const list = Object.entries(projects).map(([key, proj]) => ({ - key, - name: proj.name || '', - nodeId: proj.node_id || '', - customer: proj.customer || '', - keywords: proj.keywords || [], - detailFolder: proj.detail_folder || '', - background: proj.background || '', - })); - return { success: true, projects: list, error: '' }; - } catch (err) { - return { success: false, projects: [], error: err.message }; - } - }, - - /** - * Match project by keywords - * Ported from Python doc_creator.py match_customer_to_project() - */ - 'state.manager.v1.StateManagerService/MatchProject': async (ctx) => { - const { text } = ctx.request; - try { - const parsed = await loadProjects(); - const projects = parsed.projects || {}; - - let bestMatch = null; - let bestScore = 0; - - for (const [key, proj] of Object.entries(projects)) { - const keywords = proj.keywords || []; - for (const kw of keywords) { - if (text.includes(kw)) { - const score = kw.length / Math.max(text.length, 1); - if (score > bestScore) { - bestScore = score; - bestMatch = { - key, - name: proj.name || '', - nodeId: proj.node_id || '', - customer: proj.customer || '', - keywords, - detailFolder: proj.detail_folder || '', - background: proj.background || '', - }; - } - } - } - } - - return { - success: true, - project: bestMatch || { key: '', name: '', nodeId: '', customer: '', keywords: [], detailFolder: '', background: '' }, - confidence: bestMatch ? Math.min(bestScore * 5, 1.0) : 0, - error: '', - }; - } catch (err) { - return { success: false, confidence: 0, error: err.message }; - } - }, - - // ====== Weekly Pending Buffer (migrated to memory engine commitment type) ====== - - /** - * Add a pending weekly summary item - * Internally delegates to Remember(type=commitment, key=weekly_item:{date}:{hash}) - */ - 'state.manager.v1.StateManagerService/AddPendingItem': async (ctx) => { - await initPromise; - const { summary, category, date } = ctx.request; - const itemDate = date || new Date().toISOString().slice(0, 10); - const itemCategory = category || 'Communication'; - const key = `weekly_item:${itemDate}:${(summary || '').slice(0, 20)}`; - const value = JSON.stringify({ - summary: summary || '', - category: itemCategory, - date: itemDate, - }); - - try { - const result = await memoryEngine.remember({ - type: 'commitment', - key, - value, - ttl_seconds: 30 * 86400, // 30 days - }); - - // Count current pending total - const all = await memoryEngine.recall({ prefix: 'weekly_item:', type: 'commitment', limit: 100 }); - - return { success: result.success, pendingCount: all.entries.length, error: result.error || '' }; - } catch (err) { - return { success: false, pendingCount: 0, error: err.message }; - } - }, - - /** - * Get all pending sync items - * Internally delegates to Recall(prefix=weekly_item:, type=commitment) - */ - 'state.manager.v1.StateManagerService/GetPendingItems': async () => { - await initPromise; - try { - const result = await memoryEngine.recall({ prefix: 'weekly_item:', type: 'commitment', limit: 100 }); - const items = (result.entries || []).map((entry) => { - try { - const parsed = JSON.parse(entry.value); - return { - summary: parsed.summary || '', - category: parsed.category || '', - date: parsed.date || '', - }; - } catch { - return { summary: entry.value || '', category: '', date: '' }; - } - }); - return { success: true, items, error: '' }; - } catch (err) { - return { success: false, items: [], error: err.message }; - } - }, - - /** - * Clear pending buffer (called after successful sync) - * Internally delegates to Forget(prefix=weekly_item:, type=commitment) + records last_synced - */ - 'state.manager.v1.StateManagerService/ClearPending': async () => { - await initPromise; - try { - // Get current entry count (for returning clearedCount) - const all = await memoryEngine.recall({ prefix: 'weekly_item:', type: 'commitment', limit: 100 }); - const clearedCount = all.entries.length; - - // Batch delete all weekly_item entries - await memoryEngine.forget({ prefix: 'weekly_item:', type: 'commitment' }); - - // Record last_synced time - await memoryEngine.remember({ - type: 'commitment', - key: 'weekly:last_synced', - value: JSON.stringify({ last_synced: new Date().toISOString() }), - ttl_seconds: 30 * 86400, - }); - - return { success: true, clearedCount, error: '' }; - } catch (err) { - return { success: false, clearedCount: 0, error: err.message }; - } - }, - - // ====== Meeting Tracking (migrated to memory engine commitment type) ====== - - /** - * Register a meeting - * Internally delegates to Remember(type=commitment, key=meeting:{eventId}) - */ - 'state.manager.v1.StateManagerService/RegisterMeeting': async (ctx) => { - await initPromise; - const { eventId, summary, startTime, endTime, attendees } = ctx.request; - - try { - // Check if already exists - const existing = await memoryEngine.recall({ key: `meeting:${eventId}`, type: 'commitment' }); - - if (existing.entries && existing.entries.length > 0) { - // Already exists, skip but update last_check - return { success: true, isNew: false, error: '' }; - } - - // New meeting, write to memory engine - const value = JSON.stringify({ - eventId: eventId || '', - summary: summary || '', - startTime: startTime || '', - endTime: endTime || '', - attendees: attendees || [], - state: 'registered', - registeredAt: new Date().toISOString(), - }); - - await memoryEngine.remember({ - type: 'commitment', - key: `meeting:${eventId}`, - value, - ttl_seconds: 30 * 86400, // 30 days - }); - - return { success: true, isNew: true, error: '' }; - } catch (err) { - return { success: false, isNew: false, error: err.message }; - } - }, - - /** - * Get pending meetings (ended but no minutes submitted yet) - * Internally delegates to Recall(prefix=meeting:, type=commitment) + filtering - */ - 'state.manager.v1.StateManagerService/GetPendingMeetings': async (ctx) => { - await initPromise; - const graceHours = ctx.request.graceHours || 24; - - try { - const result = await memoryEngine.recall({ prefix: 'meeting:', type: 'commitment', limit: 200 }); - const now = new Date(); - const graceMs = graceHours * 3600 * 1000; - - const meetings = (result.entries || []) - .map((entry) => { - try { - return JSON.parse(entry.value); - } catch { - return null; - } - }) - .filter((m) => { - if (!m || m.state !== 'registered') return false; - if (!m.endTime && !m.end_time) return false; - const endTime = new Date(m.endTime || m.end_time); - return (now - endTime) > graceMs; - }) - .map((m) => ({ - eventId: m.eventId || m.event_id || '', - summary: m.summary || '', - startTime: m.startTime || m.start_time || '', - endTime: m.endTime || m.end_time || '', - attendees: m.attendees || [], - state: m.state || 'registered', - })); - - return { success: true, meetings, error: '' }; - } catch (err) { - return { success: false, meetings: [], error: err.message }; - } - }, - - /** - * Update meeting state - * Internally delegates to Recall + Remember to update memory entry - */ - 'state.manager.v1.StateManagerService/UpdateMeetingState': async (ctx) => { - await initPromise; - const { eventId, newState } = ctx.request; - - if (!VALID_MEETING_STATES.includes(newState)) { - return { success: false, error: `Invalid state "${newState}", valid values: ${VALID_MEETING_STATES.join(', ')}` }; - } - - try { - const existing = await memoryEngine.recall({ key: `meeting:${eventId}`, type: 'commitment' }); - - if (!existing.entries || existing.entries.length === 0) { - return { success: false, error: `Meeting "${eventId}" not found` }; - } - - const entry = existing.entries[0]; - let meeting; - try { - meeting = JSON.parse(entry.value); - } catch { - return { success: false, error: `Meeting "${eventId}" data corrupted` }; - } - - meeting.state = newState; - meeting.updatedAt = new Date().toISOString(); - - await memoryEngine.remember({ - type: 'commitment', - key: `meeting:${eventId}`, - value: JSON.stringify(meeting), - ttl_seconds: 30 * 86400, - }); - - return { success: true, error: '' }; - } catch (err) { - return { success: false, error: err.message }; - } - }, - - // ====== User Config ====== - - /** - * Read user personal config - * Config file: config/user-config.json (filled by deployer per actual environment) - */ - 'state.manager.v1.StateManagerService/GetUserConfig': async () => { - try { - const raw = await readFile(config.userConfigPath, 'utf-8'); - const userConfig = JSON.parse(raw); - return { success: true, config: userConfig, error: '' }; - } catch (err) { - if (err.code === 'ENOENT') { - return { - success: false, - config: null, - error: `User config file not found: ${config.userConfigPath}, please create and fill in personal config`, - }; - } - return { success: false, config: null, error: `Failed to read config: ${err.message}` }; - } - }, - // ====== Agent Memory ====== /** - * Write a memory entry + * Write a memory entry (upsert with dedup and TTL) */ 'state.manager.v1.StateManagerService/Remember': async (ctx) => { - await initPromise; // Ensure initialization is complete + await initPromise; const result = await memoryEngine.remember(ctx.request); // Log to event log (non-blocking) @@ -500,7 +113,7 @@ const service = defineService({ }, /** - * Delete memory + * Delete memory (exact or batch prefix delete) */ 'state.manager.v1.StateManagerService/Forget': async (ctx) => { await initPromise; @@ -545,7 +158,7 @@ const service = defineService({ // ====== Distillation Engine ====== /** - * Manually trigger a full distillation + * Trigger a distillation pass (compress old/low-priority memories) */ 'state.manager.v1.StateManagerService/Distill': async () => { await initPromise; @@ -589,7 +202,7 @@ const service = defineService({ // ====== Task State Machine ====== /** - * Create a task + * Create a task with optional steps */ 'state.manager.v1.StateManagerService/CreateTask': async (ctx) => { await initPromise; @@ -615,7 +228,7 @@ const service = defineService({ }, /** - * Update task state + * Update task state (enforces valid transitions) */ 'state.manager.v1.StateManagerService/UpdateTask': async (ctx) => { await initPromise; @@ -642,7 +255,7 @@ const service = defineService({ }, /** - * Add/update a step + * Add or update a step within a task */ 'state.manager.v1.StateManagerService/UpdateStep': async (ctx) => { await initPromise; @@ -658,7 +271,7 @@ const service = defineService({ }, /** - * Get a single task + * Get a single task by ID */ 'state.manager.v1.StateManagerService/GetTask': async (ctx) => { await initPromise; @@ -672,7 +285,7 @@ const service = defineService({ }, /** - * List tasks (with filtering) + * List tasks with filtering */ 'state.manager.v1.StateManagerService/ListTasks': async (ctx) => { await initPromise; @@ -714,7 +327,7 @@ const service = defineService({ // ====== Event Log ====== /** - * Log an event + * Append an event record */ 'state.manager.v1.StateManagerService/LogEvent': async (ctx) => { await initPromise; @@ -731,7 +344,7 @@ const service = defineService({ }, /** - * Query events + * Query events with time-range and type filters */ 'state.manager.v1.StateManagerService/QueryEvents': async (ctx) => { await initPromise; @@ -778,40 +391,39 @@ const service = defineService({ }; }, - // ====== General KV Storage ====== + // ====== General KV Store ====== /** - * Read state + * Read a key-value pair */ 'state.manager.v1.StateManagerService/GetState': async (ctx) => { + await initPromise; const { key } = ctx.request; - const filePath = join(config.dataDir, 'kv_state.json'); - try { - const data = await loadJSON(filePath, {}); - return { success: true, value: JSON.stringify(data[key] ?? null), error: '' }; + const value = await memoryEngine.recall({ key }); + if (value.entries && value.entries.length > 0) { + return { success: true, value: value.entries[0].value || '', error: '' }; + } + return { success: true, value: '', error: '' }; } catch (err) { return { success: false, value: '', error: err.message }; } }, /** - * Write state + * Write a key-value pair (upsert) */ 'state.manager.v1.StateManagerService/SetState': async (ctx) => { + await initPromise; const { key, value } = ctx.request; - const filePath = join(config.dataDir, 'kv_state.json'); - try { - const data = await loadJSON(filePath, {}); - try { - data[key] = JSON.parse(value); - } catch { - data[key] = value; - } - await saveJSON(filePath, data); - - return { success: true, error: '' }; + const result = await memoryEngine.remember({ + key, + value, + type: 'kv_store', + ttl_seconds: 0, // never expire + }); + return { success: result.success, error: result.error || '' }; } catch (err) { return { success: false, error: err.message }; } diff --git a/services/state-manager/config.schema.json b/services/state-manager/config.schema.json index 61a910b0..5a16d152 100644 --- a/services/state-manager/config.schema.json +++ b/services/state-manager/config.schema.json @@ -3,18 +3,18 @@ "properties": { "dataDir": { "type": "string", - "description": "Data directory path, can be overridden via DATA_DIR env var", + "description": "Data directory path for persistent storage, can be overridden via DATA_DIR env var", "default": "./data" }, - "projectsConfigPath": { + "tenant": { "type": "string", - "description": "Project config file path, can be overridden via PROJECTS_CONFIG env var", - "default": "./config/projects.yaml" + "description": "Tenant identifier for multi-tenant key namespacing, can be overridden via TENANT env var", + "default": "default" }, - "userConfigPath": { + "userId": { "type": "string", - "description": "User personal config file path (identity/kanban/CRM/instance mapping), can be overridden via USER_CONFIG env var. New users only need to edit this one file during deployment.", - "default": "./config/user-config.json" + "description": "User identifier for memory isolation, can be overridden via USER_ID env var", + "default": "default" } } } diff --git a/services/state-manager/config/user-config.example.json b/services/state-manager/config/user-config.example.json deleted file mode 100644 index b5dbb047..00000000 --- a/services/state-manager/config/user-config.example.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "_README": "Personal config file — change this one file when switching users/environments. All agents read via GetUserConfig.", - "identity": { - "user_id": "YOUR_LOGIN_ID", - "display_name": "Your Name", - "role": "Your role (pre-sales/sales/tech-support)", - "crm_user_id": "CRM user ID (look up via ListUsers on first use, can leave empty for agent auto-lookup)", - "default_sales_user_id": "Default sales userId", - "default_sales_name": "Default sales name", - "team": "Your team name" - }, - "kanban": { - "base_id": "AITable base ID (from alidocs URL nodeId)", - "table_id": "AITable table ID", - "instance": "OctoBus instance name (e.g. aitable-instance)", - "fields": { - "start_date": "Start date field ID", - "end_date": "Due date field ID", - "task_type": "Task type field ID", - "task_desc": "Task description field ID", - "sales": "Sales field ID", - "owner": "Owner field ID", - "notes": "Description field ID", - "customer_l1": "Customer tier field ID", - "customer_l2": "Sub-department field ID", - "status": "Status field ID", - "presale": "Pre-sales field ID", - "crm_link": "CRM project link field ID" - }, - "task_types": { - "communication": [{"id": "option ID", "name": "Communication"}], - "documentation": [{"id": "option ID", "name": "Documentation"}], - "bidding": [{"id": "option ID", "name": "Bidding"}], - "poc": [{"id": "option ID", "name": "POC & Others"}] - }, - "statuses": { - "completed": [{"id": "option ID", "name": "Done"}], - "in_progress": [{"id": "option ID", "name": "In Progress"}] - }, - "customers": { - "customers": [ - {"id": "option ID", "name": "Customer Name"} - ] - } - }, - "instances": { - "aitable": "aitable-instance", - "doc": "doc-instance", - "calendar": "calendar-instance", - "todo": "todo-instance", - "message": "message-instance", - "crm": "crm-instance", - "state": "state-instance" - }, - "knowledge_file": "/opt/knowledge/company-products.md", - "company_name": "Your Company", - "company_english": "CompanyNameEn" -} diff --git a/services/state-manager/docs/v3-design.md b/services/state-manager/docs/v3-design.md new file mode 100644 index 00000000..f0c3a4c4 --- /dev/null +++ b/services/state-manager/docs/v3-design.md @@ -0,0 +1,170 @@ +# Agent Memory & Distillation Engine — v3 Design + +> v3 = 通用化。从"6 个智能体的状态层"升级为"社区可用的 AI 记忆基础设施"。 + +--- + +## 一、设计目标 + +| 目标 | 说明 | +|------|------| +| **通用性** | 不绑定任何业务场景（CRM、钉钉、看板等），任何 AI agent 都能直接用 | +| **记忆即服务** | 记忆不是 KV 存储，是带类型/蒸馏/去重的认知基础设施 | +| **蒸馏是核心差异** | 自动压缩防止记忆膨胀，这是区别于 Redis/SQLite 的本质特征 | +| **API 稳定** | 17 个通用 RPC，加能力不重构，换存储引擎不动 API | + +--- + +## 二、架构 + +``` +┌──────────────────────────────────────────────────────────┐ +│ gRPC / Connect API │ +│ 17 个通用 RPC：记忆(4) + 蒸馏(2) + 任务(6) + 事件(3) │ +│ + KV(2) │ +├──────────────────────────────────────────────────────────┤ +│ Memory Engine │ Distillation Engine │ Task Machine │ +│ Remember/Recall │ 周期压缩 │ 状态机 │ +│ Forget + 去重 │ 模式提取 │ 步骤追踪 │ +│ TTL 过期 │ 摘要合并 │ 超时检测 │ +├──────────────────────────────────────────────────────────┤ +│ Type Registry │ TTL Manager │ Event Log │ +│ 5 类默认 + 可扩展 │ 后台扫描清理 │ 追加写入 │ +├──────────────────────────────────────────────────────────┤ +│ Storage Adapter（可插拔） │ +│ JSON File ← SQLite ← etcd ← Redis │ +└──────────────────────────────────────────────────────────┘ +``` + +**核心原则**：API 稳定，实现可换。换存储引擎、加蒸馏策略，API 层不动。 + +--- + +## 三、17 个通用 RPC + +### 3.1 记忆引擎（4 个） + +| RPC | 功能 | 设计要点 | +|-----|------|---------| +| Remember | 写入记忆 | upsert + 去重窗口 + TTL + 类型化默认值 | +| Recall | 读取记忆 | 精确匹配 / 前缀匹配 / 类型过滤 | +| Forget | 删除记忆 | 精确删除 / 前缀批量删除 | +| GetMemoryStats | 记忆统计 | 条数、类型分布、调用量、淘汰数 | + +### 3.2 蒸馏引擎（2 个） + +| RPC | 功能 | 设计要点 | +|-----|------|---------| +| Distill | 触发蒸馏 | action_log → 模式摘要；session_summary → 月度合并；correction 升降级 | +| GetDistillStats | 蒸馏统计 | 运行次数、压缩量、上次运行时间 | + +### 3.3 任务状态机（6 个） + +| RPC | 功能 | 设计要点 | +|-----|------|---------| +| CreateTask | 创建任务 | 可选步骤列表、优先级、截止日期 | +| UpdateTask | 更新状态 | 合法转换表校验：pending→running→done/failed/cancelled/timed_out | +| UpdateStep | 更新步骤 | 步骤级别状态追踪 | +| GetTask | 查询单个 | 含步骤详情 + 状态历史 | +| ListTasks | 列表查询 | 按状态/负责人/优先级/截止日过滤 | +| GetTaskStats | 任务统计 | 各状态计数、创建/完成/失败/超时累计 | + +### 3.4 事件日志（3 个） + +| RPC | 功能 | 设计要点 | +|-----|------|---------| +| LogEvent | 记录事件 | 追加写入，不删除不修改 | +| QueryEvents | 查询事件 | 按类型/角色/级别/时间范围过滤 | +| GetEventStats | 事件统计 | 总数、轮转数、最早/最新事件 | + +### 3.5 KV 存储（2 个） + +| RPC | 功能 | 设计要点 | +|-----|------|---------| +| GetState | 读取 KV | 底层委托记忆引擎 | +| SetState | 写入 KV | 底层委托记忆引擎，永不过期 | + +--- + +## 四、五类记忆结构 + +| 类型 | 默认 TTL | 去重窗口 | 最大条数 | 淘汰策略 | 典型场景 | +|------|---------|---------|---------|---------|---------| +| entity_cache | 7 天 | 无 | 200 | LRU | 缓存昂贵查询结果（API 返回值、外部 ID） | +| action_log | 7 天 | 24 小时 | 1000 | 最旧优先 | 操作审计 + 幂等去重（防止重复执行写操作） | +| user_correction | 永久 | 无 | 100 | 压缩 | 用户纠正记录（最高价值记忆，永久保留） | +| commitment | 30 天 | 无 | 100 | LRU | 待办承诺（用户说"下周跟进"） | +| session_summary | 30 天 | 无 | 50 | 最旧优先 | 会话关键决策 + 待办清单 | + +**扩展新类型**：在 TypeRegistry 加一行，API 立即生效。蒸馏引擎也读注册表决定压缩策略。 + +--- + +## 五、蒸馏引擎——核心差异 + +蒸馏是状态层从"被动存储"变成"主动认知引擎"的关键： + +### 三级蒸馏 + +| 级别 | 机制 | 触发方式 | +|------|------|---------| +| 被动淘汰 | TTL 过期 + 容量上限 LRU | 自动（后台扫描） | +| 主动压缩 | action_log 周期模式提取（~10:1 压缩比） | Distill() RPC 或定时调用 | +| 周期汇总 | session_summary 月度合并 | Distill() RPC | + +### 蒸馏策略 + +- **action_log → 模式摘要**：按 action/entity 统计频率，生成 Top5 模式，删除原始条目 +- **session_summary → 月度合并**：合并 topics/decisions/pending/entities，TTL=90 天 +- **user_correction 容量管理**：接近上限时，旧条目降级为 action_log（保留但不占高价值槽位） + +--- + +## 六、业务逻辑归属 + +**v3 的核心设计决策：OctoBus 只做通用能力，业务逻辑在 agent 层。** + +| 层级 | 职责 | 举例 | +|------|------|------| +| **OctoBus 服务** | 通用基础设施 | Remember/Recall/Forget、Distill、Task、EventLog | +| **Agent Prompt** | 业务编排 | "查 CRM → Remember(type=entity_cache)"、"用户纠正 → Remember(type=user_correction)" | +| **Agent Skill** | 复合操作 | 用 Recall + Remember 组合实现项目匹配、周报同步、会议追踪 | + +### 迁移示例 + +| 旧便捷 RPC | 通用 API 组合 | +|-----------|-------------| +| MatchProject(text) | Recall(prefix="project:") → agent LLM 判断匹配 | +| AddPendingItem(summary) | Remember(type=commitment, key="weekly_item:...") | +| RegisterMeeting(eventId) | Remember(type=commitment, key="meeting:{id}") | +| UpdateMeetingState(id, state) | Recall(key="meeting:{id}") + Remember 更新 | +| GetUserConfig() | GetState(key="user_config") 或环境变量 | + +--- + +## 七、多租户设计（ADR-004） + +所有记忆 key 遵守 `{tenant}:{type}:{key}` 命名规范： +- 当前 tenant 从环境变量 `TENANT` 读取（默认 `default`） +- userId 从 `USER_ID` 读取（默认 `default`） +- 将来加鉴权：API 层加前缀校验，存储引擎不动 + +--- + +## 八、路线图 + +``` +v1 MVP（已完成） v2 增强（已完成） v3 通用化（本次） v4 远期 +───────────────────────────────────────────────────────────────────────────── +Remember/Recall 蒸馏引擎 ✅ 剥离业务 RPC ✅ 租户鉴权 +Forget 任务状态机 ✅ 17 通用 RPC ✅ SQLite 迁移 +5 类记忆行为 事件日志 ✅ 通用配置 ✅ 语义搜索(embedding) +GetMemoryStats ✅ 28 个 RPC ✅ v3 设计文档 ✅ 商业化 API + 操作审计 ✅ js-yaml 依赖移除 ✅ + 优雅停机 ✅ +└─ 解决"有无" └─ 体现"蒸馏"价值 └─ 社区可用 └─ 生产级闭环 +``` + +--- + +*本文档随代码演进持续更新。每次架构变更必须同步修改。* diff --git a/services/state-manager/lib/distillation-engine.js b/services/state-manager/lib/distillation-engine.js index fc71a907..bfa31b15 100644 --- a/services/state-manager/lib/distillation-engine.js +++ b/services/state-manager/lib/distillation-engine.js @@ -4,7 +4,7 @@ * Core value: makes memory more than just "store and retrieve" — * it continuously compresses and refines. * Three levels of distillation: - * 1. action_log → session_summary (weekly summary, ~10:1 compression) + * 1. action_log → session_summary (periodic summary, ~10:1 compression) * 2. session_summary → monthly_summary (monthly rollup, ~10:1 compression) * 3. user_correction pattern extraction (multiple corrections for same key → upgrade to permanent preference) * @@ -16,13 +16,13 @@ export class DistillationEngine { /** * @param {import('./memory-engine.js').MemoryEngine} memoryEngine * @param {object} opts - * @param {number} opts.weeklyWindowDays - action_log weekly summary window (default 7 days) + * @param {number} opts.summaryWindowDays - action_log periodic summary window (default 7 days) * @param {number} opts.monthlyWindowDays - session_summary monthly rollup window (default 30 days) * @param {number} opts.correctionThreshold - Number of corrections for same key to trigger upgrade (default 3) */ constructor(memoryEngine, opts = {}) { this.engine = memoryEngine; - this.weeklyWindowDays = opts.weeklyWindowDays || 7; + this.summaryWindowDays = opts.summaryWindowDays || 7; this.monthlyWindowDays = opts.monthlyWindowDays || 30; this.correctionThreshold = opts.correctionThreshold || 3; @@ -52,7 +52,7 @@ export class DistillationEngine { }; try { - // 1. action_log weekly summary + // 1. action_log periodic summary summary.actionLog = await this._distillActionLogs(); // 2. session_summary monthly rollup @@ -75,7 +75,7 @@ export class DistillationEngine { return { ...this._stats }; } - // ─── 1. Action Log Weekly Summary ─── + // ─── 1. Action Log Periodic Summary ─── /** * Compress action_log entries within 7 days into 1 session_summary @@ -83,7 +83,7 @@ export class DistillationEngine { */ async _distillActionLogs() { const now = new Date(); - const windowMs = this.weeklyWindowDays * 86400 * 1000; + const windowMs = this.summaryWindowDays * 86400 * 1000; const cutoff = new Date(now.getTime() - windowMs); // Collect action_log entries within the window @@ -209,7 +209,7 @@ export class DistillationEngine { const frequency = { avgPerDay: Math.round(avgPerDay * 10) / 10, activeDays: dailyArr.length, - totalDays: this.weeklyWindowDays, + totalDays: this.summaryWindowDays, }; // Pattern detection diff --git a/services/state-manager/lib/memory-engine.js b/services/state-manager/lib/memory-engine.js index 296a84ce..43573e40 100644 --- a/services/state-manager/lib/memory-engine.js +++ b/services/state-manager/lib/memory-engine.js @@ -76,18 +76,13 @@ export class MemoryEngine { } /** - * 从 user-config 加载 tenant 和 userId + * Set tenant and userId from environment or config + * @param {string} tenant + * @param {string} userId */ - async loadIdentity(userConfigPath) { - try { - const { readFile } = await import('node:fs/promises'); - const raw = await readFile(userConfigPath, 'utf-8'); - const config = JSON.parse(raw); - this.tenant = config.company_english || config.companyName || 'default'; - this.userId = config.identity?.user_id || 'shared'; - } catch { - // 配置文件不存在时用默认值 - } + setIdentity(tenant, userId) { + this.tenant = tenant || 'default'; + this.userId = userId || 'default'; } // ─── Remember ─── diff --git a/services/state-manager/lib/type-registry.js b/services/state-manager/lib/type-registry.js index 595b9b5b..0d2f037c 100644 --- a/services/state-manager/lib/type-registry.js +++ b/services/state-manager/lib/type-registry.js @@ -22,7 +22,7 @@ const TYPE_REGISTRY = { defaultConfidence: 1.0, maxEntries: 200, evictPolicy: 'lru', - description: 'Query result cache (CRM IDs, project details, etc. — expensive query results)', + description: 'Query result cache (expensive API call results, external ID lookups, etc.)', label: 'Entity Cache', }, action_log: { @@ -31,7 +31,7 @@ const TYPE_REGISTRY = { defaultConfidence: 1.0, maxEntries: 1000, evictPolicy: 'oldest', - description: 'Action log (todo creation, kanban writes, etc. — write operation records for idempotent dedup)', + description: 'Action log (write operation records for idempotent dedup and audit trail)', label: 'Action Log', }, user_correction: { diff --git a/services/state-manager/package.json b/services/state-manager/package.json index 28c15d52..eeba1f4e 100644 --- a/services/state-manager/package.json +++ b/services/state-manager/package.json @@ -1,15 +1,14 @@ { "name": "octobus-state-manager", "displayName": "State Manager", - "version": "0.2.0", - "description": "Agent State Manager — Memory Engine + Distillation + Task State Machine + Event Log", + "version": "3.0.0", + "description": "Agent Memory & Distillation Engine — typed memory with auto-compression, task tracking, event logging", "private": true, "type": "module", "bin": { "state-manager": "bin/state-service.js" }, "dependencies": { - "@chaitin-ai/octobus-sdk": "^0.5.0", - "js-yaml": "^4.1.0" + "@chaitin-ai/octobus-sdk": "^0.5.0" } } diff --git a/services/state-manager/proto/state.proto b/services/state-manager/proto/state.proto index d52dd46b..f7202be9 100644 --- a/services/state-manager/proto/state.proto +++ b/services/state-manager/proto/state.proto @@ -1,292 +1,69 @@ syntax = "proto3"; package state.manager.v1; -// Agent State Manager -// OctoBus service package: state-manager -> Agent State Manager -// Provides: project config, weekly buffer, meeting tracking, user config, memory engine, distillation engine, task state machine, event log, general storage +// Agent State Manager — Generic Memory & State Service +// OctoBus service package: state-manager +// +// Provides generic building blocks for AI agent state management: +// - Memory Engine: Typed key-value storage with TTL, dedup, and prefix queries +// - Distillation Engine: Automatic memory compression to prevent unbounded growth +// - Task State Machine: Structured task lifecycle (pending→running→done/failed) +// - Event Log: Append-only event recording with time-range queries +// - KV Store: Simple key-value persistence +// +// Business-specific logic (project matching, weekly reports, meeting tracking, +// CRM integration, etc.) should be implemented at the agent/skill layer using +// the generic primitives provided here. service StateManagerService { - // ====== Project Config ====== - // Get project config by key - rpc GetProject(GetProjectRequest) returns (GetProjectResponse); - // List all projects - rpc ListProjects(ListProjectsRequest) returns (ListProjectsResponse); - // Match project by keywords - rpc MatchProject(MatchProjectRequest) returns (MatchProjectResponse); - - // ====== Weekly Pending Buffer (migrated to memory engine, internally delegates to Remember/Recall/Forget) ====== - // Add a pending weekly summary item -> Remember(type=commitment, key=weekly_item:...) - rpc AddPendingItem(AddPendingItemRequest) returns (AddPendingItemResponse); - // Get all pending sync items -> Recall(prefix=weekly_item:, type=commitment) - rpc GetPendingItems(GetPendingItemsRequest) returns (GetPendingItemsResponse); - // Clear pending buffer -> Forget(prefix=weekly_item:, type=commitment) - rpc ClearPending(ClearPendingRequest) returns (ClearPendingResponse); - - // ====== Meeting Tracking (migrated to memory engine, internally delegates to Remember/Recall/Forget) ====== - // Register a meeting -> Remember(type=commitment, key=meeting:{eventId}) - rpc RegisterMeeting(RegisterMeetingRequest) returns (RegisterMeetingResponse); - // Get pending meetings -> Recall(prefix=meeting:, type=commitment) + filtering - rpc GetPendingMeetings(GetPendingMeetingsRequest) returns (GetPendingMeetingsResponse); - // Update meeting state -> Recall + Remember update - rpc UpdateMeetingState(UpdateMeetingStateRequest) returns (UpdateMeetingStateResponse); - - // ====== User Config ====== - // Read user personal config (identity/kanban/CRM/defaults) - rpc GetUserConfig(GetUserConfigRequest) returns (GetUserConfigResponse); - // ====== Agent Memory ====== - // Write a memory entry + // Write a memory entry (upsert with dedup and TTL) rpc Remember(RememberRequest) returns (RememberResponse); // Read memory (exact match or prefix match) rpc Recall(RecallRequest) returns (RecallResponse); - // Delete memory + // Delete memory (exact or batch prefix delete) rpc Forget(ForgetRequest) returns (ForgetResponse); // Get memory engine runtime statistics rpc GetMemoryStats(GetMemoryStatsRequest) returns (GetMemoryStatsResponse); // ====== Distillation Engine ====== - // Manually trigger a full distillation pass + // Trigger a distillation pass (compress old/low-priority memories) rpc Distill(DistillRequest) returns (DistillResponse); // Get distillation statistics rpc GetDistillStats(GetDistillStatsRequest) returns (GetDistillStatsResponse); // ====== Task State Machine ====== - // Create a task + // Create a task with optional steps rpc CreateTask(CreateTaskRequest) returns (CreateTaskResponse); - // Update task state + // Update task state (enforces valid transitions) rpc UpdateTask(UpdateTaskRequest) returns (UpdateTaskResponse); - // Add/update a step + // Add or update a step within a task rpc UpdateStep(UpdateStepRequest) returns (UpdateStepResponse); - // Get a single task + // Get a single task by ID rpc GetTask(GetTaskRequest) returns (GetTaskResponse); - // List tasks (with filtering) + // List tasks with filtering rpc ListTasks(ListTasksRequest) returns (ListTasksResponse); // Get task statistics rpc GetTaskStats(GetTaskStatsRequest) returns (GetTaskStatsResponse); // ====== Event Log ====== - // Log an event + // Append an event record rpc LogEvent(LogEventRequest) returns (LogEventResponse); - // Query events + // Query events with time-range and type filters rpc QueryEvents(QueryEventsRequest) returns (QueryEventsResponse); // Get event log statistics rpc GetEventStats(GetEventStatsRequest) returns (GetEventStatsResponse); - // ====== General KV ====== + // ====== General KV Store ====== + // Read a key-value pair rpc GetState(GetStateRequest) returns (GetStateResponse); + // Write a key-value pair (upsert) rpc SetState(SetStateRequest) returns (SetStateResponse); } -// ====== Project Config ====== - -message GetProjectRequest { - string key = 1; // Project key (e.g. yidong_ai_plus) -} - -message ProjectConfig { - string key = 1; - string name = 2; - string node_id = 3; - string customer = 4; - repeated string keywords = 5; - string detail_folder = 6; - string background = 7; // Customer background (pinned info) -} - -message GetProjectResponse { - bool success = 1; - ProjectConfig project = 2; - string error = 3; -} - -message ListProjectsRequest {} - -message ListProjectsResponse { - bool success = 1; - repeated ProjectConfig projects = 2; - string error = 3; -} - -message MatchProjectRequest { - string text = 1; // Text to match (e.g. calendar title or note content) -} - -message MatchProjectResponse { - bool success = 1; - ProjectConfig project = 2; // Matched project (may be empty) - float confidence = 3; // Match confidence 0-1 - string error = 4; -} - -// ====== Weekly Pending Buffer ====== - -message AddPendingItemRequest { - string summary = 1; // One-line summary (max 30 chars) - string category = 2; // Category: Communication / Documentation / Bidding / POC & Others - string date = 3; // Date YYYY-MM-DD -} - -message AddPendingItemResponse { - bool success = 1; - int32 pending_count = 2; // Current pending total - string error = 3; -} - -message GetPendingItemsRequest {} - -message PendingItem { - string summary = 1; - string category = 2; - string date = 3; -} - -message GetPendingItemsResponse { - bool success = 1; - repeated PendingItem items = 2; - string error = 3; -} - -message ClearPendingRequest {} - -message ClearPendingResponse { - bool success = 1; - int32 cleared_count = 2; // Number of entries cleared - string error = 3; -} - -// ====== Meeting Tracking ====== - -message RegisterMeetingRequest { - string event_id = 1; // Calendar event ID - string summary = 2; // Meeting title - string start_time = 3; // ISO time - string end_time = 4; // ISO time - repeated string attendees = 5; -} - -message RegisterMeetingResponse { - bool success = 1; - bool is_new = 2; // Whether newly registered (false = already exists) - string error = 3; -} - -message GetPendingMeetingsRequest { - int32 grace_hours = 1; // Hours past end time to be considered pending (default 24) -} - -message MeetingInfo { - string event_id = 1; - string summary = 2; - string start_time = 3; - string end_time = 4; - repeated string attendees = 5; - string state = 6; // registered / notes_submitted / doc_created / reminded -} - -message GetPendingMeetingsResponse { - bool success = 1; - repeated MeetingInfo meetings = 2; - string error = 3; -} - -message UpdateMeetingStateRequest { - string event_id = 1; - string new_state = 2; // notes_submitted / doc_created / reminded -} - -message UpdateMeetingStateResponse { - bool success = 1; - string error = 2; -} - -// ====== User Config ====== - -message GetUserConfigRequest {} - -message KanbanFieldConfig { - string start_date = 1; // field ID for Start Date - string end_date = 2; // field ID for Due Date - string task_type = 3; // field ID for Task Type - string task_desc = 4; // field ID for Task Description - string sales = 5; // field ID for Sales - string owner = 6; // field ID for Owner - string notes = 7; // field ID for Description - string customer_l1 = 8; // field ID for Customer Tier - string customer_l2 = 9; // field ID for Sub-department - string status = 10; // field ID for Status - string presale = 11; // field ID for Pre-sales - string crm_link = 12; // field ID for CRM Project Link -} - -message KanbanOptionConfig { - string id = 1; - string name = 2; -} - -message KanbanTaskTypeConfig { - repeated KanbanOptionConfig communication = 1; // Communication - repeated KanbanOptionConfig documentation = 2; // Documentation - repeated KanbanOptionConfig bidding = 3; // Bidding - repeated KanbanOptionConfig poc = 4; // POC & Others -} - -message KanbanStatusConfig { - repeated KanbanOptionConfig completed = 1; // Done - repeated KanbanOptionConfig in_progress = 2; // In Progress -} - -message KanbanCustomerConfig { - repeated KanbanOptionConfig customers = 1; -} - -message KanbanConfig { - string base_id = 1; - string table_id = 2; - string instance = 3; // OctoBus instance name (e.g. "aitable-instance") - KanbanFieldConfig fields = 4; - KanbanTaskTypeConfig task_types = 5; - KanbanStatusConfig statuses = 6; - KanbanCustomerConfig customers = 7; -} - -message IdentityConfig { - string user_id = 1; // Login name (e.g. "user01") - string display_name = 2; // Display name (e.g. "Alice") - string role = 3; // Role (e.g. "Pre-sales") - string crm_user_id = 4; // CRM system user ID (look up via ListUsers on first use) - string default_sales_user_id = 5;// Default sales userId (e.g. "sales01") - string default_sales_name = 6; // Default sales name (e.g. "Bob") - string team = 7; // Team name (e.g. "Enterprise Pre-sales") -} - -message OctoBusInstanceConfig { - string aitable = 1; // e.g. "aitable-instance" - string doc = 2; // e.g. "doc-instance" - string calendar = 3; // e.g. "calendar-instance" - string todo = 4; // e.g. "todo-instance" - string message = 5; // e.g. "message-instance" - string crm = 6; // e.g. "crm-instance" - string state = 7; // e.g. "state-instance" -} - -message UserConfig { - IdentityConfig identity = 1; - KanbanConfig kanban = 2; - OctoBusInstanceConfig instances = 3; - string knowledge_file = 4; // Product knowledge file path (e.g. "/opt/knowledge/company-products.md") - string company_name = 5; // Company name (e.g. "Acme Corp") - string company_english = 6; // Company English name (e.g. "Acme") -} - -message GetUserConfigResponse { - bool success = 1; - UserConfig config = 2; - string error = 3; -} - // ====== Agent Memory ====== message RememberRequest { - string key = 1; // Memory key (e.g. "crm_user:self") + string key = 1; // Memory key (e.g. "customer:acme_corp") string value = 2; // JSON string string type = 3; // entity_cache | action_log | user_correction | commitment | session_summary int32 ttl_seconds = 4; // 0=use type default, -1=never expire @@ -445,7 +222,7 @@ message TaskInfo { } message CreateTaskRequest { - string task_id = 1; // Optional, auto-generated + string task_id = 1; // Optional, auto-generated if empty string name = 2; string description = 3; string assignee = 4; From dcabe7a922386f123ff694b926fa6db0d4d1ea0c Mon Sep 17 00:00:00 2001 From: "jianhua.wang" Date: Sun, 5 Jul 2026 20:13:20 +0800 Subject: [PATCH 4/9] =?UTF-8?q?docs:=20=E6=96=B0=E5=A2=9E=20v3=20=E5=BC=80?= =?UTF-8?q?=E5=8F=91=E8=B7=AF=E7=BA=BF=E5=9B=BE=20=E2=80=94=206=20?= =?UTF-8?q?=E4=B8=AA=E9=98=B6=E6=AE=B5=E4=BB=8E=E6=B5=8B=E8=AF=95=E8=A6=86?= =?UTF-8?q?=E7=9B=96=E5=88=B0=E8=AF=AD=E4=B9=89=E6=90=9C=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/state-manager/docs/v3-roadmap.md | 201 ++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 services/state-manager/docs/v3-roadmap.md diff --git a/services/state-manager/docs/v3-roadmap.md b/services/state-manager/docs/v3-roadmap.md new file mode 100644 index 00000000..2f1a26c6 --- /dev/null +++ b/services/state-manager/docs/v3-roadmap.md @@ -0,0 +1,201 @@ +# state-manager v3 开发路线图 + +> 目标：从"我的 6 个智能体能用"变成"社区任何 AI agent 都能直接用的记忆基础设施" + +--- + +## 当前差距分析 + +| 维度 | 当前状态 | 通用工具需要 | 差距 | +|------|---------|------------|------| +| **测试** | 0 个测试文件（测试脚本只在本地） | 单元测试 + 集成测试覆盖 | 🔴 大 | +| **SDK** | 无，用户必须用 grpcurl | Python/TypeScript SDK | 🔴 大 | +| **自动蒸馏** | 需要手动调 Distill() RPC | 后台定时自动蒸馏 | 🟡 中 | +| **存储引擎** | 仅 JSON 文件 | 至少支持 SQLite | 🟡 中 | +| **多租户** | 环境变量注入，无鉴权 | API 层鉴权 + 数据隔离 | 🟡 中 | +| **可观测性** | console.log | 健康检查 + Prometheus metrics | 🟡 中 | +| **Type 扩展** | 改源码加行 | 运行时 registerType() API | 🟢 小 | +| **文档** | README + 设计文档 | Quick Start + Tutorial + API Reference | 🟢 小 | + +--- + +## 开发阶段 + +### 阶段 1：测试覆盖（1-2 周） + +**为什么先做测试**：开源第一原则——没测试的代码没人敢用，也没人敢贡献。 + +**具体任务**： + +| # | 任务 | 工作量 | 说明 | +|---|------|--------|------| +| 1.1 | 搭建测试框架 | 0.5 天 | 用 Node.js 内置 `node:test`（零依赖），加 `npm test` 脚本 | +| 1.2 | memory-engine 单元测试 | 1 天 | Remember/Recall/Forget + 去重/TTL/容量淘汰，约 20 个用例 | +| 1.3 | distillation-engine 单元测试 | 1 天 | action_log 压缩 + session_summary 合并 + correction 升降级，约 15 个用例 | +| 1.4 | task-machine 单元测试 | 1 天 | 状态转换校验 + 步骤追踪 + 超时检测，约 15 个用例 | +| 1.5 | event-log 单元测试 | 0.5 天 | 追加写入 + 轮转 + 范围查询，约 10 个用例 | +| 1.6 | type-registry 单元测试 | 0.5 天 | resolve 默认值 + 自定义覆盖 + registerType，约 8 个用例 | +| 1.7 | 集成测试 | 1 天 | bin/state-service.js 的 handler 端到端测试（mock gRPC），约 17 个用例 | + +**验收标准**：`npm test` 全部通过，覆盖率 > 80% + +--- + +### 阶段 2：运行时扩展 + 自动蒸馏（1 周） + +**目标**：不用改源码就能加记忆类型，蒸馏自动运行不需要手动触发。 + +| # | 任务 | 工作量 | 说明 | +|---|------|--------|------| +| 2.1 | registerType() API | 1 天 | 新增 RPC `RegisterType(name, config)`，运行时动态注册记忆类型，写入 type-registry.json 持久化 | +| 2.2 | ListTypes RPC | 0.5 天 | 列出所有已注册类型（含默认 5 种 + 自定义），方便 agent 发现可用类型 | +| 2.3 | 自动蒸馏调度器 | 1.5 天 | config 里加 `distill_interval_seconds`（默认 3600），后台定时器自动调 Distill()。也支持通过 OctoBus Loader cron 触发 | +| 2.4 | 蒸馏策略可配置 | 1 天 | 当前压缩阈值硬编码，改为 TypeRegistry 里每种类型配 `distill_policy`：`{ min_entries, compression_ratio, target_ttl }` | + +**proto 新增**： +```protobuf +rpc RegisterType(RegisterTypeRequest) returns (RegisterTypeResponse); +rpc ListTypes(ListTypesRequest) returns (ListTypesResponse); +``` + +**验收标准**：`RegisterType("alert", { defaultTTL: 86400, maxEntries: 50 })` → 后续 `Remember(type="alert")` 可用；蒸馏每小时自动运行一次。 + +--- + +### 阶段 3：SQLite 存储引擎（1-2 周） + +**为什么需要 SQLite**：JSON 文件不适合高频写入和多条目场景。SQLite 是零配置的升级路径。 + +| # | 任务 | 工作量 | 说明 | +|---|------|--------|------| +| 3.1 | StorageAdapter 接口抽象 | 1 天 | 抽取 interface：`loadAll()`, `saveEntry()`, `deleteEntry()`, `query()`, `waitForPendingWrites()`。当前 JSON 实现作为 `JsonStorageAdapter` | +| 3.2 | SqliteStorageAdapter 实现 | 2 天 | 用 `better-sqlite3`（同步 API，性能好）。每类记忆一张表，前缀索引用 FTS5 | +| 3.3 | 配置切换 | 0.5 天 | config.schema.json 加 `storage.engine`：`"json" | "sqlite"`，默认 json | +| 3.4 | 数据迁移工具 | 1 天 | `migrate-to-sqlite.js` 脚本：读取 JSON 文件 → 写入 SQLite | +| 3.5 | 性能基准测试 | 0.5 天 | 对比 JSON vs SQLite 在 1K/10K/100K 条目下的读写延迟 | + +**验收标准**：`STORAGE_ENGINE=sqlite` 启动后，所有 17 个 RPC 行为不变，1 万条记忆 Recall < 50ms。 + +--- + +### 阶段 4：Agent SDK（2 周） + +**目标**：Python 和 TypeScript 开发者 3 行代码就能用记忆能力。 + +### 4.1 Python SDK + +```python +from state_manager import MemoryClient + +mem = MemoryClient("state-prod", capset="dev") + +# 记住 +mem.remember("customer:acme", {"id": "123", "name": "Acme Corp"}, type="entity_cache") + +# 回忆 +result = mem.recall(prefix="customer:") + +# 忘记 +mem.forget(prefix="customer:old_") +``` + +| # | 任务 | 工作量 | +|---|------|--------| +| 4.1.1 | gRPC client 生成（from proto） | 0.5 天 | +| 4.1.2 | MemoryClient 封装 | 1 天 | +| 4.1.3 | 类型提示 + PyPI 发布 | 0.5 天 | + +### 4.2 TypeScript SDK + +```typescript +import { MemoryClient } from "@chaitin-ai/state-manager-client"; + +const mem = new MemoryClient({ instance: "state-prod", capset: "dev" }); +await mem.remember("customer:acme", { id: "123" }, { type: "entity_cache" }); +``` + +| # | 任务 | 工作量 | +|---|------|--------| +| 4.2.1 | 从 proto 生成 TS 类型 | 0.5 天 | +| 4.2.2 | MemoryClient 封装 | 1 天 | +| 4.2.3 | npm 发布 | 0.5 天 | + +**验收标准**：`pip install chaitin-state-manager` + 3 行代码 = 能 Remember/Recall。 + +--- + +### 阶段 5：多租户 + 可观测性（2 周） + +| # | 任务 | 工作量 | 说明 | +|---|------|--------|------| +| 5.1 | gRPC 拦截器鉴权 | 2 天 | 从 gRPC metadata 提取 tenant/userId，校验 key 前缀匹配 | +| 5.2 | 数据隔离验证 | 1 天 | tenant A 的 Recall 看不到 tenant B 的数据 | +| 5.3 | Health Check RPC | 0.5 天 | `HealthCheck()` → 存储引擎连通性 + 条目数 + 上次蒸馏时间 | +| 5.4 | Prometheus metrics | 1 天 | `state_remember_total`、`state_recall_total`、`state_distill_runs`、`state_entries` | +| 5.5 | 结构化日志 | 1 天 | 替换 console.log → pino/winston，JSON 格式输出 | + +**验收标准**：两个 tenant 的 agent 跑在同一实例上，数据完全隔离；`curl /metrics` 暴露 Prometheus 指标。 + +--- + +### 阶段 6：语义搜索（远期，4+ 周） + +> 这个是 v4 远期目标，需要 embedding 服务。当前先留接口。 + +| # | 任务 | 说明 | +|---|------|------| +| 6.1 | RecallSemantic RPC | 输入自然语言查询 → embedding → 向量搜索 → 返回 top-K 相关记忆 | +| 6.2 | Embedding 集成 | 调外部 embedding API（OpenAI / 本地模型），写入时自动生成向量 | +| 6.3 | 向量存储 | SQLite + vec 扩展，或独立向量库（Qdrant / Chroma） | + +**proto 预留**： +```protobuf +rpc RecallSemantic(RecallSemanticRequest) returns (RecallResponse); +``` + +--- + +## 时间线总览 + +``` +Week 1-2 │ 阶段 1：测试覆盖（70+ 用例，覆盖率 > 80%） +Week 3 │ 阶段 2：registerType + 自动蒸馏（19 个 RPC） +Week 4-5 │ 阶段 3：SQLite 存储引擎（可插拔 StorageAdapter） +Week 6-7 │ 阶段 4：Python + TypeScript SDK +Week 8-9 │ 阶段 5：多租户鉴权 + 可观测性 +Week 10+ │ 阶段 6：语义搜索（远期） +``` + +## 你的 6 个智能体迁移计划 + +在阶段 2 完成后（自动蒸馏就绪），就可以开始迁移 agent prompt： + +| 步骤 | 改动 | 风险 | +|------|------|------| +| 1 | VM 部署 v3 为 `state-v3` 实例（与 state-prod 并行） | 低 | +| 2 | 选 1 个低风险 agent（如 daily-summary）切到 state-v3 | 低 | +| 3 | 把 prompt 中的 MatchProject → Recall + LLM 判断 | 中（需要调 prompt） | +| 4 | AddPendingItem/ClearPending → Remember/Forget | 低（1:1 映射） | +| 5 | RegisterMeeting/UpdateMeetingState → Remember/Recall 组合 | 中 | +| 6 | GetUserConfig → GetState 或环境变量 | 低 | +| 7 | 全部 agent 验证通过后，替换 state-prod | 低 | + +**关键原则**：每步迁移后跑一轮完整业务验证，确认智能体行为不退化。 + +--- + +## 每个阶段的 PR 计划 + +| 阶段 | GitHub PR | 仓库 | +|------|-----------|------| +| 1 | test: add unit + integration tests for all modules | OctoBus | +| 2 | feat: RegisterType/ListTypes RPC + auto-distill scheduler | OctoBus | +| 3 | feat: SQLite storage adapter (pluggable) | OctoBus | +| 4a | feat: Python SDK for state-manager | 新仓库 chaitin/state-manager-python | +| 4b | feat: TypeScript SDK for state-manager | 新仓库 chaitin/state-manager-ts | +| 5 | feat: multi-tenant auth + health check + metrics | OctoBus | +| 6 | feat: semantic recall with embedding | OctoBus | + +--- + +*每次完成一个阶段，更新此文档和 v3-design.md。* From 5a75119cf9853e69d8e88c17ca651a4722138580 Mon Sep 17 00:00:00 2001 From: "jianhua.wang" Date: Sun, 5 Jul 2026 23:47:01 +0800 Subject: [PATCH 5/9] =?UTF-8?q?fix:=20Monk=20Scan=20=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E4=B8=8E=E5=B9=B6=E5=8F=91=E4=BF=AE=E5=A4=8D=20=E2=80=94=20?= =?UTF-8?q?=E5=91=BD=E4=BB=A4=E6=B3=A8=E5=85=A5/=E8=B7=AF=E5=BE=84?= =?UTF-8?q?=E9=81=8D=E5=8E=86/=E7=AB=9E=E6=80=81/=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - doc-service.js: 所有参数(nodeId/blockId/startIndex/endIndex/parentFolder/level/pageSize/limit)统一 shellEscape 转义，修复远程命令注入 - safety-check.js: backupDocument 对 nodeId 做安全清洗，防止路径遍历 - dws-runner.js: dwsPath 拼接进 shell 命令前 shellEscape 转义 - calendar/message/todo/aitable service: runDws 改为读取 DWS_PATH 环境变量，修复硬编码 dws - todo/aitable/drive service: 数值参数(limit/maxResults)使用 parseInt + shellEscape 防止注入 - memory-engine.js: 新增 per-type Promise 串行队列(_enqueueWrite)，确保容量检查→淘汰→写入原子化 - task-machine.js: _save 引入 Promise 链锁 + tmp 路径加时间戳，防止并发覆盖 - event-log.js: _rotate 前先 _save flush 内存 + _save 串行化 + tmp 路径加时间戳 - memory-index.js: _reconstructKey/_findFullKey 改为 O(1)(存储 _fullKey) + _deleteTrie 回溯清理空分支 --- services/dingtalk__aitable/bin/service.js | 5 ++- services/dingtalk__calendar/bin/service.js | 3 +- services/dingtalk__doc/bin/doc-service.js | 30 ++++++------- services/dingtalk__doc/lib/dws-runner.js | 2 +- services/dingtalk__doc/lib/safety-check.js | 3 +- services/dingtalk__drive/bin/drive-service.js | 2 +- services/dingtalk__message/bin/service.js | 3 +- services/dingtalk__todo/bin/service.js | 4 +- services/state-manager/lib/event-log.js | 41 +++++++++++------- services/state-manager/lib/memory-engine.js | 42 +++++++++++++------ services/state-manager/lib/memory-index.js | 33 +++++++++------ services/state-manager/lib/task-machine.js | 37 +++++++++------- 12 files changed, 124 insertions(+), 81 deletions(-) diff --git a/services/dingtalk__aitable/bin/service.js b/services/dingtalk__aitable/bin/service.js index 3eccee6f..09c626ca 100644 --- a/services/dingtalk__aitable/bin/service.js +++ b/services/dingtalk__aitable/bin/service.js @@ -27,8 +27,9 @@ function loadKanbanConfig(configPath) { } function runDws(command, timeout = 60000) { + const dwsPath = process.env.DWS_PATH || 'dws'; return new Promise((resolve) => { - execFile('sh', ['-c', `dws ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + execFile('sh', ['-c', `${dwsPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, (error, stdout) => { const raw = stdout.trim(); let data = null; @@ -81,7 +82,7 @@ const service = defineService({ 'dingtalk.aitable.v1.AITableService/QueryRecords': async (ctx) => { const { baseId, tableId, limit } = ctx.request; - const cmd = `aitable record list --base-id ${shellEscape(baseId)} --table-id ${shellEscape(tableId)} --limit ${limit || 10}`; + const cmd = `aitable record list --base-id ${shellEscape(baseId)} --table-id ${shellEscape(tableId)} --limit ${shellEscape(String(parseInt(limit || 10, 10)))}`; const res = await runDws(cmd); if (!res.success) return { success: false, records: [], error: res.error }; diff --git a/services/dingtalk__calendar/bin/service.js b/services/dingtalk__calendar/bin/service.js index 41f604cb..eb92d0f4 100644 --- a/services/dingtalk__calendar/bin/service.js +++ b/services/dingtalk__calendar/bin/service.js @@ -6,8 +6,9 @@ import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; import { execFile } from 'child_process'; function runDws(command, timeout = 60000) { + const dwsPath = process.env.DWS_PATH || 'dws'; return new Promise((resolve) => { - execFile('sh', ['-c', `dws ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + execFile('sh', ['-c', `${dwsPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, (error, stdout) => { const raw = stdout.trim(); let data = null; diff --git a/services/dingtalk__doc/bin/doc-service.js b/services/dingtalk__doc/bin/doc-service.js index ac363580..adbc727f 100644 --- a/services/dingtalk__doc/bin/doc-service.js +++ b/services/dingtalk__doc/bin/doc-service.js @@ -39,7 +39,7 @@ const service = defineService({ 'dingtalk.doc.v1.DocService/ReadDoc': async (ctx) => { const { nodeId } = ctx.request; const result = await runDws( - `doc read --node-id ${nodeId}`, + `doc read --node-id ${shellEscape(nodeId)}`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); // dws doc read 返回 JSON 对象: {markdown, nodeId, docUrl, ...} @@ -67,7 +67,7 @@ const service = defineService({ const tmpFile = await writeTempFile(cleanContent); try { const result = await runDws( - `doc update --node-id ${nodeId} --mode overwrite --content-file ${shellEscape(tmpFile)}`, + `doc update --node-id ${shellEscape(nodeId)} --mode overwrite --content-file ${shellEscape(tmpFile)}`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); return { success: result.success, error: result.error }; @@ -86,7 +86,7 @@ const service = defineService({ try { let cmd = `doc create --title ${shellEscape(title)} --content-file ${shellEscape(tmpFile)}`; if (parentFolder) { - cmd += ` --folder-id ${parentFolder}`; + cmd += ` --folder-id ${shellEscape(parentFolder)}`; } const result = await runDws(cmd, { dwsPath: config.dwsPath, timeout: config.dwsTimeout }); @@ -122,7 +122,7 @@ const service = defineService({ // 2. 读取文档 const readResult = await runDws( - `doc read --node-id ${nodeId}`, + `doc read --node-id ${shellEscape(nodeId)}`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); if (!readResult.success) { @@ -156,7 +156,7 @@ const service = defineService({ const tmpFile = await writeTempFile(cleanContent); try { const writeResult = await runDws( - `doc update --node-id ${nodeId} --mode overwrite --content-file ${shellEscape(tmpFile)}`, + `doc update --node-id ${shellEscape(nodeId)} --mode overwrite --content-file ${shellEscape(tmpFile)}`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); if (!writeResult.success) { @@ -184,7 +184,7 @@ const service = defineService({ 'dingtalk.doc.v1.DocService/ListBlocks': async (ctx) => { const { nodeId, startIndex, endIndex } = ctx.request; const result = await runDws( - `doc block list --node ${nodeId} --start-index ${startIndex} --end-index ${endIndex}`, + `doc block list --node ${shellEscape(nodeId)} --start-index ${shellEscape(String(startIndex))} --end-index ${shellEscape(String(endIndex))}`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); @@ -223,7 +223,7 @@ const service = defineService({ const tmpFile = await writeTempFile(content); try { const result = await runDws( - `doc block update --node ${nodeId} --block-id ${blockId} --content-file ${shellEscape(tmpFile)}`, + `doc block update --node ${shellEscape(nodeId)} --block-id ${shellEscape(blockId)} --content-file ${shellEscape(tmpFile)}`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); return { success: result.success, error: result.error }; @@ -239,9 +239,9 @@ const service = defineService({ const { nodeId, afterBlockId, blockType, content, level } = ctx.request; const tmpFile = await writeTempFile(content); try { - let cmd = `doc block insert --node ${nodeId} --after-block-id ${afterBlockId} --type ${blockType} --content-file ${shellEscape(tmpFile)}`; + let cmd = `doc block insert --node ${shellEscape(nodeId)} --after-block-id ${shellEscape(afterBlockId)} --type ${shellEscape(blockType)} --content-file ${shellEscape(tmpFile)}`; if (level && blockType === 'heading') { - cmd += ` --level ${level}`; + cmd += ` --level ${shellEscape(String(level))}`; } const result = await runDws(cmd, { dwsPath: config.dwsPath, timeout: config.dwsTimeout }); const newBlockId = result.data?.blockId || result.data?.id || ''; @@ -258,7 +258,7 @@ const service = defineService({ const { keyword, maxResults } = ctx.request; const limit = maxResults || 10; const result = await runDws( - `doc search --keyword ${shellEscape(keyword)} --limit ${limit}`, + `doc search --keyword ${shellEscape(keyword)} --limit ${shellEscape(String(limit))}`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); @@ -294,7 +294,7 @@ const service = defineService({ // 读取文档，找到上次简报 const readResult = await runDws( - `doc read --node-id ${nodeId}`, + `doc read --node-id ${shellEscape(nodeId)}`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); if (!readResult.success) { @@ -346,7 +346,7 @@ const service = defineService({ // ── 辅助函数 ── const listBlocks = async (start, end) => { const result = await runDws( - `doc block list --node ${nodeId} --start-index ${start} --end-index ${end}`, + `doc block list --node ${shellEscape(nodeId)} --start-index ${shellEscape(String(start))} --end-index ${shellEscape(String(end))}`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); if (!result.success) return []; @@ -373,7 +373,7 @@ const service = defineService({ const updateBlock = async (blockId, text) => { const safeText = shellEscape(text); const result = await runDws( - `doc block update --node ${nodeId} --block-id ${blockId} --type orderedList --text ${safeText} --fix-jsonml`, + `doc block update --node ${shellEscape(nodeId)} --block-id ${shellEscape(blockId)} --type orderedList --text ${safeText} --fix-jsonml`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); return result.success; @@ -382,7 +382,7 @@ const service = defineService({ const insertBlockAfter = async (refBlockId, text) => { const safeText = shellEscape(text); const result = await runDws( - `doc block insert --node ${nodeId} --ref-block ${refBlockId} --type orderedList --text ${safeText} --fix-jsonml`, + `doc block insert --node ${shellEscape(nodeId)} --ref-block ${shellEscape(refBlockId)} --type orderedList --text ${safeText} --fix-jsonml`, { dwsPath: config.dwsPath, timeout: config.dwsTimeout } ); return result.success; @@ -585,7 +585,7 @@ const service = defineService({ cmd += ` --folder ${shellEscape(folderId)}`; } if (pageSize && pageSize > 0) { - cmd += ` --page-size ${pageSize}`; + cmd += ` --page-size ${shellEscape(String(pageSize))}`; } const result = await runDws(cmd, { dwsPath: config.dwsPath, timeout: config.dwsTimeout }); const rawItems = result.data?.nodes || result.data?.items || (Array.isArray(result.data) ? result.data : []); diff --git a/services/dingtalk__doc/lib/dws-runner.js b/services/dingtalk__doc/lib/dws-runner.js index 3b853339..b23597ba 100644 --- a/services/dingtalk__doc/lib/dws-runner.js +++ b/services/dingtalk__doc/lib/dws-runner.js @@ -59,7 +59,7 @@ export async function runDws(command, options = {}) { const fullCommand = `${command} --yes`; return new Promise((resolve) => { - execFile('sh', ['-c', `${dwsPath} ${fullCommand}`], { + execFile('sh', ['-c', `${shellEscape(dwsPath)} ${fullCommand}`], { timeout, maxBuffer: 10 * 1024 * 1024, // 10MB }, (error, stdout, stderr) => { diff --git a/services/dingtalk__doc/lib/safety-check.js b/services/dingtalk__doc/lib/safety-check.js index 093a1c53..9d359389 100644 --- a/services/dingtalk__doc/lib/safety-check.js +++ b/services/dingtalk__doc/lib/safety-check.js @@ -52,7 +52,8 @@ export async function backupDocument(nodeId, content, backupDir) { await mkdir(backupDir, { recursive: true }); const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - const filename = `${nodeId}_${timestamp}.md`; + const safeNodeId = String(nodeId).replace(/[^a-zA-Z0-9_-]/g, '_'); + const filename = `${safeNodeId}_${timestamp}.md`; const filepath = join(backupDir, filename); await writeFile(filepath, content, 'utf-8'); diff --git a/services/dingtalk__drive/bin/drive-service.js b/services/dingtalk__drive/bin/drive-service.js index b1e07530..08a32196 100644 --- a/services/dingtalk__drive/bin/drive-service.js +++ b/services/dingtalk__drive/bin/drive-service.js @@ -112,7 +112,7 @@ const service = defineService({ let cmd = `drive list`; if (spaceId) cmd += ` --space-id ${shellEscape(spaceId)}`; if (folderId) cmd += ` --folder ${shellEscape(folderId)}`; - if (maxResults) cmd += ` --max-results ${maxResults}`; + if (maxResults) cmd += ` --max-results ${shellEscape(String(parseInt(maxResults, 10)))}`; if (nextToken) cmd += ` --next-token ${shellEscape(nextToken)}`; const res = await runDws(cmd); diff --git a/services/dingtalk__message/bin/service.js b/services/dingtalk__message/bin/service.js index 48c836b3..6336c81f 100644 --- a/services/dingtalk__message/bin/service.js +++ b/services/dingtalk__message/bin/service.js @@ -6,8 +6,9 @@ import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; import { execFile } from 'child_process'; function runDws(command, timeout = 60000) { + const dwsPath = process.env.DWS_PATH || 'dws'; return new Promise((resolve) => { - execFile('sh', ['-c', `dws ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + execFile('sh', ['-c', `${dwsPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, (error, stdout) => { const raw = stdout.trim(); let data = null; diff --git a/services/dingtalk__todo/bin/service.js b/services/dingtalk__todo/bin/service.js index be4b8d58..37fd490d 100644 --- a/services/dingtalk__todo/bin/service.js +++ b/services/dingtalk__todo/bin/service.js @@ -7,7 +7,7 @@ import { defineService, runServiceMain } from '@chaitin-ai/octobus-sdk'; import { execFile } from 'child_process'; function runDws(command, timeout = 60000) { - const dwsPath = 'dws'; + const dwsPath = process.env.DWS_PATH || 'dws'; return new Promise((resolve) => { execFile('sh', ['-c', `${dwsPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, (error, stdout) => { @@ -48,7 +48,7 @@ const service = defineService({ 'dingtalk.todo.v1.TodoService/ListTodos': async (ctx) => { const { isDone, limit } = ctx.request; - let cmd = `todo task list --size ${limit || 10}`; + let cmd = `todo task list --size ${parseInt(limit || 10, 10)}`; if (isDone === true) cmd += ` --status true`; else if (isDone === false) cmd += ` --status false`; diff --git a/services/state-manager/lib/event-log.js b/services/state-manager/lib/event-log.js index f3eeb71d..2cae18b7 100644 --- a/services/state-manager/lib/event-log.js +++ b/services/state-manager/lib/event-log.js @@ -35,6 +35,7 @@ export class EventLog { // 内存中的事件缓冲（最近的事件） this.events = []; this._dirty = false; + this._saveChain = Promise.resolve(); // 串行化并发 _save 调用 // 统计 this._stats = { @@ -88,7 +89,8 @@ export class EventLog { await this._rotate(); } - // 立即持久化（事件日志要求可靠写入） + // 立即持久化（事件日志要求可靠写入，串行化防并发损坏） + this._dirty = true; await this._save(); return { success: true, event_id: eventId, error: '' }; @@ -193,6 +195,9 @@ export class EventLog { */ async _rotate() { try { + // 先落盘内存中的事件，防止轮转时丢数据 + await this._save(); + // 递增轮转文件编号 for (let i = this.maxFiles - 1; i >= 1; i--) { const from = join(this.dataDir, `event_log.${i}.json`); @@ -234,21 +239,25 @@ export class EventLog { } async _save() { - try { - await mkdir(this.dataDir, { recursive: true }); - const data = { - version: 1, - updatedAt: new Date().toISOString(), - eventCount: this.events.length, - events: this.events, - }; - const tmpPath = this.filePath + '.tmp'; - await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); - await rename(tmpPath, this.filePath); - this._dirty = false; - } catch (err) { - console.error('[EventLog] save error:', err.message); - } + // 串行化并发写入，防止同一 tmp 文件冲突 + this._saveChain = this._saveChain.then(async () => { + try { + await mkdir(this.dataDir, { recursive: true }); + const data = { + version: 1, + updatedAt: new Date().toISOString(), + eventCount: this.events.length, + events: this.events, + }; + const tmpPath = `${this.filePath}.${Date.now()}.tmp`; + await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); + await rename(tmpPath, this.filePath); + this._dirty = false; + } catch (err) { + console.error('[EventLog] save error:', err.message); + } + }).catch(() => {}); + return this._saveChain; } } diff --git a/services/state-manager/lib/memory-engine.js b/services/state-manager/lib/memory-engine.js index 43573e40..1e98e8ed 100644 --- a/services/state-manager/lib/memory-engine.js +++ b/services/state-manager/lib/memory-engine.js @@ -42,6 +42,8 @@ export class MemoryEngine { totalEvictions: 0, totalDedups: 0, }; + // per-type Promise 串行队列，确保容量检查→淘汰→写入原子化 + this._writeQueues = new Map(); } /** @@ -85,6 +87,20 @@ export class MemoryEngine { this.userId = userId || 'default'; } + /** + * 串行化 per-type 写入操作，防止并发竞态导致容量超限 + * @param {string} type + * @param {() => Promise} fn + */ + async _enqueueWrite(type, fn) { + if (!this._writeQueues.has(type)) { + this._writeQueues.set(type, Promise.resolve()); + } + const chain = this._writeQueues.get(type).then(fn, fn); + this._writeQueues.set(type, chain.catch(() => {})); + return chain; + } + // ─── Remember ─── /** @@ -99,7 +115,7 @@ export class MemoryEngine { const fullKey = this._fullKey(resolved.type, resolved.key); const now = new Date(); - // 去重检查 + // 去重检查（读操作，无需串行化） const existing = this.index.get(fullKey); if (existing && resolved.dedupWindowSec > 0) { const createdMs = new Date(existing.createdAt).getTime(); @@ -135,20 +151,22 @@ export class MemoryEngine { protected: resolved.protected, }; - // 容量检查 + 淘汰 + // 容量检查 + 淘汰 + 写入 + 持久化 在 per-type 队列中串行化 let evictedCount = 0; - const currentCount = this.index.getTypeCount(resolved.type); - const maxEntries = resolved.maxEntries; - if (!existing && currentCount >= maxEntries) { - evictedCount = await this._evict(resolved.type, resolved.evictPolicy, currentCount - maxEntries + 1); - this._stats.totalEvictions += evictedCount; - } + await this._enqueueWrite(resolved.type, async () => { + const currentCount = this.index.getTypeCount(resolved.type); + const maxEntries = resolved.maxEntries; + if (!existing && currentCount >= maxEntries) { + evictedCount = await this._evict(resolved.type, resolved.evictPolicy, currentCount - maxEntries + 1); + this._stats.totalEvictions += evictedCount; + } - // 写入索引 - this.index.set(fullKey, entry); + // 写入索引 + this.index.set(fullKey, entry); - // 持久化 - await this._persistType(resolved.type); + // 持久化 + await this._persistType(resolved.type); + }); this._log('remember', { key: resolved.key, type: resolved.type, result: existing ? 'updated' : 'created', evicted: evictedCount }); return { success: true, unchanged: false, evictedCount, error: '' }; diff --git a/services/state-manager/lib/memory-index.js b/services/state-manager/lib/memory-index.js index 96660ce1..61ac47dc 100644 --- a/services/state-manager/lib/memory-index.js +++ b/services/state-manager/lib/memory-index.js @@ -36,6 +36,9 @@ export class MemoryIndex { set(fullKey, entry) { const old = this.entries.get(fullKey); + // Store fullKey on the entry for O(1) reconstruction + entry._fullKey = fullKey; + // 更新扁平 Map this.entries.set(fullKey, entry); @@ -294,18 +297,28 @@ export class MemoryIndex { _deleteTrie(fullKey) { const segments = fullKey.split(':'); let node = this.trie; - const path = [this.trie]; + const path = [{ node: this.trie, key: null }]; for (const seg of segments) { if (!node.has(seg)) return; node = node.get(seg); - path.push(node); + path.push({ node, key: seg }); } // 删除叶子 entry node.delete('__entry__'); - // 回溯清理空分支（可选优化，MVP 不做） + // 回溯清理空分支（防止 trie 内存泄漏） + for (let i = path.length - 1; i >= 1; i--) { + const { node: currentNode, key: currentKey } = path[i]; + if (currentNode.size === 0) { + // 当前节点已空，从父节点中删除 + path[i - 1].node.delete(currentKey); + } else { + // 该节点还有其他分支，停止回溯 + break; + } + } } _collectLeaves(node, results) { @@ -321,19 +334,13 @@ export class MemoryIndex { } _reconstructKey(entry) { - // 从 entry 的 metadata 重建 fullKey - // 这是一个简化实现：遍历 entries Map 反查 - for (const [k, v] of this.entries) { - if (v === entry) return k; - } - return ''; + // O(1): fullKey is stored on the entry at set() time + return entry._fullKey || ''; } _findFullKey(entry) { - for (const [k, v] of this.entries) { - if (v.createdAt === entry.createdAt && v.type === entry.type) return k; - } - return ''; + // O(1): fullKey is stored on the entry at set() time + return entry._fullKey || ''; } _ensureTtlSorted() { diff --git a/services/state-manager/lib/task-machine.js b/services/state-manager/lib/task-machine.js index 29b88552..722e36f2 100644 --- a/services/state-manager/lib/task-machine.js +++ b/services/state-manager/lib/task-machine.js @@ -37,6 +37,7 @@ export class TaskMachine { this.checkIntervalMs = opts.checkIntervalMs || 60000; this._timer = null; this._dirty = false; + this._saveChain = Promise.resolve(); // 串行化并发 _save 调用 // Statistics this._stats = { @@ -394,22 +395,26 @@ export class TaskMachine { } async _save() { - try { - await mkdir(dirname(this.filePath), { recursive: true }); - const data = { - version: 1, - updatedAt: new Date().toISOString(), - taskCount: this.tasks.size, - tasks: Object.fromEntries(this.tasks), - }; - const tmpPath = this.filePath + '.tmp'; - await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); - const { rename } = await import('node:fs/promises'); - await rename(tmpPath, this.filePath); - this._dirty = false; - } catch (err) { - console.error('[TaskMachine] save error:', err.message); - } + // 串行化并发写入，防止更新丢失 + this._saveChain = this._saveChain.then(async () => { + try { + await mkdir(dirname(this.filePath), { recursive: true }); + const data = { + version: 1, + updatedAt: new Date().toISOString(), + taskCount: this.tasks.size, + tasks: Object.fromEntries(this.tasks), + }; + const tmpPath = `${this.filePath}.${Date.now()}.tmp`; + await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); + const { rename } = await import('node:fs/promises'); + await rename(tmpPath, this.filePath); + this._dirty = false; + } catch (err) { + console.error('[TaskMachine] save error:', err.message); + } + }).catch(() => {}); + return this._saveChain; } } From 45ca04ffc43fd4c35aae59671a5f72c0dfd70951 Mon Sep 17 00:00:00 2001 From: "jianhua.wang" Date: Mon, 6 Jul 2026 04:48:41 +0800 Subject: [PATCH 6/9] =?UTF-8?q?fix(security):=20address=20MonkeyScan=20fin?= =?UTF-8?q?dings=20=E2=80=94=20dwsPath=20shellEscape,=20silent=20error=20s?= =?UTF-8?q?wallowing,=20null=20safety?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes for the latest audit round (2026-07-05 16:01-16:02 UTC): Command injection (4 services): - aitable/calendar/message/todo/bin/service.js: shell-escape dwsPath from process.env.DWS_PATH in local runDws() Null safety: - dingtalk__doc/lib/dws-runner.js: shellEscape(str) now handles null/undefined with String(str ?? ''), preventing TypeError crashes from missing RPC params Data integrity — EventLog: - _save(): removed .catch(() => {}); errors now propagate to callers while the queue itself recovers so subsequent writes aren't blocked - _rotate(): aborts rotation if _save() fails; keeps events in memory for retry on next logEvent call Data integrity — MemoryEngine: - _enqueueWrite(): fixed .then(fn,fn) dirty-write pattern; prior write failures are logged but current write still proceeds (memory is truth) Data integrity — TaskMachine: - _save(): same pattern as EventLog — errors propagate to callers while queue recovers Co-Authored-By: Claude --- services/dingtalk__aitable/bin/service.js | 3 +- services/dingtalk__calendar/bin/service.js | 3 +- services/dingtalk__doc/lib/dws-runner.js | 3 +- services/dingtalk__message/bin/service.js | 3 +- services/dingtalk__todo/bin/service.js | 3 +- services/state-manager/lib/event-log.js | 46 +++++++++++---------- services/state-manager/lib/memory-engine.js | 18 +++++++- services/state-manager/lib/task-machine.js | 42 ++++++++++--------- 8 files changed, 73 insertions(+), 48 deletions(-) diff --git a/services/dingtalk__aitable/bin/service.js b/services/dingtalk__aitable/bin/service.js index 09c626ca..e74ae84c 100644 --- a/services/dingtalk__aitable/bin/service.js +++ b/services/dingtalk__aitable/bin/service.js @@ -28,8 +28,9 @@ function loadKanbanConfig(configPath) { function runDws(command, timeout = 60000) { const dwsPath = process.env.DWS_PATH || 'dws'; + const escapedPath = `'${dwsPath.replace(/'/g, "'\\''")}'`; return new Promise((resolve) => { - execFile('sh', ['-c', `${dwsPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + execFile('sh', ['-c', `${escapedPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, (error, stdout) => { const raw = stdout.trim(); let data = null; diff --git a/services/dingtalk__calendar/bin/service.js b/services/dingtalk__calendar/bin/service.js index eb92d0f4..5a4f3339 100644 --- a/services/dingtalk__calendar/bin/service.js +++ b/services/dingtalk__calendar/bin/service.js @@ -7,8 +7,9 @@ import { execFile } from 'child_process'; function runDws(command, timeout = 60000) { const dwsPath = process.env.DWS_PATH || 'dws'; + const escapedPath = `'${dwsPath.replace(/'/g, "'\\''")}'`; return new Promise((resolve) => { - execFile('sh', ['-c', `${dwsPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + execFile('sh', ['-c', `${escapedPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, (error, stdout) => { const raw = stdout.trim(); let data = null; diff --git a/services/dingtalk__doc/lib/dws-runner.js b/services/dingtalk__doc/lib/dws-runner.js index b23597ba..367147ea 100644 --- a/services/dingtalk__doc/lib/dws-runner.js +++ b/services/dingtalk__doc/lib/dws-runner.js @@ -124,5 +124,6 @@ export async function cleanupTempFile(filePath) { * Shell 转义 */ export function shellEscape(str) { - return `'${str.replace(/'/g, "'\\''")}'`; + const s = String(str ?? ''); + return `'${s.replace(/'/g, "'\\''")}'`; } diff --git a/services/dingtalk__message/bin/service.js b/services/dingtalk__message/bin/service.js index 6336c81f..aea4b914 100644 --- a/services/dingtalk__message/bin/service.js +++ b/services/dingtalk__message/bin/service.js @@ -7,8 +7,9 @@ import { execFile } from 'child_process'; function runDws(command, timeout = 60000) { const dwsPath = process.env.DWS_PATH || 'dws'; + const escapedPath = `'${dwsPath.replace(/'/g, "'\\''")}'`; return new Promise((resolve) => { - execFile('sh', ['-c', `${dwsPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + execFile('sh', ['-c', `${escapedPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, (error, stdout) => { const raw = stdout.trim(); let data = null; diff --git a/services/dingtalk__todo/bin/service.js b/services/dingtalk__todo/bin/service.js index 37fd490d..32122e81 100644 --- a/services/dingtalk__todo/bin/service.js +++ b/services/dingtalk__todo/bin/service.js @@ -8,8 +8,9 @@ import { execFile } from 'child_process'; function runDws(command, timeout = 60000) { const dwsPath = process.env.DWS_PATH || 'dws'; + const escapedPath = `'${dwsPath.replace(/'/g, "'\\''")}'`; return new Promise((resolve) => { - execFile('sh', ['-c', `${dwsPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, + execFile('sh', ['-c', `${escapedPath} ${command} --yes --format json`], { timeout, maxBuffer: 10*1024*1024 }, (error, stdout) => { const raw = stdout.trim(); let data = null; diff --git a/services/state-manager/lib/event-log.js b/services/state-manager/lib/event-log.js index 2cae18b7..5f2fa495 100644 --- a/services/state-manager/lib/event-log.js +++ b/services/state-manager/lib/event-log.js @@ -216,12 +216,14 @@ export class EventLog { // 当前文件不存在，忽略 } - // 清空内存 + // 清空内存（only after successful save + file rotation） this.events = []; this._stats.totalRotated++; this._dirty = false; } catch (err) { - console.error('[EventLog] rotation error:', err.message); + // _save rejected — keep events in memory, do NOT rotate/clear. + // The next logEvent call will retry persistence. + console.error('[EventLog] rotation aborted (save failed, events held in memory):', err.message); } } @@ -239,25 +241,27 @@ export class EventLog { } async _save() { - // 串行化并发写入，防止同一 tmp 文件冲突 - this._saveChain = this._saveChain.then(async () => { - try { - await mkdir(this.dataDir, { recursive: true }); - const data = { - version: 1, - updatedAt: new Date().toISOString(), - eventCount: this.events.length, - events: this.events, - }; - const tmpPath = `${this.filePath}.${Date.now()}.tmp`; - await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); - await rename(tmpPath, this.filePath); - this._dirty = false; - } catch (err) { - console.error('[EventLog] save error:', err.message); - } - }).catch(() => {}); - return this._saveChain; + // 串行化并发写入，防止同一 tmp 文件冲突。 + // The returned promise propagates errors so callers (_rotate, logEvent, + // shutdown) can detect persistence failures. The queue itself recovers + // after a failure so subsequent writes are not permanently blocked. + const writePromise = this._saveChain.then(async () => { + await mkdir(this.dataDir, { recursive: true }); + const data = { + version: 1, + updatedAt: new Date().toISOString(), + eventCount: this.events.length, + events: this.events, + }; + const tmpPath = `${this.filePath}.${Date.now()}.tmp`; + await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); + await rename(tmpPath, this.filePath); + this._dirty = false; + }); + this._saveChain = writePromise.catch((err) => { + console.error('[EventLog] save error (events held in memory, queue continues):', err.message); + }); + return writePromise; } } diff --git a/services/state-manager/lib/memory-engine.js b/services/state-manager/lib/memory-engine.js index 1e98e8ed..38fb54be 100644 --- a/services/state-manager/lib/memory-engine.js +++ b/services/state-manager/lib/memory-engine.js @@ -96,8 +96,22 @@ export class MemoryEngine { if (!this._writeQueues.has(type)) { this._writeQueues.set(type, Promise.resolve()); } - const chain = this._writeQueues.get(type).then(fn, fn); - this._writeQueues.set(type, chain.catch(() => {})); + // Chain the current write onto the queue. If the previous write failed, + // log the error but still attempt the current write — in-memory state + // is the source of truth and must not be silently discarded. + // The per-write Promise returned to the caller propagates errors; + // the queue-level catch prevents a single failure from blocking the + // entire queue for subsequent writes. + const chain = this._writeQueues.get(type).then( + () => fn(), + (err) => { + console.error(`[MemoryEngine] prior write for type "${type}" failed, continuing with current write:`, err.message); + return fn(); + } + ); + this._writeQueues.set(type, chain.catch((err) => { + console.error(`[MemoryEngine] write queue error for type "${type}" (data held in memory, queue continues):`, err.message); + })); return chain; } diff --git a/services/state-manager/lib/task-machine.js b/services/state-manager/lib/task-machine.js index 722e36f2..4b56283f 100644 --- a/services/state-manager/lib/task-machine.js +++ b/services/state-manager/lib/task-machine.js @@ -395,26 +395,28 @@ export class TaskMachine { } async _save() { - // 串行化并发写入，防止更新丢失 - this._saveChain = this._saveChain.then(async () => { - try { - await mkdir(dirname(this.filePath), { recursive: true }); - const data = { - version: 1, - updatedAt: new Date().toISOString(), - taskCount: this.tasks.size, - tasks: Object.fromEntries(this.tasks), - }; - const tmpPath = `${this.filePath}.${Date.now()}.tmp`; - await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); - const { rename } = await import('node:fs/promises'); - await rename(tmpPath, this.filePath); - this._dirty = false; - } catch (err) { - console.error('[TaskMachine] save error:', err.message); - } - }).catch(() => {}); - return this._saveChain; + // 串行化并发写入，防止更新丢失。 + // The returned promise propagates errors so callers (updateTask, createTask) + // can detect persistence failures. The queue itself recovers after a + // failure so subsequent writes are not permanently blocked. + const writePromise = this._saveChain.then(async () => { + await mkdir(dirname(this.filePath), { recursive: true }); + const data = { + version: 1, + updatedAt: new Date().toISOString(), + taskCount: this.tasks.size, + tasks: Object.fromEntries(this.tasks), + }; + const tmpPath = `${this.filePath}.${Date.now()}.tmp`; + await writeFile(tmpPath, JSON.stringify(data, null, 2), 'utf-8'); + const { rename } = await import('node:fs/promises'); + await rename(tmpPath, this.filePath); + this._dirty = false; + }); + this._saveChain = writePromise.catch((err) => { + console.error('[TaskMachine] save error (tasks held in memory, queue continues):', err.message); + }); + return writePromise; } } From 9c65d44b200bca5981641b07cbfb1244d93edc03 Mon Sep 17 00:00:00 2001 From: "jianhua.wang" Date: Mon, 6 Jul 2026 04:56:11 +0800 Subject: [PATCH 7/9] fix(state-manager): wrap _checkTimeouts _save() with try/catch The background timer _checkTimeouts calls _save() without error handling. Since _save() now propagates errors (instead of silently swallowing them), an unhandled rejection would crash the Node.js process. Wrap with try/catch to log the error without crashing. Found by: MonkeyScan re-scan of commit 4259540 (PR #479) Co-Authored-By: Claude --- services/state-manager/lib/task-machine.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/state-manager/lib/task-machine.js b/services/state-manager/lib/task-machine.js index 4b56283f..21e9a7c5 100644 --- a/services/state-manager/lib/task-machine.js +++ b/services/state-manager/lib/task-machine.js @@ -353,7 +353,11 @@ export class TaskMachine { if (changed) { this._dirty = true; - await this._save(); + try { + await this._save(); + } catch (err) { + console.error('[TaskMachine] timeout check save failed (tasks held in memory):', err.message); + } } } From 7e694dede710f76d92a7ed7f89be486bb1c8f33f Mon Sep 17 00:00:00 2001 From: "jianhua.wang" Date: Tue, 7 Jul 2026 09:49:11 +0800 Subject: [PATCH 8/9] =?UTF-8?q?fix(security):=20queryEvents=20=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E8=BD=AE=E8=BD=AC=E6=96=87=E4=BB=B6=20+=20doc-memory?= =?UTF-8?q?=20=E5=8E=9F=E5=AD=90=E5=86=99=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - event-log: queryEvents 新增轮转文件(1..N)读取，历史事件不丢失 - doc-memory: save() 改为原子写入(tmp+rename)，防止并发损坏 MonkeyScan findings addressed: EventLog.queryEvents + doc-memory 并发写入 Co-Authored-By: Claude --- services/dingtalk__doc/lib/doc-memory.js | 7 +++++-- services/state-manager/lib/event-log.js | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/services/dingtalk__doc/lib/doc-memory.js b/services/dingtalk__doc/lib/doc-memory.js index 87ba1a05..cf271fe6 100644 --- a/services/dingtalk__doc/lib/doc-memory.js +++ b/services/dingtalk__doc/lib/doc-memory.js @@ -39,12 +39,15 @@ export class DocMemory { } /** - * 保存记忆到文件 + * 保存记忆到文件（原子写入：先写临时文件再 rename，防止并发损坏） */ async save() { if (!this.cache) return; await mkdir(dirname(this.filePath), { recursive: true }); - await writeFile(this.filePath, JSON.stringify(this.cache, null, 2), 'utf-8'); + const { rename } = await import('node:fs/promises'); + const tmpPath = `${this.filePath}.${Date.now()}.tmp`; + await writeFile(tmpPath, JSON.stringify(this.cache, null, 2), 'utf-8'); + await rename(tmpPath, this.filePath); } /** diff --git a/services/state-manager/lib/event-log.js b/services/state-manager/lib/event-log.js index 5f2fa495..23b04d96 100644 --- a/services/state-manager/lib/event-log.js +++ b/services/state-manager/lib/event-log.js @@ -114,9 +114,24 @@ export class EventLog { * @returns {object} { success, events, total, error } */ async queryEvents(filter = {}) { + // 1. 先查内存中的事件 let results = [...this.events]; - // 按事件类型过滤 + // 2. 读取轮转文件中的历史事件 + const { readFile } = await import('node:fs/promises'); + for (let i = 1; i <= this.maxFiles; i++) { + const rotatedPath = join(this.dataDir, `event_log.${i}.json`); + try { + const raw = await readFile(rotatedPath, 'utf-8'); + const parsed = JSON.parse(raw); + const events = Array.isArray(parsed) ? parsed : (parsed.events || []); + results.push(...events); + } catch { + // 文件不存在，跳过 + } + } + + // 3. 按事件类型过滤 if (filter.event_type) { const types = filter.event_type.split(','); results = results.filter(e => types.includes(e.event_type)); From 63a3a9dde848eed4b7058f883825270e0508f9aa Mon Sep 17 00:00:00 2001 From: "jianhua.wang" Date: Thu, 9 Jul 2026 14:41:57 +0800 Subject: [PATCH 9/9] =?UTF-8?q?fix(security):=20MonkeyScan=20=E5=AE=A1?= =?UTF-8?q?=E8=AE=A1=E4=BF=AE=E5=A4=8D=20=E2=80=94=20=E5=B9=B6=E5=8F=91?= =?UTF-8?q?=E7=AB=9E=E6=80=81/=E5=8E=9F=E5=AD=90=E5=86=99=E5=85=A5/?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E9=9D=99=E9=BB=98=E5=90=9E=E6=B2=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit doc-memory.js: - save() 加 try/finally 清理 rename 失败残留的临时文件 - 临时文件名加随机后缀，防并发碰撞 - get()/set() 经 Promise 链锁串行化 load→modify→save，防 TOCTOU 后写覆盖 - load() 区分 ENOENT(正常跳过) vs 其他错误(记日志)，不静默吞没 - rename/rm 改静态 import event-log.js: - queryEvents 读轮转文件 catch 区分 ENOENT vs 其他(warn 日志) - _load() 同样区分 ENOENT，避免静默丢历史 - _rotate 内层 rename catch 记录非 ENOENT 错误 v3-design.md: - 新增「八、可靠性保证」章节，文档化并发串行化/原子写入/错误处理/轮转保护保证 Co-Authored-By: Claude --- services/dingtalk__doc/lib/doc-memory.js | 58 +++++++++++++++++------- services/state-manager/docs/v3-design.md | 40 +++++++++++++++- services/state-manager/lib/event-log.js | 28 ++++++++---- 3 files changed, 101 insertions(+), 25 deletions(-) diff --git a/services/dingtalk__doc/lib/doc-memory.js b/services/dingtalk__doc/lib/doc-memory.js index cf271fe6..f5e7fd9e 100644 --- a/services/dingtalk__doc/lib/doc-memory.js +++ b/services/dingtalk__doc/lib/doc-memory.js @@ -13,7 +13,7 @@ * 通过 STATE_DATA_DIR 环境变量注入，消除独立 dataDir 配置。 */ -import { readFile, writeFile, mkdir } from 'fs/promises'; +import { readFile, writeFile, mkdir, rename, rm } from 'fs/promises'; import { dirname, join } from 'path'; export class DocMemory { @@ -21,6 +21,9 @@ export class DocMemory { this.dataDir = stateDataDir; this.filePath = join(stateDataDir, 'doc_memory.json'); this.cache = null; + // 串行化所有读/写，防止并发 load→modify→save 的 TOCTOU 竞态 + // （两个并发 set 会各自读到旧缓存、各自写回，后写覆盖前写） + this._chain = Promise.resolve(); } /** @@ -32,43 +35,66 @@ export class DocMemory { try { const raw = await readFile(this.filePath, 'utf-8'); this.cache = JSON.parse(raw); - } catch { + } catch (err) { + // 仅 ENOENT（文件不存在）属正常首次启动；其他错误（权限不足、JSON 损坏、 + // 磁盘故障）需记录，避免静默吞没导致数据丢失无可排查 + if (err.code !== 'ENOENT') { + console.error(`[DocMemory] load failed (${err.code || err.name}): ${err.message}; starting with empty cache`); + } this.cache = {}; } return this.cache; } /** - * 保存记忆到文件（原子写入：先写临时文件再 rename，防止并发损坏） + * 保存记忆到文件（原子写入：先写临时文件再 rename） + * - 临时文件名含随机后缀，防并发碰撞 + * - try/finally 确保 rename 失败时清理残留临时文件 */ async save() { if (!this.cache) return; await mkdir(dirname(this.filePath), { recursive: true }); - const { rename } = await import('node:fs/promises'); - const tmpPath = `${this.filePath}.${Date.now()}.tmp`; - await writeFile(tmpPath, JSON.stringify(this.cache, null, 2), 'utf-8'); - await rename(tmpPath, this.filePath); + const tmpPath = `${this.filePath}.${Date.now()}.${Math.random().toString(36).slice(2, 8)}.tmp`; + try { + await writeFile(tmpPath, JSON.stringify(this.cache, null, 2), 'utf-8'); + await rename(tmpPath, this.filePath); + } finally { + // rename 失败或 writeFile 失败都清理临时文件；force 忽略已不存在 + await rm(tmpPath, { force: true }).catch(() => {}); + } } /** * 获取指定文档的记忆 */ async get(nodeId) { - const memory = await this.load(); - return memory[nodeId] || null; + // 经串行链：等在途写完成后再读，避免读到半写状态 + const next = this._chain.then(async () => { + const memory = await this.load(); + return memory[nodeId] || null; + }); + // 链自身恢复，一次失败不永久阻塞后续调用 + this._chain = next.catch(() => {}); + return next; } /** * 更新文档记忆 */ async set(nodeId, { headerLength, lastBriefTitle }) { - const memory = await this.load(); - memory[nodeId] = { - header_length: headerLength, - last_brief_title: lastBriefTitle, - last_updated: new Date().toISOString(), - }; - await this.save(); + // 串行化 load→modify→save，防止并发 set 互相覆盖 + const next = this._chain.then(async () => { + const memory = await this.load(); + memory[nodeId] = { + header_length: headerLength, + last_brief_title: lastBriefTitle, + last_updated: new Date().toISOString(), + }; + await this.save(); + }); + // 链自身恢复，一次失败不永久阻塞后续调用；返回原 promise 让调用方感知错误 + this._chain = next.catch(() => {}); + return next; } /** diff --git a/services/state-manager/docs/v3-design.md b/services/state-manager/docs/v3-design.md index f0c3a4c4..f0955e93 100644 --- a/services/state-manager/docs/v3-design.md +++ b/services/state-manager/docs/v3-design.md @@ -151,7 +151,45 @@ --- -## 八、路线图 +## 八、可靠性保证 + +状态层作为持久化基础设施，对并发写入、崩溃恢复、错误可观测性有明确保证。以下机制覆盖 EventLog、DocMemory 及 StorageAdapter： + +### 8.1 并发写入串行化（防 TOCTOU） + +所有 read-modify-write 路径经 Promise 链锁串行化，防止两个并发写各自读到旧状态、各自写回导致后写覆盖前写： + +| 组件 | 链锁 | 保护范围 | +|------|------|---------| +| EventLog | `_saveChain` | `logEvent`/`_save`/`_rotate` 串行落盘 | +| DocMemory | `_chain` | `get`/`set` 的 load→modify→save 串行 | +| StorageAdapter | 内部链锁 | 所有 JSON 文件读写 | + +链锁自身具备**失败恢复**：一次写入失败不会永久阻塞后续调用（链尾 `.catch(() => {})` 恢复链，同时原 promise 向调用方传播错误）。 + +### 8.2 原子写入 + 临时文件清理 + +写入采用「先写临时文件再 rename」： +- 临时文件名含 `Date.now()` + 随机后缀，防并发碰撞 +- `try/finally` 确保 rename 失败时 `rm(tmpPath, {force:true})` 清理残留，避免长期积累 `.tmp` + +进程在写入过程中崩溃时，目标文件要么是旧完整内容、要么是新完整内容，不会出现半写损坏。 + +### 8.3 错误不静默吞没 + +文件操作 `catch` 区分 `ENOENT` 与其他错误： +- **ENOENT**（文件不存在）：首次启动的正常情况，静默跳过 +- **其他**（EACCES/ENOSPC/JSON 损坏等）：输出 `warn`/`error` 日志，避免数据丢失或查询不完整而无从排查 + +### 8.4 轮转数据保护 + +`_rotate()` 先 `await this._save()` 落盘内存事件，再执行文件重命名与清空： +- `_save()` 失败 → `_rotate` 捕获异常、**不清空内存**、记日志，下次 `logEvent` 重试持久化 +- 避免磁盘满/权限不足时未落盘事件被轮转清空导致永久丢失 + +--- + +## 九、路线图 ``` v1 MVP（已完成） v2 增强（已完成） v3 通用化（本次） v4 远期 diff --git a/services/state-manager/lib/event-log.js b/services/state-manager/lib/event-log.js index 23b04d96..933763b7 100644 --- a/services/state-manager/lib/event-log.js +++ b/services/state-manager/lib/event-log.js @@ -126,8 +126,12 @@ export class EventLog { const parsed = JSON.parse(raw); const events = Array.isArray(parsed) ? parsed : (parsed.events || []); results.push(...events); - } catch { - // 文件不存在，跳过 + } catch (err) { + // 仅 ENOENT（文件不存在）属正常跳过；其他错误（权限不足、JSON 损坏、 + // 磁盘故障）需记录，避免静默吞没导致查询结果不完整且无可排查 + if (err.code !== 'ENOENT') { + console.warn(`[EventLog] failed to read rotated log ${rotatedPath} (${err.code || err.name}): ${err.message}`); + } } } @@ -219,16 +223,21 @@ export class EventLog { const to = join(this.dataDir, `event_log.${i + 1}.json`); try { await rename(from, to); - } catch { - // 文件不存在，跳过 + } catch (err) { + // 仅 ENOENT 属正常（该编号文件尚未生成）；其他错误需记录 + if (err.code !== 'ENOENT') { + console.warn(`[EventLog] rotate rename ${from} -> ${to} failed (${err.code || err.name}): ${err.message}`); + } } } // 当前文件 → .1 try { await rename(this.filePath, join(this.dataDir, 'event_log.1.json')); - } catch { - // 当前文件不存在，忽略 + } catch (err) { + if (err.code !== 'ENOENT') { + console.warn(`[EventLog] rotate rename current file failed (${err.code || err.name}): ${err.message}`); + } } // 清空内存（only after successful save + file rotation） @@ -250,8 +259,11 @@ export class EventLog { if (Array.isArray(data.events)) { this.events = data.events; } - } catch { - // 文件不存在，从空开始 + } catch (err) { + // 仅 ENOENT 属正常首次启动；其他错误（权限/损坏）需记录，避免静默丢历史 + if (err.code !== 'ENOENT') { + console.warn(`[EventLog] failed to load ${this.filePath} (${err.code || err.name}): ${err.message}; starting empty`); + } } }