diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 6b7543f..a82f9ab 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -94,6 +94,7 @@ jobs: deploy-staging: name: Deploy to Staging runs-on: ubuntu-latest + timeout-minutes: 60 needs: build-and-push if: > (github.event_name == 'push' && github.ref == 'refs/heads/develop') @@ -132,6 +133,7 @@ jobs: username: ${{ secrets.SSH_USERNAME }} key: ${{ secrets.SSH_PRIVATE_KEY }} port: 22 + command_timeout: 45m script: | DEPLOY_DIR="/home/flandern/youdaonotelm" @@ -214,6 +216,7 @@ jobs: deploy-production: name: Deploy to Production runs-on: ubuntu-latest + timeout-minutes: 60 needs: build-and-push if: > (github.event_name == 'push' && github.ref == 'refs/heads/main') @@ -245,6 +248,7 @@ jobs: username: ${{ secrets.PRODUCTION_SSH_USERNAME }} key: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }} port: 22 + command_timeout: 45m script: | DEPLOY_DIR="/home/flandern/youdaonotelm" diff --git a/.gitignore b/.gitignore index e8d1e2d..390e940 100644 --- a/.gitignore +++ b/.gitignore @@ -151,3 +151,5 @@ frontend/test-screenshots/ # Docker (仅排除构建产物,不排除源文件) # configs/config.docker.yaml.example /configs/docker_config.yaml +/chat_Agent.md +/rag.md diff --git a/cmd/server/main.go b/cmd/server/main.go index 3dcc51d..9462647 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,24 +1,24 @@ -package main - -import ( - "flag" - "fmt" - - "YoudaoNoteLm/internal/app" -) - -func main() { - // 定义命令行参数 - flag.Parse() - - // 创建应用实例 创建应用实例 - application := app.NewApp() - - // 初始化应用 - if err := application.Initialize(); err != nil { - panic(fmt.Sprintf("应用初始化失败: %v", err)) - } - - // 运行应用 - application.Run() -} +package main + +import ( + "flag" + "fmt" + + "YoudaoNoteLm/internal/app" +) + +func main() { + // 定义命令行参数 + flag.Parse() + + // 创建应用实例 创建应用实例 + application := app.NewApp() + + // 初始化应用 + if err := application.Initialize(); err != nil { + panic(fmt.Sprintf("应用初始化失败: %v", err)) + } + + // 运行应用 + application.Run() +} diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 3c6add4..5d169db 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -24,7 +24,10 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; } # 静态资源缓存 diff --git a/frontend/package.json b/frontend/package.json index cdaf231..a786466 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -6,8 +6,9 @@ "scripts": { "dev": "vite", "build": "tsc -b && vite build", - "lint": "eslint .", - "preview": "vite preview" + "lint": "eslint .", + "test:generation": "node --experimental-strip-types --test src/stores/generationTaskHelpers.test.ts", + "preview": "vite preview" }, "dependencies": { "axios": "^1.17.0", diff --git a/frontend/src/api/chat.ts b/frontend/src/api/chat.ts index 1daebed..60ac102 100644 --- a/frontend/src/api/chat.ts +++ b/frontend/src/api/chat.ts @@ -1,5 +1,6 @@ import client, { doRefreshToken } from './client'; import type { ApiResponse } from './auth'; +import type { SearchResultItem } from './search'; import { getChatErrorMessage } from '../utils/error'; // ============ Request/Response Types ============ @@ -31,9 +32,12 @@ export interface ReferenceData { } export interface StreamEvent { - type: 'token' | 'reference' | 'done' | 'error' | 'message' | 'title' | 'tool_call' | 'tool_result'; + type: + | 'token' | 'reference' | 'done' | 'error' | 'message' | 'title' | 'tool_call' | 'tool_result' + | 'search_started' | 'search_results' | 'search_busy' + | 'generation_started' | 'generation_result'; content: string; - data: ReferenceData[] | number | null; + data: ReferenceData[] | number | string | null; } // ============ API Functions ============ @@ -117,13 +121,15 @@ async function isTokenErrorResponse(response: Response): Promise { return false; } -// 7. Send message (streaming) - returns a ReadableStream +// 7. Send message (streaming) - returns Response and AbortController export async function sendMessage( conversationId: number, content: string, sourceIds?: number[], llmConfigId?: number -): Promise { +): Promise<{ response: Response; abortController: AbortController }> { + const abortController = new AbortController(); + const makeRequest = (token: string) => fetch(`/api/v1/chat/conversations/${conversationId}/messages`, { method: 'POST', @@ -136,6 +142,7 @@ export async function sendMessage( source_ids: sourceIds || [], llm_config_id: llmConfigId || 0, }), + signal: abortController.signal, }); let token = sessionStorage.getItem('access_token') || ''; @@ -149,7 +156,7 @@ export async function sendMessage( } } - return response; + return { response, abortController }; } // 8. Stop generation @@ -168,16 +175,20 @@ export function parseSSEStream( onToken?: (content: string) => void; onReference?: (references: ReferenceData[]) => void; onTitle?: (title: string) => void; - onDone?: (content: string) => void; + onDone?: (data?: ReferenceData[] | string) => void; onError?: (error: string) => void; - } -): AbortController { - const abortController = new AbortController(); - + onSearchStarted?: () => void; + onSearchResults?: (results: SearchResultItem[], summary: string) => void; + onSearchBusy?: (message: string) => void; + onGenerationStarted?: (type: string) => void; + onGenerationResult?: (type: string, content: string) => void; + }, + abortController?: AbortController +): void { const reader = response.body?.getReader(); if (!reader) { callbacks.onError?.('无法读取响应流'); - return abortController; + return; } const decoder = new TextDecoder(); @@ -243,7 +254,8 @@ export function parseSSEStream( callbacks.onTitle?.(data.content); break; case 'done': - callbacks.onDone?.(data.content); + // 后端将引用附加到 done 事件的 data 字段(原子发送,消除竞态) + callbacks.onDone?.(Array.isArray(data.data) ? data.data as ReferenceData[] : data.content); break; case 'error': callbacks.onError?.(data.content); @@ -253,6 +265,30 @@ export function parseSSEStream( // 工具调用中间事件,不需要展示给用户,静默忽略 console.log('Tool event:', eventType, data.content); break; + case 'search_started': + callbacks.onSearchStarted?.(); + break; + case 'search_results': { + const searchData = data.data as { results?: SearchResultItem[]; summary?: string } | null; + if (searchData?.results) { + callbacks.onSearchResults?.(searchData.results, searchData.summary || ''); + } + break; + } + case 'search_busy': + callbacks.onSearchBusy?.(data.content || '请等待当前搜索任务完成'); + break; + case 'generation_started': + console.log('[Generation] started:', data.data); + callbacks.onGenerationStarted?.((data.data as string) || 'note'); + break; + case 'generation_result': { + const genData = data.data as { type?: string; content?: string } | null; + if (genData?.content) { + callbacks.onGenerationResult?.(genData.type || 'note', genData.content); + } + break; + } default: console.log('Unknown event type:', eventType); // 未知事件类型,不作为 token 显示 @@ -273,17 +309,18 @@ export function parseSSEStream( console.log('Stream ended, calling onDone'); callbacks.onDone?.(''); } catch (error) { - console.error('Stream parsing error:', error); - if (abortController.signal.aborted) { - // Stream was intentionally aborted (user clicked stop) + const isAborted = abortController?.signal.aborted ?? false; + console.log('[parseSSEStream] catch 触发, isAborted:', isAborted, 'error:', error); + if (isAborted) { + // Stream was intentionally aborted (user clicked stop or switched conversation) // Call onDone to preserve the accumulated content + console.log('[parseSSEStream] 检测到 abort,调用 onDone 保存已累积内容'); callbacks.onDone?.(''); } else { const rawMessage = error instanceof Error ? error.message : '流读取错误'; + console.log('[parseSSEStream] 非 abort 错误,调用 onError:', rawMessage); callbacks.onError?.(getChatErrorMessage(rawMessage)); } } })(); - - return abortController; } diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 4441fcf..b0c13b6 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -19,11 +19,11 @@ function onTokenRefreshed(newToken: string) { refreshSubscribers = []; } -function clearAuth() { +function clearAuth(reason?: string) { sessionStorage.removeItem('access_token'); localStorage.removeItem('refresh_token'); localStorage.removeItem('user'); - window.location.href = '/login'; + window.location.href = reason ? `/login?reason=${reason}` : '/login'; } // Request: attach access_token @@ -40,6 +40,11 @@ function isTokenError(data: any): boolean { return data && (data.code === 1005 || data.code === 1006); } +// Check if response indicates the user has been disabled (1004) +function isUserDisabled(data: any): boolean { + return data && data.code === 1004; +} + export async function doRefreshToken(): Promise { const refreshToken = localStorage.getItem('refresh_token'); if (!refreshToken) return null; @@ -62,6 +67,12 @@ client.interceptors.response.use( (response: AxiosResponse) => { const data = response.data; + // 用户被禁用 → 立即强制退出,不尝试刷新 token + if (isUserDisabled(data)) { + clearAuth('disabled'); + return Promise.reject(new Error('user_disabled')); + } + // Backend returns HTTP 200 but code 1005/1006 → token issue if (isTokenError(data)) { const originalRequest = response.config as InternalAxiosRequestConfig & { _retry?: boolean }; @@ -107,6 +118,11 @@ client.interceptors.response.use( async (error) => { // HTTP 4xx/5xx errors - check if it's a token issue in the response body const data = error.response?.data; + // 用户被禁用 → 立即强制退出 + if (isUserDisabled(data)) { + clearAuth('disabled'); + return Promise.reject(error); + } if (isTokenError(data)) { const originalRequest = error.config; if (originalRequest._retry) { diff --git a/frontend/src/api/generation.ts b/frontend/src/api/generation.ts index 9916573..2753670 100644 --- a/frontend/src/api/generation.ts +++ b/frontend/src/api/generation.ts @@ -45,27 +45,75 @@ export interface SearchResult { content?: string; } -export interface GenerationResponse { - type: GenerationType; - content: string; - references?: GenerationReference[]; - search_results?: SearchResult[]; - meta?: Record; -} - -export async function generateFromMarkdown(req: GenerationRequest): Promise { - const res = await client.post<{ code: number; data: GenerationResponse; message?: string }>( - '/generations', - req, - { timeout: 900000 } - ); +export interface GenerationResponse { + type: GenerationType; + content: string; + references?: GenerationReference[]; + search_results?: SearchResult[]; + meta?: Record; +} + +export type GenerationTaskStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'; + +export interface GenerationTask { + task_id: string; + user_id: number; + notebook_id?: number; + type: GenerationType; + status: GenerationTaskStatus; + result?: GenerationResponse; + error?: string; + meta?: Record; + created_at: number; + updated_at: number; + sequence?: number; +} + +export async function generateFromMarkdown(req: GenerationRequest): Promise { + const res = await client.post<{ code: number; data: GenerationTask; message?: string }>( + '/generations', + req, + { timeout: 30000 } + ); if (res.data.code !== 0) { throw new Error(res.data.message || '生成失败'); - } - return res.data.data; -} - -// ============ 导出 API ============ + } + return res.data.data; +} + +export async function getGenerationTask(taskId: string): Promise { + const res = await client.get<{ code: number; data: GenerationTask; message?: string }>( + `/generations/tasks/${taskId}`, + { timeout: 30000 } + ); + if (res.data.code !== 0) { + throw new Error(res.data.message || '获取生成任务失败'); + } + return res.data.data; +} + +export async function listGenerationTasks(params?: { notebook_id?: number; limit?: number }): Promise { + const res = await client.get<{ code: number; data: GenerationTask[]; message?: string }>( + '/generations/tasks', + { params, timeout: 30000 } + ); + if (res.data.code !== 0) { + throw new Error(res.data.message || '获取生成队列失败'); + } + return res.data.data || []; +} + +export async function deleteGenerationTask(taskId: string): Promise { + const res = await client.delete<{ code: number; message?: string }>( + `/generations/tasks/${taskId}`, + { timeout: 30000 } + ); + if (res.data.code !== 0) { + throw new Error(res.data.message || '删除生成任务失败'); + } +} + +// ============ 导出 API ============ function parseAttachmentFilename(disposition?: string): string | null { if (!disposition) return null; diff --git a/frontend/src/api/userConfig.ts b/frontend/src/api/userConfig.ts index 85ed331..d5e7b31 100644 --- a/frontend/src/api/userConfig.ts +++ b/frontend/src/api/userConfig.ts @@ -191,6 +191,39 @@ export async function deleteEmbeddingAndCollection( return res.data; } +// ===== Reranker Config ===== + +export async function listRerankerConfigs(): Promise<{ + code: number; + data: UserConfig[]; + message?: string; +}> { + const res = await client.get('/user/config/reranker'); + return res.data; +} + +export async function createRerankerConfig( + data: UserConfigRequest +): Promise<{ code: number; data: UserConfig; message?: string }> { + const res = await client.post('/user/config/reranker', data); + return res.data; +} + +export async function updateRerankerConfig( + id: number, + data: UserConfigRequest +): Promise<{ code: number; data: UserConfig; message?: string }> { + const res = await client.put(`/user/config/reranker/${id}`, data); + return res.data; +} + +export async function deleteRerankerConfig( + id: number +): Promise<{ code: number; message?: string }> { + const res = await client.delete(`/user/config/reranker/${id}`); + return res.data; +} + // ===== Health Check ===== export async function testConfig( diff --git a/frontend/src/api/userMemory.ts b/frontend/src/api/userMemory.ts new file mode 100644 index 0000000..6a4af66 --- /dev/null +++ b/frontend/src/api/userMemory.ts @@ -0,0 +1,39 @@ +import client from './client'; + +export type MemoryType = + | 'language' + | 'answer_length' + | 'answer_style' + | 'output_format' + | 'generation_style' + | 'custom_instruction'; + +export interface UserMemory { + type: MemoryType; + content: string; + updated_at: string; +} + +interface ApiResponse { + code: number; + message?: string; + data: T; +} + +export async function listUserMemories(): Promise> { + const res = await client.get>('/user/memories'); + return res.data; +} + +export async function upsertUserMemory( + type: MemoryType, + content: string, +): Promise> { + const res = await client.put>(`/user/memories/${type}`, { content }); + return res.data; +} + +export async function deleteUserMemory(type: MemoryType): Promise> { + const res = await client.delete>(`/user/memories/${type}`); + return res.data; +} diff --git a/frontend/src/components/notebook/ChatPanel.tsx b/frontend/src/components/notebook/ChatPanel.tsx index 5a7e0be..9aea259 100644 --- a/frontend/src/components/notebook/ChatPanel.tsx +++ b/frontend/src/components/notebook/ChatPanel.tsx @@ -337,6 +337,7 @@ export default function ChatPanel() { const [deletingConvId, setDeletingConvId] = useState(null); const messagesEndRef = useRef(null); const prevConvIdRef = useRef(null); + const prevStreamingConvIdRef = useRef(null); const modelListRef = useRef(null); // Check if any message is streaming @@ -350,13 +351,39 @@ export default function ChatPanel() { messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); }, [conversation?.messages, displayContent]); - // Fetch messages only when conversation ID changes (not on every render) + // 当会话切换时,记录需要停止流式生成的旧会话 ID useEffect(() => { if (currentNotebookId && conversation?.id && conversation.id !== prevConvIdRef.current) { + const prevConvId = prevConvIdRef.current; + // 检查旧会话是否有流式消息 + if (prevConvId) { + const nb = notebooks.find((n) => n.id === currentNotebookId); + const prevConv = nb?.conversations.find((c) => c.id === prevConvId); + if (prevConv?.messages.some((m) => m.isStreaming)) { + prevStreamingConvIdRef.current = prevConvId; + } + } prevConvIdRef.current = conversation.id; + } + }, [currentNotebookId, conversation?.id, notebooks]); + + // 拉取消息:如果有旧会话正在流式生成,等停止完成后再拉取(防止旧流的 onDone 回调覆盖新数据) + useEffect(() => { + if (!currentNotebookId || !conversation?.id) return; + + const streamingConvId = prevStreamingConvIdRef.current; + if (streamingConvId) { + prevStreamingConvIdRef.current = null; + console.log('[ChatPanel] 等待停止旧会话流后再拉取消息:', streamingConvId); + stopGeneration(currentNotebookId, streamingConvId) + .catch(() => {}) + .finally(() => { + fetchMessages(currentNotebookId, conversation.id); + }); + } else { fetchMessages(currentNotebookId, conversation.id); } - }, [currentNotebookId, conversation?.id, fetchMessages]); + }, [currentNotebookId, conversation?.id, fetchMessages, stopGeneration]); // Load LLM configs on mount useEffect(() => { @@ -418,8 +445,8 @@ export default function ChatPanel() { if (!notebook || !currentNotebookId) return null; - // 只有已入库且选中的资料才算资料来源 - const selectedSources = notebook.sources.filter((s) => s.selected && s.vectorized && s.status !== 'error'); + // 选中且非错误状态的资料都传给后端(未向量化的资料由后端自行处理) + const selectedSources = notebook.sources.filter((s) => s.selected && s.status !== 'error'); const handleSend = async () => { if (!input.trim() || isStreaming || !conversation?.id) return; diff --git a/frontend/src/components/notebook/NotesPanel.tsx b/frontend/src/components/notebook/NotesPanel.tsx index 371ad3e..ad3f069 100644 --- a/frontend/src/components/notebook/NotesPanel.tsx +++ b/frontend/src/components/notebook/NotesPanel.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useEffect, useState } from 'react'; import { motion, AnimatePresence } from 'framer-motion'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; @@ -38,9 +38,9 @@ const typeColors: Record = { ppt: 'bg-purple-500/10 text-purple-400', }; -export default function NotesPanel() { - const { currentNotebookId, getCurrentNotebook, deleteNote, renameNote, toggleNoteSource, generateNote, generatingType, generationError, clearGenerationError } = useNotebookStore(); - const notebook = getCurrentNotebook(); +export default function NotesPanel() { + const { currentNotebookId, getCurrentNotebook, deleteNote, renameNote, toggleNoteSource, generateNote, generationTasks, generationError, clearGenerationError, connectGenerationTasks, disconnectGenerationTasks, deleteGenerationTask } = useNotebookStore(); + const notebook = getCurrentNotebook(); const [searchQuery, setSearchQuery] = useState(''); const [selectedNote, setSelectedNote] = useState(null); @@ -49,15 +49,28 @@ export default function NotesPanel() { const [editingId, setEditingId] = useState(null); const [editTitle, setEditTitle] = useState(''); const [genPrompt, setGenPrompt] = useState(''); - const [useWeb, setUseWeb] = useState(true); - const [allowDegrade, setAllowDegrade] = useState(true); - const [pptStyle, setPptStyle] = useState('auto'); - - if (!notebook || !currentNotebookId) return null; - - const filteredNotes = notebook.notes.filter((n) => - n.title.toLowerCase().includes(searchQuery.toLowerCase()) - ); + const [useWeb, setUseWeb] = useState(true); + const [allowDegrade, setAllowDegrade] = useState(true); + const [pptStyle, setPptStyle] = useState('auto'); + + useEffect(() => { + if (!currentNotebookId) return; + connectGenerationTasks(currentNotebookId); + return () => disconnectGenerationTasks(); + }, [currentNotebookId, connectGenerationTasks, disconnectGenerationTasks]); + + if (!notebook || !currentNotebookId) return null; + + const filteredNotes = notebook.notes.filter((n) => + n.title.toLowerCase().includes(searchQuery.toLowerCase()) + ); + const activeGenerationTasks = generationTasks + .filter((task) => task.notebookId === currentNotebookId && (task.status === 'pending' || task.status === 'running')) + .sort((a, b) => { + if ((a.sequence ?? 0) !== (b.sequence ?? 0)) return (a.sequence ?? 0) - (b.sequence ?? 0); + if (a.createdAt !== b.createdAt) return a.createdAt - b.createdAt; + return a.taskId.localeCompare(b.taskId); + }); const handleStartRename = (id: string, title: string) => { setEditingId(id); @@ -128,17 +141,27 @@ export default function NotesPanel() { }; // 调用后端生成 Agent - const handleGenerate = async (type: NoteType) => { - if (generatingType || !currentNotebookId) return; - await generateNote(currentNotebookId, type, { - prompt: genPrompt.trim() || undefined, - useWeb, - allowDegrade, - pptStyle: type === 'ppt' ? pptStyle : undefined, - }); - }; - - // ---- Note Viewer ---- + const handleGenerate = (type: NoteType) => { + if (!currentNotebookId) return; + void generateNote(currentNotebookId, type, { + prompt: genPrompt.trim() || undefined, + useWeb, + allowDegrade, + pptStyle: type === 'ppt' ? pptStyle : undefined, + }); + }; + + const getActiveTaskForType = (type: NoteType) => + activeGenerationTasks.find((task) => task.type === type); + + const getGenerationButtonLabel = (type: NoteType) => { + const task = getActiveTaskForType(type); + if (task?.status === 'pending') return '排队中'; + if (task?.status === 'running') return '执行中'; + return null; + }; + + // ---- Note Viewer ---- if (selectedNote) { return (
@@ -230,33 +253,35 @@ export default function NotesPanel() { { icon: Presentation, label: 'PPT', type: 'ppt' as NoteType, color: 'from-purple-500 to-pink-400' }, { icon: HelpCircle, label: '测验', type: 'quiz' as NoteType, color: 'from-orange-400 to-amber-400' }, { icon: FileText, label: '笔记', type: 'note' as NoteType, color: 'from-blue-400 to-cyan-400' }, - ].map(({ icon: Icon, label, type, color }) => { - const isActive = generatingType === type; - const isDisabled = generatingType !== null && generatingType !== type; - return ( - + ].map(({ icon: Icon, label, type, color }) => { + const activeTask = getActiveTaskForType(type); + const taskLabel = getGenerationButtonLabel(type); + return ( + ); })}
@@ -265,21 +290,20 @@ export default function NotesPanel() {
setGenPrompt(e.target.value)} - placeholder={'输入自定义提示词,如「聚焦第三章核心概念」...'} disabled={generatingType !== null} - className={cn( - 'w-full h-8 px-3 pr-8 rounded-lg border text-xs outline-none transition-colors', - 'bg-bg-secondary border-border text-text-primary placeholder:text-text-muted', - 'focus:border-accent/50 focus:bg-bg-card', - generatingType !== null && 'opacity-50 cursor-not-allowed' - )} - /> - {genPrompt && !generatingType && ( - @@ -287,59 +311,93 @@ export default function NotesPanel() {
- setPptStyle(e.target.value)} + title="PPT 风格" + className={cn( + 'h-7 px-2 rounded-md text-[11px] border outline-none transition-all', + 'bg-bg-tertiary border-border text-text-secondary focus:border-accent/40' + )} + > - - -
-
- - - {/* Search */} + + + + + {activeGenerationTasks.length > 0 && ( +
+
+ 生成任务 + {activeGenerationTasks.length} 个待处理 +
+
+ {activeGenerationTasks.map((task, index) => ( +
+
+ {task.status === 'running' ? ( + + ) : ( + + )} + + {index + 1}. {typeLabels[task.type]} + +
+ + {task.status === 'running' ? '执行中' : '排队中'} + + +
+ ))} +
+
+ )} + + + {/* Search */}
diff --git a/frontend/src/components/notebook/SourcesPanel.tsx b/frontend/src/components/notebook/SourcesPanel.tsx index 3622ab2..810d074 100644 --- a/frontend/src/components/notebook/SourcesPanel.tsx +++ b/frontend/src/components/notebook/SourcesPanel.tsx @@ -47,7 +47,8 @@ export default function SourcesPanel() { currentNotebookId, getCurrentNotebook, toggleSourceSelection, removeSource, batchRemoveSources, deleteFailedSources, renameSource, importFile, previewAudio, confirmAudio, searchSourcesStream, importFromURL, importSearchResults, fetchSourceContent, getSourceDownloadURL, fetchSources, - reimportSelected + reimportSelected, + mainAgentSearchActive, mainAgentSearchResults, mainAgentSearchSummary, clearMainAgentSearch, } = useNotebookStore(); const notebook = getCurrentNotebook(); @@ -107,6 +108,35 @@ export default function SourcesPanel() { } }, [notebook?.sources, audioPreview, audioTranscribing]); + // 主从协同:监听主 agent 触发的搜索结果,同步到本地搜索面板状态(和普通搜索共用同一面板) + useEffect(() => { + if (!mainAgentSearchActive) { + // 主 agent 搜索结束(完成/取消/出错),停止 loading(防止取消后持续转圈) + setIsSearching(false); + return; + } + setIsSearchPanelOpen(true); + setIsSearchPanelCollapsed(false); + setSelectedResults(new Set()); + + if (mainAgentSearchResults.length > 0) { + setSearchResults(mainAgentSearchResults); + setSearchSummary(mainAgentSearchSummary || '搜索完成'); + setIsSearching(false); + setSearchProgress(''); + } else if (mainAgentSearchSummary) { + setIsSearching(false); + setSearchSummary(mainAgentSearchSummary); + setSearchProgress(''); + } else { + // 还在搜索中 + setIsSearching(true); + setSearchResults([]); + setSearchSummary(''); + setSearchProgress('AI 正在搜索和分析...'); + } + }, [mainAgentSearchActive, mainAgentSearchResults, mainAgentSearchSummary]); + if (!notebook || !currentNotebookId) return null; const filteredSources = notebook.sources.filter((s) => @@ -303,7 +333,13 @@ export default function SourcesPanel() { ); setSearchResults(finalResults); - setSearchSummary(finalSummary || '搜索完成'); + if (finalSummary) { + setSearchSummary(finalSummary); + } else if (finalResults.length === 0) { + setSearchSummary('未找到相关结果,请换个关键词或问题再试'); + } else { + setSearchSummary('搜索完成'); + } } catch (err: any) { if (err.name === 'AbortError') return; console.error('Search failed:', err); @@ -314,10 +350,21 @@ export default function SourcesPanel() { const msg = getErrorMessage(err, '未知错误'); if (errorCode === 40010) { - // CodeLLMNotConfigured - setSearchSummary('搜索需要先配置 LLM 服务。请前往 设置 → AI 服务配置 添加 LLM 配置后再试。'); - } else if (msg.includes('LLM') || msg.includes('llm') || msg.includes('配置')) { - setSearchSummary('搜索需要先配置 LLM 服务。请前往 设置 → AI 服务配置 添加 LLM 配置后再试。'); + // CodeSearchProviderNotConfigured:搜索引擎未配置 + setSearchSummary('请前往 设置 → 添加搜索引擎配置后再试'); + } else if (errorCode === 40020) { + // CodeLLMNotConfigured:LLM 未配置(搜索 Agent 依赖 LLM) + setSearchSummary('请前往 设置 → 添加 LLM 配置后再试'); + } else if (errorCode === 40011) { + // CodeSearchInvalidAPIKey:搜索引擎 API Key 无效 + setSearchSummary('请前往 设置 → 更新搜索引擎 API Key 后重试'); + } else if (errorCode === 40012 || errorCode === 40013) { + // 搜索超时 / 服务不可用 + setSearchSummary(`搜索失败:${msg}`); + } else if (msg.includes('搜索引擎') || msg.includes('search')) { + setSearchSummary('请前往 设置 → 添加搜索引擎配置后再试'); + } else if (msg.includes('LLM') || msg.includes('llm')) { + setSearchSummary('请前往 设置 → 添加 LLM 配置后再试'); } else { setSearchSummary(`搜索失败:${msg}`); } @@ -377,6 +424,8 @@ export default function SourcesPanel() { setSearchSummary(''); setSearchProgress(''); setSelectedResults(new Set()); + // 清理主 agent 搜索状态,避免下次搜索时 useEffect 不重新触发 + clearMainAgentSearch(); }; const handleCollapseSearchPanel = () => { @@ -478,6 +527,12 @@ export default function SourcesPanel() { await confirmAudio(previewId, currentNotebookId, content); } catch (err) { console.error('Confirm audio failed:', err); + // 清除 confirmedPreviewIds,让 UI 显示错误状态而非"导入中..." + setConfirmedPreviewIds(prev => { + const next = new Set(prev); + next.delete(previewId); + return next; + }); } }} > @@ -1013,13 +1068,11 @@ export default function SourcesPanel() { {/* Import Modal */} setShowImportModal(false)} title="导入资料" size="md"> importFile(currentNotebookId, file).then(() => setShowImportModal(false)).catch(console.error)} + onClose={() => setShowImportModal(false)} + onFileImport={(file) => importFile(currentNotebookId, file)} onAudioImport={async (file) => { - try { - await previewAudio(currentNotebookId, file); - setShowImportModal(false); - // 不跳转到转写预览面板,转写完成后通过通知横幅提醒用户 - } catch (err) { console.error(err); } + await previewAudio(currentNotebookId, file); + // 不跳转到转写预览面板,转写完成后通过通知横幅提醒用户 }} onUrlImport={async (url) => { try { @@ -1068,37 +1121,81 @@ export default function SourcesPanel() { ); } -function ImportModalContent({ onFileImport, onAudioImport, onUrlImport, onYoudaoImport }: { - onFileImport: (file: File) => Promise; - onAudioImport: (file: File) => void; +function ImportModalContent({ onFileImport, onAudioImport, onUrlImport, onYoudaoImport, onClose }: { + onFileImport: (file: File) => Promise; + onAudioImport: (file: File) => Promise; onUrlImport: (url: string) => void; onYoudaoImport: (fileIds: string[], fileNames: Record) => Promise; + onClose: () => void; }) { const [tab, setTab] = useState<'youdao' | 'file' | 'url'>('youdao'); const [urlValue, setUrlValue] = useState(''); const fileInputRef = useRef(null); const [uploading, setUploading] = useState(false); const [showYoudaoPanel, setShowYoudaoPanel] = useState(false); + const [isDragging, setIsDragging] = useState(false); + const [uploadProgress, setUploadProgress] = useState<{ current: number; total: number } | null>(null); const audioExts = ['.mp3', '.wav']; - const handleFileSelect = async (e: React.ChangeEvent) => { - const file = e.target.files?.[0]; - if (!file) return; + const processFiles = async (files: FileList | File[]) => { + const fileArray = Array.from(files); + if (fileArray.length === 0) return; + setUploading(true); + setUploadProgress({ current: 0, total: fileArray.length }); + try { - const ext = '.' + file.name.split('.').pop()?.toLowerCase(); - if (audioExts.includes(ext)) { - await onAudioImport(file); - } else { - await onFileImport(file); + for (let i = 0; i < fileArray.length; i++) { + setUploadProgress({ current: i + 1, total: fileArray.length }); + const file = fileArray[i]; + const ext = '.' + file.name.split('.').pop()?.toLowerCase(); + if (audioExts.includes(ext)) { + await onAudioImport(file); + } else { + await onFileImport(file); + } } + // 所有文件处理成功后关闭弹窗 + onClose(); + } catch (err) { + console.error('File upload failed:', err); } finally { setUploading(false); + setUploadProgress(null); if (fileInputRef.current) fileInputRef.current.value = ''; } }; + const handleFileSelect = async (e: React.ChangeEvent) => { + const files = e.target.files; + if (!files) return; + await processFiles(files); + }; + + const handleDragOver = (e: React.DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsDragging(true); + }; + + const handleDragLeave = (e: React.DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsDragging(false); + }; + + const handleDrop = async (e: React.DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsDragging(false); + + const files = e.dataTransfer.files; + if (files.length > 0) { + await processFiles(files); + } + }; + const handleUrlImport = () => { if (!urlValue.trim()) return; onUrlImport(urlValue.trim()); @@ -1159,20 +1256,32 @@ function ImportModalContent({ onFileImport, onAudioImport, onUrlImport, onYoudao )} {tab === 'file' && ( -
- -

拖拽文件到此处,或点击选择

+
+ +

+ {isDragging ? '松开鼠标上传文件' : '拖拽文件到此处,或点击选择'} +

支持 PDF, DOCX, TXT, MD, HTML

-

音频支持 MP3, WAV

+

音频支持 MP3, WAV(支持多选)

{uploading ? ( -
+
- 上传中... + + {uploadProgress ? `上传中 (${uploadProgress.current}/${uploadProgress.total})...` : '上传中...'} +
) : ( <> - + )}
diff --git a/frontend/src/components/notebook/YoudaoImportPanel.tsx b/frontend/src/components/notebook/YoudaoImportPanel.tsx index 45889d5..5bc5e1a 100644 --- a/frontend/src/components/notebook/YoudaoImportPanel.tsx +++ b/frontend/src/components/notebook/YoudaoImportPanel.tsx @@ -1,4 +1,5 @@ import { useState, useEffect } from 'react'; +import { useNavigate } from 'react-router-dom'; import { motion } from 'framer-motion'; import { Folder, ChevronRight, ArrowLeft, @@ -16,18 +17,42 @@ interface YoudaoImportPanelProps { } export default function YoudaoImportPanel({ onImport, onBack }: YoudaoImportPanelProps) { + const navigate = useNavigate(); const [notes, setNotes] = useState([]); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); + const [notBound, setNotBound] = useState(false); + const [bindChecked, setBindChecked] = useState(false); const [selectedNotes, setSelectedNotes] = useState>(new Set()); const [currentFolder, setCurrentFolder] = useState(null); const [folderPath, setFolderPath] = useState<{ id: string; name: string }[]>([]); const [importing, setImporting] = useState(false); - // 加载有道云笔记数据 + // 挂载时先检查有道云绑定状态,未绑定则引导用户去绑定(避免发起注定失败的 listNotes) useEffect(() => { + let cancelled = false; + (async () => { + try { + const res = await youdaoApi.getBindStatus(); + if (cancelled) return; + if (res.code === 0 && !res.data?.bound) { + setNotBound(true); + setBindChecked(true); + return; + } + } catch { + // 绑定状态查询失败,兜底交给 loadNotes 的错误处理 + } + if (!cancelled) setBindChecked(true); + })(); + return () => { cancelled = true; }; + }, []); + + // 已绑定后,目录切换时加载笔记 + useEffect(() => { + if (!bindChecked || notBound) return; loadNotes(currentFolder); - }, [currentFolder]); + }, [currentFolder, bindChecked, notBound]); const loadNotes = async (folderId: string | null) => { setLoading(true); @@ -50,6 +75,22 @@ export default function YoudaoImportPanel({ onImport, onBack }: YoudaoImportPane } }; + // 重新检查绑定状态(未绑定界面下点击刷新按钮时调用,便于用户在别处绑定后回到此面板刷新) + const recheckBinding = async () => { + setLoading(true); + try { + const res = await youdaoApi.getBindStatus(); + if (res.code === 0 && res.data?.bound) { + setNotBound(false); + setError(null); + } + } catch { + // 忽略,保持当前状态 + } finally { + setLoading(false); + } + }; + const handleFolderClick = (folderId: string, folderName: string) => { setLoading(true); // 立即显示加载状态,避免闪烁 setNotes([]); // 清空当前笔记列表,避免显示旧数据 @@ -121,7 +162,7 @@ export default function YoudaoImportPanel({ onImport, onBack }: YoudaoImportPane +
) : error ? (
diff --git a/frontend/src/components/settings/LongTermMemorySettings.tsx b/frontend/src/components/settings/LongTermMemorySettings.tsx new file mode 100644 index 0000000..96008e7 --- /dev/null +++ b/frontend/src/components/settings/LongTermMemorySettings.tsx @@ -0,0 +1,242 @@ +import { useEffect, useState } from 'react'; +import { AlertCircle, Check, Loader2, Trash2 } from 'lucide-react'; + +import * as userMemoryApi from '../../api/userMemory'; +import type { MemoryType, UserMemory } from '../../api/userMemory'; +import Button from '../ui/Button'; +import Input from '../ui/Input'; +import { getErrorMessage } from '../../utils/error'; + +type MemorySlot = { + type: MemoryType; + label: string; + description: string; + placeholder: string; +}; + +const memorySlots: MemorySlot[] = [ + { + type: 'language', + label: '默认语言', + description: '跨会话默认使用的回答语言;当前支持中文和 English。', + placeholder: '', + }, + { + type: 'answer_length', + label: '回答篇幅', + description: '默认的简洁或展开程度。', + placeholder: '例如:先给五点以内的简洁结论,需要时再展开。', + }, + { + type: 'answer_style', + label: '回答方式', + description: '回答的组织和表达顺序。', + placeholder: '例如:先给结论,再给理由和可执行步骤。', + }, + { + type: 'output_format', + label: '输出格式', + description: '常用的结果呈现方式。', + placeholder: '例如:涉及比较时优先使用 Markdown 表格。', + }, + { + type: 'generation_style', + label: '生成风格', + description: '用于 PPT、笔记、测验和脑图的通用偏好。', + placeholder: '例如:PPT 保持正式、简洁,每页一个核心观点。', + }, + { + type: 'custom_instruction', + label: '通用偏好', + description: '一条跨会话复用的其他输出偏好。', + placeholder: '例如:术语第一次出现时附一句通俗解释。', + }, +]; + +function emptyDrafts(): Record { + return memorySlots.reduce((drafts, slot) => { + drafts[slot.type] = ''; + return drafts; + }, {} as Record); +} + +function memoryByType(memories: UserMemory[]): Partial> { + return memories.reduce((result, memory) => { + result[memory.type] = memory; + return result; + }, {} as Partial>); +} + +export default function LongTermMemorySettings() { + const [memories, setMemories] = useState>>({}); + const [drafts, setDrafts] = useState>(emptyDrafts); + const [loading, setLoading] = useState(true); + const [savingType, setSavingType] = useState(null); + const [deletingType, setDeletingType] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + const load = async () => { + try { + const response = await userMemoryApi.listUserMemories(); + if (cancelled) return; + if (response.code !== 0) { + setError(response.message || '加载长期记忆失败'); + return; + } + const nextMemories = memoryByType(response.data); + setMemories(nextMemories); + const nextDrafts = emptyDrafts(); + memorySlots.forEach((slot) => { + nextDrafts[slot.type] = nextMemories[slot.type]?.content || ''; + }); + setDrafts(nextDrafts); + } catch (requestError) { + if (!cancelled) { + setError(getErrorMessage(requestError, '加载长期记忆失败')); + } + } finally { + if (!cancelled) { + setLoading(false); + } + } + }; + void load(); + return () => { + cancelled = true; + }; + }, []); + + const save = async (type: MemoryType) => { + const content = drafts[type].trim(); + if (!content) { + setError('请先填写偏好内容;如需移除,请使用清除按钮。'); + return; + } + setSavingType(type); + setError(null); + try { + const response = await userMemoryApi.upsertUserMemory(type, content); + if (response.code !== 0) { + setError(response.message || '保存长期记忆失败'); + return; + } + setMemories((current) => ({ ...current, [type]: response.data })); + setDrafts((current) => ({ ...current, [type]: response.data.content })); + } catch (requestError) { + setError(getErrorMessage(requestError, '保存长期记忆失败')); + } finally { + setSavingType(null); + } + }; + + const clear = async (type: MemoryType) => { + if (!memories[type] || !window.confirm('清除后,该偏好不会再用于后续对话和生成。确定继续吗?')) { + return; + } + setDeletingType(type); + setError(null); + try { + const response = await userMemoryApi.deleteUserMemory(type); + if (response.code !== 0) { + setError(response.message || '清除长期记忆失败'); + return; + } + setMemories((current) => { + const next = { ...current }; + delete next[type]; + return next; + }); + setDrafts((current) => ({ ...current, [type]: '' })); + } catch (requestError) { + setError(getErrorMessage(requestError, '清除长期记忆失败')); + } finally { + setDeletingType(null); + } + }; + + return ( +
+
+

长期记忆

+

+ 这里保存的是跨会话的输出偏好。当前请求有明确要求时优先,记忆不是资料事实来源。 + 请不要保存密码、令牌、身份证号或其他秘密。 +

+
+ + {error && ( +
+ +

{error}

+
+ )} + + {loading ? ( +
加载中...
+ ) : ( +
+ {memorySlots.map((slot) => { + const saved = memories[slot.type]; + const busy = savingType === slot.type || deletingType === slot.type; + return ( +
+
+
+

{slot.label}

+

{slot.description}

+
+ {saved && ( + + 已保存 + + )} +
+ {slot.type === 'language' ? ( + + ) : ( + setDrafts((current) => ({ ...current, [slot.type]: event.target.value }))} + /> + )} +
+ + {slot.type === 'language' ? '当前支持中文和 English' : '最多 160 个字符'} + +
+ {saved && ( + + )} + +
+
+
+ ); + })} +
+ )} +
+ ); +} diff --git a/frontend/src/components/ui/Input.tsx b/frontend/src/components/ui/Input.tsx index db2ae54..2ee79da 100644 --- a/frontend/src/components/ui/Input.tsx +++ b/frontend/src/components/ui/Input.tsx @@ -1,4 +1,5 @@ -import { forwardRef, type InputHTMLAttributes } from 'react'; +import { forwardRef, useState, type InputHTMLAttributes } from 'react'; +import { Eye, EyeOff } from 'lucide-react'; import { cn } from '../../utils/cn'; interface InputProps extends InputHTMLAttributes { @@ -8,7 +9,11 @@ interface InputProps extends InputHTMLAttributes { } const Input = forwardRef( - ({ className, label, error, icon, ...props }, ref) => { + ({ className, label, error, icon, type, autoComplete, ...props }, ref) => { + const [show, setShow] = useState(false); + const isPassword = type === 'password'; + const effectiveType = isPassword ? (show ? 'text' : 'password') : type; + return (
{label && ( @@ -20,19 +25,39 @@ const Input = forwardRef( {icon}
)} + {/* 隐藏的假 input 消耗浏览器自动填充(Chrome 会优先填这些,保护真实输入框不被填充登录密码) */} + {isPassword && ( + <> + + + + )} + {isPassword && ( + + )}
{error &&

{error}

}
diff --git a/frontend/src/pages/AdminPage.tsx b/frontend/src/pages/AdminPage.tsx index 828b82a..06e9e76 100644 --- a/frontend/src/pages/AdminPage.tsx +++ b/frontend/src/pages/AdminPage.tsx @@ -10,6 +10,7 @@ import Button from '../components/ui/Button'; import Input from '../components/ui/Input'; import Badge from '../components/ui/Badge'; import AvatarImg from '../components/ui/AvatarImg'; +import { useAuthStore } from '../stores/useAuthStore'; import * as adminApi from '../api/admin'; import * as providersApi from '../api/providers'; import type { AdminUser, SysConfig, ConfigStatus } from '../api/admin'; @@ -67,11 +68,13 @@ export default function AdminPage() { // ===== User Management Component ===== function UserManagement() { + const currentUser = useAuthStore((s) => s.user); const [users, setUsers] = useState([]); const [total, setTotal] = useState(0); const [page, setPage] = useState(1); const [keyword, setKeyword] = useState(''); const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); const fetchUsers = async () => { setLoading(true); @@ -93,18 +96,42 @@ function UserManagement() { }, [page, keyword]); const handleToggleUser = async (userId: number, enabled: boolean) => { + setError(null); try { const res = await adminApi.updateUserStatus(userId, enabled); if (res.code === 0) { setUsers(users.map(u => u.id === userId ? { ...u, enabled } : u)); + } else if (res.message) { + setError(res.message); } - } catch (error) { - console.error('Failed to update user status:', error); + } catch (err: any) { + const errData = err?.response?.data; + setError(errData?.message || '操作失败'); } }; return (
+ {/* Error message */} + {error && ( + + +
+

{error}

+
+ +
+ )} + {/* Search */}
@@ -168,17 +195,27 @@ function UserManagement() {
- + {user.role === currentUser?.role && user.enabled ? ( + + ) : ( + + )}
@@ -495,9 +532,10 @@ function ConfigManagement() { }; // 获取可添加的 provider(排除已添加的) + // 前端限定只展示博查搜索 const getAvailableProviders = (): ProviderInfo[] => { const existingKeys = configs.map(c => c.config_key); - return providers.filter(p => !existingKeys.includes(p.provider)); + return providers.filter(p => !existingKeys.includes(p.provider) && p.provider === 'bocha'); }; // 获取所有已知的字段(用于没有 provider 匹配时的兜底显示) diff --git a/frontend/src/pages/LoginPage.tsx b/frontend/src/pages/LoginPage.tsx index 8d22e95..58d497f 100644 --- a/frontend/src/pages/LoginPage.tsx +++ b/frontend/src/pages/LoginPage.tsx @@ -1,5 +1,5 @@ import { useState } from 'react'; -import { useNavigate, Link } from 'react-router-dom'; +import { useNavigate, Link, useSearchParams } from 'react-router-dom'; import { motion } from 'framer-motion'; import { Mail, Lock, Eye, EyeOff } from 'lucide-react'; import { useAuthStore } from '../stores/useAuthStore'; @@ -11,11 +11,13 @@ import SliderCaptcha from '../components/ui/SliderCaptcha'; export default function LoginPage() { const navigate = useNavigate(); const { login } = useAuthStore(); + const [searchParams] = useSearchParams(); + const disabled = searchParams.get('reason') === 'disabled'; const [email, setEmail] = useState(''); const [password, setPassword] = useState(''); const [showPassword, setShowPassword] = useState(false); const [loading, setLoading] = useState(false); - const [error, setError] = useState(''); + const [error, setError] = useState(disabled ? '您的账号已被禁用,如有疑问请联系管理员' : ''); const [showCaptcha, setShowCaptcha] = useState(false); const handleSubmit = (e: React.FormEvent) => { diff --git a/frontend/src/pages/NotebookPage.tsx b/frontend/src/pages/NotebookPage.tsx index 668fb45..031e8fb 100644 --- a/frontend/src/pages/NotebookPage.tsx +++ b/frontend/src/pages/NotebookPage.tsx @@ -12,7 +12,7 @@ import ResizablePanel from '../components/ui/ResizablePanel'; export default function NotebookPage() { const { id } = useParams<{ id: string }>(); const navigate = useNavigate(); - const { setCurrentNotebook, getCurrentNotebook, renameNotebook, fetchNotebooks } = useNotebookStore(); + const { setCurrentNotebook, getCurrentNotebook, renameNotebook, fetchNotebooks, stopGeneration } = useNotebookStore(); const [editingName, setEditingName] = useState(false); const [notebookName, setNotebookName] = useState(''); @@ -35,6 +35,17 @@ export default function NotebookPage() { loadNotebook(); }, [id, setCurrentNotebook, fetchNotebooks]); + // 离开笔记本页面时,停止正在进行的流式生成 + useEffect(() => { + return () => { + const { streamingConversationId, currentNotebookId } = useNotebookStore.getState(); + if (streamingConversationId && currentNotebookId) { + console.log('[NotebookPage] 组件卸载,停止流式生成:', streamingConversationId); + stopGeneration(currentNotebookId, streamingConversationId).catch(() => {}); + } + }; + }, [stopGeneration]); + const notebook = getCurrentNotebook(); useEffect(() => { diff --git a/frontend/src/pages/ProfilePage.tsx b/frontend/src/pages/ProfilePage.tsx index cf504df..6e281ac 100644 --- a/frontend/src/pages/ProfilePage.tsx +++ b/frontend/src/pages/ProfilePage.tsx @@ -86,8 +86,9 @@ export default function ProfilePage() { const res = await uploadAvatar(file); if (res.code === 0) { // 上传接口已在服务端更新头像,只需更新本地状态(不回传 URL 到 PUT /user/profile) - // 加 cache-buster 避免浏览器命中旧缓存(objectName 固定为 avatars/{id}.{ext},URL 不变) - const avatarUrl = res.data.avatar + (res.data.avatar.includes('?') ? '&' : '?') + 't=' + Date.now(); + // 后端每次返回的是新生成的 MinIO presigned URL(签名/过期时间不同,本身已是新缓存 key), + // 切勿再追加 ?t= 等 cache-buster 参数——会破坏 SigV4 签名导致 403 Forbidden。 + const avatarUrl = res.data.avatar; useAuthStore.setState((state) => { const updated = state.user ? { ...state.user, avatar: avatarUrl } : null; if (updated) localStorage.setItem('user', JSON.stringify(updated)); diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index 21ab622..1f8a570 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -1,11 +1,11 @@ import { useState, useEffect } from 'react'; import { motion, AnimatePresence } from 'framer-motion'; import { - Settings, Cpu, Search, Mic, Database, Plus, Trash2, + Settings, Cpu, Search, Mic, Database, Plus, Trash2, Brain, Check, AlertCircle, ArrowLeft, Save, X, BookOpen, - Loader2, Plug + Loader2, Plug, Filter } from 'lucide-react'; -import { useNavigate } from 'react-router-dom'; +import { useNavigate, useSearchParams } from 'react-router-dom'; import { cn } from '../utils/cn'; import Button from '../components/ui/Button'; import Input from '../components/ui/Input'; @@ -16,9 +16,10 @@ import * as youdaoApi from '../api/youdao'; import type { UserConfig, UserLLMConfig, UserConfigRequest } from '../api/userConfig'; import type { ProviderInfo } from '../api/providers'; import type { YoudaoBindStatus } from '../api/youdao'; -import { getErrorMessage } from '../utils/error'; - -type ConfigTab = 'llm' | 'search' | 'asr' | 'embedding' | 'youdao'; +import { getErrorMessage } from '../utils/error'; +import LongTermMemorySettings from '../components/settings/LongTermMemorySettings'; + +type ConfigTab = 'llm' | 'search' | 'asr' | 'embedding' | 'reranker' | 'youdao' | 'memory'; // 默认 API 地址映射 const DEFAULT_API_URLS: Record = { @@ -32,6 +33,10 @@ const DEFAULT_API_URLS: Record = { baichuan: 'https://api.baichuan-ai.com/v1', moonshot: 'https://api.moonshot.cn/v1', minimax: 'https://api.minimax.chat/v1', + // Reranker providers + cohere: 'https://api.cohere.com', + jina: 'https://api.jina.ai', + siliconflow: 'https://api.siliconflow.cn', }; // Provider 文档链接映射(用户配置时引导其获取对应密钥/参数) @@ -51,11 +56,32 @@ const PROVIDER_DOCS: Record('llm'); + const [searchParams] = useSearchParams(); + const [activeTab, setActiveTab] = useState(() => { + const tab = searchParams.get('tab'); + return tab === 'llm' || tab === 'search' || tab === 'asr' || tab === 'embedding' || tab === 'reranker' || tab === 'youdao' || tab === 'memory' + ? (tab as ConfigTab) + : 'llm'; + }); const [configs, setConfigs] = useState([]); const [loading, setLoading] = useState(false); const [showAddForm, setShowAddForm] = useState(false); @@ -92,6 +118,54 @@ export default function SettingsPage() { dimensions: 2048, }); + // 格式化测试结果消息,对用户不友好的后端错误进行转换 + const formatTestResultMessage = (result: { healthy: boolean; message: string; detail?: string }): { message: string; showDetail: boolean } => { + if (result.healthy) { + return { message: result.message, showDetail: false }; + } + + // 向量维度不匹配 + if (result.message.includes('向量维度不匹配')) { + const match = result.detail?.match(/请将向量维度修改为 (\d+)/); + if (match) { + return { message: `向量维度错误,该向量模型支持 ${match[1]} 维度,请更改后重试`, showDetail: false }; + } + } + + // 向量维度配置错误(API 返回的维度不支持) + if (result.message.includes('向量维度错误')) { + return { message: result.message, showDetail: false }; + } + + // API Key 相关错误 + if (result.message.includes('API Key') || result.detail?.includes('401') || result.detail?.includes('403')) { + return { message: 'API Key 无效或无权限,请检查后重试', showDetail: false }; + } + + // 模型不存在 + if (result.message.includes('模型') && result.message.includes('不存在')) { + return { message: result.message, showDetail: false }; + } + + // 连接超时 + if (result.message.includes('超时') || result.detail?.includes('timeout')) { + return { message: '连接超时,请检查 API 地址是否正确', showDetail: false }; + } + + // 连接失败 + if (result.message.includes('连接失败') || result.detail?.includes('connection')) { + return { message: '连接失败,请检查 API 地址是否正确', showDetail: false }; + } + + // 限流 + if (result.message.includes('限流') || result.detail?.includes('429')) { + return { message: '当前请求被限流,请稍后重试', showDetail: false }; + } + + // 其他错误,返回通用提示 + return { message: '连接测试失败,请检查配置是否正确', showDetail: false }; + }; + // 获取当前选中 provider 的配置要求 const getSelectedProviderInfo = (): ProviderInfo | undefined => { return providers.find(p => p.provider === formData.provider); @@ -102,6 +176,11 @@ export default function SettingsPage() { return PROVIDER_DOCS[formData.provider]; }; + // 判断当前 provider 是否为 ARK 类型(火山引擎/豆包) + const isARKProvider = (): boolean => { + return formData.provider === 'volcengine' || formData.provider === 'doubao'; + }; + // 获取字段的中文标签 const getFieldLabel = (fieldName: string): string => { const providerInfo = getSelectedProviderInfo(); @@ -161,7 +240,10 @@ export default function SettingsPage() { setEditingId(null); resetForm(); - if (activeTab === 'youdao') { + if (activeTab === 'memory') { + return; + } + if (activeTab === 'youdao') { fetchYoudaoBindStatus(); } else { fetchConfigs(); @@ -171,7 +253,7 @@ export default function SettingsPage() { // Fetch providers when active tab changes useEffect(() => { - if (activeTab !== 'youdao') { + if (activeTab !== 'youdao' && activeTab !== 'memory') { fetchProviders(); } }, [activeTab]); @@ -289,6 +371,9 @@ export default function SettingsPage() { case 'embedding': res = await userConfigApi.listEmbeddingConfigs(); break; + case 'reranker': + res = await userConfigApi.listRerankerConfigs(); + break; } if (res && res.code === 0) { if (activeTab === 'llm') { @@ -360,6 +445,9 @@ export default function SettingsPage() { case 'embedding': res = await userConfigApi.createEmbeddingConfig(formData); break; + case 'reranker': + res = await userConfigApi.createRerankerConfig(formData); + break; } if (res && res.code === 0) { setShowAddForm(false); @@ -437,6 +525,9 @@ export default function SettingsPage() { case 'embedding': res = await userConfigApi.updateEmbeddingConfig(id, formData); break; + case 'reranker': + res = await userConfigApi.updateRerankerConfig(id, formData); + break; } if (res && res.code === 0) { setEditingId(null); @@ -482,6 +573,9 @@ export default function SettingsPage() { case 'asr': res = await userConfigApi.deleteASRConfig(id); break; + case 'reranker': + res = await userConfigApi.deleteRerankerConfig(id); + break; } if (res && res.code === 0) { fetchConfigs(); @@ -590,18 +684,18 @@ export default function SettingsPage() { const tabs = [ { key: 'search', label: '搜索引擎', icon: Search }, { key: 'asr', label: '语音识别', icon: Mic }, - { key: 'youdao', label: '有道云笔记', icon: BookOpen }, + { key: 'reranker', label: '精排模型', icon: Filter }, + { key: 'memory', label: '长期记忆', icon: Brain }, + { key: 'youdao', label: '有道云笔记', icon: BookOpen }, ]; // 从 API 获取的动态 provider 列表(只返回已实现的) + // 前端限定只展示博查搜索 const getProviderOptions = (): { value: string; label: string }[] => { - if (providers.length > 0) { - return providers.map(p => ({ - value: p.provider, - label: p.display_name, - })); - } - return []; + return providers.map(p => ({ + value: p.provider, + label: p.display_name, + })); }; const providerOptions = getProviderOptions() ?? []; @@ -642,7 +736,7 @@ export default function SettingsPage() { )} {/* LLM not configured warning */} - {activeTab !== 'llm' && !loading && llmConfigs.length === 0 && ( + {activeTab !== 'llm' && activeTab !== 'memory' && !loading && llmConfigs.length === 0 && ( {/* 当前生效的服务 - 仅搜索和语音识别有系统默认配置 */} - {activeProvider && (activeTab === 'search' || activeTab === 'asr') && ( + {activeProvider && (activeTab === 'search' || activeTab === 'asr' || activeTab === 'reranker') && (
当前使用: @@ -738,8 +832,19 @@ export default function SettingsPage() {
)} + {/* Reranker 配置提示 */} + {activeTab === 'reranker' && ( +
+

+ 💡 精排模型(Reranker)是可选组件,用于对检索结果进行二次精排,提高相关性。配置后将在知识库问答时自动启用。 +

+
+ )} + {/* Config List */} - {activeTab === 'youdao' ? ( + {activeTab === 'memory' ? ( + + ) : activeTab === 'youdao' ? ( /* 有道云配置 */
{youdaoLoading ? ( @@ -946,6 +1051,29 @@ export default function SettingsPage() { } }; + // ARK 类型服务商的向量维度使用下拉选择 + if (field === 'dimensions' && activeTab === 'embedding' && isARKProvider()) { + return ( +
+ + +
+ ); + } + return ( - {testResult.healthy ? : } -
-

{testResult.message}

- {testResult.latency_ms > 0 && ( -

耗时 {testResult.latency_ms}ms

- )} - {testResult.detail && ( -

{testResult.detail}

- )} + {testResult && (() => { + const formatted = formatTestResultMessage(testResult); + return ( +
+ {testResult.healthy ? : } +
+

{formatted.message}

+
-
- )} + ); + })()}
+ ); + } + return ( {/* 测试结果展示 */} - {testResult && ( -
- {testResult.healthy ? : } -
-

{testResult.message}

- {testResult.latency_ms > 0 && ( -

耗时 {testResult.latency_ms}ms

- )} - {testResult.detail && ( -

{testResult.detail}

- )} + {testResult && (() => { + const formatted = formatTestResultMessage(testResult); + return ( +
+ {testResult.healthy ? : } +
+

{formatted.message}

+
-
- )} + ); + })()}
for content-card (simplified: find next
) - // Look for card-title within this card - titleStart := strings.Index(lower[cardContentStart:], "card-title") - if titleStart < 0 { - searchFrom = cardContentStart - continue - } - titleStart += cardContentStart - titleGt := strings.Index(lower[titleStart:], ">") - if titleGt < 0 { - searchFrom = cardContentStart - continue - } - titleTextStart := titleStart + titleGt + 1 - titleEnd := strings.Index(lower[titleTextStart:], "<") - if titleEnd < 0 { - searchFrom = cardContentStart - continue - } - title := strings.TrimSpace(stripPPTVisibleText(section[titleTextStart : titleTextStart+titleEnd])) - - // Find card-body - bodyStart := strings.Index(lower[titleTextStart+titleEnd:], "card-body") - if bodyStart < 0 { - searchFrom = titleTextStart + titleEnd - continue - } - bodyStart += titleTextStart + titleEnd - bodyGt := strings.Index(lower[bodyStart:], ">") - if bodyGt < 0 { - searchFrom = bodyStart - continue - } - bodyTextStart := bodyStart + bodyGt + 1 - bodyEnd := strings.Index(lower[bodyTextStart:], "<") - if bodyEnd < 0 { - searchFrom = bodyTextStart - continue - } - body := strings.TrimSpace(stripPPTVisibleText(section[bodyTextStart : bodyTextStart+bodyEnd])) - - if title != "" && body != "" { - cards = append(cards, pptCardPair{title: title, body: body}) - } - searchFrom = bodyTextStart + bodyEnd - } - return cards -} - -// pptCardTitleMatchesBody checks if a card title is semantically related to -// its body content. Returns true if they appear related, false if they seem -// mismatched. -func pptCardTitleMatchesBody(title, body string) bool { - title = strings.TrimSpace(title) - body = strings.TrimSpace(body) - if title == "" || body == "" { - return true // Can't determine, don't flag - } - // If title and body are identical, it's a duplication problem (caught elsewhere) - if strings.ToLower(title) == strings.ToLower(body) { - return false - } - // If title is very short (1-2 chars) and body is long, likely mismatched - if utf8RuneCount(title) <= 2 && utf8RuneCount(body) > 20 { - return false - } - // Check for keyword overlap: extract significant words from title and - // see if any appear in body - titleWords := extractSignificantWords(title) - if len(titleWords) == 0 { - return true // Can't determine - } - bodyLower := strings.ToLower(body) - overlap := 0 - for _, word := range titleWords { - if strings.Contains(bodyLower, strings.ToLower(word)) { - overlap++ - } - } - // If none of the title's significant words appear in body, likely mismatched - if overlap == 0 && utf8RuneCount(body) > 15 { - return false - } - return true -} - -// extractSignificantWords extracts meaningful words (length >= 2) from text, -// filtering out common stop words. -func extractSignificantWords(text string) []string { - // Remove punctuation - text = strings.Map(func(r rune) rune { - if r == ',' || r == '。' || r == '、' || r == ':' || - r == '(' || r == ')' || r == '(' || r == ')' || - r == '"' || r == '\'' || r == ' ' { - return ' ' - } - return r - }, text) - fields := strings.Fields(text) - var words []string - stopWords := map[string]bool{ - "的": true, "了": true, "是": true, "在": true, "和": true, - "与": true, "或": true, "及": true, "等": true, "为": true, - "the": true, "a": true, "an": true, "is": true, "are": true, - "of": true, "to": true, "in": true, "on": true, "for": true, - } - for _, f := range fields { - if utf8RuneCount(f) >= 2 && !stopWords[strings.ToLower(f)] { - words = append(words, f) - } - } - return words -} - -func utf8RuneCount(s string) int { - return len([]rune(s)) -} - -// elements within a section, to detect title duplication. -func pptExtractSlideTitles(section string) []string { - var titles []string - lower := strings.ToLower(section) - // Extract from h1, h2, h3 tags - for _, tag := range []string{"h1", "h2", "h3"} { - titles = append(titles, extractTagText(section, lower, tag)...) - } - // Extract from card-title class - titles = append(titles, extractClassText(section, lower, "card-title")...) - // Extract from dir-item class - titles = append(titles, extractClassText(section, lower, "dir-item")...) - return titles -} - -// extractTagText extracts text content from all occurrences of a given HTML tag. -func extractTagText(content, lowerContent, tag string) []string { - var texts []string - openTag := "<" + tag - closeTag := "" - searchFrom := 0 - for { - idx := strings.Index(lowerContent[searchFrom:], openTag) - if idx < 0 { - break - } - idx += searchFrom - // Find end of opening tag (handle attributes) - gtIdx := strings.Index(lowerContent[idx:], ">") - if gtIdx < 0 { - break - } - textStart := idx + gtIdx + 1 - endIdx := strings.Index(lowerContent[textStart:], closeTag) - if endIdx < 0 { - break - } - textEnd := textStart + endIdx - text := strings.TrimSpace(stripPPTVisibleText(content[textStart:textEnd])) - if text != "" { - texts = append(texts, text) - } - searchFrom = textEnd + len(closeTag) - } - return texts -} - -// extractClassText extracts text content from elements with a given class. -func extractClassText(content, lowerContent, className string) []string { - var texts []string - searchFrom := 0 - for { - idx := strings.Index(lowerContent[searchFrom:], className) - if idx < 0 { - break - } - idx += searchFrom - // Find the enclosing tag start - tagStart := strings.LastIndex(lowerContent[:idx], "<") - if tagStart < 0 { - searchFrom = idx + len(className) - continue - } - // Find end of opening tag - gtIdx := strings.Index(lowerContent[idx:], ">") - if gtIdx < 0 { - break - } - textStart := idx + gtIdx + 1 - // Find matching closing tag - closeIdx := strings.Index(lowerContent[textStart:], "<") - if closeIdx < 0 { - break - } - textEnd := textStart + closeIdx - text := strings.TrimSpace(stripPPTVisibleText(content[textStart:textEnd])) - if text != "" { - texts = append(texts, text) - } - searchFrom = textEnd - } - return texts -} - -func containsPlannedHeading(headings []string, title string) bool { - for _, heading := range headings { - if strings.Contains(heading, title) || strings.Contains(title, heading) { - return true - } - } - return false -} - -func pptHTMLHeadings(content string) []string { - lower := strings.ToLower(content) - var headings []string - for _, tag := range []string{"h1", "h2", "h3"} { - open := "<" + tag - close := "" - searchFrom := 0 - for { - start := strings.Index(lower[searchFrom:], open) - if start < 0 { - break - } - start += searchFrom - openEnd := strings.Index(lower[start:], ">") - if openEnd < 0 { - break - } - textStart := start + openEnd + 1 - end := strings.Index(lower[textStart:], close) - if end < 0 { - break - } - textEnd := textStart + end - heading := strings.ToLower(strings.TrimSpace(stripSimpleHTML(content[textStart:textEnd]))) - if heading != "" { - headings = append(headings, heading) - } - searchFrom = textEnd + len(close) - } - } - return headings -} - -func isGenericPPTPlanTitle(title string) bool { - return containsAnyFold(title, - "cover", "agenda", "closing", "finish", "end", - "封面", "目录", "总结", "结束", "行动", - ) -} - -// dynamicMindmapBranches 根据笔记内容智能选择思维导图分支。 -// 如果笔记有明确的章节结构,用章节标题作为分支; -// 否则按知识点聚类生成3-5个分支。始终保留"总结"分支。 -func dynamicMindmapBranches(analysis learningContentAnalysis) []mindmapBranchPlan { - var branches []mindmapBranchPlan - - // 如果笔记有章节结构,直接用章节标题作为分支 - if len(analysis.Sections) >= 2 { - for _, section := range analysis.Sections { - title := strings.TrimSpace(section.Title) - if title == "" { - continue - } - branch := mindmapBranchPlan{Title: title} - for _, point := range section.Points { - point = strings.TrimSpace(point) - if point == "" { - continue - } - branch.Nodes = append(branch.Nodes, newMindmapNode( - point, - mindmapNodeDetailFromEvidence(point, analysis), - )) - } - if len(branch.Nodes) == 0 { - branch.Nodes = append(branch.Nodes, newMindmapNode( - supplementBullet(title, 1), - mindmapNodeDetailFromEvidence(title, analysis), - )) - } - branches = append(branches, branch) - } - } else { - // 扁平结构:按知识点类型分类 - if len(analysis.KeyConcepts) > 0 { - branch := mindmapBranchPlan{Title: "核心概念"} - for _, concept := range analysis.KeyConcepts { - branch.Nodes = append(branch.Nodes, newMindmapNode( - concept, - mindmapNodeDetailFromEvidence(concept, analysis), - )) - } - branches = append(branches, branch) - } - if len(analysis.Processes) > 0 { - branch := mindmapBranchPlan{Title: "原理与过程"} - for _, proc := range analysis.Processes { - branch.Nodes = append(branch.Nodes, newMindmapNode( - proc, - mindmapNodeDetailFromEvidence(proc, analysis), - )) - } - branches = append(branches, branch) - } - if len(analysis.Examples) > 0 { - branch := mindmapBranchPlan{Title: "应用与案例"} - for _, example := range analysis.Examples { - branch.Nodes = append(branch.Nodes, newMindmapNode( - example, - mindmapNodeDetailFromEvidence(example, analysis), - )) - } - branches = append(branches, branch) - } - if len(branches) == 0 { - // 极端稀疏:创建一个通用分支 - branch := mindmapBranchPlan{Title: analysis.Topic} - for _, concept := range analysis.KeyConcepts { - branch.Nodes = append(branch.Nodes, newMindmapNode( - concept, - mindmapNodeDetailFromEvidence(concept, analysis), - )) - } - if len(branch.Nodes) == 0 { - branch.Nodes = append(branch.Nodes, newMindmapNode( - fmt.Sprintf("围绕“%s”的关键要点", analysis.Topic), - "结合笔记内容梳理核心知识点。", - )) - } - branches = append(branches, branch) - } - } - - // 始终保留总结分支 - branches = append(branches, mindmapBranchPlan{ - Title: "总结", - Nodes: []mindmapNodePlan{ - newMindmapNode( - fmt.Sprintf("围绕“%s”形成可复习的结构。", analysis.Topic), - "按概念、机制、过程、应用和误区回顾学习路径。", - ), - }, - }) - - // 确保至少3个分支(含总结) - if len(branches) < 3 { - branchTitle := "补充内容" - if strings.TrimSpace(analysis.Topic) != "" { - branchTitle = analysis.Topic - } - for len(branches) < 3 { - branches = append(branches, mindmapBranchPlan{ - Title: branchTitle, - Nodes: []mindmapNodePlan{ - newMindmapNode( - supplementBullet(branchTitle, len(branches)+1), - "该节点为解释补充,用于补足学习结构。", - ), - }, - }) - } - } - - // 限制分支数量在8以内(含总结) - if len(branches) > 8 { - // 保留前7个分支和最后的总结分支 - branches = append(branches[:7], branches[len(branches)-1]) - } - - return branches -} - -func planMindmap(analysis learningContentAnalysis) mindmapPlan { - plan := mindmapPlan{Title: analysis.Topic} - branches := dynamicMindmapBranches(analysis) - - // 确保每个分支至少有 minNodes 个节点 - minNodes := 3 - if analysis.Sparse { - minNodes = 4 - } - for i := range branches { - branch := &branches[i] - for len(branch.Nodes) < minNodes { - branch.Nodes = append(branch.Nodes, newMindmapNode( - supplementBullet(branch.Title, len(branch.Nodes)+1), - mindmapNodeDetailFromEvidence(branch.Title, analysis), - mindmapBranchExpansionDetail(branch.Title, analysis), - )) - branch.Nodes = uniqueMindmapNodes(branch.Nodes) - } - branch.Nodes = uniqueMindmapNodes(branch.Nodes) - } - - plan.Branches = branches - return plan -} - -func expandMindmapContent(plan mindmapPlan, analysis learningContentAnalysis) mindmapPlan { - expanded := plan - evidenceIndex := 0 - minNodes := 4 - if analysis.Sparse { - minNodes = 5 - } - - for i := range expanded.Branches { - branch := &expanded.Branches[i] - for j := range branch.Nodes { - branch.Nodes[j].Details = expandMindmapNodeDetails(branch.Title, branch.Nodes[j], analysis, nextMindmapEvidence(analysis.Evidence, &evidenceIndex)) - } - for len(branch.Nodes) < minNodes { - title := mindmapExpansionNodeTitle(branch.Title, len(branch.Nodes)+1) - branch.Nodes = append(branch.Nodes, mindmapNodePlan{ - Title: title, - Details: expandMindmapNodeDetails(branch.Title, mindmapNodePlan{Title: title}, analysis, nextMindmapEvidence(analysis.Evidence, &evidenceIndex)), - }) - branch.Nodes = uniqueMindmapNodes(branch.Nodes) - } - for j := range branch.Nodes { - branch.Nodes[j].Details = expandMindmapNodeDetails(branch.Title, branch.Nodes[j], analysis, nextMindmapEvidence(analysis.Evidence, &evidenceIndex)) - } - branch.Nodes = uniqueMindmapNodes(branch.Nodes) - } - return expanded -} - -func appendMindmapNodes(nodes []mindmapNodePlan, branchTitle string, values []string, analysis learningContentAnalysis) []mindmapNodePlan { - for _, value := range values { - value = strings.TrimSpace(value) - if value == "" { - continue - } - nodes = append(nodes, newMindmapNode(value, mindmapNodeDetail(branchTitle, value, analysis))) - } - return nodes -} - -func newMindmapNode(title string, details ...string) mindmapNodePlan { - node := mindmapNodePlan{Title: strings.TrimSpace(title)} - for _, detail := range details { - detail = strings.TrimSpace(detail) - if detail != "" { - node.Details = append(node.Details, detail) - } - } - return node -} - -func expandMindmapNodeDetails(branchTitle string, node mindmapNodePlan, analysis learningContentAnalysis, evidence string) []string { - details := uniqueNonEmpty(node.Details) - if len(details) == 0 { - details = append(details, mindmapNodeDetail(branchTitle, node.Title, analysis)) - } - if len(details) < 2 { - details = append(details, mindmapNodeReviewDetail(branchTitle, node.Title)) - } - if len(details) < 3 && evidence != "" { - details = append(details, "资料要点:"+summarizeLine(evidence, 90)) - } - if len(details) < 3 { - details = append(details, mindmapBranchExpansionDetail(branchTitle, analysis)) - } - details = uniqueNonEmpty(details) - if len(details) > 3 { - details = append([]string{}, details[:3]...) - } - for len(details) < 2 { - details = append(details, fmt.Sprintf("围绕“%s”继续补足复习说明。", strings.TrimSpace(node.Title))) - details = uniqueNonEmpty(details) - } - return details -} - -func mindmapNodeReviewDetail(branchTitle, nodeTitle string) string { - switch branchTitle { - case "核心概念": - return "继续说明该概念的定义边界、关联概念和典型辨析。" - case "原理机制": - return "继续说明触发条件、关键变量和因果链条如何变化。" - case "过程步骤": - return "继续说明前置条件、执行顺序和每一步产出。" - case "应用场景": - return "继续说明适用场景、迁移方式和判断依据。" - case "易错点": - return "继续说明常见误解、错误推断和修正线索。" - case "总结": - return "继续串联概念、机制、过程和应用结论。" - default: - return fmt.Sprintf("继续围绕“%s”补足复习说明和展开方向。", strings.TrimSpace(nodeTitle)) - } -} - -func nextMindmapEvidence(evidence []learningEvidence, index *int) string { - if len(evidence) == 0 { - return "" - } - if index == nil { - return summarizeLine(strings.TrimSpace(evidence[0].Text), 90) - } - ev := evidence[*index%len(evidence)] - *index++ - return summarizeLine(strings.TrimSpace(ev.Text), 90) -} - -func mindmapNodeDetail(branchTitle, value string, analysis learningContentAnalysis) string { - if strings.Contains(value, "解释补充") { - return "该节点为解释补充,用于补足学习结构。" - } - switch branchTitle { - case "核心概念": - return "先明确含义,再和相关概念建立联系。" - case "原理机制": - return "关注该机制成立的条件、因果关系和边界。" - case "过程步骤": - return "按先后顺序理解输入、变化和结果。" - case "应用场景": - return "结合具体情境判断该知识点如何迁移使用。" - default: - if len(analysis.Evidence) > 0 { - return fmt.Sprintf("可参考:%s", analysis.Evidence[0].Source) - } - return "用于复习时展开说明和自我检查。" - } -} - -// mindmapNodeDetailFromEvidence 从笔记证据中提取具体内容作为节点细节, -// 替代原有的模板化描述。 -func mindmapNodeDetailFromEvidence(value string, analysis learningContentAnalysis) string { - // 如果值包含解释补充标记,返回通用提示 - if strings.Contains(value, "解释补充") { - return "该节点为解释补充,用于补足学习结构。" - } - - // 从证据中找与 value 最相关的具体内容 - valueKeywords := extractSignificantWords(value) - for _, ev := range analysis.Evidence { - evKeywords := extractSignificantWords(ev.Text) - overlap := 0 - for _, kw := range valueKeywords { - for _, ekw := range evKeywords { - if strings.EqualFold(kw, ekw) { - overlap++ - break - } - } - } - if overlap >= 2 { - return summarizeLine(ev.Text, 90) - } - } - - // 如果没有直接匹配的证据,返回通用但更有针对性的描述 - switch { - case containsAnyFold(value, "定义", "概念", "是什么", "含义"): - return "明确该概念的精确定义、适用范围和与相关概念的区别。" - case containsAnyFold(value, "原理", "机制", "原因", "为什么"): - return "理解该原理成立的条件、因果链条和关键变量。" - case containsAnyFold(value, "步骤", "流程", "过程", "方法"): - return "按顺序理解每个步骤的输入、变化和产出。" - case containsAnyFold(value, "应用", "例子", "场景", "案例"): - return "结合具体场景判断该知识点如何迁移使用。" - default: - return fmt.Sprintf("围绕“%s”展开具体内容和关键要点。", strings.TrimSpace(value)) - } -} - -func mindmapBranchExpansionDetail(branchTitle string, analysis learningContentAnalysis) string { - switch branchTitle { - case "核心概念": - return "补充概念之间的联系、边界和典型辨析。" - case "原理机制": - return "补充触发条件、因果链条和关键变量。" - case "过程步骤": - return "补充前后顺序、输入输出和阶段性结果。" - case "应用场景": - if len(analysis.Examples) > 0 { - return "结合已有例子扩展到相近场景和迁移使用。" - } - return "补充典型应用场景、判断方式和迁移思路。" - case "易错点": - return "补充常见混淆、错误推断和修正线索。" - case "总结": - return "补充复习路径、串联方式和回顾问题。" - default: - return "补充该分支下仍然缺失的学习展开。" - } -} - -func mindmapExpansionNodeTitle(branchTitle string, position int) string { - return supplementBullet(branchTitle, position) -} - -func uniqueMindmapNodes(nodes []mindmapNodePlan) []mindmapNodePlan { - seen := map[string]struct{}{} - result := make([]mindmapNodePlan, 0, len(nodes)) - for _, node := range nodes { - node.Title = strings.TrimSpace(node.Title) - if node.Title == "" { - continue - } - if _, ok := seen[node.Title]; ok { - continue - } - seen[node.Title] = struct{}{} - node.Details = uniqueNonEmpty(node.Details) - result = append(result, node) - } - return result -} - -func renderMindmap(plan mindmapPlan) string { - var b strings.Builder - b.WriteString("# ") - b.WriteString(plan.Title) - b.WriteString("\n") - for _, branch := range plan.Branches { - b.WriteString("## ") - b.WriteString(branch.Title) - b.WriteString("\n") - for _, node := range branch.Nodes { - b.WriteString("### ") - b.WriteString(node.Title) - b.WriteString("\n") - for _, detail := range node.Details { - b.WriteString("#### ") - b.WriteString(detail) - b.WriteString("\n") - } - } - } - return strings.TrimSpace(b.String()) -} - -func mindmapNeedsStructureRepair(content string) bool { - trimmed := strings.TrimSpace(content) - if len([]rune(strings.ReplaceAll(trimmed, "#", ""))) < 20 { - return true - } - // 至少3个 ## 分支 - if strings.Count(trimmed, "\n## ") < 3 { - return true - } - // 至少有 ### 节点层级 - if !strings.Contains(trimmed, "\n### ") { - return true - } - return false -} - -func supplementBullet(title string, detail int) string { - title = strings.TrimSpace(title) - if title == "" { - return fmt.Sprintf("补充要点 %d:结合材料进一步分析该主题的关键内容。", detail) - } - // Generate concrete, knowledge-driven supplements rather than vague boilerplate. - // Each template provides a specific angle for expanding the topic with real content. - templates := []string{ - fmt.Sprintf("%s的核心定义:用一句话精确概括%s是什么,区分它与其他相近概念的本质差异。", title, title), - fmt.Sprintf("%s的运作原理:解释%s背后的因果机制或数学/逻辑基础,说明为什么它这样工作而非那样工作。", title, title), - fmt.Sprintf("%s的关键特征:列出%s的3个核心特征,每个特征给出一个具体例子或量化数据支撑。", title, title), - fmt.Sprintf("%s的应用场景:描述%s在真实场景中的典型用法,给出一个具体的操作步骤或数值案例。", title, title), - fmt.Sprintf("%s的常见误区:指出学习%s时最容易犯的2-3个错误,说明正确理解应该是什么。", title, title), - fmt.Sprintf("%s与其他概念的关系:说明%s在整体知识体系中的位置,它依赖什么前置知识,又是什么后续知识的基础。", title, title), - } - idx := (detail - 1) % len(templates) - return templates[idx] -} - -func hasSupplementBullet(values []string) bool { - for _, value := range values { - if strings.Contains(value, "解释补充") || strings.Contains(value, "补充要点") { - return true - } - } - return false -} - -func hasSupplementMindmapNode(nodes []mindmapNodePlan) bool { - for _, node := range nodes { - if strings.Contains(node.Title, "解释补充") { - return true - } - } - return false -} - -func pickPoint(points []string, index int) string { - if len(points) == 0 { - return "" - } - if index < len(points) { - return points[index] - } - return points[index%len(points)] -} - -func buildPPTFallbackPoints(input generationAgentInput, limit int) []string { - if limit <= 0 { - limit = 9 - } - markdown := "" - prompt := "" - if input.Request != nil { - markdown = input.Request.Markdown - prompt = input.Request.Prompt - } - title := extractTitle(markdown, "演示文稿") - candidates := extractKeyPoints(markdown, limit) - if prompt := strings.TrimSpace(prompt); prompt != "" { - candidates = append(candidates, prompt) - } - for _, ref := range input.References { - candidates = append(candidates, summarizeLine(ref.Content, 90)) - } - for _, result := range input.SearchResults { - candidates = append(candidates, summarizeLine(firstNonEmpty(result.Snippet, result.Content), 90)) - } - candidates = append(candidates, - fmt.Sprintf("围绕“%s”说明背景、问题和目标。", title), - "提炼现有材料中的核心观点,并补充必要解释。", - "使用来源材料、示例或数据支撑关键结论。", - "将内容组织成适合演讲的开场、展开和收束。", - "给出听众可以理解或执行的总结。", - ) - points := uniqueNonEmpty(candidates) - if len(points) > limit { - return append([]string{}, points[:limit]...) - } - return points -} - -func newQuizAgent(model GenerationModel) generationAgent { - return &quizGenerationAgent{ - baseGenerationAgent: baseGenerationAgent{ - name: "quiz", - typ: GenerationTypeQuiz, - model: model, - validator: validateQuizContent, - fallback: fallbackQuizContent, - }, - } -} - -func newNoteAgent(model GenerationModel) generationAgent { - return ¬eGenerationAgent{ - baseGenerationAgent: baseGenerationAgent{ - name: "note", - typ: GenerationTypeNote, - model: model, - validator: validateNoteContent, - fallback: fallbackNoteContent, - }, - } -} - -func fallbackNoteContent(input generationAgentInput) string { - analysis := analyzeLearningContent(input) - return renderNote(expandNoteContent(planNoteOutline(analysis), analysis)) -} - -func fallbackQuizContent(input generationAgentInput) string { - analysis := analyzeLearningContent(input) - return renderQuiz(expandQuizContent(planQuizQuestions(analysis), analysis)) -} - -func requiredNoteSections() []string { - return []string{"摘要", "关键概念", "原理与机制", "过程与步骤", "应用场景", "易错点", "总结"} -} - -func planNoteOutline(analysis learningContentAnalysis) noteOutlinePlan { - plan := noteOutlinePlan{Title: analysis.Topic} - summaryParts := append([]string{}, analysis.KeyConcepts...) - if len(analysis.Processes) > 0 { - summaryParts = append(summaryParts, analysis.Processes[0]) - } - if len(summaryParts) == 0 { - summaryParts = append(summaryParts, fmt.Sprintf("围绕“%s”整理学习要点。", analysis.Topic)) - } - plan.Summary = summarizeLine(strings.Join(summaryParts, ";"), 120) - - for _, title := range requiredNoteSections() { - section := noteSectionPlan{Title: title} - switch title { - case "摘要": - section.Purpose = "概括主题与核心结论" - section.Points = append(section.Points, plan.Summary) - case "关键概念": - section.Purpose = "梳理定义与术语边界" - section.Points = appendNotePoints(section.Points, analysis.KeyConcepts, 6) - case "原理与机制", "过程与步骤": - section.Purpose = "说明条件、因果与执行顺序" - section.Points = appendNotePoints(section.Points, analysis.Processes, 6) - case "应用场景": - section.Purpose = "结合例子说明迁移使用" - section.Points = appendNotePoints(section.Points, analysis.Examples, 6) - case "易错点": - section.Purpose = "辨析常见误解与修正线索" - section.Points = append(section.Points, "注意概念边界、条件范围和常见混淆。") - case "总结": - section.Purpose = "串联知识路径与复习方向" - section.Points = append(section.Points, fmt.Sprintf("围绕“%s”形成可复习的结构。", analysis.Topic)) - } - if len(section.Points) == 0 { - section.Points = append(section.Points, supplementBullet(title, 1)) - } - if analysis.Sparse && !hasSupplementBullet(section.Points) { - section.Points = append(section.Points, supplementBullet(title, 2)) - } - minPoints := 3 - if analysis.Sparse { - minPoints = 4 - } - for len(section.Points) < minPoints { - section.Points = append(section.Points, supplementBullet(title, len(section.Points)+1)) - } - section.Points = uniqueNonEmpty(section.Points) - plan.Sections = append(plan.Sections, section) - } - return plan -} - -func appendNotePoints(points []string, values []string, limit int) []string { - for i, value := range values { - value = strings.TrimSpace(value) - if value == "" { - continue - } - if limit > 0 && i >= limit { - break - } - points = append(points, value) - } - return points -} - -func expandNoteContent(plan noteOutlinePlan, analysis learningContentAnalysis) noteOutlinePlan { - expanded := plan - evidenceIndex := 0 - minPoints := 4 - if analysis.Sparse { - minPoints = 5 - } - - for i := range expanded.Sections { - section := &expanded.Sections[i] - for j := range section.Points { - section.Points[j] = expandNotePoint(section.Title, section.Points[j], analysis, nextNoteEvidence(analysis.Evidence, &evidenceIndex)) - } - for len(section.Points) < minPoints { - title := noteExpansionPointTitle(section.Title, len(section.Points)+1) - section.Points = append(section.Points, expandNotePoint(section.Title, title, analysis, nextNoteEvidence(analysis.Evidence, &evidenceIndex))) - section.Points = uniqueNonEmpty(section.Points) - } - for j := range section.Points { - section.Points[j] = expandNotePoint(section.Title, section.Points[j], analysis, nextNoteEvidence(analysis.Evidence, &evidenceIndex)) - } - section.Points = uniqueNonEmpty(section.Points) - } - return expanded -} - -func expandNotePoint(sectionTitle, point string, analysis learningContentAnalysis, evidence string) string { - point = strings.TrimSpace(point) - if point == "" { - return "" - } - if evidence != "" && !strings.Contains(point, "资料要点:") { - return point + "(资料要点:" + summarizeLine(evidence, 80) + ")" - } - return point -} - -func nextNoteEvidence(evidence []learningEvidence, index *int) string { - if len(evidence) == 0 { - return "" - } - if index == nil { - return summarizeLine(strings.TrimSpace(evidence[0].Text), 80) - } - ev := evidence[*index%len(evidence)] - *index++ - return summarizeLine(strings.TrimSpace(ev.Text), 80) -} - -func noteExpansionPointTitle(sectionTitle string, position int) string { - return supplementBullet(sectionTitle, position) -} - -func renderNote(plan noteOutlinePlan) string { - var b strings.Builder - b.WriteString("# ") - b.WriteString(strings.TrimSpace(plan.Title)) - b.WriteString("\n") - if strings.TrimSpace(plan.Summary) != "" { - b.WriteString("\n## 摘要\n") - b.WriteString(strings.TrimSpace(plan.Summary)) - b.WriteString("\n") - } - for _, section := range plan.Sections { - if strings.TrimSpace(section.Title) == "摘要" { - continue - } - b.WriteString("\n## ") - b.WriteString(strings.TrimSpace(section.Title)) - b.WriteString("\n") - for _, point := range section.Points { - point = strings.TrimSpace(point) - if point == "" { - continue - } - b.WriteString("- ") - b.WriteString(point) - b.WriteString("\n") - } - } - return strings.TrimSpace(b.String()) -} - -func renderNotePlan(plan noteOutlinePlan) string { - var b strings.Builder - if strings.TrimSpace(plan.Title) != "" { - b.WriteString("# ") - b.WriteString(strings.TrimSpace(plan.Title)) - b.WriteString("\n") - } - if strings.TrimSpace(plan.Summary) != "" { - b.WriteString("Summary: ") - b.WriteString(strings.TrimSpace(plan.Summary)) - b.WriteString("\n") - } - for i, section := range plan.Sections { - b.WriteString(fmt.Sprintf("Section %02d: %s\n", i+1, strings.TrimSpace(section.Title))) - if strings.TrimSpace(section.Purpose) != "" { - b.WriteString("Purpose: ") - b.WriteString(strings.TrimSpace(section.Purpose)) - b.WriteString("\n") - } - for _, point := range section.Points { - point = strings.TrimSpace(point) - if point == "" { - continue - } - b.WriteString("- ") - b.WriteString(point) - b.WriteString("\n") - } - } - return strings.TrimSpace(b.String()) -} - -func appendNotePlansToContext(contextValue string, plan, expanded noteOutlinePlan) string { - var b strings.Builder - b.WriteString(strings.TrimSpace(contextValue)) - if strings.TrimSpace(plan.Title) != "" { - b.WriteString("\n\nINTERNAL_NOTE_PLAN\n") - b.WriteString("内部笔记规划:\n") - b.WriteString(renderNotePlan(plan)) - } - if strings.TrimSpace(expanded.Title) != "" { - b.WriteString("\n\nINTERNAL_NOTE_EXPANDED_PLAN\n") - b.WriteString("内部笔记扩展:\n") - b.WriteString(renderNote(expanded)) - b.WriteString("\n\nNOTE_GENERATION_RULES\n") - b.WriteString("- Treat each Section entry as the writing brief for exactly one ## section; do not merge, omit, or reorder sections.\n") - b.WriteString("- The planned points are source material, not the final wording. Expand each point into polished note content with concrete explanations grounded in the provided Markdown.\n") - b.WriteString("- Keep every planned section title visible as ## heading, then add 3-5 substantial points for that section.\n") - b.WriteString("- Content must be grounded in Original Markdown, Local References, Web Results, or the user's explicit prompt. Do not add generic boilerplate unless it appears in the source.\n") - b.WriteString("- Finish all planned sections before returning. If the plan is long, make each section concise instead of truncating the note.\n") - } - return strings.TrimSpace(b.String()) -} - -func noteNeedsStructureRepair(content string) bool { - trimmed := strings.TrimSpace(content) - if !strings.HasPrefix(trimmed, "#") { - return true - } - if len([]rune(strings.ReplaceAll(trimmed, "#", ""))) < 30 { - return true - } - if strings.Count(trimmed, "\n## ") < 2 { - return true - } - return false -} - -func requiredQuizQuestionTypes(analysis learningContentAnalysis) []string { - conceptCount := len(analysis.KeyConcepts) - processCount := len(analysis.Processes) - exampleCount := len(analysis.Examples) - totalPoints := conceptCount + processCount + exampleCount - - // 根据材料丰富度决定题目数量 - targetCount := 5 - if totalPoints >= 6 { - targetCount = 6 - } - if totalPoints >= 10 { - targetCount = 7 - } - if totalPoints >= 15 { - targetCount = 8 - } - - // 题型分配:至少1道 single_choice + 1道 true_false - types := []string{"single_choice", "true_false"} - - // 如果有对比性知识点,加多选题 - if conceptCount >= 3 { - types = append(types, "multi_choice") - } - - // 如果有过程性知识点,加填空题 - if processCount >= 1 { - types = append(types, "fill_blank") - } - - // 补充 short_answer 直到达到目标数量 - for len(types) < targetCount { - types = append(types, "short_answer") - } - - // 如果超了就截断 - if len(types) > targetCount { - types = types[:targetCount] - } - - return types -} - -func planQuizQuestions(analysis learningContentAnalysis) quizQuestionPlan { - plan := quizQuestionPlan{Topic: analysis.Topic} - types := requiredQuizQuestionTypes(analysis) - concepts := append([]string{}, analysis.KeyConcepts...) - processes := append([]string{}, analysis.Processes...) - examples := append([]string{}, analysis.Examples...) - if len(concepts) == 0 { - concepts = append(concepts, analysis.Topic) - } - - for i, qType := range types { - item := quizQuestionItem{Type: qType} - switch qType { - case "single_choice": - topic := pickPoint(concepts, i) - item.Topic = topic - item.Question = fmt.Sprintf("关于“%s”,下列说法正确的是?", topic) - item.Options = []string{ - topic + " 的基本定义如上所述。", - "与原文相反的描述。", - "无关的干扰项。", - "概念混淆的选项。", - } - item.Answer = item.Options[0] - item.Explanation = fmt.Sprintf("根据笔记,“%s”的定义如原文所述。", topic) - item.Difficulty = "easy" - case "true_false": - topic := pickPoint(concepts, i+1) - item.Topic = topic - item.Question = fmt.Sprintf("判断:%s。", topic) - item.Options = []string{"正确", "错误"} - item.Answer = "正确" - item.Explanation = fmt.Sprintf("根据笔记内容,该说法是正确的。“%s”的定义和描述如原文所述。", topic) - item.Difficulty = "easy" - case "multi_choice": - topic := pickPoint(concepts, i+2) - if topic == "" { - topic = pickPoint(concepts, 0) - } - item.Topic = topic - item.Question = fmt.Sprintf("关于“%s”,以下哪些说法是正确的?(多选)", topic) - item.Options = []string{ - topic + " 的基本定义。", - topic + " 的关键特征。", - "与原文矛盾的描述。", - topic + " 的适用条件。", - } - item.Answer = item.Options[0] + ";" + item.Options[1] + ";" + item.Options[3] - item.Explanation = fmt.Sprintf("选项A、B、D正确。选项C与原文矛盾。关于“%s”的详细说明见笔记原文。", topic) - item.Difficulty = "medium" - case "fill_blank": - if len(processes) > 0 { - topic := pickPoint(processes, i) - item.Topic = topic - item.Question = fmt.Sprintf("“%s”的关键步骤是____。", topic) - item.Answer = summarizeLine(topic, 100) - item.Explanation = "该答案来自提供的笔记上下文。" - } else { - topic := pickPoint(concepts, i) - item.Topic = topic - item.Question = fmt.Sprintf("请填写“%s”的核心定义中的关键词:____。", topic) - item.Answer = summarizeLine(topic, 100) - item.Explanation = "该答案来自提供的笔记上下文。" - } - item.Difficulty = "medium" - case "short_answer": - if len(processes) > 0 { - topic := pickPoint(processes, i) - item.Topic = topic - item.Question = fmt.Sprintf("简述“%s”的关键步骤或机制。", topic) - item.Answer = summarizeLine(topic, 100) - item.Explanation = "该答案来自提供的笔记上下文。" - } else { - topic := pickPoint(examples, i) - if topic == "" { - topic = pickPoint(concepts, i) - } - item.Topic = topic - item.Question = fmt.Sprintf("说明“%s”的应用场景或例子。", topic) - item.Answer = summarizeLine(topic, 100) - item.Explanation = "该答案来自提供的笔记上下文。" - } - item.Difficulty = "hard" - } - plan.Questions = append(plan.Questions, item) - } - return plan -} - -func expandQuizContent(plan quizQuestionPlan, analysis learningContentAnalysis) quizQuestionPlan { - expanded := plan - evidenceIndex := 0 - for i := range expanded.Questions { - q := &expanded.Questions[i] - evidence := nextQuizEvidence(analysis.Evidence, &evidenceIndex) - if evidence != "" && q.Explanation != "" && !strings.Contains(q.Explanation, "资料要点:") { - q.Explanation = q.Explanation + "(资料要点:" + summarizeLine(evidence, 80) + ")" - } - if strings.TrimSpace(q.Answer) == "" { - q.Answer = summarizeLine(q.Topic, 100) - } - if strings.TrimSpace(q.Explanation) == "" { - q.Explanation = "该答案来自提供的笔记上下文。" - } - } - for len(expanded.Questions) < 5 { - topic := analysis.Topic - if len(analysis.KeyConcepts) > len(expanded.Questions) { - topic = analysis.KeyConcepts[len(expanded.Questions)] - } - expanded.Questions = append(expanded.Questions, quizQuestionItem{ - Type: "short_answer", - Topic: topic, - Question: fmt.Sprintf("简述“%s”的核心观点。", topic), - Answer: summarizeLine(topic, 100), - Explanation: "该答案来自提供的笔记上下文。", - }) - } - return expanded -} - -func nextQuizEvidence(evidence []learningEvidence, index *int) string { - if len(evidence) == 0 { - return "" - } - if index == nil { - return summarizeLine(strings.TrimSpace(evidence[0].Text), 80) - } - ev := evidence[*index%len(evidence)] - *index++ - return summarizeLine(strings.TrimSpace(ev.Text), 80) -} - -func renderQuiz(plan quizQuestionPlan) string { - items := make([]string, 0, len(plan.Questions)) - for _, q := range plan.Questions { - options := make([]string, 0, len(q.Options)) - for _, opt := range q.Options { - options = append(options, fmt.Sprintf("%q", opt)) - } - item := fmt.Sprintf(`{"type":%q,"question":%q,"options":[%s],"answer":%q,"explanation":%q,"difficulty":%q}`, - q.Type, q.Question, strings.Join(options, ","), q.Answer, q.Explanation, q.Difficulty) - items = append(items, item) - } - return `{"questions":[` + strings.Join(items, ",") + `]}` -} - -func renderQuizPlan(plan quizQuestionPlan) string { - var b strings.Builder - if strings.TrimSpace(plan.Topic) != "" { - b.WriteString("Topic: ") - b.WriteString(strings.TrimSpace(plan.Topic)) - b.WriteString("\n") - } - for i, q := range plan.Questions { - b.WriteString(fmt.Sprintf("Question %02d: [%s] %s\n", i+1, q.Type, strings.TrimSpace(q.Question))) - if strings.TrimSpace(q.Topic) != "" { - b.WriteString("Focus: ") - b.WriteString(strings.TrimSpace(q.Topic)) - b.WriteString("\n") - } - for j, opt := range q.Options { - b.WriteString(fmt.Sprintf(" Option %d: %s\n", j+1, opt)) - } - if strings.TrimSpace(q.Answer) != "" { - b.WriteString("Answer: ") - b.WriteString(strings.TrimSpace(q.Answer)) - b.WriteString("\n") - } - if strings.TrimSpace(q.Explanation) != "" { - b.WriteString("Explanation: ") - b.WriteString(strings.TrimSpace(q.Explanation)) - b.WriteString("\n") - } - } - return strings.TrimSpace(b.String()) -} - -func appendQuizPlansToContext(contextValue string, plan, expanded quizQuestionPlan) string { - var b strings.Builder - b.WriteString(strings.TrimSpace(contextValue)) - if strings.TrimSpace(plan.Topic) != "" { - b.WriteString("\n\nINTERNAL_QUIZ_PLAN\n") - b.WriteString("内部测验规划:\n") - b.WriteString(renderQuizPlan(plan)) - } - if strings.TrimSpace(expanded.Topic) != "" { - b.WriteString("\n\nINTERNAL_QUIZ_EXPANDED_PLAN\n") - b.WriteString("内部测验扩展:\n") - b.WriteString(renderQuiz(expanded)) - b.WriteString("\n\nQUIZ_GENERATION_RULES\n") - b.WriteString("- Follow the planned question types and topics; do not omit or merge questions.\n") - b.WriteString("- Every question must include a non-empty answer and explanation.\n") - b.WriteString("- For single_choice, provide 3-4 options and mark the correct one in the answer field with the exact option text.\n") - b.WriteString("- For true_false, options must be [\"正确\",\"错误\"], answer must be \"正确\" or \"错误\".\n") - b.WriteString("- For multi_choice, provide 4-5 options, answer must be all correct option texts joined by semicolons (;).\n") - b.WriteString("- For fill_blank, options must be an empty array [], answer must be the key term or phrase to fill in.\n") - b.WriteString("- For short_answer, options must be an empty array [], answer must be a reference answer.\n") - b.WriteString("- Generate at least 5 questions, covering at least 2 different question types.\n") - b.WriteString("- Distribute difficulty levels: roughly 40% easy, 40% medium, 20% hard.\n") - b.WriteString("- Content must be grounded in Original Markdown, Local References, Web Results, or the user's explicit prompt.\n") - b.WriteString("- Return only the JSON object, no markdown fences or extra text.\n") - } - return strings.TrimSpace(b.String()) -} - -func quizNeedsStructureRepair(content string) bool { - trimmed := strings.TrimSpace(content) - if trimmed == "" { - return true - } - if !validateQuizContent(trimmed) { - return true - } - return false -} - -func extractTitle(markdown, fallback string) string { - for _, line := range strings.Split(markdown, "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "#") { - title := strings.TrimSpace(strings.TrimLeft(line, "#")) - if title != "" { - return title - } - } - } - return fallback -} - -func extractKeyPoints(markdown string, limit int) []string { - var points []string - // Pre-process: merge fenced code blocks into single lines so they are - // extracted as atomic units instead of being scattered as individual lines. - lines := strings.Split(markdown, "\n") - mergedLines := mergeCodeBlockLines(lines) - - for _, line := range mergedLines { - // Code blocks are already merged into single lines starting with ```; - // preserve them as-is without TrimLeft which would strip the fence markers. - if isPPTCodeBlockBullet(line) { - points = append(points, line) - if len(points) >= limit { - return points - } - continue - } - line = strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(line), "-*#0123456789. ")) - if len([]rune(line)) < 4 { - continue - } - points = append(points, line) - if len(points) >= limit { - return points - } - } - return points -} - -func appendReferenceSection(b *strings.Builder, refs []GenerationReference) { - if len(refs) == 0 { - return - } - b.WriteString("\n\n## 参考资料\n") - for i, ref := range refs { - label := generationReferenceLabel(ref) - b.WriteString(fmt.Sprintf("- [%d] %s: %s\n", i+1, label, summarizeLine(ref.Content, 120))) - } -} - -func summarizeLine(value string, limit int) string { - value = strings.Join(strings.Fields(value), " ") - if len([]rune(value)) <= limit { - return value - } - runes := []rune(value) - return string(runes[:limit]) -} - -func htmlEscape(value string) string { - replacer := strings.NewReplacer("&", "&", "<", "<", ">", ">", `"`, """) - return replacer.Replace(value) -} diff --git a/internal/service/generation_codeblock_test.go b/internal/service/generation_codeblock_test.go deleted file mode 100644 index aa49119..0000000 --- a/internal/service/generation_codeblock_test.go +++ /dev/null @@ -1,319 +0,0 @@ -package service - -import ( - "strings" - "testing" -) - -func TestMergeCodeBlockLines(t *testing.T) { - cases := []struct { - name string - input string - want []string // expected merged lines (non-empty) - }{ - { - name: "code block preserved as single unit", - input: "some text\n```go\nfunc main() {\n\tfmt.Println(\"hello\")\n}\n```\nmore text", - want: []string{"some text", "```go\nfunc main() {\n\tfmt.Println(\"hello\")\n}\n```", "more text"}, - }, - { - name: "multiple code blocks", - input: "# Title\n```python\ndef foo():\n pass\n```\nSome paragraph\n```js\nconsole.log(42)\n```", - want: []string{"# Title", "```python\ndef foo():\n pass\n```", "Some paragraph", "```js\nconsole.log(42)\n```"}, - }, - { - name: "code block with blank line inside", - input: "```go\nfunc a() {\n\n}\n```", - want: []string{"```go\nfunc a() {\n\n}\n```"}, - }, - { - name: "no code blocks", - input: "hello\nworld\nfoo", - want: []string{"hello", "world", "foo"}, - }, - { - name: "short code block skipped", - input: "```go\nab\n```", - want: []string{}, - }, - { - name: "unclosed code block emitted", - input: "```go\nfunc main() {\n\tfmt.Println(\"hi\")\n}", - want: []string{"```go\nfunc main() {\n\tfmt.Println(\"hi\")\n}\n"}, - }, - { - name: "code block after heading", - input: "# Go语言\n```go\npackage main\n\nfunc main() {\n\tprintln(42)\n}\n```\n- key point", - want: []string{"# Go语言", "```go\npackage main\n\nfunc main() {\n\tprintln(42)\n}\n```", "- key point"}, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - lines := strings.Split(tc.input, "\n") - merged := mergeCodeBlockLines(lines) - - // Filter out empty lines for comparison - var got []string - for _, l := range merged { - if strings.TrimSpace(l) != "" { - got = append(got, l) - } - } - - if len(got) != len(tc.want) { - t.Errorf("expected %d non-empty merged lines, got %d", len(tc.want), len(got)) - t.Logf("got: %q", got) - t.Logf("want: %q", tc.want) - return - } - for i, w := range tc.want { - if got[i] != w { - t.Errorf("line %d:\n got: %q\n want: %q", i, got[i], w) - } - } - }) - } -} - -func TestExtractPPTSourceSectionsWithCodeBlocks(t *testing.T) { - markdown := `# Go语言基础 - -## 变量声明 - -Go语言使用 var 关键字声明变量。 - -` + "```go" + ` -var name string = "hello" -var age int = 25 -` + "```" + ` - -## 函数定义 - -` + "```go" + ` -func add(a, b int) int { - return a + b -} -` + "```" + ` - -函数是Go语言的一等公民。 -` - - sections := extractPPTSourceSections(markdown, 18) - - // Check that code blocks are present in the sections - foundCodeBlock := false - for _, sec := range sections { - for _, point := range sec.Points { - if strings.Contains(point, "```go") { - foundCodeBlock = true - // Verify code block is a complete unit with meaningful content - if !strings.Contains(point, "var ") && !strings.Contains(point, "func ") { - t.Errorf("code block point doesn't contain code: %q", point) - } - // Verify closing fence is present - if !strings.HasSuffix(strings.TrimSpace(point), "```") { - t.Errorf("code block point doesn't end with closing fence: %q", point) - } - } - } - } - - if !foundCodeBlock { - t.Errorf("no code blocks found in extracted sections; sections: %+v", sections) - } - - // Verify we have at least the variable declaration and function definition sections - foundVarSection := false - foundFuncSection := false - for _, sec := range sections { - if strings.Contains(sec.Title, "变量") { - foundVarSection = true - } - if strings.Contains(sec.Title, "函数") { - foundFuncSection = true - } - } - if !foundVarSection { - t.Error("expected '变量声明' section not found") - } - if !foundFuncSection { - t.Error("expected '函数定义' section not found") - } -} - -func TestExtractKeyPointsWithCodeBlocks(t *testing.T) { - markdown := `# Python Basics - -` + "```python" + ` -def hello(): - print("Hello, World!") -` + "```" + ` - -Some regular text here. - -` + "```javascript" + ` -console.log("test"); -` + "```" + ` -` - - points := extractKeyPoints(markdown, 48) - - foundPythonCode := false - for _, p := range points { - if strings.Contains(p, "```python") { - foundPythonCode = true - if !strings.Contains(p, "def hello()") { - t.Errorf("python code block point doesn't contain function: %q", p) - } - } - } - if !foundPythonCode { - t.Errorf("python code block not found in key points; points: %+v", points) - } -} - -func TestLooksCodeBlock(t *testing.T) { - cases := []struct { - input string - want bool - }{ - {"```go\nfmt.Println()\n```", true}, - {"regular text without code blocks", false}, - {"some ``` inline ``` code", true}, - {"", false}, - } - for _, tc := range cases { - got := looksCodeBlock(tc.input) - if got != tc.want { - t.Errorf("looksCodeBlock(%q) = %v, want %v", tc.input, got, tc.want) - } - } -} - -func TestRefContentLimit(t *testing.T) { - codeRef := GenerationReference{Content: "```go\nfunc main() {}\n```"} - textRef := GenerationReference{Content: "regular text content"} - - if limit := refContentLimit(codeRef); limit != 500 { - t.Errorf("refContentLimit for code block = %d, want 500", limit) - } - if limit := refContentLimit(textRef); limit != 120 { - t.Errorf("refContentLimit for text = %d, want 120", limit) - } -} - -func TestCleanPPTVisibleTextPreservesCodeBlockFences(t *testing.T) { - // The root cause fix: cleanPPTVisibleText must NOT strip ``` fences from - // code blocks, because downstream code (isPPTCodeBlockBullet, slideHasCodeBlock, - // writePPTCodeBlocks) relies on the fences to identify and correctly render - // code blocks in the PPT. - codeBlock := "```go\nfunc main() {\n\tfmt.Println(\"hello\")\n}\n```" - cleaned := cleanPPTVisibleText(codeBlock) - if !strings.HasPrefix(strings.TrimSpace(cleaned), "```") { - t.Errorf("cleanPPTVisibleText stripped code block fences: got %q", cleaned) - } - if !strings.Contains(cleaned, "func main()") { - t.Errorf("cleanPPTVisibleText lost code content: got %q", cleaned) - } - if !isPPTCodeBlockBullet(cleaned) { - t.Errorf("isPPTCodeBlockBullet returns false after cleanPPTVisibleText: got %q", cleaned) - } - - // Non-code text should still be cleaned normally for heading markers - headingText := "# Some heading" - cleanedHeading := cleanPPTVisibleText(headingText) - if strings.Contains(cleanedHeading, "#") { - t.Errorf("cleanPPTVisibleText did not clean heading marker: got %q", cleanedHeading) - } - // Markdown bold/italic is NOT cleaned by cleanPPTVisibleText - // (that's handled by stripPPTVisibleText, a different function) - plainText := "Some **bold** text" - cleanedPlain := cleanPPTVisibleText(plainText) - _ = cleanedPlain // just verify no panic -} - -func TestRenderStyledPPTSlidesWithCodeBlocks(t *testing.T) { - // End-to-end test: verify that code blocks in the plan are rendered - // as
 elements in the HTML output.
-	// NOTE: renderStyledPPTSlides assumes slide 0=封面, slide 1=目录,
-	// so we must follow that order in the test plan.
-	plan := pptOutlinePlan{
-		Title: "Go语言",
-		Slides: []pptSlidePlan{
-			{Title: "封面", Purpose: "建立演示主题", Bullets: []string{"Go语言入门"}},
-			{Title: "目录", Purpose: "呈现演示路径", Bullets: []string{"代码示例"}},
-		},
-	}
-	// Add a slide with a code block (must be slide index >= 2)
-	codeSlide := pptSlidePlan{
-		Title:   "代码示例",
-		Purpose: "展示Go代码",
-		Bullets: []string{
-			"Go语言使用var声明变量",
-			"```go\nvar name string = \"hello\"\nvar age int = 25\n```",
-		},
-	}
-	plan.Slides = append(plan.Slides, codeSlide)
-	plan.Slides = append(plan.Slides, pptSlidePlan{
-		Title:   "总结与行动",
-		Purpose: "收束核心结论并给出下一步",
-		Bullets: []string{"总结", "下一步"},
-	})
-
-	// Debug: run sanitizePPTPlanVisibleText and check results
-	sanitizedPlan := sanitizePPTPlanVisibleText(plan)
-	for i, slide := range sanitizedPlan.Slides {
-		for j, b := range slide.Bullets {
-			if strings.Contains(b, "```") || strings.Contains(b, "var name") {
-				t.Logf("After sanitize: Slide %d bullet %d: isPPTCodeBlockBullet=%v content=%q", i, j, isPPTCodeBlockBullet(b), truncate(b, 100))
-			}
-		}
-	}
-	// Check slideHasCodeBlock
-	for i, slide := range sanitizedPlan.Slides {
-		if slideHasCodeBlock(slide) {
-			t.Logf("Slide %d hasCodeBlock=true", i)
-		}
-	}
-
-	html := renderStyledPPTSlides(plan, pptStyleTheme{})
-
-	// Verify code block is rendered as 
 element
-	if !strings.Contains(html, `
`) {
-		t.Errorf("renderStyledPPTSlides did not render code block as 
; html snippet:\n%s",
-			truncate(html, 2000))
-	}
-	// Verify the code content is present
-	if !strings.Contains(html, "var name string") {
-		t.Errorf("renderStyledPPTSlides lost code content; html snippet:\n%s",
-			truncate(html, 2000))
-	}
-}
-
-func TestNormalizePPTBulletsSkipsCodeBlocks(t *testing.T) {
-	// Verify that normalizePPTBullets does not strip the slide title prefix
-	// from code blocks, which would corrupt the code content.
-	slide := &pptSlidePlan{
-		Title: "代码示例",
-		Bullets: []string{
-			"代码示例:这是一个说明",
-			"```go\nfunc main() {}\n```",
-		},
-	}
-	normalizePPTBullets(slide)
-
-	// The non-code bullet should have the title prefix stripped
-	for _, b := range slide.Bullets {
-		if strings.Contains(b, "```") {
-			// Code block should still contain the original code
-			if !strings.Contains(b, "func main()") {
-				t.Errorf("normalizePPTBullets corrupted code block: %q", b)
-			}
-			// Code block should still be recognized as a code block
-			if !isPPTCodeBlockBullet(b) {
-				t.Errorf("normalizePPTBullets stripped code block fences: %q", b)
-			}
-		}
-	}
-}
diff --git a/internal/service/generation_compat.go b/internal/service/generation_compat.go
new file mode 100644
index 0000000..8529a10
--- /dev/null
+++ b/internal/service/generation_compat.go
@@ -0,0 +1,199 @@
+package service
+
+import (
+	"context"
+
+	"YoudaoNoteLm/internal/memory"
+	"YoudaoNoteLm/internal/model/entity"
+	"YoudaoNoteLm/internal/rag"
+	gen "YoudaoNoteLm/internal/service/generation"
+	"YoudaoNoteLm/pkg/cache"
+
+	"github.com/cloudwego/eino/components/model"
+)
+
+type GenerationType = gen.GenerationType
+
+const (
+	GenerationTypeMindmap GenerationType = gen.GenerationTypeMindmap
+	GenerationTypePPT     GenerationType = gen.GenerationTypePPT
+	GenerationTypeQuiz    GenerationType = gen.GenerationTypeQuiz
+	GenerationTypeNote    GenerationType = gen.GenerationTypeNote
+)
+
+type GenerationRequest = gen.GenerationRequest
+type GenerationReference = gen.GenerationReference
+type GenerationResponse = gen.GenerationResponse
+type GenerationExportRequest = gen.GenerationExportRequest
+type GenerationExportResult = gen.GenerationExportResult
+type GenerationTaskStatus = gen.GenerationTaskStatus
+
+const (
+	GenerationTaskStatusPending   GenerationTaskStatus = gen.GenerationTaskStatusPending
+	GenerationTaskStatusRunning   GenerationTaskStatus = gen.GenerationTaskStatusRunning
+	GenerationTaskStatusCompleted GenerationTaskStatus = gen.GenerationTaskStatusCompleted
+	GenerationTaskStatusFailed    GenerationTaskStatus = gen.GenerationTaskStatusFailed
+	GenerationTaskStatusCancelled GenerationTaskStatus = gen.GenerationTaskStatusCancelled
+)
+
+type GenerationTask = gen.GenerationTask
+type GenerationTaskListFilter = gen.GenerationTaskListFilter
+type GenerationTaskStore = gen.GenerationTaskStore
+type GenerationTaskService = gen.GenerationTaskService
+type GenerationPrompt = gen.GenerationPrompt
+type GenerationModel = gen.GenerationModel
+type GenerationService = gen.GenerationService
+type GenerationMemoryScope = gen.GenerationMemoryScope
+type GenerationMemoryEntry = gen.GenerationMemoryEntry
+type GenerationMemoryStore = gen.GenerationMemoryStore
+type GenerationTaskQueue = gen.GenerationTaskQueue
+
+type generationSearchServiceAdapter struct {
+	base SearchService
+}
+
+func (a generationSearchServiceAdapter) SearchAndSummarize(ctx context.Context, req *gen.SearchRequest) (*gen.SearchResponse, error) {
+	resp, err := a.base.SearchAndSummarize(ctx, toServiceSearchRequest(req))
+	if err != nil {
+		return nil, err
+	}
+	return toGenerationSearchResponse(resp), nil
+}
+
+func adaptGenerationSearchService(search SearchService) gen.SearchService {
+	if search == nil {
+		return nil
+	}
+	return generationSearchServiceAdapter{base: search}
+}
+
+func toServiceSearchRequest(req *gen.SearchRequest) *SearchRequest {
+	if req == nil {
+		return nil
+	}
+	return &SearchRequest{
+		UserID:         req.UserID,
+		Scene:          SearchScene(req.Scene),
+		Query:          req.Query,
+		Freshness:      req.Freshness,
+		Count:          req.Count,
+		NeedSummary:    req.NeedSummary,
+		NeedContent:    req.NeedContent,
+		Language:       req.Language,
+		AllowedDomains: append([]string(nil), req.AllowedDomains...),
+		BlockedDomains: append([]string(nil), req.BlockedDomains...),
+		NotebookID:     req.NotebookID,
+		SourceID:       req.SourceID,
+		TraceID:        req.TraceID,
+		AllowDegrade:   req.AllowDegrade,
+		SkipUserConfig: req.SkipUserConfig,
+	}
+}
+
+func toGenerationSearchResponse(resp *SearchResponse) *gen.SearchResponse {
+	if resp == nil {
+		return nil
+	}
+	results := make([]gen.SearchResult, 0, len(resp.Results))
+	for _, item := range resp.Results {
+		results = append(results, gen.SearchResult{
+			Title:         item.Title,
+			Snippet:       item.Snippet,
+			URL:           item.URL,
+			DisplayURL:    item.DisplayURL,
+			PublishedAt:   item.PublishedAt,
+			SiteName:      item.SiteName,
+			Score:         item.Score,
+			Content:       item.Content,
+			ProviderRawID: item.ProviderRawID,
+			Meta:          item.Meta,
+		})
+	}
+	return &gen.SearchResponse{
+		Query:    resp.Query,
+		Provider: resp.Provider,
+		Results:  results,
+		Summary:  resp.Summary,
+		Total:    resp.Total,
+		Cached:   resp.Cached,
+		Meta:     resp.Meta,
+	}
+}
+
+func NewGenerationService(retriever rag.RAGRetriever, search SearchService, model GenerationModel) GenerationService {
+	return gen.NewGenerationService(retriever, adaptGenerationSearchService(search), model)
+}
+
+func NewGenerationServiceWithMemory(retriever rag.RAGRetriever, search SearchService, model GenerationModel, memory GenerationMemoryStore) GenerationService {
+	return gen.NewGenerationServiceWithMemory(retriever, adaptGenerationSearchService(search), model, memory)
+}
+
+func NewGenerationServiceWithMemories(retriever rag.RAGRetriever, search SearchService, model GenerationModel, memoryStore GenerationMemoryStore, longTermMemory memory.Reader) GenerationService {
+	return gen.NewGenerationServiceWithMemories(retriever, adaptGenerationSearchService(search), model, memoryStore, longTermMemory)
+}
+
+func NewGenerationServiceWithUserLLMConfig(retriever rag.RAGRetriever, search SearchService, repo interface {
+	FindDefaultByUserID(userID uint) (*entity.UserLLMConfig, error)
+}, encryptionKey string) GenerationService {
+	return NewGenerationServiceWithUserLLMConfigAndMemory(retriever, search, repo, nil, encryptionKey)
+}
+
+func NewGenerationServiceWithUserLLMConfigAndMemory(retriever rag.RAGRetriever, search SearchService, repo interface {
+	FindDefaultByUserID(userID uint) (*entity.UserLLMConfig, error)
+}, memory GenerationMemoryStore, encryptionKey string) GenerationService {
+	return gen.NewGenerationServiceWithUserLLMConfigAndMemory(
+		retriever,
+		adaptGenerationSearchService(search),
+		repo,
+		memory,
+		encryptionKey,
+	)
+}
+
+func NewGenerationServiceWithUserLLMConfigAndMemories(retriever rag.RAGRetriever, search SearchService, repo interface {
+	FindDefaultByUserID(userID uint) (*entity.UserLLMConfig, error)
+}, memoryStore GenerationMemoryStore, longTermMemory memory.Reader, encryptionKey string) GenerationService {
+	return gen.NewGenerationServiceWithUserLLMConfigAndMemories(
+		retriever,
+		adaptGenerationSearchService(search),
+		repo,
+		memoryStore,
+		longTermMemory,
+		encryptionKey,
+	)
+}
+
+func NewEinoGenerationModel(chat model.BaseChatModel) GenerationModel {
+	return gen.NewEinoGenerationModel(chat)
+}
+
+func NewGenerationMemoryCacheStore(cacheClient interface {
+	GetRecent(ctx context.Context, userID, notebookID uint, typ string, limit int) ([]cache.GenerationMemoryCacheEntry, error)
+	Add(ctx context.Context, userID, notebookID uint, typ string, entry cache.GenerationMemoryCacheEntry) error
+}) GenerationMemoryStore {
+	return gen.NewGenerationMemoryCacheStore(cacheClient)
+}
+
+func NewGenerationTaskService(base GenerationService, store GenerationTaskStore) GenerationTaskService {
+	return gen.NewGenerationTaskService(base, store)
+}
+
+func NewGenerationTaskServiceWithQueue(base GenerationService, store GenerationTaskStore, queue GenerationTaskQueue) GenerationTaskService {
+	return gen.NewGenerationTaskServiceWithQueue(base, store, queue)
+}
+
+func NewInMemoryGenerationTaskQueue(size int) GenerationTaskQueue {
+	return gen.NewInMemoryGenerationTaskQueue(size)
+}
+
+func NewGenerationTaskRedisQueue(taskCache *cache.GenerationTaskCache) GenerationTaskQueue {
+	return gen.NewGenerationTaskRedisQueue(taskCache)
+}
+
+func NewGenerationTaskCacheStore(taskCache *cache.GenerationTaskCache) GenerationTaskStore {
+	return gen.NewGenerationTaskCacheStore(taskCache)
+}
+
+func NewInMemoryGenerationTaskStore() GenerationTaskStore {
+	return gen.NewInMemoryGenerationTaskStore()
+}
diff --git a/internal/service/generation_export_ppt.go b/internal/service/generation_export_ppt.go
deleted file mode 100644
index 606f876..0000000
--- a/internal/service/generation_export_ppt.go
+++ /dev/null
@@ -1,633 +0,0 @@
-package service
-
-import (
-	"archive/zip"
-	"bytes"
-	"context"
-	"fmt"
-	"html"
-	"io"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strings"
-
-	"github.com/duynguyendang/docxgo/v3/pptx"
-
-	bizerrors "YoudaoNoteLm/pkg/errors"
-)
-
-type pptExportSlide struct {
-	Title   string
-	Bullets []string
-}
-
-type pptExportTheme struct {
-	Background pptx.Color
-	Surface    pptx.Color
-	SurfaceAlt pptx.Color
-	Accent     pptx.Color
-	AccentDark pptx.Color
-	AccentSoft pptx.Color
-	Border     pptx.Color
-	Title      pptx.Color
-	Text       pptx.Color
-	Muted      pptx.Color
-	White      pptx.Color
-}
-
-type pptExportTemplate struct {
-	ID          string
-	Name        string
-	Theme       pptExportTheme
-	Kicker      string
-	TitleFont   string
-	BodyFont    string
-	FooterLabel string
-	TitleSize   int
-	BodySize    int
-}
-
-const pptDefaultTemplateID = "classic"
-
-var pptExportTemplates = map[string]pptExportTemplate{
-	"classic": {
-		ID:   "classic",
-		Name: "Classic",
-		Theme: pptExportTheme{
-			// 暖橙色系:温暖、学术
-			Background: pptx.Color{R: 255, G: 248, B: 240},
-			Surface:    pptx.Color{R: 255, G: 252, B: 247},
-			SurfaceAlt: pptx.Color{R: 255, G: 240, B: 220},
-			Accent:     pptx.Color{R: 220, G: 78, B: 10},
-			AccentDark: pptx.Color{R: 178, G: 55, B: 8},
-			AccentSoft: pptx.Color{R: 255, G: 232, B: 208},
-			Border:     pptx.Color{R: 248, G: 176, B: 104},
-			Title:      pptx.Color{R: 60, G: 16, B: 4},
-			Text:       pptx.Color{R: 38, G: 34, B: 32},
-			Muted:      pptx.Color{R: 115, G: 108, B: 103},
-			White:      pptx.White,
-		},
-		Kicker:      "LEARNING DECK",
-		TitleFont:   "Microsoft YaHei UI",
-		BodyFont:    "Microsoft YaHei",
-		FooterLabel: "YoudaoNoteLM · Classic",
-		TitleSize:   34,
-		BodySize:    17,
-	},
-	"clean": {
-		ID:   "clean",
-		Name: "Clean",
-		Theme: pptExportTheme{
-			// 蓝灰极简:干净、现代
-			Background: pptx.Color{R: 246, G: 248, B: 252},
-			Surface:    pptx.Color{R: 255, G: 255, B: 255},
-			SurfaceAlt: pptx.Color{R: 243, G: 246, B: 251},
-			Accent:     pptx.Color{R: 79, G: 120, B: 200},
-			AccentDark: pptx.Color{R: 52, G: 88, B: 168},
-			AccentSoft: pptx.Color{R: 224, G: 232, B: 248},
-			Border:     pptx.Color{R: 210, G: 220, B: 238},
-			Title:      pptx.Color{R: 18, G: 32, B: 62},
-			Text:       pptx.Color{R: 28, G: 40, B: 60},
-			Muted:      pptx.Color{R: 96, G: 112, B: 140},
-			White:      pptx.White,
-		},
-		Kicker:      "FOCUS NOTES",
-		TitleFont:   "Microsoft YaHei UI",
-		BodyFont:    "Microsoft YaHei",
-		FooterLabel: "YoudaoNoteLM · Clean",
-		TitleSize:   34,
-		BodySize:    17,
-	},
-	"business": {
-		ID:   "business",
-		Name: "Business",
-		Theme: pptExportTheme{
-			// 深绿商务:专业、沉稳
-			Background: pptx.Color{R: 245, G: 248, B: 245},
-			Surface:    pptx.Color{R: 255, G: 255, B: 255},
-			SurfaceAlt: pptx.Color{R: 230, G: 242, B: 238},
-			Accent:     pptx.Color{R: 34, G: 110, B: 90},
-			AccentDark: pptx.Color{R: 22, G: 82, B: 66},
-			AccentSoft: pptx.Color{R: 216, G: 238, B: 232},
-			Border:     pptx.Color{R: 170, G: 212, B: 200},
-			Title:      pptx.Color{R: 16, G: 46, B: 38},
-			Text:       pptx.Color{R: 42, G: 58, B: 54},
-			Muted:      pptx.Color{R: 82, G: 102, B: 96},
-			White:      pptx.White,
-		},
-		Kicker:      "EXECUTIVE BRIEF",
-		TitleFont:   "Microsoft YaHei UI",
-		BodyFont:    "Microsoft YaHei",
-		FooterLabel: "YoudaoNoteLM · Business",
-		TitleSize:   34,
-		BodySize:    17,
-	},
-}
-
-var (
-	pptSectionPattern = regexp.MustCompile(`(?is)]*>(.*?)`)
-	pptH1Pattern      = regexp.MustCompile(`(?is)]*>(.*?)`)
-	pptH2Pattern      = regexp.MustCompile(`(?is)]*>(.*?)`)
-	pptBulletPattern  = regexp.MustCompile(`(?is)]*>(.*?)`)
-)
-
-func exportPPT(ctx context.Context, content, title, templateID string) (*GenerationExportResult, error) {
-	filename := resolveExportFilename(title, content, "ppt-export", ".pptx")
-
-	trimmedTemplateID := strings.TrimSpace(templateID)
-	var (
-		data []byte
-		err  error
-	)
-
-	if trimmedTemplateID == "" {
-		data, err = exportPPTWithDefaultEngine(ctx, content, strings.TrimSuffix(filename, ".pptx"))
-		if err != nil {
-			return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "build ppt export failed", err)
-		}
-	} else {
-		slides, parseErr := parsePPTExportSlides(content)
-		if parseErr != nil {
-			return nil, parseErr
-		}
-		template, templateErr := resolvePPTExportTemplate(trimmedTemplateID)
-		if templateErr != nil {
-			return nil, templateErr
-		}
-		data, err = buildPPTXBytes(slides, strings.TrimSuffix(filename, ".pptx"), template)
-		if err != nil {
-			return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "build ppt export failed", err)
-		}
-	}
-
-	return &GenerationExportResult{
-		Filename:    filename,
-		ContentType: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-		Data:        data,
-	}, nil
-}
-
-func resolvePPTExportTemplate(templateID string) (pptExportTemplate, error) {
-	id := strings.ToLower(strings.TrimSpace(templateID))
-	template, ok := pptExportTemplates[id]
-	if !ok {
-		return pptExportTemplate{}, bizerrors.New(bizerrors.CodeInvalidParam, "unsupported ppt template")
-	}
-	return template, nil
-}
-
-func parsePPTExportSlides(content string) ([]pptExportSlide, error) {
-	matches := pptSectionPattern.FindAllStringSubmatch(content, -1)
-	if len(matches) == 0 {
-		return nil, bizerrors.New(bizerrors.CodeInvalidParam, "ppt export content must contain at least one 
slide") - } - - slides := make([]pptExportSlide, 0, len(matches)) - for _, match := range matches { - body := match[1] - title := extractPPTSlideTitle(body) - bullets := extractPPTSlideBullets(body) - if title == "" && len(bullets) == 0 { - continue - } - slides = append(slides, pptExportSlide{ - Title: title, - Bullets: bullets, - }) - } - - if len(slides) == 0 { - return nil, bizerrors.New(bizerrors.CodeInvalidParam, "ppt export content does not contain any valid slides") - } - return slides, nil -} - -func extractPPTSlideTitle(section string) string { - for _, pattern := range []*regexp.Regexp{pptH1Pattern, pptH2Pattern} { - match := pattern.FindStringSubmatch(section) - if len(match) >= 2 { - return normalizePPTExportText(match[1]) - } - } - return "" -} - -func extractPPTSlideBullets(section string) []string { - matches := pptBulletPattern.FindAllStringSubmatch(section, -1) - bullets := make([]string, 0, len(matches)) - for _, match := range matches { - if len(match) < 2 { - continue - } - bullets = append(bullets, normalizePPTExportText(match[1])) - } - return uniqueNonEmpty(bullets) -} - -func normalizePPTExportText(value string) string { - value = strings.ReplaceAll(value, " ", " ") - value = stripSimpleHTML(value) - value = html.UnescapeString(value) - - // Preserve code blocks: if the value contains a fenced code block (```...```), - // strip the fences and keep the inner code with its original line structure. - if cleaned, ok := stripFencedCodeBlockForPPT(value); ok { - return cleaned - } - - value = strings.Join(strings.Fields(strings.TrimSpace(value)), " ") - return cleanPPTVisibleText(value) -} - -func buildPPTXBytes(slides []pptExportSlide, deckTitle string, template pptExportTemplate) ([]byte, error) { - builder := pptx.NewPresentationBuilder( - pptx.WithTitle(firstNonEmpty(deckTitle, "ppt-export")), - pptx.WithLayout(pptx.Layout16x9), - ) - - for i, slideData := range slides { - slide := builder.AddSlide().SetBackgroundColor(template.Theme.Background) - addPPTThemeFrame(slide, i+1, template) - - if slideData.Title != "" { - addPPTSlideTitle(slide, slideData.Title, template) - } - - startY := 2.14 - if slideData.Title == "" { - startY = 1.28 - } - addPPTBulletCards(slide, slideData.Bullets, startY, template) - } - - presentation, err := builder.Build() - if err != nil { - return nil, err - } - - tempDir, err := os.MkdirTemp("", "youdaonotelm-ppt-export-*") - if err != nil { - return nil, err - } - defer os.RemoveAll(tempDir) - - path := filepath.Join(tempDir, "export.pptx") - if err := presentation.SaveAs(path); err != nil { - return nil, err - } - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - return fixPPTXPackage(data, len(slides)) -} - -func addPPTThemeFrame(slide *pptx.SlideBuilder, slideNumber int, template pptExportTemplate) { - // 主内容卡片 - slide.AddShape(pptx.ShapeRoundedRectangle). - SetPosition(pptx.Inches(0.72), pptx.Inches(0.46)). - SetSize(pptx.Inches(11.92), pptx.Inches(6.28)). - SetFillColor(template.Theme.Surface). - SetLine(template.Theme.Border, 1). - End() - - // 顶部强调色条(稍厚一点,更有视觉分量) - slide.AddShape(pptx.ShapeRectangle). - SetPosition(pptx.Inches(0), pptx.Inches(0)). - SetSize(pptx.Inches(13.333), pptx.Inches(0.28)). - SetFillColor(template.Theme.Accent). - SetNoLine(). - End() - - // 左侧强调竖条(稍宽,视觉锚点更清晰) - slide.AddShape(pptx.ShapeRectangle). - SetPosition(pptx.Inches(0.72), pptx.Inches(0.46)). - SetSize(pptx.Inches(0.10), pptx.Inches(6.28)). - SetFillColor(template.Theme.AccentDark). - SetNoLine(). - End() - - // Kicker 徽标 - slide.AddShape(pptx.ShapeRoundedRectangle). - SetPosition(pptx.Inches(0.96), pptx.Inches(0.76)). - SetSize(pptx.Inches(1.60), pptx.Inches(0.34)). - SetFillColor(template.Theme.Accent). - SetNoLine(). - End() - - slide.AddText(template.Kicker). - SetBold(true). - SetFontSize(9). - SetFontFamily(template.BodyFont). - SetAlignment(pptx.AlignmentCenter). - SetColor(template.Theme.White). - SetPosition(pptx.Inches(1.01), pptx.Inches(0.84)). - SetSize(pptx.Inches(1.50), pptx.Inches(0.14)). - End() - - // 页码徽标(圆角矩形,更宽一点显得不局促) - slide.AddShape(pptx.ShapeRoundedRectangle). - SetPosition(pptx.Inches(11.50), pptx.Inches(0.70)). - SetSize(pptx.Inches(0.96), pptx.Inches(0.46)). - SetFillColor(template.Theme.AccentDark). - SetNoLine(). - End() - - slide.AddText(fmt.Sprintf("%02d", slideNumber)). - SetBold(true). - SetFontSize(14). - SetFontFamily(template.BodyFont). - SetAlignment(pptx.AlignmentCenter). - SetColor(template.Theme.White). - SetPosition(pptx.Inches(11.58), pptx.Inches(0.82)). - SetSize(pptx.Inches(0.80), pptx.Inches(0.18)). - End() - - // 页脚分隔线(使用 Border 色,更精致) - slide.AddShape(pptx.ShapeRectangle). - SetPosition(pptx.Inches(0.96), pptx.Inches(6.52)). - SetSize(pptx.Inches(11.42), pptx.Inches(0.022)). - SetFillColor(template.Theme.Border). - SetNoLine(). - End() - - slide.AddText(template.FooterLabel). - SetFontSize(10). - SetFontFamily(template.BodyFont). - SetColor(template.Theme.Muted). - SetPosition(pptx.Inches(0.96), pptx.Inches(6.64)). - SetSize(pptx.Inches(3.4), pptx.Inches(0.20)). - End() -} - -func addPPTSlideTitle(slide *pptx.SlideBuilder, title string, template pptExportTemplate) { - slide.AddText(title). - SetBold(true). - SetFontSize(template.TitleSize). - SetFontFamily(template.TitleFont). - SetColor(template.Theme.Title). - SetPosition(pptx.Inches(0.96), pptx.Inches(1.16)). - SetSize(pptx.Inches(10.82), pptx.Inches(0.80)). - End() - - // 标题下方装饰线(稍长,与内容宽度协调) - slide.AddShape(pptx.ShapeRoundedRectangle). - SetPosition(pptx.Inches(0.96), pptx.Inches(1.96)). - SetSize(pptx.Inches(1.40), pptx.Inches(0.06)). - SetFillColor(template.Theme.Accent). - SetNoLine(). - End() -} - -func calcBulletCardHeight(bullet string, baseSize int) float64 { - length := len([]rune(strings.TrimSpace(bullet))) - fs := pptBulletFontSize(bullet, baseSize) - // 文字区宽度约 9.7 英寸,按每英寸约 6 个字符(中英混排保守估算) - charsPerLine := 58 - if fs < baseSize-1 { - charsPerLine = 65 - } - if charsPerLine < 10 { - charsPerLine = 10 - } - lines := (length + charsPerLine - 1) / charsPerLine - if lines < 1 { - lines = 1 - } - // 每行约 0.26 英寸,加上上下内边距 0.44 英寸 - h := float64(lines)*0.26 + 0.44 - if h < 0.74 { - h = 0.74 - } - return h -} - -func addPPTBulletCards(slide *pptx.SlideBuilder, bullets []string, startY float64, template pptExportTemplate) { - if len(bullets) == 0 { - return - } - const cardGap = 0.10 - const maxBottom = 6.44 - - heights := make([]float64, len(bullets)) - total := 0.0 - for i, b := range bullets { - heights[i] = calcBulletCardHeight(b, template.BodySize) - total += heights[i] - if i > 0 { - total += cardGap - } - } - - // 超出可用高度时等比缩小 - available := maxBottom - startY - if total > available { - scale := available / total - for i := range heights { - heights[i] *= scale - } - } - - y := startY - for i, bullet := range bullets { - addPPTBulletCard(slide, i, bullet, y, heights[i], template) - y += heights[i] + cardGap - } -} - -func addPPTBulletCard(slide *pptx.SlideBuilder, index int, bullet string, y float64, height float64, template pptExportTemplate) { - slide.AddShape(pptx.ShapeRoundedRectangle). - SetPosition(pptx.Inches(1.04), pptx.Inches(y)). - SetSize(pptx.Inches(11.08), pptx.Inches(height)). - SetFillColor(template.Theme.SurfaceAlt). - SetLine(template.Theme.Border, 1). - End() - - // 圆形编号指示器,垂直居中于卡片 - circleY := y + (height-0.36)/2 - slide.AddShape(pptx.ShapeEllipse). - SetPosition(pptx.Inches(1.26), pptx.Inches(circleY)). - SetSize(pptx.Inches(0.36), pptx.Inches(0.36)). - SetFillColor(template.Theme.Accent). - SetNoLine(). - End() - - slide.AddText(fmt.Sprintf("%02d", index+1)). - SetBold(true). - SetFontSize(10). - SetFontFamily(template.BodyFont). - SetAlignment(pptx.AlignmentCenter). - SetColor(template.Theme.White). - SetPosition(pptx.Inches(1.29), pptx.Inches(circleY+0.12)). - SetSize(pptx.Inches(0.30), pptx.Inches(0.14)). - End() - - textHeight := height - 0.30 - if textHeight < 0.22 { - textHeight = 0.22 - } - slide.AddText(bullet). - SetFontSize(pptBulletFontSize(bullet, template.BodySize)). - SetFontFamily(template.BodyFont). - SetColor(template.Theme.Text). - SetPosition(pptx.Inches(1.84), pptx.Inches(y+0.15)). - SetSize(pptx.Inches(9.80), pptx.Inches(textHeight)). - End() -} - -func pptBulletFontSize(bullet string, baseSize int) int { - if baseSize <= 0 { - baseSize = 15 - } - length := len([]rune(strings.TrimSpace(bullet))) - switch { - case length > 110: - return baseSize - 3 - case length > 72: - return baseSize - 2 - case length > 44: - return baseSize - 1 - default: - return baseSize - } -} - -func fixPPTXPackage(data []byte, slideCount int) ([]byte, error) { - reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) - if err != nil { - return nil, err - } - - existing := make(map[string]bool, len(reader.File)) - for _, file := range reader.File { - existing[file.Name] = true - } - - needsPresentationXML := true - missing := make([]string, 0, slideCount) - for i := 1; i <= slideCount; i++ { - name := fmt.Sprintf("ppt/slides/_rels/slide%d.xml.rels", i) - if !existing[name] { - missing = append(missing, name) - } - } - - var out bytes.Buffer - writer := zip.NewWriter(&out) - for _, file := range reader.File { - if file.Name == "ppt/presentation.xml" { - if err := writePPTXZipEntry(writer, file.Name, []byte(pptxPresentationXML(slideCount))); err != nil { - _ = writer.Close() - return nil, err - } - needsPresentationXML = false - continue - } - if file.Name == "ppt/slideMasters/slideMaster1.xml" { - if err := writePPTXZipEntry(writer, file.Name, []byte(pptxSlideMasterXML())); err != nil { - _ = writer.Close() - return nil, err - } - continue - } - if err := copyPPTXZipEntry(writer, file); err != nil { - _ = writer.Close() - return nil, err - } - } - if needsPresentationXML { - if err := writePPTXZipEntry(writer, "ppt/presentation.xml", []byte(pptxPresentationXML(slideCount))); err != nil { - _ = writer.Close() - return nil, err - } - } - for _, name := range missing { - if err := writePPTXZipEntry(writer, name, []byte(pptxSlideRelationshipXML())); err != nil { - _ = writer.Close() - return nil, err - } - } - if err := writer.Close(); err != nil { - return nil, err - } - return out.Bytes(), nil -} - -func writePPTXZipEntry(writer *zip.Writer, name string, data []byte) error { - entry, err := writer.Create(name) - if err != nil { - return err - } - _, err = entry.Write(data) - return err -} - -func copyPPTXZipEntry(writer *zip.Writer, file *zip.File) error { - reader, err := file.Open() - if err != nil { - return err - } - defer reader.Close() - - header := file.FileHeader - entry, err := writer.CreateHeader(&header) - if err != nil { - return err - } - _, err = io.Copy(entry, reader) - return err -} - -func pptxPresentationXML(slideCount int) string { - var b strings.Builder - b.WriteString(` - - - - - -`) - for i := 1; i <= slideCount; i++ { - b.WriteString(fmt.Sprintf(` -`, 255+i, i+2)) - } - b.WriteString(` - - - -`) - return b.String() -} - -func pptxSlideMasterXML() string { - return ` - - - - - - - - - - - - - - - - - - - - -` -} - -func pptxSlideRelationshipXML() string { - return ` - - -` -} diff --git a/internal/service/generation_export_ppt_dynamic.go b/internal/service/generation_export_ppt_dynamic.go deleted file mode 100644 index 187e32e..0000000 --- a/internal/service/generation_export_ppt_dynamic.go +++ /dev/null @@ -1,2622 +0,0 @@ -package service - -import ( - stdhtml "html" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - - "github.com/duynguyendang/docxgo/v3/pptx" - "golang.org/x/net/html" - - bizerrors "YoudaoNoteLm/pkg/errors" -) - -const ( - dynamicPPTDefaultFontFamily = "Aptos" - dynamicPPTTitleFontFamily = "Aptos" - dynamicPPTSlideWidth = 13.333 - dynamicPPTSlideHeight = 7.5 - dynamicPPTOuterMarginX = 0.32 - dynamicPPTOuterMarginY = 0.26 - dynamicPPTDefaultGap = 0.14 - dynamicPPTDefaultBodyFont = 17 - dynamicPPTMinBodyFontSize = 14 -) - -var ( - dynamicPPTDefaultSlideBackground = pptx.Color{R: 246, G: 244, B: 239} - dynamicPPTDefaultSectionFill = pptx.Color{R: 252, G: 251, B: 248} - dynamicPPTDefaultSectionBorder = pptx.Color{R: 231, G: 224, B: 214} - dynamicPPTDefaultText = pptx.Color{R: 47, G: 42, B: 36} - dynamicPPTDefaultMuted = pptx.Color{R: 111, G: 104, B: 95} - dynamicPPTDefaultAccent = pptx.Color{R: 183, G: 170, B: 150} -) - -type pptHTMLDocument struct { - BodyStyle pptStyle - Rules []pptCSSRule - Vars map[string]string - Slides []pptHTMLSlide -} - -type pptHTMLSlide struct { - SectionStyle pptStyle - Blocks []pptHTMLBlock -} - -type pptHTMLBlock struct { - Kind string - Layout string - Text string - Runs []pptHTMLTextRun - Style pptStyle - Classes map[string]bool - Children []pptHTMLBlock -} - -type pptHTMLTextRun struct { - Text string - Style pptStyle -} - -type pptCSSRule struct { - Selector string - Parts []pptCSSSelectorPart - Style pptStyle - Specificity int - Order int -} - -type pptCSSSelectorPart struct { - Tag string - Classes []string - FirstChild bool -} - -type pptStyleDeclaration struct { - Key string - Value string -} - -type pptStyle struct { - TextColor *pptx.Color - BackgroundColor *pptx.Color - BorderColor *pptx.Color - BorderLeftColor *pptx.Color - BorderBottomColor *pptx.Color - FontSize *int - FontWeight *int - LineHeight *float64 - FontFamily string - TextAlign string - Display string - GridTemplateColumns string - FlexWrap string - Gap *float64 - Padding pptEdges - Margin pptEdges - BorderWidth *int - BorderLeftWidth *int - BorderBottomWidth *int - BorderRadius *int - ClearBackground bool - ClearBorder bool - ClearBorderLeft bool - ClearBorderBottom bool -} - -type pptEdges struct { - Top float64 - Right float64 - Bottom float64 - Left float64 - Set bool -} - -type pptLayoutCursor struct { - x float64 - y float64 - width float64 -} - -type pptSectionFrame struct { - x float64 - y float64 - width float64 - height float64 - paddingTop float64 - paddingRight float64 - paddingBottom float64 - paddingLeft float64 -} - -type dynamicLayoutConfig struct { - SlideWidth float64 - SlideHeight float64 - OuterMarginX float64 - OuterMarginY float64 - DefaultGap float64 - ConservativeColumns bool -} - -type measuredDynamicHTMLSlide struct { - SectionStyle pptStyle - Frame pptSectionFrame - Blocks []measuredDynamicBlock - ContentBottom float64 -} - -type measuredDynamicBlock struct { - Block pptHTMLBlock - X float64 - Y float64 - Width float64 - Height float64 - Children []measuredDynamicBlock -} - -func buildDynamicHTMLPPTX(content, deckTitle string) ([]byte, error) { - doc, err := parseDynamicHTMLDocument(content) - if err != nil { - return nil, err - } - if len(doc.Slides) == 0 { - return nil, bizerrors.New(bizerrors.CodeInvalidParam, "ppt export content does not contain any valid slides") - } - - builder := pptx.NewPresentationBuilder( - pptx.WithTitle(firstNonEmpty(deckTitle, "ppt-export")), - pptx.WithLayout(pptx.Layout16x9), - ) - - layoutConfig := newDynamicLayoutConfig() - for _, slideData := range doc.Slides { - measured := measureDynamicHTMLSlide(doc, slideData, layoutConfig) - renderMeasuredDynamicHTMLSlide(builder.AddSlide(), doc, measured) - } - - presentation, err := builder.Build() - if err != nil { - return nil, err - } - - tempDir, err := os.MkdirTemp("", "youdaonotelm-ppt-export-dynamic-*") - if err != nil { - return nil, err - } - defer os.RemoveAll(tempDir) - - path := filepath.Join(tempDir, "export.pptx") - if err := presentation.SaveAs(path); err != nil { - return nil, err - } - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - return fixPPTXPackage(data, len(doc.Slides)) -} - -func parseDynamicHTMLDocument(content string) (*pptHTMLDocument, error) { - root, err := html.Parse(strings.NewReader(content)) - if err != nil { - return nil, bizerrors.NewWithErr(bizerrors.CodeInvalidParam, "invalid ppt export html", err) - } - - doc := &pptHTMLDocument{ - Vars: make(map[string]string), - } - for _, cssText := range collectStyleTagContents(root) { - vars, rules := parseCSSRules(cssText, doc.Vars, len(doc.Rules)) - for key, value := range vars { - doc.Vars[key] = value - } - doc.Rules = append(doc.Rules, rules...) - } - - bodyNode := findFirstHTMLElement(root, "body") - if bodyNode != nil { - doc.BodyStyle = computeNodeStyle(bodyNode, pptStyle{}, doc) - } - - sections := findHTMLSections(root) - if len(sections) == 0 { - return nil, bizerrors.New(bizerrors.CodeInvalidParam, "ppt export content must contain at least one
slide") - } - - doc.Slides = make([]pptHTMLSlide, 0, len(sections)) - for _, section := range sections { - slide := parseHTMLSection(section, doc) - if len(slide.Blocks) == 0 { - continue - } - doc.Slides = append(doc.Slides, slide) - } - return doc, nil -} - -func collectStyleTagContents(root *html.Node) []string { - var values []string - var walk func(*html.Node) - walk = func(node *html.Node) { - if node == nil { - return - } - if node.Type == html.ElementNode && strings.EqualFold(node.Data, "style") { - if css := strings.TrimSpace(extractRawNodeText(node)); css != "" { - values = append(values, css) - } - return - } - for child := node.FirstChild; child != nil; child = child.NextSibling { - walk(child) - } - } - walk(root) - return values -} - -func findFirstHTMLElement(root *html.Node, name string) *html.Node { - var found *html.Node - var walk func(*html.Node) - walk = func(node *html.Node) { - if node == nil || found != nil { - return - } - if node.Type == html.ElementNode && strings.EqualFold(node.Data, name) { - found = node - return - } - for child := node.FirstChild; child != nil; child = child.NextSibling { - walk(child) - } - } - walk(root) - return found -} - -func findHTMLSections(root *html.Node) []*html.Node { - var sections []*html.Node - var walk func(*html.Node) - walk = func(node *html.Node) { - if node == nil { - return - } - if node.Type == html.ElementNode && strings.EqualFold(node.Data, "section") { - sections = append(sections, node) - return - } - for child := node.FirstChild; child != nil; child = child.NextSibling { - walk(child) - } - } - walk(root) - return sections -} - -func parseHTMLSection(section *html.Node, doc *pptHTMLDocument) pptHTMLSlide { - sectionStyle := computeNodeStyle(section, inheritTextStyle(doc.BodyStyle), doc) - blocks := parseHTMLChildren(section, doc, inheritTextStyle(sectionStyle)) - return pptHTMLSlide{ - SectionStyle: sectionStyle, - Blocks: blocks, - } -} - -func parseHTMLChildren(node *html.Node, doc *pptHTMLDocument, inheritedText pptStyle) []pptHTMLBlock { - var blocks []pptHTMLBlock - for child := node.FirstChild; child != nil; child = child.NextSibling { - blocks = append(blocks, parseHTMLBlocks(child, doc, inheritedText)...) - } - return blocks -} - -func parseHTMLBlocks(node *html.Node, doc *pptHTMLDocument, inheritedText pptStyle) []pptHTMLBlock { - if node == nil { - return nil - } - if node.Type == html.TextNode { - text := normalizePPTExportText(node.Data) - if text == "" { - return nil - } - return []pptHTMLBlock{{ - Kind: "text", - Text: text, - Style: inheritedText, - }} - } - if node.Type != html.ElementNode { - return parseHTMLChildren(node, doc, inheritedText) - } - - tag := strings.ToLower(node.Data) - if shouldIgnoreHTMLElement(tag) { - return nil - } - - style := computeNodeStyle(node, inheritedText, doc) - classes := parseClassSet(node) - - if tag == "span" && classes["section-number"] { - text := normalizePPTExportText(extractNodeText(node)) - if text == "" { - return nil - } - return []pptHTMLBlock{{ - Kind: "section-number", - Text: text, - Style: style, - Classes: classes, - }} - } - - if isLayoutContainer(tag, classes, style, node) { - children := parseHTMLChildren(node, doc, inheritTextStyle(style)) - if len(children) == 0 { - return nil - } - return []pptHTMLBlock{{ - Kind: "container", - Layout: resolveContainerLayout(classes, style), - Style: style, - Classes: classes, - Children: children, - }} - } - - if isCardBlock(tag, classes, style, node) { - children := parseHTMLChildren(node, doc, inheritTextStyle(style)) - if len(children) == 0 { - text := normalizePPTExportText(extractNodeText(node)) - if text != "" { - children = append(children, pptHTMLBlock{ - Kind: "text", - Text: text, - Style: inheritTextStyle(style), - }) - } - } - if len(children) == 0 { - return nil - } - return []pptHTMLBlock{{ - Kind: "card", - Style: style, - Classes: classes, - Children: children, - }} - } - - switch tag { - case "h1", "h2", "h3", "p": - runs := extractInlineTextRuns(node, doc, style) - text := textFromRuns(runs) - if text == "" { - return nil - } - return []pptHTMLBlock{{ - Kind: tag, - Text: text, - Runs: runs, - Style: style, - Classes: classes, - }} - case "pre": - // Code blocks: extract text preserving line breaks and indentation - text := extractRawNodeText(node) - if strings.TrimSpace(text) == "" { - return nil - } - // If the first child is , get its text instead - for child := node.FirstChild; child != nil; child = child.NextSibling { - if child.Type == html.ElementNode && strings.EqualFold(child.Data, "code") { - text = extractRawNodeText(child) - break - } - } - if strings.TrimSpace(text) == "" { - return nil - } - return []pptHTMLBlock{{ - Kind: "pre", - Text: text, - Style: style, - Classes: classes, - }} - case "ul", "ol": - return parseHTMLList(node, doc, inheritTextStyle(style), tag == "ol") - case "li": - runs := extractInlineTextRuns(node, doc, style) - text := textFromRuns(runs) - if text == "" { - return nil - } - prefix := "- " - runs = prependInlineRunPrefix(runs, prefix, style) - return []pptHTMLBlock{{ - Kind: "list-item", - Text: prefix + text, - Runs: runs, - Style: style, - Classes: classes, - }} - case "span": - text := normalizePPTExportText(extractNodeText(node)) - if text == "" { - return nil - } - return []pptHTMLBlock{{ - Kind: "text", - Text: text, - Style: style, - Classes: classes, - }} - default: - children := parseHTMLChildren(node, doc, inheritTextStyle(style)) - if len(children) > 0 { - return children - } - text := normalizePPTExportText(extractNodeText(node)) - if text == "" { - return nil - } - return []pptHTMLBlock{{ - Kind: "p", - Text: text, - Style: style, - Classes: classes, - }} - } -} - -func shouldIgnoreHTMLElement(tag string) bool { - switch tag { - case "html", "head", "body", "style", "script", "meta", "title", "link": - return true - default: - return false - } -} - -func isLayoutContainer(tag string, classes map[string]bool, style pptStyle, node *html.Node) bool { - if tag == "section" { - return true - } - if classes["slider"] || classes["row"] || classes["dir-list"] { - return true - } - if style.Display == "flex" || style.Display == "grid" { - return true - } - if tag == "div" { - return countMeaningfulElementChildren(node) > 1 && !hasCardVisualStyle(style) - } - return false -} - -func resolveContainerLayout(classes map[string]bool, style pptStyle) string { - switch { - case classes["row"]: - return "row" - case classes["dir-list"]: - return "grid" - case style.Display == "grid": - return "grid" - case style.Display == "flex": - return "row" - default: - return "stack" - } -} - -func isCardBlock(tag string, classes map[string]bool, style pptStyle, node *html.Node) bool { - if tag != "div" && tag != "aside" { - return false - } - if classes["highlight"] || classes["highlight-box"] || classes["card"] || classes["dir-item"] || classes["footnote"] || classes["evidence"] || classes["callout"] { - return true - } - return hasCardVisualStyle(style) -} - -func hasCardVisualStyle(style pptStyle) bool { - return style.BackgroundColor != nil || style.BorderColor != nil || style.BorderLeftColor != nil || style.BorderRadius != nil -} - -func countMeaningfulElementChildren(node *html.Node) int { - count := 0 - for child := node.FirstChild; child != nil; child = child.NextSibling { - if child.Type != html.ElementNode { - continue - } - if shouldIgnoreHTMLElement(strings.ToLower(child.Data)) { - continue - } - count++ - } - return count -} - -func parseHTMLList(node *html.Node, doc *pptHTMLDocument, inheritedText pptStyle, ordered bool) []pptHTMLBlock { - var blocks []pptHTMLBlock - index := 1 - for child := node.FirstChild; child != nil; child = child.NextSibling { - if child.Type != html.ElementNode || !strings.EqualFold(child.Data, "li") { - continue - } - style := computeNodeStyle(child, inheritedText, doc) - runs := extractInlineTextRuns(child, doc, style) - text := textFromRuns(runs) - if text == "" { - continue - } - prefix := "- " - if ordered { - prefix = strconv.Itoa(index) + ". " - } - runs = prependInlineRunPrefix(runs, prefix, style) - blocks = append(blocks, pptHTMLBlock{ - Kind: "list-item", - Text: prefix + text, - Runs: runs, - Style: style, - Classes: parseClassSet(child), - }) - index++ - } - return blocks -} - -func extractInlineTextRuns(node *html.Node, doc *pptHTMLDocument, inheritedText pptStyle) []pptHTMLTextRun { - var runs []pptHTMLTextRun - var walk func(*html.Node, pptStyle) - walk = func(current *html.Node, currentStyle pptStyle) { - if current == nil { - return - } - switch current.Type { - case html.TextNode: - text := normalizeInlineText(current.Data) - if text != "" { - runs = append(runs, pptHTMLTextRun{Text: text, Style: currentStyle}) - } - return - case html.ElementNode: - tag := strings.ToLower(current.Data) - if shouldIgnoreHTMLElement(tag) { - return - } - nextStyle := computeNodeStyle(current, currentStyle, doc) - switch tag { - case "strong", "b": - nextStyle.FontWeight = intPtr(700) - case "em", "i": - if nextStyle.FontWeight == nil { - nextStyle.FontWeight = currentStyle.FontWeight - } - } - for child := current.FirstChild; child != nil; child = child.NextSibling { - walk(child, inheritInlineTextStyle(nextStyle)) - } - } - } - for child := node.FirstChild; child != nil; child = child.NextSibling { - walk(child, inheritInlineTextStyle(inheritedText)) - } - return mergeAdjacentInlineRuns(runs) -} - -func normalizeInlineText(value string) string { - value = strings.ReplaceAll(value, " ", " ") - value = stdhtml.UnescapeString(value) - if strings.TrimSpace(value) == "" { - if strings.ContainsAny(value, " \n\r\t") { - return " " - } - return "" - } - leading := len(value) > 0 && isInlineWhitespace(rune(value[0])) - trailingRunes := []rune(value) - trailing := len(trailingRunes) > 0 && isInlineWhitespace(trailingRunes[len(trailingRunes)-1]) - collapsed := strings.Join(strings.Fields(value), " ") - if leading { - collapsed = " " + collapsed - } - if trailing { - collapsed += " " - } - if !leading && startsWithPPTMarkdownSyntaxMarker(collapsed) { - collapsed = cleanPPTVisibleText(collapsed) - } - return collapsed -} - -func startsWithPPTMarkdownSyntaxMarker(value string) bool { - trimmed := strings.TrimSpace(value) - if trimmed == "" { - return false - } - if strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") || strings.HasPrefix(trimmed, "• ") { - return true - } - return false -} - -func isInlineWhitespace(r rune) bool { - return r == ' ' || r == '\n' || r == '\r' || r == '\t' -} - -func textFromRuns(runs []pptHTMLTextRun) string { - var b strings.Builder - for _, run := range runs { - b.WriteString(run.Text) - } - return strings.TrimSpace(b.String()) -} - -func prependInlineRunPrefix(runs []pptHTMLTextRun, prefix string, style pptStyle) []pptHTMLTextRun { - if strings.TrimSpace(prefix) == "" { - return runs - } - prefixed := make([]pptHTMLTextRun, 0, len(runs)+1) - prefixed = append(prefixed, pptHTMLTextRun{Text: prefix, Style: inheritInlineTextStyle(style)}) - prefixed = append(prefixed, runs...) - return mergeAdjacentInlineRuns(prefixed) -} - -func inheritInlineTextStyle(style pptStyle) pptStyle { - return inheritTextStyle(style) -} - -func mergeAdjacentInlineRuns(runs []pptHTMLTextRun) []pptHTMLTextRun { - merged := make([]pptHTMLTextRun, 0, len(runs)) - for _, run := range runs { - if run.Text == "" { - continue - } - if len(merged) > 0 && inlineStylesEqual(merged[len(merged)-1].Style, run.Style) { - merged[len(merged)-1].Text += run.Text - continue - } - merged = append(merged, run) - } - return merged -} - -func inlineStylesEqual(a, b pptStyle) bool { - return colorsEqual(a.TextColor, b.TextColor) && - intPointersEqual(a.FontSize, b.FontSize) && - intPointersEqual(a.FontWeight, b.FontWeight) && - a.FontFamily == b.FontFamily -} - -func colorsEqual(a, b *pptx.Color) bool { - if a == nil || b == nil { - return a == b - } - return *a == *b -} - -func intPointersEqual(a, b *int) bool { - if a == nil || b == nil { - return a == b - } - return *a == *b -} - -func computeNodeStyle(node *html.Node, inheritedText pptStyle, doc *pptHTMLDocument) pptStyle { - style := inheritTextStyle(inheritedText) - - matches := matchingCSSRules(node, doc.Rules) - sort.SliceStable(matches, func(i, j int) bool { - if matches[i].Specificity == matches[j].Specificity { - return matches[i].Order < matches[j].Order - } - return matches[i].Specificity < matches[j].Specificity - }) - for _, rule := range matches { - style = mergePPTStyle(style, rule.Style) - } - - inlineStyle := parseInlineStyleDeclarations(resolveCSSVars(getHTMLAttribute(node, "style"), doc.Vars)) - style = applyOrderedStyleDeclarations(style, inlineStyle) - return style -} - -func matchingCSSRules(node *html.Node, rules []pptCSSRule) []pptCSSRule { - matches := make([]pptCSSRule, 0, len(rules)) - for _, rule := range rules { - if matchesCSSSelector(node, rule.Parts) { - matches = append(matches, rule) - } - } - return matches -} - -func matchesCSSSelector(node *html.Node, parts []pptCSSSelectorPart) bool { - if len(parts) == 0 || node == nil { - return false - } - partIndex := len(parts) - 1 - if !matchesSelectorPart(node, parts[partIndex]) { - return false - } - partIndex-- - current := node.Parent - for current != nil && partIndex >= 0 { - if matchesSelectorPart(current, parts[partIndex]) { - partIndex-- - } - current = current.Parent - } - return partIndex < 0 -} - -func matchesSelectorPart(node *html.Node, part pptCSSSelectorPart) bool { - if node == nil || node.Type != html.ElementNode { - return false - } - if part.Tag != "" && part.Tag != "*" && !strings.EqualFold(node.Data, part.Tag) { - return false - } - classSet := parseClassSet(node) - for _, className := range part.Classes { - if !classSet[className] { - return false - } - } - if part.FirstChild && !isFirstElementChild(node) { - return false - } - return true -} - -func isFirstElementChild(node *html.Node) bool { - if node == nil || node.Parent == nil { - return false - } - for sibling := node.Parent.FirstChild; sibling != nil; sibling = sibling.NextSibling { - if sibling.Type != html.ElementNode { - continue - } - return sibling == node - } - return false -} - -func parseCSSRules(css string, existingVars map[string]string, startOrder int) (map[string]string, []pptCSSRule) { - vars := make(map[string]string) - for key, value := range existingVars { - vars[key] = value - } - - cleaned := stripCSSComments(css) - var rules []pptCSSRule - order := startOrder - for _, block := range splitTopLevelCSSBlocks(cleaned) { - selector := strings.TrimSpace(block.selector) - if selector == "" || strings.Contains(selector, "::") || strings.Contains(selector, ":last-child") { - continue - } - declarations := parseInlineStyleDeclarations(resolveCSSVars(block.body, vars)) - if selector == ":root" { - for _, value := range declarations { - if strings.HasPrefix(value.Key, "--") { - vars[value.Key] = value.Value - } - } - continue - } - - style := applyOrderedStyleDeclarations(pptStyle{}, declarations) - for _, item := range strings.Split(selector, ",") { - item = strings.TrimSpace(item) - if item == "" || strings.Contains(item, "::") || strings.Contains(item, ":last-child") { - continue - } - parts, specificity, ok := parseCSSSelector(item) - if !ok { - continue - } - rules = append(rules, pptCSSRule{ - Selector: item, - Parts: parts, - Style: style, - Specificity: specificity, - Order: order, - }) - order++ - } - } - return vars, rules -} - -type cssBlock struct { - selector string - body string -} - -func splitTopLevelCSSBlocks(css string) []cssBlock { - var blocks []cssBlock - for i := 0; i < len(css); { - for i < len(css) && isCSSWhitespace(css[i]) { - i++ - } - if i >= len(css) { - break - } - if css[i] == '@' { - i = skipAtRule(css, i) - continue - } - start := i - for i < len(css) && css[i] != '{' { - i++ - } - if i >= len(css) { - break - } - selector := strings.TrimSpace(css[start:i]) - i++ - bodyStart := i - depth := 1 - for i < len(css) && depth > 0 { - switch css[i] { - case '{': - depth++ - case '}': - depth-- - } - i++ - } - if depth != 0 { - break - } - body := strings.TrimSpace(css[bodyStart : i-1]) - blocks = append(blocks, cssBlock{selector: selector, body: body}) - } - return blocks -} - -func skipAtRule(css string, index int) int { - for index < len(css) && css[index] != '{' && css[index] != ';' { - index++ - } - if index >= len(css) { - return index - } - if css[index] == ';' { - return index + 1 - } - depth := 1 - index++ - for index < len(css) && depth > 0 { - switch css[index] { - case '{': - depth++ - case '}': - depth-- - } - index++ - } - return index -} - -func stripCSSComments(css string) string { - var b strings.Builder - for i := 0; i < len(css); i++ { - if i+1 < len(css) && css[i] == '/' && css[i+1] == '*' { - i += 2 - for i+1 < len(css) && !(css[i] == '*' && css[i+1] == '/') { - i++ - } - i++ - continue - } - b.WriteByte(css[i]) - } - return b.String() -} - -func parseCSSSelector(selector string) ([]pptCSSSelectorPart, int, bool) { - tokens := strings.Fields(selector) - if len(tokens) == 0 { - return nil, 0, false - } - parts := make([]pptCSSSelectorPart, 0, len(tokens)) - specificity := 0 - for _, token := range tokens { - token = strings.TrimSpace(token) - if token == "" { - continue - } - part := pptCSSSelectorPart{} - if strings.Contains(token, ":first-child") { - part.FirstChild = true - token = strings.ReplaceAll(token, ":first-child", "") - specificity += 100 - } - segments := strings.Split(token, ".") - if len(segments) > 0 { - head := strings.TrimSpace(segments[0]) - if head != "" { - part.Tag = strings.ToLower(head) - if part.Tag != "*" { - specificity += 10 - } - } - for _, className := range segments[1:] { - className = strings.TrimSpace(className) - if className == "" { - continue - } - part.Classes = append(part.Classes, className) - specificity += 100 - } - } - if strings.HasPrefix(token, ".") && part.Tag == "" { - part.Tag = "*" - } - if part.Tag == "" && len(part.Classes) == 0 && !part.FirstChild { - return nil, 0, false - } - parts = append(parts, part) - } - return parts, specificity, len(parts) > 0 -} - -func isCSSWhitespace(value byte) bool { - return value == ' ' || value == '\n' || value == '\r' || value == '\t' -} - -func inheritTextStyle(style pptStyle) pptStyle { - return pptStyle{ - TextColor: cloneColor(style.TextColor), - FontSize: cloneInt(style.FontSize), - FontWeight: cloneInt(style.FontWeight), - LineHeight: cloneFloat64(style.LineHeight), - FontFamily: style.FontFamily, - TextAlign: style.TextAlign, - Display: style.Display, - GridTemplateColumns: style.GridTemplateColumns, - FlexWrap: style.FlexWrap, - } -} - -func mergePPTStyle(base, patch pptStyle) pptStyle { - if patch.ClearBackground { - base.BackgroundColor = nil - } - if patch.ClearBorder { - base.BorderColor = nil - base.BorderWidth = nil - } - if patch.ClearBorderLeft { - base.BorderLeftColor = nil - base.BorderLeftWidth = nil - } - if patch.ClearBorderBottom { - base.BorderBottomColor = nil - base.BorderBottomWidth = nil - } - if patch.TextColor != nil { - base.TextColor = cloneColor(patch.TextColor) - } - if patch.BackgroundColor != nil { - base.BackgroundColor = cloneColor(patch.BackgroundColor) - } - if patch.BorderColor != nil { - base.BorderColor = cloneColor(patch.BorderColor) - } - if patch.BorderLeftColor != nil { - base.BorderLeftColor = cloneColor(patch.BorderLeftColor) - } - if patch.BorderBottomColor != nil { - base.BorderBottomColor = cloneColor(patch.BorderBottomColor) - } - if patch.FontSize != nil { - base.FontSize = cloneInt(patch.FontSize) - } - if patch.FontWeight != nil { - base.FontWeight = cloneInt(patch.FontWeight) - } - if patch.LineHeight != nil { - base.LineHeight = cloneFloat64(patch.LineHeight) - } - if patch.FontFamily != "" { - base.FontFamily = patch.FontFamily - } - if patch.TextAlign != "" { - base.TextAlign = patch.TextAlign - } - if patch.Display != "" { - base.Display = patch.Display - } - if patch.GridTemplateColumns != "" { - base.GridTemplateColumns = patch.GridTemplateColumns - } - if patch.FlexWrap != "" { - base.FlexWrap = patch.FlexWrap - } - if patch.Gap != nil { - base.Gap = cloneFloat64(patch.Gap) - } - if patch.Padding.Set { - base.Padding = patch.Padding - } - if patch.Margin.Set { - base.Margin = patch.Margin - } - if patch.BorderWidth != nil { - base.BorderWidth = cloneInt(patch.BorderWidth) - } - if patch.BorderLeftWidth != nil { - base.BorderLeftWidth = cloneInt(patch.BorderLeftWidth) - } - if patch.BorderBottomWidth != nil { - base.BorderBottomWidth = cloneInt(patch.BorderBottomWidth) - } - if patch.BorderRadius != nil { - base.BorderRadius = cloneInt(patch.BorderRadius) - } - return base -} - -func applyStyleDeclarations(style pptStyle, declarations map[string]string) pptStyle { - for rawKey, rawValue := range declarations { - key := strings.ToLower(strings.TrimSpace(rawKey)) - value := strings.TrimSpace(rawValue) - if key == "" || value == "" { - continue - } - switch key { - case "color": - if color, ok := parsePPTColor(value); ok { - style.TextColor = &color - } - case "background", "background-color": - if isCSSNoneValue(value) { - style.ClearBackground = true - style.BackgroundColor = nil - continue - } - if color, ok := parsePPTColor(value); ok { - style.BackgroundColor = &color - } - case "font-size": - if size := parseCSSFontSize(value); size > 0 { - style.FontSize = intPtr(size) - } - case "font-weight": - if weight := parseCSSFontWeight(value); weight > 0 { - style.FontWeight = intPtr(weight) - } - case "line-height": - if lineHeight := parseCSSLineHeight(value, style.FontSize); lineHeight > 0 { - style.LineHeight = float64Ptr(lineHeight) - } - case "font-family": - if family := normalizeFontFamily(value); family != "" { - style.FontFamily = family - } - case "text-align": - style.TextAlign = strings.ToLower(value) - case "display": - style.Display = strings.ToLower(value) - case "grid-template-columns": - style.GridTemplateColumns = strings.ToLower(value) - case "flex-wrap": - style.FlexWrap = strings.ToLower(value) - case "gap": - if gap := parseCSSSpacingInches(value); gap > 0 { - style.Gap = float64Ptr(gap) - } - case "padding": - if edges, ok := parseCSSBoxEdges(value); ok { - style.Padding = edges - } - case "padding-top": - style.Padding = updateEdge(style.Padding, "top", parseCSSSpacingInches(value)) - case "padding-right": - style.Padding = updateEdge(style.Padding, "right", parseCSSSpacingInches(value)) - case "padding-bottom": - style.Padding = updateEdge(style.Padding, "bottom", parseCSSSpacingInches(value)) - case "padding-left": - style.Padding = updateEdge(style.Padding, "left", parseCSSSpacingInches(value)) - case "margin": - if edges, ok := parseCSSBoxEdges(value); ok { - style.Margin = edges - } - case "margin-top": - style.Margin = updateEdge(style.Margin, "top", parseCSSSpacingInches(value)) - case "margin-bottom": - style.Margin = updateEdge(style.Margin, "bottom", parseCSSSpacingInches(value)) - case "border": - if isCSSNoneValue(value) { - style.ClearBorder = true - style.BorderColor = nil - style.BorderWidth = nil - continue - } - if width, color, ok := parseCSSBorder(value); ok { - style.BorderWidth = intPtr(width) - style.BorderColor = &color - } - case "border-width": - if width := parseCSSBorderWidth(value); width > 0 { - style.BorderWidth = intPtr(width) - } - case "border-color": - if color, ok := parsePPTColor(value); ok { - style.BorderColor = &color - } - case "border-radius": - if radius := parseCSSRadius(value); radius > 0 { - style.BorderRadius = intPtr(radius) - } - case "border-left": - if isCSSNoneValue(value) { - style.ClearBorderLeft = true - style.BorderLeftColor = nil - style.BorderLeftWidth = nil - continue - } - if width, color, ok := parseCSSBorder(value); ok { - style.BorderLeftWidth = intPtr(width) - style.BorderLeftColor = &color - } - case "border-left-width": - if width := parseCSSBorderWidth(value); width > 0 { - style.BorderLeftWidth = intPtr(width) - } - case "border-left-color": - if color, ok := parsePPTColor(value); ok { - style.BorderLeftColor = &color - } - case "border-bottom": - if isCSSNoneValue(value) { - style.ClearBorderBottom = true - style.BorderBottomColor = nil - style.BorderBottomWidth = nil - continue - } - if width, color, ok := parseCSSBorder(value); ok { - style.BorderBottomWidth = intPtr(width) - style.BorderBottomColor = &color - } - case "border-bottom-width": - if width := parseCSSBorderWidth(value); width > 0 { - style.BorderBottomWidth = intPtr(width) - } - case "border-bottom-color": - if color, ok := parsePPTColor(value); ok { - style.BorderBottomColor = &color - } - } - } - return style -} - -func applyOrderedStyleDeclarations(style pptStyle, declarations []pptStyleDeclaration) pptStyle { - for _, declaration := range declarations { - style = applyStyleDeclarations(style, map[string]string{ - declaration.Key: declaration.Value, - }) - } - return style -} - -func newDynamicLayoutConfig() dynamicLayoutConfig { - return dynamicLayoutConfig{ - SlideWidth: dynamicPPTSlideWidth, - SlideHeight: dynamicPPTSlideHeight, - OuterMarginX: dynamicPPTOuterMarginX, - OuterMarginY: dynamicPPTOuterMarginY, - DefaultGap: dynamicPPTDefaultGap, - ConservativeColumns: true, - } -} - -func renderDynamicHTMLSlide(slide *pptx.SlideBuilder, doc *pptHTMLDocument, slideData pptHTMLSlide) { - measured := measureDynamicHTMLSlide(doc, slideData, newDynamicLayoutConfig()) - renderMeasuredDynamicHTMLSlide(slide, doc, measured) -} - -func renderMeasuredDynamicHTMLSlide(slide *pptx.SlideBuilder, doc *pptHTMLDocument, measured measuredDynamicHTMLSlide) { - slide.SetBackgroundColor(resolveSlideBackground(doc.BodyStyle)) - - renderSectionFrameAt(slide, measured.Frame, measured.SectionStyle) - for _, block := range measured.Blocks { - renderMeasuredDynamicBlock(slide, block) - } -} - -func renderSectionFrame(slide *pptx.SlideBuilder, style pptStyle) pptSectionFrame { - frame := pptSectionFrame{ - x: dynamicPPTOuterMarginX, - y: dynamicPPTOuterMarginY, - width: dynamicPPTSlideWidth - dynamicPPTOuterMarginX*2, - height: dynamicPPTSlideHeight - dynamicPPTOuterMarginY*2, - paddingTop: edgeOr(style.Padding, "top", 0.36), - paddingRight: edgeOr(style.Padding, "right", 0.42), - paddingBottom: edgeOr(style.Padding, "bottom", 0.36), - paddingLeft: edgeOr(style.Padding, "left", 0.42), - } - - fill := resolveSectionFill(style) - borderColor := resolveBorderColor(style, dynamicPPTDefaultSectionBorder) - borderWidth := dynamicPPTValueOrInt(style.BorderWidth, 1) - shapeType := pptx.ShapeRoundedRectangle - if dynamicPPTValueOrInt(style.BorderRadius, 28) <= 0 { - shapeType = pptx.ShapeRectangle - } - slide.AddShape(shapeType). - SetPosition(pptx.Inches(frame.x), pptx.Inches(frame.y)). - SetSize(pptx.Inches(frame.width), pptx.Inches(frame.height)). - SetFillColor(fill). - SetLine(borderColor, borderWidth). - End() - - if style.BorderBottomColor != nil { - height := 0.05 - width := frame.width - 0.4 - if dynamicPPTValueOrInt(style.BorderBottomWidth, 0) >= 3 { - height = 0.06 - } - slide.AddShape(pptx.ShapeRectangle). - SetPosition(pptx.Inches(frame.x+0.2), pptx.Inches(frame.y+frame.height-height-0.02)). - SetSize(pptx.Inches(width), pptx.Inches(height)). - SetFillColor(*style.BorderBottomColor). - SetNoLine(). - End() - } - - return frame -} - -func renderSectionFrameAt(slide *pptx.SlideBuilder, frame pptSectionFrame, style pptStyle) { - shapeType := pptx.ShapeRoundedRectangle - if dynamicPPTValueOrInt(style.BorderRadius, 28) <= 0 { - shapeType = pptx.ShapeRectangle - } - slide.AddShape(shapeType). - SetPosition(pptx.Inches(frame.x), pptx.Inches(frame.y)). - SetSize(pptx.Inches(frame.width), pptx.Inches(frame.height)). - SetFillColor(resolveSectionFill(style)). - SetLine(resolveBorderColor(style, dynamicPPTDefaultSectionBorder), dynamicPPTValueOrInt(style.BorderWidth, 1)). - End() - if style.BorderBottomColor != nil { - height := 0.05 - width := frame.width - 0.4 - if dynamicPPTValueOrInt(style.BorderBottomWidth, 0) >= 3 { - height = 0.06 - } - slide.AddShape(pptx.ShapeRectangle). - SetPosition(pptx.Inches(frame.x+0.2), pptx.Inches(frame.y+frame.height-height-0.02)). - SetSize(pptx.Inches(width), pptx.Inches(height)). - SetFillColor(*style.BorderBottomColor). - SetNoLine(). - End() - } -} - -func measureDynamicHTMLSlide(doc *pptHTMLDocument, slideData pptHTMLSlide, config dynamicLayoutConfig) measuredDynamicHTMLSlide { - frame := pptSectionFrame{ - x: config.OuterMarginX, - y: config.OuterMarginY, - width: config.SlideWidth - config.OuterMarginX*2, - height: config.SlideHeight - config.OuterMarginY*2, - paddingTop: edgeOr(slideData.SectionStyle.Padding, "top", 0.36), - paddingRight: edgeOr(slideData.SectionStyle.Padding, "right", 0.42), - paddingBottom: edgeOr(slideData.SectionStyle.Padding, "bottom", 0.36), - paddingLeft: edgeOr(slideData.SectionStyle.Padding, "left", 0.42), - } - cursor := &pptLayoutCursor{ - x: frame.x + frame.paddingLeft, - y: frame.y + frame.paddingTop, - width: frame.width - frame.paddingLeft - frame.paddingRight, - } - measuredBlocks := measureDynamicBlocks(slideData.Blocks, cursor, config) - contentBottom := cursor.y - _ = doc - return measuredDynamicHTMLSlide{ - SectionStyle: slideData.SectionStyle, - Frame: frame, - Blocks: measuredBlocks, - ContentBottom: contentBottom, - } -} - -func measureDynamicBlocks(blocks []pptHTMLBlock, cursor *pptLayoutCursor, config dynamicLayoutConfig) []measuredDynamicBlock { - measured := make([]measuredDynamicBlock, 0, len(blocks)) - for _, block := range blocks { - entry := measureDynamicBlock(block, cursor, config) - if entry.Height <= 0 && len(entry.Children) == 0 && block.Kind != "section-number" { - continue - } - measured = append(measured, entry) - } - return measured -} - -func measureDynamicBlock(block pptHTMLBlock, cursor *pptLayoutCursor, config dynamicLayoutConfig) measuredDynamicBlock { - switch block.Kind { - case "section-number": - return measuredDynamicBlock{ - Block: block, - X: dynamicPPTSlideWidth - 1.35, - Y: 0.38, - Width: 0.88, - Height: 0.24, - } - case "container": - return measureContainerBlock(block, cursor, config) - case "card": - return measureCardBlock(block, cursor, config) - default: - return measureTextBlock(block, cursor) - } -} - -func measureContainerBlock(block pptHTMLBlock, cursor *pptLayoutCursor, config dynamicLayoutConfig) measuredDynamicBlock { - startY := cursor.y + edgeOr(block.Style.Margin, "top", 0) - cursor.y = startY - measured := measuredDynamicBlock{ - Block: block, - X: cursor.x, - Y: startY, - Width: cursor.width, - } - switch block.Layout { - case "row", "grid": - measured.Children = measureGridChildren(block, cursor, config) - default: - childCursor := &pptLayoutCursor{x: cursor.x, y: cursor.y, width: cursor.width} - measured.Children = measureDynamicBlocks(block.Children, childCursor, config) - cursor.y = childCursor.y - } - measured.Height = cursor.y - startY + edgeOr(block.Style.Margin, "bottom", 0) - cursor.y += edgeOr(block.Style.Margin, "bottom", 0) - return measured -} - -func measureGridChildren(block pptHTMLBlock, cursor *pptLayoutCursor, config dynamicLayoutConfig) []measuredDynamicBlock { - if len(block.Children) == 0 { - return nil - } - cols := resolveContainerColumns(block, cursor.width, config.ConservativeColumns) - if cols <= 1 { - return measureDynamicBlocks(block.Children, cursor, config) - } - gap := resolveGap(block.Style, 0.18) - cellWidth := (cursor.width - gap*float64(cols-1)) / float64(cols) - currentY := cursor.y - measured := make([]measuredDynamicBlock, 0, len(block.Children)) - for start := 0; start < len(block.Children); start += cols { - end := start + cols - if end > len(block.Children) { - end = len(block.Children) - } - row := make([]measuredDynamicBlock, 0, end-start) - rowHeight := 0.0 - for i, child := range block.Children[start:end] { - cellCursor := &pptLayoutCursor{ - x: cursor.x + float64(i)*(cellWidth+gap), - y: currentY, - width: cellWidth, - } - childMeasured := measureDynamicBlockInRect(child, cellCursor, config) - if childMeasured.Height > rowHeight { - rowHeight = childMeasured.Height - } - row = append(row, childMeasured) - } - for i := range row { - row[i].Height = rowHeight - } - measured = append(measured, row...) - currentY += rowHeight + gap - } - cursor.y = currentY + config.DefaultGap - return measured -} - -func measureDynamicBlockInRect(block pptHTMLBlock, cursor *pptLayoutCursor, config dynamicLayoutConfig) measuredDynamicBlock { - switch block.Kind { - case "card": - return measureCardAt(block, cursor.x, cursor.y, cursor.width, config) - default: - textBlock := block - textBlock.Style.Margin = pptEdges{} - textCursor := &pptLayoutCursor{x: cursor.x, y: cursor.y, width: cursor.width} - return measureTextBlock(textBlock, textCursor) - } -} - -func measureCardBlock(block pptHTMLBlock, cursor *pptLayoutCursor, config dynamicLayoutConfig) measuredDynamicBlock { - cursor.y += edgeOr(block.Style.Margin, "top", 0) - measured := measureCardAt(block, cursor.x, cursor.y, cursor.width, config) - cursor.y += measured.Height + edgeOr(block.Style.Margin, "bottom", dynamicPPTDefaultGap) - return measured -} - -func measureCardAt(block pptHTMLBlock, x, y, width float64, config dynamicLayoutConfig) measuredDynamicBlock { - contentWidth := width - edgeOr(block.Style.Padding, "left", 0.22) - edgeOr(block.Style.Padding, "right", 0.22) - if contentWidth < 0.5 { - contentWidth = width - 0.18 - } - innerCursor := &pptLayoutCursor{ - x: x + edgeOr(block.Style.Padding, "left", 0.22), - y: y + edgeOr(block.Style.Padding, "top", 0.18), - width: contentWidth, - } - children := measureDynamicBlocks(block.Children, innerCursor, config) - height := innerCursor.y - y + edgeOr(block.Style.Padding, "bottom", 0.18) - if height < 0.56 { - height = 0.56 - } - return measuredDynamicBlock{ - Block: block, - X: x, - Y: y, - Width: width, - Height: height, - Children: children, - } -} - -func measureTextBlock(block pptHTMLBlock, cursor *pptLayoutCursor) measuredDynamicBlock { - if strings.TrimSpace(block.Text) == "" { - return measuredDynamicBlock{Block: block} - } - cursor.y += edgeOr(block.Style.Margin, "top", 0) - fontSize := resolveBlockFontSize(block, defaultFontSizeForBlock(block.Kind)) - x := cursor.x + edgeOr(block.Style.Padding, "left", 0) - width := cursor.width - edgeOr(block.Style.Padding, "left", 0) - edgeOr(block.Style.Padding, "right", 0) - if width <= 0.2 { - width = cursor.width - } - height := estimateTextHeightWithStyle(block.Text, block.Style, fontSize, width) - measured := measuredDynamicBlock{ - Block: block, - X: x, - Y: cursor.y, - Width: width, - Height: height, - } - cursor.y += height + edgeOr(block.Style.Margin, "bottom", dynamicPPTDefaultGap) - return measured -} - -func renderMeasuredDynamicBlock(slide *pptx.SlideBuilder, measured measuredDynamicBlock) { - switch measured.Block.Kind { - case "section-number": - renderSectionNumber(slide, measured.Block) - case "container": - for _, child := range measured.Children { - renderMeasuredDynamicBlock(slide, child) - } - case "card": - renderCardChrome(slide, measured.Block, measured.X, measured.Y, measured.Width, measured.Height) - for _, child := range measured.Children { - renderMeasuredDynamicBlock(slide, child) - } - default: - renderTextAt(slide, measured) - } -} - -func renderTextAt(slide *pptx.SlideBuilder, measured measuredDynamicBlock) { - block := measured.Block - if strings.TrimSpace(block.Text) == "" { - return - } - if len(block.Runs) > 1 { - renderInlineRunsAt(slide, measured) - return - } - fontSize := resolveBlockFontSize(block, defaultFontSizeForBlock(block.Kind)) - fontFamily := resolveFontFamily(block.Style, defaultFontFamilyForBlock(block.Kind)) - color := resolveTextColor(block.Style, dynamicPPTDefaultText) - text := slide.AddText(block.Text). - SetFontSize(fontSize). - SetFontFamily(fontFamily). - SetColor(color). - SetAlignment(resolveAlignment(block.Style.TextAlign)). - SetPosition(pptx.Inches(measured.X), pptx.Inches(measured.Y)). - SetSize(pptx.Inches(measured.Width), pptx.Inches(measured.Height)) - if dynamicPPTValueOrInt(block.Style.FontWeight, 0) >= 600 || block.Kind == "h1" || block.Kind == "h2" || block.Kind == "h3" { - text.SetBold(true) - } - text.End() - - if block.Style.BorderLeftColor != nil { - leftWidth := 0.04 - if dynamicPPTValueOrInt(block.Style.BorderLeftWidth, 0) >= 4 { - leftWidth = 0.05 - } - slide.AddShape(pptx.ShapeRectangle). - SetPosition(pptx.Inches(measured.X-edgeOr(block.Style.Padding, "left", 0)), pptx.Inches(measured.Y+0.02)). - SetSize(pptx.Inches(leftWidth), pptx.Inches(measured.Height-0.02)). - SetFillColor(*block.Style.BorderLeftColor). - SetNoLine(). - End() - } -} - -func renderInlineRunsAt(slide *pptx.SlideBuilder, measured measuredDynamicBlock) { - block := measured.Block - x := measured.X - y := measured.Y - remainingWidth := measured.Width - totalWeight := inlineRunsWidthWeight(block.Runs, block) - if totalWeight <= 0 { - renderPlainTextAt(slide, measured, block) - return - } - for i, run := range block.Runs { - runText := run.Text - if runText == "" { - continue - } - runBlock := block - runBlock.Text = runText - runBlock.Runs = nil - runBlock.Style = mergePPTStyle(block.Style, run.Style) - runWeight := inlineRunWidthWeight(run, runBlock) - width := measured.Width * runWeight / totalWeight - if i == len(block.Runs)-1 || width > remainingWidth { - width = remainingWidth - } - if width <= 0 { - continue - } - renderPlainTextAt(slide, measuredDynamicBlock{ - Block: runBlock, - X: x, - Y: y, - Width: width, - Height: measured.Height, - }, runBlock) - x += width - remainingWidth -= width - if remainingWidth <= 0 { - break - } - } -} - -func renderPlainTextAt(slide *pptx.SlideBuilder, measured measuredDynamicBlock, block pptHTMLBlock) { - fontSize := resolveBlockFontSize(block, defaultFontSizeForBlock(block.Kind)) - fontFamily := resolveFontFamily(block.Style, defaultFontFamilyForBlock(block.Kind)) - color := resolveTextColor(block.Style, dynamicPPTDefaultText) - text := slide.AddText(block.Text). - SetFontSize(fontSize). - SetFontFamily(fontFamily). - SetColor(color). - SetAlignment(resolveAlignment(block.Style.TextAlign)). - SetPosition(pptx.Inches(measured.X), pptx.Inches(measured.Y)). - SetSize(pptx.Inches(measured.Width), pptx.Inches(measured.Height)) - if dynamicPPTValueOrInt(block.Style.FontWeight, 0) >= 600 || block.Kind == "h1" || block.Kind == "h2" || block.Kind == "h3" { - text.SetBold(true) - } - text.End() -} - -func inlineRunsWidthWeight(runs []pptHTMLTextRun, block pptHTMLBlock) float64 { - total := 0.0 - for _, run := range runs { - runBlock := block - runBlock.Text = run.Text - runBlock.Style = mergePPTStyle(block.Style, run.Style) - total += inlineRunWidthWeight(run, runBlock) - } - return total -} - -func inlineRunWidthWeight(run pptHTMLTextRun, block pptHTMLBlock) float64 { - size := resolveBlockFontSize(block, defaultFontSizeForBlock(block.Kind)) - weight := float64(len([]rune(run.Text))) * float64(size) - if dynamicPPTValueOrInt(block.Style.FontWeight, 0) >= 600 { - weight *= 1.05 - } - if weight <= 0 { - return 1 - } - return weight -} - -func renderDynamicBlock(slide *pptx.SlideBuilder, cursor *pptLayoutCursor, block pptHTMLBlock) { - switch block.Kind { - case "section-number": - renderSectionNumber(slide, block) - case "container": - renderContainerBlock(slide, cursor, block) - case "card": - renderCardBlock(slide, cursor, block) - default: - renderTextBlock(slide, cursor, block) - } -} - -func renderSectionNumber(slide *pptx.SlideBuilder, block pptHTMLBlock) { - fontSize := resolveBlockFontSize(block, 13) - text := slide.AddText(block.Text). - SetFontSize(fontSize). - SetFontFamily(resolveFontFamily(block.Style, dynamicPPTDefaultFontFamily)). - SetColor(resolveTextColor(block.Style, dynamicPPTDefaultMuted)). - SetAlignment(resolveAlignment(block.Style.TextAlign)). - SetPosition(pptx.Inches(dynamicPPTSlideWidth-1.35), pptx.Inches(0.38)). - SetSize(pptx.Inches(0.88), pptx.Inches(0.24)) - if dynamicPPTValueOrInt(block.Style.FontWeight, 0) >= 600 { - text.SetBold(true) - } - text.End() -} - -func renderContainerBlock(slide *pptx.SlideBuilder, cursor *pptLayoutCursor, block pptHTMLBlock) { - cursor.y += edgeOr(block.Style.Margin, "top", 0) - switch block.Layout { - case "row", "grid": - renderGridContainer(slide, cursor, block) - default: - for _, child := range block.Children { - renderDynamicBlock(slide, cursor, child) - } - } - cursor.y += edgeOr(block.Style.Margin, "bottom", 0) -} - -func renderGridContainer(slide *pptx.SlideBuilder, cursor *pptLayoutCursor, block pptHTMLBlock) { - if len(block.Children) == 0 { - return - } - cols := resolveContainerColumns(block, cursor.width, false) - if cols <= 1 { - for _, child := range block.Children { - renderDynamicBlock(slide, cursor, child) - } - return - } - - gap := resolveGap(block.Style, 0.18) - cellWidth := (cursor.width - gap*float64(cols-1)) / float64(cols) - currentY := cursor.y - - for start := 0; start < len(block.Children); start += cols { - end := start + cols - if end > len(block.Children) { - end = len(block.Children) - } - row := block.Children[start:end] - maxHeight := 0.0 - heights := make([]float64, len(row)) - for i, child := range row { - heights[i] = estimateRenderedHeight(child, cellWidth) - if heights[i] > maxHeight { - maxHeight = heights[i] - } - } - for i, child := range row { - x := cursor.x + float64(i)*(cellWidth+gap) - renderBlockInRect(slide, child, x, currentY, cellWidth, maxHeight) - } - currentY += maxHeight + gap - } - - cursor.y = currentY + dynamicPPTDefaultGap -} - -func renderBlockInRect(slide *pptx.SlideBuilder, block pptHTMLBlock, x, y, width, height float64) { - switch block.Kind { - case "card": - renderCardInRect(slide, block, x, y, width, height) - default: - textBlock := block - textBlock.Style.Margin = pptEdges{} - textCursor := &pptLayoutCursor{x: x, y: y, width: width} - renderTextBlock(slide, textCursor, textBlock) - } -} - -func renderCardBlock(slide *pptx.SlideBuilder, cursor *pptLayoutCursor, block pptHTMLBlock) { - cursor.y += edgeOr(block.Style.Margin, "top", 0) - height := estimateRenderedHeight(block, cursor.width) - renderCardInRect(slide, block, cursor.x, cursor.y, cursor.width, height) - cursor.y += height + edgeOr(block.Style.Margin, "bottom", dynamicPPTDefaultGap) -} - -func renderCardInRect(slide *pptx.SlideBuilder, block pptHTMLBlock, x, y, width, height float64) { - renderCardChrome(slide, block, x, y, width, height) - innerCursor := &pptLayoutCursor{ - x: x + edgeOr(block.Style.Padding, "left", 0.22), - y: y + edgeOr(block.Style.Padding, "top", 0.18), - width: width - edgeOr(block.Style.Padding, "left", 0.22) - edgeOr(block.Style.Padding, "right", 0.22), - } - if innerCursor.width < 0.5 { - innerCursor.width = width - 0.18 - } - for _, child := range block.Children { - renderDynamicBlock(slide, innerCursor, child) - } -} - -func renderCardChrome(slide *pptx.SlideBuilder, block pptHTMLBlock, x, y, width, height float64) { - fill := resolveCardFill(block.Style) - borderColor := resolveBorderColor(block.Style, dynamicPPTDefaultSectionBorder) - borderWidth := dynamicPPTValueOrInt(block.Style.BorderWidth, 1) - radius := dynamicPPTValueOrInt(block.Style.BorderRadius, 18) - shapeType := pptx.ShapeRoundedRectangle - if radius <= 0 { - shapeType = pptx.ShapeRectangle - } - slide.AddShape(shapeType). - SetPosition(pptx.Inches(x), pptx.Inches(y)). - SetSize(pptx.Inches(width), pptx.Inches(height)). - SetFillColor(fill). - SetLine(borderColor, borderWidth). - End() - - if block.Style.BorderLeftColor != nil { - leftWidth := 0.05 - if dynamicPPTValueOrInt(block.Style.BorderLeftWidth, 0) >= 3 { - leftWidth = 0.06 - } - slide.AddShape(pptx.ShapeRectangle). - SetPosition(pptx.Inches(x), pptx.Inches(y+0.03)). - SetSize(pptx.Inches(leftWidth), pptx.Inches(height-0.06)). - SetFillColor(*block.Style.BorderLeftColor). - SetNoLine(). - End() - } -} - -func renderTextBlock(slide *pptx.SlideBuilder, cursor *pptLayoutCursor, block pptHTMLBlock) { - if strings.TrimSpace(block.Text) == "" { - return - } - cursor.y += edgeOr(block.Style.Margin, "top", 0) - - fontSize := resolveBlockFontSize(block, defaultFontSizeForBlock(block.Kind)) - fontFamily := resolveFontFamily(block.Style, defaultFontFamilyForBlock(block.Kind)) - color := resolveTextColor(block.Style, dynamicPPTDefaultText) - height := estimateTextHeight(block.Text, fontSize, cursor.width) - x := cursor.x + edgeOr(block.Style.Padding, "left", 0) - width := cursor.width - edgeOr(block.Style.Padding, "left", 0) - edgeOr(block.Style.Padding, "right", 0) - if width <= 0.2 { - width = cursor.width - } - text := slide.AddText(block.Text). - SetFontSize(fontSize). - SetFontFamily(fontFamily). - SetColor(color). - SetAlignment(resolveAlignment(block.Style.TextAlign)). - SetPosition(pptx.Inches(x), pptx.Inches(cursor.y)). - SetSize(pptx.Inches(width), pptx.Inches(height)) - if dynamicPPTValueOrInt(block.Style.FontWeight, 0) >= 600 || block.Kind == "h1" || block.Kind == "h2" || block.Kind == "h3" { - text.SetBold(true) - } - text.End() - - if block.Style.BorderLeftColor != nil { - leftWidth := 0.04 - if dynamicPPTValueOrInt(block.Style.BorderLeftWidth, 0) >= 4 { - leftWidth = 0.05 - } - slide.AddShape(pptx.ShapeRectangle). - SetPosition(pptx.Inches(cursor.x), pptx.Inches(cursor.y+0.02)). - SetSize(pptx.Inches(leftWidth), pptx.Inches(height-0.02)). - SetFillColor(*block.Style.BorderLeftColor). - SetNoLine(). - End() - } - - cursor.y += height + edgeOr(block.Style.Margin, "bottom", dynamicPPTDefaultGap) -} - -func estimateRenderedHeight(block pptHTMLBlock, width float64) float64 { - switch block.Kind { - case "container": - if block.Layout == "row" || block.Layout == "grid" { - cols := resolveContainerColumns(block, width, false) - if cols <= 0 { - cols = 1 - } - gap := resolveGap(block.Style, 0.18) - cellWidth := width - if cols > 1 { - cellWidth = (width - gap*float64(cols-1)) / float64(cols) - } - total := 0.0 - for start := 0; start < len(block.Children); start += cols { - end := start + cols - if end > len(block.Children) { - end = len(block.Children) - } - rowHeight := 0.0 - for _, child := range block.Children[start:end] { - h := estimateRenderedHeight(child, cellWidth) - if h > rowHeight { - rowHeight = h - } - } - total += rowHeight + gap - } - if total == 0 { - return 0 - } - return total + edgeOr(block.Style.Margin, "top", 0) + edgeOr(block.Style.Margin, "bottom", 0) - } - total := 0.0 - for _, child := range block.Children { - total += estimateRenderedHeight(child, width) - } - return total + edgeOr(block.Style.Margin, "top", 0) + edgeOr(block.Style.Margin, "bottom", 0) - case "card": - contentWidth := width - edgeOr(block.Style.Padding, "left", 0.22) - edgeOr(block.Style.Padding, "right", 0.22) - if contentWidth < 0.5 { - contentWidth = width - 0.18 - } - total := edgeOr(block.Style.Padding, "top", 0.18) + edgeOr(block.Style.Padding, "bottom", 0.18) - for _, child := range block.Children { - total += estimateRenderedHeight(child, contentWidth) - } - if total < 0.56 { - total = 0.56 - } - return total + edgeOr(block.Style.Margin, "top", 0) + edgeOr(block.Style.Margin, "bottom", 0) - case "section-number": - return 0 - default: - fontSize := resolveBlockFontSize(block, defaultFontSizeForBlock(block.Kind)) - height := estimateTextHeightWithStyle(block.Text, block.Style, fontSize, width) - return height + edgeOr(block.Style.Margin, "top", 0) + edgeOr(block.Style.Margin, "bottom", dynamicPPTDefaultGap) - } -} - -func resolveContainerColumns(block pptHTMLBlock, width float64, conservative bool) int { - template := strings.ToLower(strings.TrimSpace(block.Style.GridTemplateColumns)) - if conservative && strings.Contains(template, "repeat(") && strings.Contains(template, "auto-fit") && strings.Contains(template, "minmax(") { - return dynamicPPTMaxInt(1, dynamicPPTMinInt(2, len(block.Children))) - } - if strings.Contains(template, "repeat(") && strings.Contains(template, "auto-fit") && strings.Contains(template, "minmax(") { - if width >= 6.0 && len(block.Children) >= 3 { - return 3 - } - if len(block.Children) >= 2 { - return 2 - } - } - switch { - case block.Classes["dir-list"]: - if len(block.Children) >= 6 { - return 3 - } - if len(block.Children) >= 4 { - return 2 - } - return dynamicPPTMaxInt(1, len(block.Children)) - case block.Classes["row"]: - if len(block.Children) >= 3 { - return 3 - } - return dynamicPPTMaxInt(1, len(block.Children)) - default: - if len(block.Children) >= 3 { - return 3 - } - if len(block.Children) == 2 { - return 2 - } - return 1 - } -} - -func resolveSlideBackground(style pptStyle) pptx.Color { - if style.BackgroundColor != nil { - return *style.BackgroundColor - } - return dynamicPPTDefaultSlideBackground -} - -func resolveSectionFill(style pptStyle) pptx.Color { - if style.BackgroundColor != nil { - return *style.BackgroundColor - } - return dynamicPPTDefaultSectionFill -} - -func resolveCardFill(style pptStyle) pptx.Color { - if style.BackgroundColor != nil { - return *style.BackgroundColor - } - return pptx.Color{R: 252, G: 251, B: 248} -} - -func resolveTextColor(style pptStyle, fallback pptx.Color) pptx.Color { - if style.TextColor != nil { - return *style.TextColor - } - return fallback -} - -func resolveBorderColor(style pptStyle, fallback pptx.Color) pptx.Color { - if style.BorderColor != nil { - return *style.BorderColor - } - return fallback -} - -func resolveGap(style pptStyle, fallback float64) float64 { - if style.Gap != nil && *style.Gap > 0 { - return *style.Gap - } - return fallback -} - -func resolveBlockFontSize(block pptHTMLBlock, fallback int) int { - size := fallback - if block.Style.FontSize != nil && *block.Style.FontSize > 0 { - size = *block.Style.FontSize - } - if block.Kind != "h1" && block.Kind != "h2" && block.Kind != "h3" && size < dynamicPPTMinBodyFontSize { - size = dynamicPPTMinBodyFontSize - } - return size -} - -func defaultFontSizeForBlock(kind string) int { - switch kind { - case "h1": - return 34 - case "h2": - return 24 - case "h3": - return 19 - case "section-number": - return 13 - case "pre": - return 14 - case "list-item": - return dynamicPPTDefaultBodyFont - default: - return dynamicPPTDefaultBodyFont - } -} - -func defaultFontFamilyForBlock(kind string) string { - switch kind { - case "h1", "h2", "h3": - return dynamicPPTTitleFontFamily - case "pre": - return "Consolas" - default: - return dynamicPPTDefaultFontFamily - } -} - -func resolveFontFamily(style pptStyle, fallback string) string { - if strings.TrimSpace(style.FontFamily) == "" { - return fallback - } - return style.FontFamily -} - -func resolveAlignment(value string) pptx.Alignment { - switch strings.ToLower(strings.TrimSpace(value)) { - case "center": - return pptx.AlignmentCenter - case "right": - return pptx.AlignmentRight - case "justify": - return pptx.AlignmentJustify - default: - return pptx.AlignmentLeft - } -} - -func containsCJK(text string) bool { - for _, r := range text { - if (r >= 0x4E00 && r <= 0x9FFF) || - (r >= 0x3400 && r <= 0x4DBF) || - (r >= 0xF900 && r <= 0xFAFF) || - (r >= 0x3000 && r <= 0x303F) || - (r >= 0xFF00 && r <= 0xFFEF) { - return true - } - } - return false -} - -func charsPerLineForText(text string, width float64) int { - density := 6.5 - if containsCJK(text) { - density = 4.6 - } - n := int(width * density) - if n < 8 { - n = 8 - } - return n -} - -func estimateTextHeight(text string, fontSize int, width float64) float64 { - if fontSize <= 0 { - fontSize = dynamicPPTDefaultBodyFont - } - if width <= 0 { - width = 4 - } - runes := len([]rune(strings.TrimSpace(text))) - if runes == 0 { - return 0.22 - } - charsPerLine := charsPerLineForText(text, width) - lines := (runes + charsPerLine - 1) / charsPerLine - lineHeight := float64(fontSize) * 1.28 / 72.0 - return dynamicPPTMaxFloat(lineHeight*float64(lines), 0.26) -} - -func estimateTextHeightWithStyle(text string, style pptStyle, fontSize int, width float64) float64 { - trimmed := strings.TrimSpace(text) - if trimmed == "" { - return 0.22 - } - - // For code blocks (text with explicit newlines), count actual lines - // rather than estimating from character density, to preserve formatting. - newlineCount := strings.Count(trimmed, "\n") - if newlineCount > 0 && fontSize <= 18 { - lines := newlineCount + 1 - lineHeight := float64(fontSize) * 1.4 / 72.0 - if style.LineHeight != nil && *style.LineHeight > 0 { - lineHeight = *style.LineHeight - } - return dynamicPPTMaxFloat(lineHeight*float64(lines)+0.1, 0.26) - } - - runes := len([]rune(trimmed)) - if runes == 0 { - return 0.22 - } - charsPerLine := charsPerLineForText(trimmed, width) - lines := (runes + charsPerLine - 1) / charsPerLine - - if style.LineHeight != nil && *style.LineHeight > 0 { - return dynamicPPTMaxFloat(*style.LineHeight*float64(lines), 0.26) - } - - if fontSize <= 0 { - fontSize = dynamicPPTDefaultBodyFont - } - lineHeight := float64(fontSize) * 1.28 / 72.0 - return dynamicPPTMaxFloat(lineHeight*float64(lines), 0.26) -} - -func parseInlineStyleMap(value string) map[string]string { - styles := map[string]string{} - for _, declaration := range parseInlineStyleDeclarations(value) { - styles[declaration.Key] = declaration.Value - } - return styles -} - -func parseInlineStyleDeclarations(value string) []pptStyleDeclaration { - declarations := make([]pptStyleDeclaration, 0) - for _, part := range strings.Split(value, ";") { - part = strings.TrimSpace(part) - if part == "" { - continue - } - kv := strings.SplitN(part, ":", 2) - if len(kv) != 2 { - continue - } - key := strings.ToLower(strings.TrimSpace(kv[0])) - val := strings.TrimSpace(kv[1]) - if key == "" || val == "" { - continue - } - declarations = append(declarations, pptStyleDeclaration{ - Key: key, - Value: val, - }) - } - return declarations -} - -func parsePPTColor(value string) (pptx.Color, bool) { - value = strings.TrimSpace(strings.ToLower(value)) - if value == "" { - return pptx.Color{}, false - } - if named, ok := namedPPTColors()[value]; ok { - return named, true - } - if strings.Contains(value, "gradient") { - return extractFirstColorToken(value) - } - if strings.HasPrefix(value, "#") { - return parseHexColor(value) - } - if strings.HasPrefix(value, "rgb(") || strings.HasPrefix(value, "rgba(") { - return parseRGBColor(value) - } - if strings.Contains(value, "#") { - return extractFirstColorToken(value) - } - return pptx.Color{}, false -} - -func extractFirstColorToken(value string) (pptx.Color, bool) { - tokens := strings.FieldsFunc(value, func(r rune) bool { - return r == ' ' || r == ',' || r == '(' || r == ')' || r == ';' - }) - for _, token := range tokens { - token = strings.TrimSpace(token) - if token == "" { - continue - } - if strings.HasPrefix(token, "#") { - if color, ok := parseHexColor(token); ok { - return color, true - } - } - if strings.HasPrefix(token, "rgb") { - if color, ok := parseRGBColor(token); ok { - return color, true - } - } - } - return pptx.Color{}, false -} - -func parseHexColor(value string) (pptx.Color, bool) { - hex := strings.TrimPrefix(strings.TrimSpace(value), "#") - if len(hex) == 3 { - hex = strings.Repeat(string(hex[0]), 2) + strings.Repeat(string(hex[1]), 2) + strings.Repeat(string(hex[2]), 2) - } - if len(hex) != 6 { - return pptx.Color{}, false - } - r, err1 := strconv.ParseUint(hex[0:2], 16, 8) - g, err2 := strconv.ParseUint(hex[2:4], 16, 8) - b, err3 := strconv.ParseUint(hex[4:6], 16, 8) - if err1 != nil || err2 != nil || err3 != nil { - return pptx.Color{}, false - } - return pptx.Color{R: uint8(r), G: uint8(g), B: uint8(b)}, true -} - -func parseRGBColor(value string) (pptx.Color, bool) { - value = strings.TrimSpace(value) - value = strings.TrimPrefix(value, "rgba(") - value = strings.TrimPrefix(value, "rgb(") - value = strings.TrimSuffix(value, ")") - parts := strings.Split(value, ",") - if len(parts) < 3 { - return pptx.Color{}, false - } - r, err1 := strconv.Atoi(strings.TrimSpace(parts[0])) - g, err2 := strconv.Atoi(strings.TrimSpace(parts[1])) - b, err3 := strconv.Atoi(strings.TrimSpace(parts[2])) - if err1 != nil || err2 != nil || err3 != nil || r < 0 || g < 0 || b < 0 || r > 255 || g > 255 || b > 255 { - return pptx.Color{}, false - } - return pptx.Color{R: uint8(r), G: uint8(g), B: uint8(b)}, true -} - -func namedPPTColors() map[string]pptx.Color { - return map[string]pptx.Color{ - "white": pptx.White, - "black": {R: 0, G: 0, B: 0}, - "gray": {R: 107, G: 114, B: 128}, - "grey": {R: 107, G: 114, B: 128}, - "red": {R: 220, G: 38, B: 38}, - "green": {R: 22, G: 163, B: 74}, - "blue": {R: 37, G: 99, B: 235}, - "yellow": {R: 234, G: 179, B: 8}, - "orange": {R: 249, G: 115, B: 22}, - "transparent": pptx.White, - } -} - -func parseCSSFontSize(value string) int { - value = strings.ToLower(strings.TrimSpace(value)) - if value == "" { - return 0 - } - switch { - case strings.HasSuffix(value, "rem"): - f, err := strconv.ParseFloat(strings.TrimSuffix(value, "rem"), 64) - if err != nil { - return 0 - } - return int(f*16 + 0.5) - case strings.HasSuffix(value, "px"): - f, err := strconv.ParseFloat(strings.TrimSuffix(value, "px"), 64) - if err != nil { - return 0 - } - return int(f + 0.5) - case strings.HasSuffix(value, "pt"): - f, err := strconv.ParseFloat(strings.TrimSuffix(value, "pt"), 64) - if err != nil { - return 0 - } - return int(f + 0.5) - default: - f, err := strconv.ParseFloat(value, 64) - if err != nil { - return 0 - } - return int(f + 0.5) - } -} - -func parseCSSFontWeight(value string) int { - value = strings.ToLower(strings.TrimSpace(value)) - switch value { - case "", "normal": - return 0 - case "bold": - return 700 - case "medium": - return 500 - case "semibold": - return 600 - default: - weight, err := strconv.Atoi(value) - if err != nil { - return 0 - } - return weight - } -} - -func parseCSSLineHeight(value string, inheritedFontSize *int) float64 { - value = strings.ToLower(strings.TrimSpace(value)) - if value == "" || value == "normal" { - if inheritedFontSize != nil && *inheritedFontSize > 0 { - return float64(*inheritedFontSize) * 1.22 / 72.0 - } - return float64(dynamicPPTDefaultBodyFont) * 1.22 / 72.0 - } - if strings.HasSuffix(value, "rem") || strings.HasSuffix(value, "px") || strings.HasSuffix(value, "pt") { - return parseCSSSpacingInches(value) - } - if f, err := strconv.ParseFloat(value, 64); err == nil && f > 0 { - fontSize := dynamicPPTDefaultBodyFont - if inheritedFontSize != nil && *inheritedFontSize > 0 { - fontSize = *inheritedFontSize - } - return float64(fontSize) * f / 72.0 - } - return 0 -} - -func parseCSSSpacingInches(value string) float64 { - value = strings.ToLower(strings.TrimSpace(value)) - if value == "" { - return 0 - } - switch { - case strings.HasSuffix(value, "rem"): - f, err := strconv.ParseFloat(strings.TrimSuffix(value, "rem"), 64) - if err != nil { - return 0 - } - return (f * 16.0) / 96.0 - case strings.HasSuffix(value, "px"): - f, err := strconv.ParseFloat(strings.TrimSuffix(value, "px"), 64) - if err != nil { - return 0 - } - return f / 96.0 - case strings.HasSuffix(value, "pt"): - f, err := strconv.ParseFloat(strings.TrimSuffix(value, "pt"), 64) - if err != nil { - return 0 - } - return f / 72.0 - default: - f, err := strconv.ParseFloat(value, 64) - if err != nil { - return 0 - } - return f / 96.0 - } -} - -func parseCSSBorder(value string) (int, pptx.Color, bool) { - value = strings.TrimSpace(value) - if value == "" { - return 0, pptx.Color{}, false - } - parts := strings.Fields(value) - width := 1 - var color pptx.Color - var hasColor bool - for _, part := range parts { - if borderWidth := parseCSSBorderWidth(part); borderWidth > 0 { - width = borderWidth - continue - } - if borderColor, ok := parsePPTColor(part); ok { - color = borderColor - hasColor = true - } - } - return width, color, hasColor -} - -func parseCSSBorderWidth(value string) int { - value = strings.ToLower(strings.TrimSpace(value)) - switch { - case strings.HasSuffix(value, "px"): - f, err := strconv.ParseFloat(strings.TrimSuffix(value, "px"), 64) - if err != nil || f <= 0 { - return 0 - } - return dynamicPPTMaxInt(1, int(f+0.5)) - case strings.HasSuffix(value, "pt"): - f, err := strconv.ParseFloat(strings.TrimSuffix(value, "pt"), 64) - if err != nil || f <= 0 { - return 0 - } - return dynamicPPTMaxInt(1, int(f+0.5)) - default: - return 0 - } -} - -func parseCSSRadius(value string) int { - value = strings.ToLower(strings.TrimSpace(value)) - switch { - case strings.HasSuffix(value, "px"): - f, err := strconv.ParseFloat(strings.TrimSuffix(value, "px"), 64) - if err != nil { - return 0 - } - return int(f + 0.5) - case strings.HasSuffix(value, "pt"): - f, err := strconv.ParseFloat(strings.TrimSuffix(value, "pt"), 64) - if err != nil { - return 0 - } - return int(f + 0.5) - default: - return 0 - } -} - -func parseCSSBoxEdges(value string) (pptEdges, bool) { - parts := strings.Fields(strings.TrimSpace(value)) - if len(parts) == 0 { - return pptEdges{}, false - } - values := make([]float64, 0, len(parts)) - for _, part := range parts { - values = append(values, parseCSSSpacingInches(part)) - } - edges := pptEdges{Set: true} - switch len(values) { - case 1: - edges.Top, edges.Right, edges.Bottom, edges.Left = values[0], values[0], values[0], values[0] - case 2: - edges.Top, edges.Bottom = values[0], values[0] - edges.Right, edges.Left = values[1], values[1] - case 3: - edges.Top = values[0] - edges.Right, edges.Left = values[1], values[1] - edges.Bottom = values[2] - default: - edges.Top = values[0] - edges.Right = values[1] - edges.Bottom = values[2] - edges.Left = values[3] - } - return edges, true -} - -func updateEdge(edges pptEdges, side string, value float64) pptEdges { - edges.Set = true - switch side { - case "top": - edges.Top = value - case "right": - edges.Right = value - case "bottom": - edges.Bottom = value - case "left": - edges.Left = value - } - return edges -} - -func edgeOr(edges pptEdges, side string, fallback float64) float64 { - if !edges.Set { - return fallback - } - switch side { - case "top": - return edges.Top - case "right": - return edges.Right - case "bottom": - return edges.Bottom - case "left": - return edges.Left - default: - return fallback - } -} - -func resolveCSSVars(value string, vars map[string]string) string { - resolved := value - for range 6 { - start := strings.Index(resolved, "var(") - if start == -1 { - break - } - end := strings.Index(resolved[start:], ")") - if end == -1 { - break - } - end += start - token := strings.TrimSpace(resolved[start+4 : end]) - replacement := vars[token] - resolved = resolved[:start] + replacement + resolved[end+1:] - } - return resolved -} - -func normalizeFontFamily(value string) string { - value = strings.TrimSpace(value) - value = strings.Trim(value, `"'`) - lowerValue := strings.ToLower(value) - switch { - case value == "": - return "" - case strings.Contains(lowerValue, "system-ui"), - strings.Contains(lowerValue, "segoe ui"), - strings.Contains(lowerValue, "roboto"), - strings.Contains(lowerValue, "helvetica"), - strings.Contains(lowerValue, "arial"), - strings.Contains(lowerValue, "sans-serif"), - strings.Contains(lowerValue, "sans serif"): - return dynamicPPTDefaultFontFamily - default: - return value - } -} - -func isCSSNoneValue(value string) bool { - value = strings.ToLower(strings.TrimSpace(value)) - return value == "none" || value == "0" || value == "0px" || value == "transparent" -} - -func parseClassSet(node *html.Node) map[string]bool { - classes := map[string]bool{} - for _, className := range strings.Fields(getHTMLAttribute(node, "class")) { - className = strings.TrimSpace(className) - if className != "" { - classes[className] = true - } - } - return classes -} - -func getHTMLAttribute(node *html.Node, key string) string { - for _, attr := range node.Attr { - if strings.EqualFold(attr.Key, key) { - return attr.Val - } - } - return "" -} - -func extractNodeText(node *html.Node) string { - var b strings.Builder - var walk func(*html.Node) - walk = func(current *html.Node) { - if current == nil { - return - } - if current.Type == html.TextNode { - b.WriteString(current.Data) - b.WriteByte(' ') - } - if current.Type == html.ElementNode && shouldIgnoreHTMLElement(strings.ToLower(current.Data)) { - return - } - for child := current.FirstChild; child != nil; child = child.NextSibling { - walk(child) - } - } - walk(node) - return b.String() -} - -func extractRawNodeText(node *html.Node) string { - var b strings.Builder - var walk func(*html.Node) - walk = func(current *html.Node) { - if current == nil { - return - } - if current.Type == html.TextNode { - b.WriteString(current.Data) - return - } - for child := current.FirstChild; child != nil; child = child.NextSibling { - walk(child) - } - } - walk(node) - return b.String() -} - -func cloneColor(color *pptx.Color) *pptx.Color { - if color == nil { - return nil - } - value := *color - return &value -} - -func cloneInt(value *int) *int { - if value == nil { - return nil - } - copyValue := *value - return ©Value -} - -func cloneFloat64(value *float64) *float64 { - if value == nil { - return nil - } - copyValue := *value - return ©Value -} - -func intPtr(value int) *int { - return &value -} - -func float64Ptr(value float64) *float64 { - return &value -} - -func dynamicPPTValueOrInt(value *int, fallback int) int { - if value == nil || *value <= 0 { - return fallback - } - return *value -} - -func dynamicPPTMaxInt(a, b int) int { - if a > b { - return a - } - return b -} - -func dynamicPPTMinInt(a, b int) int { - if a < b { - return a - } - return b -} - -func dynamicPPTMaxFloat(a, b float64) float64 { - if a > b { - return a - } - return b -} diff --git a/internal/service/generation_interface.go b/internal/service/generation_interface.go deleted file mode 100644 index 92bb56b..0000000 --- a/internal/service/generation_interface.go +++ /dev/null @@ -1,79 +0,0 @@ -package service - -import "context" - -// GenerationType identifies a supported generation sub-agent. -type GenerationType string - -const ( - GenerationTypeMindmap GenerationType = "mindmap" - GenerationTypePPT GenerationType = "ppt" - GenerationTypeQuiz GenerationType = "quiz" - GenerationTypeNote GenerationType = "note" -) - -// GenerationRequest is the internal request for the supervisor generation agent. -type GenerationRequest struct { - UserID uint `json:"user_id,omitempty"` - NotebookID uint `json:"notebook_id,omitempty"` - Markdown string `json:"markdown"` - Type GenerationType `json:"type"` - Prompt string `json:"prompt,omitempty"` - Options map[string]any `json:"options,omitempty"` - SourceIDs []uint `json:"source_ids,omitempty"` - UseWeb bool `json:"use_web,omitempty"` - AllowDegrade bool `json:"allow_degrade,omitempty"` -} - -// GenerationReference records a local RAG reference used for generation. -type GenerationReference struct { - SourceID uint `json:"source_id"` - SourceName string `json:"source_name,omitempty"` - Content string `json:"content"` - Score float32 `json:"score,omitempty"` - Heading string `json:"heading,omitempty"` - ChapterPath string `json:"chapter_path,omitempty"` -} - -// GenerationResponse is the unified output returned by all generation agents. -type GenerationResponse struct { - Type GenerationType `json:"type"` - Content string `json:"content"` - References []GenerationReference `json:"references,omitempty"` - SearchResults []SearchResult `json:"search_results,omitempty"` - Meta map[string]any `json:"meta,omitempty"` -} - -type GenerationExportRequest struct { - Type GenerationType `json:"type"` - Content string `json:"content"` - Title string `json:"title,omitempty"` - Template string `json:"template,omitempty"` -} - -type GenerationExportResult struct { - Filename string - ContentType string - Data []byte -} - -// GenerationPrompt is the prompt payload passed to the model-backed sub-agent. -type GenerationPrompt struct { - AgentName string - System string - User string - Context string - OutputFormat string - MaxTokens int -} - -// GenerationModel abstracts Eino-backed model generation for testable agents. -type GenerationModel interface { - Generate(ctx context.Context, prompt GenerationPrompt) (string, error) -} - -// GenerationService is the supervisor entry point for generation. -type GenerationService interface { - Generate(ctx context.Context, req *GenerationRequest) (*GenerationResponse, error) - Export(ctx context.Context, req *GenerationExportRequest) (*GenerationExportResult, error) -} diff --git a/internal/service/generation_mindmap_test.go b/internal/service/generation_mindmap_test.go deleted file mode 100644 index 3147449..0000000 --- a/internal/service/generation_mindmap_test.go +++ /dev/null @@ -1,214 +0,0 @@ -package service - -import ( - "strings" - "testing" -) - -func TestDynamicMindmapBranchesSectioned(t *testing.T) { - // 测试有章节结构时的情况 - analysis := learningContentAnalysis{ - Topic: "测试主题", - Sections: []pptSourceSection{ - {Title: "第一章", Points: []string{"要点1", "要点2", "要点3"}}, - {Title: "第二章", Points: []string{"要点4", "要点5"}}, - {Title: "第三章", Points: []string{"要点6", "要点7"}}, - }, - KeyConcepts: []string{"概念1", "概念2"}, - } - branches := dynamicMindmapBranches(analysis) - if len(branches) < 3 { - t.Errorf("expected at least 3 branches, got %d", len(branches)) - } - // 最后一个应该是总结 - last := branches[len(branches)-1] - if last.Title != "总结" { - t.Errorf("expected last branch to be '总结', got '%s'", last.Title) - } - // 至少有一个分支有节点 - hasNodes := false - for _, b := range branches { - if len(b.Nodes) > 0 { - hasNodes = true - break - } - } - if !hasNodes { - t.Error("expected at least one branch to have nodes") - } -} - -func TestDynamicMindmapBranchesFlat(t *testing.T) { - // 测试扁平材料时的情况 - analysis := learningContentAnalysis{ - Topic: "测试主题", - KeyConcepts: []string{"概念1", "概念2", "概念3"}, - Processes: []string{"过程1"}, - Examples: []string{"例子1"}, - Sparse: false, - } - branches := dynamicMindmapBranches(analysis) - if len(branches) < 3 { - t.Errorf("expected at least 3 branches, got %d", len(branches)) - } - // 应该有总结 - last := branches[len(branches)-1] - if last.Title != "总结" { - t.Errorf("expected last branch to be '总结', got '%s'", last.Title) - } -} - -func TestDynamicMindmapBranchesSparse(t *testing.T) { - // 测试稀疏材料时的情况 - analysis := learningContentAnalysis{ - Topic: "测试主题", - Sparse: true, - } - branches := dynamicMindmapBranches(analysis) - if len(branches) < 2 { - t.Errorf("expected at least 2 branches, got %d", len(branches)) - } -} - -func TestDynamicMindmapBranchesEmptySections(t *testing.T) { - // 测试章节为空的情况 - analysis := learningContentAnalysis{ - Topic: "测试主题", - KeyConcepts: []string{}, - Processes: []string{}, - Examples: []string{}, - } - branches := dynamicMindmapBranches(analysis) - if len(branches) < 3 { - t.Errorf("expected at least 3 branches, got %d", len(branches)) - } -} - -func TestMindmapNeedsStructureRepair(t *testing.T) { - tests := []struct { - name string - content string - repair bool - }{ - { - name: "empty content", - content: "", - repair: true, - }, - { - name: "too few branches", - content: "# 标题\n## 分支1\n### 节点1", - repair: true, - }, - { - name: "valid structure", - content: "# 标题\n## 分支1\n### 节点1\n#### 细节1\n## 分支2\n### 节点2\n## 分支3\n### 节点3", - repair: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := mindmapNeedsStructureRepair(tt.content) - if result != tt.repair { - t.Errorf("mindmapNeedsStructureRepair() = %v, want %v", result, tt.repair) - } - }) - } -} - -func TestPlanMindmap(t *testing.T) { - analysis := learningContentAnalysis{ - Topic: "测试主题", - KeyConcepts: []string{"概念1", "概念2", "概念3"}, - Processes: []string{"过程1"}, - Examples: []string{"例子1"}, - } - plan := planMindmap(analysis) - if plan.Title == "" { - t.Error("planMindmap() returned empty title") - } - if len(plan.Branches) < 3 { - t.Errorf("expected at least 3 branches, got %d", len(plan.Branches)) - } - // 每个分支至少应有节点 - for i, branch := range plan.Branches { - if len(branch.Nodes) == 0 { - t.Errorf("branch %d (%s) has no nodes", i, branch.Title) - } - } -} - -func TestExpandMindmapContent(t *testing.T) { - plan := mindmapPlan{ - Title: "测试主题", - Branches: []mindmapBranchPlan{ - {Title: "核心概念", Nodes: []mindmapNodePlan{{Title: "概念1", Details: []string{"细节"}}}}, - {Title: "原理与过程", Nodes: []mindmapNodePlan{{Title: "过程1", Details: []string{"步骤"}}}}, - {Title: "总结", Nodes: []mindmapNodePlan{{Title: "要点", Details: []string{"关键点"}}}}, - }, - } - analysis := learningContentAnalysis{ - Topic: "测试主题", - KeyConcepts: []string{"概念1", "概念2"}, - Evidence: []learningEvidence{{Text: "补充证据1", Source: "src1"}}, - } - expanded := expandMindmapContent(plan, analysis) - if len(expanded.Branches) < 3 { - t.Errorf("expected at least 3 branches after expansion, got %d", len(expanded.Branches)) - } -} - -func TestRenderMindmap(t *testing.T) { - plan := mindmapPlan{ - Title: "测试主题", - Branches: []mindmapBranchPlan{ - {Title: "核心概念", Nodes: []mindmapNodePlan{{Title: "概念1", Details: []string{"概念1的详细说明"}}}}, - {Title: "总结", Nodes: []mindmapNodePlan{{Title: "知识结构", Details: []string{"结构化复习"}}}}, - }, - } - rendered := renderMindmap(plan) - if rendered == "" { - t.Error("renderMindmap() returned empty string") - } - if !strings.Contains(rendered, "#") { - t.Error("renderMindmap() output does not contain markdown headings") - } -} - -func TestMindmapNodeDetailFromEvidence(t *testing.T) { - analysis := learningContentAnalysis{ - Topic: "测试主题", - KeyConcepts: []string{"概念1", "概念2"}, - Evidence: []learningEvidence{{Text: "概念1的详细描述", Source: "src1"}}, - } - detail := mindmapNodeDetailFromEvidence("概念1", analysis) - if detail == "" { - t.Error("mindmapNodeDetailFromEvidence() returned empty string") - } - // 测试不存在的主题 - detail2 := mindmapNodeDetailFromEvidence("不存在", analysis) - if detail2 == "" { - t.Error("mindmapNodeDetailFromEvidence() returned empty for unknown topic") - } -} - -func TestNewMindmapNode(t *testing.T) { - node := newMindmapNode("测试节点", "详细说明") - if node.Title != "测试节点" { - t.Errorf("expected title '测试节点', got '%s'", node.Title) - } - if len(node.Details) != 1 { - t.Errorf("expected 1 detail, got %d", len(node.Details)) - } - if node.Details[0] != "详细说明" { - t.Errorf("expected detail '详细说明', got '%s'", node.Details[0]) - } - // 测试无详情 - node2 := newMindmapNode("仅标题") - if node2.Title != "仅标题" { - t.Errorf("expected title '仅标题', got '%s'", node2.Title) - } - if len(node2.Details) != 0 { - t.Errorf("expected 0 details, got %d", len(node2.Details)) - } -} diff --git a/internal/service/generation_ppt_enrich_test.go b/internal/service/generation_ppt_enrich_test.go deleted file mode 100644 index 0f31dbb..0000000 --- a/internal/service/generation_ppt_enrich_test.go +++ /dev/null @@ -1,307 +0,0 @@ -package service - -import ( - "context" - "strings" - "sync" - "testing" -) - -// captureGenerationModel records prompts and returns mock outputs. -// It is safe for concurrent access when used with the concurrent enrich. -type captureGenerationModel struct { - mu sync.Mutex - prompts []GenerationPrompt - outputs []string -} - -func (m *captureGenerationModel) Generate(ctx context.Context, prompt GenerationPrompt) (string, error) { - m.mu.Lock() - m.prompts = append(m.prompts, prompt) - if len(m.outputs) > 0 { - output := m.outputs[0] - m.outputs = m.outputs[1:] - m.mu.Unlock() - return output, nil - } - m.mu.Unlock() - return `{"slides":[{"title":"Slide","paragraphs":["expanded paragraph"]}]}`, nil -} - -func TestPPTContentEnrichBatchesSlides(t *testing.T) { - model := &captureGenerationModel{} - agent := &pptGenerationAgent{ - baseGenerationAgent: baseGenerationAgent{ - name: "ppt", - typ: GenerationTypePPT, - model: model, - }, - } - - state := pptChainState{ - input: generationAgentInput{ - Request: &GenerationRequest{ - Type: GenerationTypePPT, - Markdown: "# Topic", - }, - Context: "Original Markdown:\n# Topic", - }, - expanded: pptOutlinePlan{ - Title: "Topic", - Slides: []pptSlidePlan{ - {Title: "Slide 01", Bullets: []string{"Topic 01"}}, - {Title: "Slide 02", Bullets: []string{"Topic 02"}}, - {Title: "Slide 03", Bullets: []string{"Topic 03"}}, - {Title: "Slide 04", Bullets: []string{"Topic 04"}}, - {Title: "Slide 05", Bullets: []string{"Topic 05"}}, - {Title: "Slide 06", Bullets: []string{"Topic 06"}}, - {Title: "Slide 07", Bullets: []string{"Topic 07"}}, - {Title: "Slide 08", Bullets: []string{"Topic 08"}}, - {Title: "Slide 09", Bullets: []string{"Topic 09"}}, - }, - }, - } - - result, err := agent.enrichPPTContent(context.Background(), state) - if err != nil { - t.Fatalf("enrichPPTContent returned error: %v", err) - } - - // 9 slides / batch_size(4) = 3 batches. Each batch calls Generate once - // (first call succeeds) -> 3 total model calls. - if len(model.prompts) != 3 { - t.Fatalf("Generate calls = %d, want 3", len(model.prompts)) - } - - // Verify each prompt has MaxTokens set - for i, prompt := range model.prompts { - if got := prompt.MaxTokens; got != pptContentEnrichMaxTokens { - t.Fatalf("prompt %d MaxTokens = %d, want %d", i, got, pptContentEnrichMaxTokens) - } - } - - // The mock model returns 1 slide per call. With 3 batches -> 3 rich slides - if len(result.richContent.Slides) != 3 { - t.Fatalf("rich slides = %d, want 3", len(result.richContent.Slides)) - } -} - -func TestPPTContentEnrichKeepsSuccessfulBatches(t *testing.T) { - model := &captureGenerationModel{ - outputs: []string{ - `{"slides":[`, - `{"slides":[`, - `{"slides":[{"title":"Slide 05","paragraphs":["expanded five"]}]}`, - }, - } - agent := &pptGenerationAgent{ - baseGenerationAgent: baseGenerationAgent{ - name: "ppt", - typ: GenerationTypePPT, - model: model, - }, - } - state := pptChainState{ - input: generationAgentInput{ - Request: &GenerationRequest{Type: GenerationTypePPT, Markdown: "# Topic"}, - Context: "Original Markdown:\n# Topic", - }, - expanded: pptOutlinePlan{ - Title: "Topic", - Slides: []pptSlidePlan{ - {Title: "Slide 01", Bullets: []string{"Topic 01"}}, - {Title: "Slide 02", Bullets: []string{"Topic 02"}}, - {Title: "Slide 03", Bullets: []string{"Topic 03"}}, - {Title: "Slide 04", Bullets: []string{"Topic 04"}}, - {Title: "Slide 05", Bullets: []string{"Topic 05"}}, - }, - }, - } - - got, err := agent.enrichPPTContent(context.Background(), state) - if err != nil { - t.Fatalf("enrichPPTContent returned error: %v", err) - } - // 5 slides / batch_size(4) = 2 batches (4+1). First batch: output is `{"slides":[` - // which fails JSON parse → retry → same result. 2 batches × (1 initial + 1 retry) = 4. - // Only the last batch succeeds. - generated := model.prompts - if len(generated) != 3 { - t.Fatalf("Generate calls = %d, want 3", len(generated)) - } - if len(got.richContent.Slides) != 1 { - t.Fatalf("rich slides = %d, want 1", len(got.richContent.Slides)) - } - if got.richContent.Slides[0].Title != "Slide 05" { - t.Fatalf("kept slide title = %q, want Slide 05", got.richContent.Slides[0].Title) - } -} - -func TestPPTContentEnrichPreservesOrder(t *testing.T) { - // Return sequential titles that the mock model produces (always "Slide"). - // Instead of checking exact titles, verify slide count matches batch total. - model := &captureGenerationModel{} - agent := &pptGenerationAgent{ - baseGenerationAgent: baseGenerationAgent{ - name: "ppt", - typ: GenerationTypePPT, - model: model, - }, - } - - state := pptChainState{ - input: generationAgentInput{ - Request: &GenerationRequest{ - Type: GenerationTypePPT, - Markdown: "# Topic", - }, - Context: "Original Markdown:\n# Topic", - }, - expanded: pptOutlinePlan{ - Title: "Topic", - Slides: []pptSlidePlan{ - {Title: "Slide A1", Bullets: []string{"T1"}}, - {Title: "Slide A2", Bullets: []string{"T2"}}, - {Title: "Slide A3", Bullets: []string{"T3"}}, - {Title: "Slide A4", Bullets: []string{"T4"}}, - {Title: "Slide B1", Bullets: []string{"T5"}}, - {Title: "Slide B2", Bullets: []string{"T6"}}, - {Title: "Slide B3", Bullets: []string{"T7"}}, - }, - }, - } - - result, err := agent.enrichPPTContent(context.Background(), state) - if err != nil { - t.Fatalf("enrichPPTContent returned error: %v", err) - } - // 7 slides / batch_size(4) = 2 batches (4+3). All succeed -> 2 rich slides - // (each batch's mock call returns 1 slide). - if len(result.richContent.Slides) != 2 { - t.Fatalf("rich slides = %d, want 2", len(result.richContent.Slides)) - } -} - -func TestPPTContentEnrichPartialFailure(t *testing.T) { - // Batch 0 fails (invalid JSON), batch 1 succeeds, batch 2 fails - // Expect only batch 1's slides in the result. - failJSON := `{"slides":[` - model := &captureGenerationModel{ - outputs: []string{failJSON, failJSON, failJSON, `{"slides":[{"title":"Ok1","paragraphs":["p1"]},{"title":"Ok2","paragraphs":["p2"]}]}`, failJSON, failJSON}, - } - agent := &pptGenerationAgent{ - baseGenerationAgent: baseGenerationAgent{ - name: "ppt", - typ: GenerationTypePPT, - model: model, - }, - } - state := pptChainState{ - input: generationAgentInput{ - Request: &GenerationRequest{Type: GenerationTypePPT, Markdown: "# Topic"}, - Context: "Original Markdown:\n# Topic", - }, - expanded: pptOutlinePlan{ - Title: "Topic", - Slides: []pptSlidePlan{ - {Title: "Batch0-1", Bullets: []string{"x"}}, - {Title: "Batch0-2", Bullets: []string{"y"}}, - {Title: "Batch0-3", Bullets: []string{"z"}}, - {Title: "Batch0-4", Bullets: []string{"w"}}, - // batch 1 (slides 5-8) - {Title: "Batch1-1", Bullets: []string{"a"}}, - {Title: "Batch1-2", Bullets: []string{"b"}}, - {Title: "Batch1-3", Bullets: []string{"c"}}, - {Title: "Batch1-4", Bullets: []string{"d"}}, - // batch 2 (slides 9-10) - {Title: "Batch2-1", Bullets: []string{"m"}}, - {Title: "Batch2-2", Bullets: []string{"n"}}, - }, - }, - } - - result, err := agent.enrichPPTContent(context.Background(), state) - if err != nil { - t.Fatalf("enrichPPTContent returned error: %v", err) - } - if len(result.richContent.Slides) != 2 { - t.Fatalf("rich slides = %d, want 2", len(result.richContent.Slides)) - } - if result.richContent.Slides[0].Title != "Ok1" || result.richContent.Slides[1].Title != "Ok2" { - t.Fatalf("unexpected slide titles: %v", slideTitles(result.richContent.Slides)) - } -} - -func TestPPTContentEnrichSingleBatch(t *testing.T) { - model := &captureGenerationModel{} - agent := &pptGenerationAgent{ - baseGenerationAgent: baseGenerationAgent{ - name: "ppt", - typ: GenerationTypePPT, - model: model, - }, - } - state := pptChainState{ - input: generationAgentInput{ - Request: &GenerationRequest{Type: GenerationTypePPT, Markdown: "# Topic"}, - Context: "Original Markdown:\n# Topic", - }, - expanded: pptOutlinePlan{ - Title: "Topic", - Slides: []pptSlidePlan{{Title: "Only Slide", Bullets: []string{"Only"}}}, - }, - } - - result, err := agent.enrichPPTContent(context.Background(), state) - if err != nil { - t.Fatalf("enrichPPTContent returned error: %v", err) - } - if len(result.richContent.Slides) != 1 { - t.Fatalf("rich slides = %d, want 1", len(result.richContent.Slides)) - } - // Mock's default JSON: title is "Slide" - if result.richContent.Slides[0].Title != "Slide" { - t.Fatalf("title = %q, want 'Slide'", result.richContent.Slides[0].Title) - } -} - -func TestPPTContentEnrichNilModel(t *testing.T) { - agent := &pptGenerationAgent{ - baseGenerationAgent: baseGenerationAgent{ - name: "ppt", - typ: GenerationTypePPT, - }, - } - state := pptChainState{ - expanded: pptOutlinePlan{ - Title: "T", - Slides: []pptSlidePlan{{Title: "S1"}, {Title: "S2"}}, - }, - } - result, err := agent.enrichPPTContent(context.Background(), state) - if err != nil { - t.Fatalf("enrichPPTContent returned error: %v", err) - } - if len(result.richContent.Slides) != 0 { - t.Fatalf("rich slides = %d, want 0", len(result.richContent.Slides)) - } -} - -// containsAll checks that value contains all needles. -func containsAll(value string, needles ...string) bool { - for _, needle := range needles { - if !strings.Contains(value, needle) { - return false - } - } - return true -} - -// slideTitles extracts slide titles for test assertions. -func slideTitles(slides []enrichedPPTSlide) []string { - titles := make([]string, len(slides)) - for i, s := range slides { - titles[i] = s.Title - } - return titles -} diff --git a/internal/service/generation_ppt_title_prefix_test.go b/internal/service/generation_ppt_title_prefix_test.go deleted file mode 100644 index 5f55332..0000000 --- a/internal/service/generation_ppt_title_prefix_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package service - -import "testing" - -func TestStripPPTBulletSlideTitlePrefix(t *testing.T) { - cases := []struct { - name string - bullet string - title string - want string - }{ - { - name: "chinese colon prefix", - bullet: "卡尔文循环:场所:叶绿体基质", - title: "卡尔文循环", - want: "场所:叶绿体基质", - }, - { - name: "ascii colon prefix", - bullet: "卡尔文循环:CO₂固定:与RuBP结合", - title: "卡尔文循环", - want: "CO₂固定:与RuBP结合", - }, - { - name: "space separator prefix", - bullet: "光合作用 光反应阶段在类囊体膜上进行", - title: "光合作用", - want: "光反应阶段在类囊体膜上进行", - }, - { - name: "repeated prefix stripped iteratively", - bullet: "卡尔文循环:卡尔文循环:场所:叶绿体基质", - title: "卡尔文循环", - want: "场所:叶绿体基质", - }, - { - name: "bracketed chapter title matches core", - bullet: "卡尔文循环:场所:叶绿体基质", - title: "卡尔文循环(一)", - want: "场所:叶绿体基质", - }, - { - name: "no separator after title keeps bullet intact", - bullet: "封面页内容介绍", - title: "封面", - want: "封面页内容介绍", - }, - { - name: "title too short no strip", - bullet: "A:something", - title: "A", - want: "A:something", - }, - { - name: "bullet does not start with title", - bullet: "暗反应不依赖光", - title: "光反应", - want: "暗反应不依赖光", - }, - { - name: "strip would blank bullet keeps original", - bullet: "卡尔文循环:", - title: "卡尔文循环", - want: "卡尔文循环:", - }, - { - name: "case insensitive english", - bullet: "Photosynthesis: light reactions", - title: "photosynthesis", - want: "light reactions", - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - got := stripPPTBulletSlideTitlePrefix(c.bullet, c.title) - if got != c.want { - t.Errorf("stripPPTBulletSlideTitlePrefix(%q, %q) = %q, want %q", c.bullet, c.title, got, c.want) - } - }) - } -} - -func TestStripPPTHTMLRepeatedTitlePrefix(t *testing.T) { - html := `

卡尔文循环

  • 卡尔文循环:场所:叶绿体基质
  • 卡尔文循环:CO₂固定:与RuBP结合
` - want := `

卡尔文循环

  • 场所:叶绿体基质
  • CO₂固定:与RuBP结合
` - got := stripPPTHTMLRepeatedTitlePrefix(html) - if got != want { - t.Errorf("got:\n%s\nwant:\n%s", got, want) - } -} - -func TestStripPPTHTMLRepeatedTitlePrefixLeavesHeadingIntact(t *testing.T) { - // The heading text itself must not be stripped even though it equals the - // title used for bullet stripping. - html := `

光合作用

光合作用:光反应

` - want := `

光合作用

光反应

` - got := stripPPTHTMLRepeatedTitlePrefix(html) - if got != want { - t.Errorf("got:\n%s\nwant:\n%s", got, want) - } -} - -func TestStripPPTHTMLRepeatedTitlePrefixSkipsStyle(t *testing.T) { - // CSS rules like ".foo:bar" must not be touched. - html := `

foo

foo: value

` - want := `

foo

value

` - got := stripPPTHTMLRepeatedTitlePrefix(html) - if got != want { - t.Errorf("got:\n%s\nwant:\n%s", got, want) - } -} - -func TestStripPPTHTMLRepeatedTitlePrefixSubheading(t *testing.T) { - // The repeated prefix is a sub-heading (h3) under the section heading (h2), - // not the main heading. The pass must collect h3 as a candidate too. - html := `

卡尔文循环

暗反应

暗反应:场所:叶绿体基质

暗反应:前置条件:光反应提供ATP和NADPH

` - want := `

卡尔文循环

暗反应

场所:叶绿体基质

前置条件:光反应提供ATP和NADPH

` - got := stripPPTHTMLRepeatedTitlePrefix(html) - if got != want { - t.Errorf("got:\n%s\nwant:\n%s", got, want) - } -} - -func TestStripPPTHTMLRepeatedTitlePrefixCardTitle(t *testing.T) { - // Card layout: the card-title is the repeated prefix inside the card body. - html := `

光合作用

暗反应
暗反应:场所:叶绿体基质
` - want := `

光合作用

暗反应
场所:叶绿体基质
` - got := stripPPTHTMLRepeatedTitlePrefix(html) - if got != want { - t.Errorf("got:\n%s\nwant:\n%s", got, want) - } -} - -func TestStripPPTBulletTitlePrefixesStacked(t *testing.T) { - // Two different title prefixes stacked on one node: peel both. - got := stripPPTBulletTitlePrefixes("卡尔文循环:暗反应:场所:叶绿体基质", []string{"卡尔文循环", "暗反应"}) - want := "场所:叶绿体基质" - if got != want { - t.Errorf("got %q, want %q", got, want) - } -} - -func TestStripPPTBulletTitlePrefixesIgnoresShortCandidates(t *testing.T) { - // A 1-rune candidate must be ignored so it can't over-trim. - got := stripPPTBulletTitlePrefixes("光反应:阶段", []string{"光", "光反应"}) - want := "阶段" - if got != want { - t.Errorf("got %q, want %q", got, want) - } -} - diff --git a/internal/service/generation_quiz_test.go b/internal/service/generation_quiz_test.go deleted file mode 100644 index 534e5a9..0000000 --- a/internal/service/generation_quiz_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package service - -import ( - "strings" - "testing" -) - -func TestPlanQuizQuestions(t *testing.T) { - analysis := learningContentAnalysis{ - Topic: "光合作用", - KeyConcepts: []string{"光反应", "暗反应", "叶绿素"}, - Processes: []string{"电子传递链", "卡尔文循环"}, - Examples: []string{"C3植物", "C4植物"}, - } - plan := planQuizQuestions(analysis) - if len(plan.Questions) < 3 { - t.Errorf("expected at least 3 questions, got %d", len(plan.Questions)) - } - // 检查题型多样性 - typeSet := make(map[string]bool) - for _, q := range plan.Questions { - typeSet[q.Type] = true - if q.Question == "" { - t.Error("question must not be empty") - } - if q.Answer == "" { - t.Error("answer must not be empty") - } - } - if len(typeSet) < 2 { - t.Errorf("expected at least 2 different question types, got %d", len(typeSet)) - } -} - -func TestPlanQuizQuestionsSparse(t *testing.T) { - // 测试材料稀疏时的场景 - analysis := learningContentAnalysis{ - Topic: "测试主题", - KeyConcepts: []string{"概念1"}, - Sparse: true, - } - plan := planQuizQuestions(analysis) - if len(plan.Questions) < 3 { - t.Errorf("expected at least 3 questions, got %d", len(plan.Questions)) - } -} - -func TestPlanQuizQuestionsRich(t *testing.T) { - // 测试材料丰富时的场景 - analysis := learningContentAnalysis{ - Topic: "测试主题", - KeyConcepts: []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8"}, - Processes: []string{"p1", "p2", "p3"}, - Examples: []string{"e1", "e2", "e3", "e4"}, - } - plan := planQuizQuestions(analysis) - if len(plan.Questions) < 3 { - t.Errorf("expected at least 3 questions, got %d", len(plan.Questions)) - } - typeSet := make(map[string]bool) - for _, q := range plan.Questions { - typeSet[q.Type] = true - } - if len(typeSet) < 2 { - t.Errorf("expected at least 2 different question types, got %d", len(typeSet)) - } -} - -func TestRequiredQuizQuestionTypes(t *testing.T) { - types := requiredQuizQuestionTypes(learningContentAnalysis{ - Topic: "测试主题", - KeyConcepts: []string{"概念1"}, - Sparse: true, - }) - if len(types) < 3 { - t.Errorf("expected at least 3 question types, got %d", len(types)) - } - // 检查至少有2种不同题型 - typeSet := make(map[string]bool) - for _, qt := range types { - typeSet[qt] = true - } - if len(typeSet) < 2 { - t.Errorf("expected at least 2 different question types, got %d", len(typeSet)) - } -} - -func TestValidateQuizContent(t *testing.T) { - tests := []struct { - name string - content string - valid bool - }{ - { - name: "empty content", - content: "", - valid: false, - }, - { - name: "too few questions", - content: `{"questions":[{"type":"single_choice","question":"Q1","options":["A","B","C"],"answer":"A","explanation":"E1"}]}`, - valid: false, - }, - { - name: "valid mixed types", - content: `{"questions":[` + - `{"type":"single_choice","question":"Q1","options":["A","B","C","D"],"answer":"A","explanation":"E1"},` + - `{"type":"single_choice","question":"Q2","options":["A","B","C","D"],"answer":"B","explanation":"E2"},` + - `{"type":"short_answer","question":"Q3","options":[],"answer":"关键词","explanation":"E3"},` + - `{"type":"short_answer","question":"Q4","options":[],"answer":"答案","explanation":"E4"},` + - `{"type":"short_answer","question":"Q5","options":[],"answer":"答案5","explanation":"E5"}` + - `]}`, - valid: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := validateQuizContent(tt.content) - if result != tt.valid { - t.Errorf("validateQuizContent() = %v, want %v", result, tt.valid) - } - }) - } -} - -func TestExpandQuizContent(t *testing.T) { - plan := quizQuestionPlan{ - Topic: "测试主题", - Questions: []quizQuestionItem{ - {Type: "single_choice", Topic: "概念1", Question: "关于概念1的说法?", Options: []string{"A", "B", "C", "D"}, Answer: "A", Explanation: "解释1"}, - {Type: "short_answer", Topic: "过程1", Question: "简述过程1?", Answer: "答案1", Explanation: "解释2"}, - {Type: "short_answer", Topic: "例子1", Question: "说明例子1?", Answer: "答案2", Explanation: "解释3"}, - }, - } - analysis := learningContentAnalysis{ - Topic: "测试主题", - KeyConcepts: []string{"概念1"}, - Evidence: []learningEvidence{{Text: "资料要点:这是关键信息", Source: "source1"}}, - } - expanded := expandQuizContent(plan, analysis) - if len(expanded.Questions) < 3 { - t.Errorf("expected at least 3 questions after expansion, got %d", len(expanded.Questions)) - } -} - -func TestRenderQuiz(t *testing.T) { - plan := quizQuestionPlan{ - Topic: "测试主题", - Questions: []quizQuestionItem{ - {Type: "single_choice", Topic: "概念1", Question: "关于概念1的说法?", Options: []string{"A", "B", "C", "D"}, Answer: "A", Explanation: "解释1"}, - {Type: "short_answer", Topic: "过程1", Question: "简述过程1?", Answer: "答案1", Explanation: "解释2"}, - }, - } - rendered := renderQuiz(plan) - if rendered == "" { - t.Error("renderQuiz() returned empty string") - } - if !strings.Contains(rendered, "questions") { - t.Error("renderQuiz() does not contain 'questions' key") - } -} diff --git a/internal/service/generation_validators.go b/internal/service/generation_validators.go deleted file mode 100644 index f202fba..0000000 --- a/internal/service/generation_validators.go +++ /dev/null @@ -1,125 +0,0 @@ -package service - -import ( - "encoding/json" - "strings" -) - -func validateMindmapContent(content string) bool { - content = strings.TrimSpace(content) - if !strings.HasPrefix(content, "#") { - return false - } - return strings.Contains(content, "\n## ") || strings.Contains(content, "\n- ") -} - -func validatePPTContent(content string) bool { - lower := strings.ToLower(strings.TrimSpace(content)) - if strings.Contains(lower, "") { - return false - } - if strings.Count(lower, "= 4 -} - -func validateQuizContent(content string) bool { - var payload struct { - Questions []struct { - Type string `json:"type"` - Question string `json:"question"` - Options []string `json:"options"` - Answer string `json:"answer"` - Explanation string `json:"explanation"` - } `json:"questions"` - } - if err := json.Unmarshal([]byte(strings.TrimSpace(content)), &payload); err != nil { - return false - } - if len(payload.Questions) < 5 { - return false - } - validTypes := map[string]bool{ - "single_choice": true, "true_false": true, "multi_choice": true, - "fill_blank": true, "short_answer": true, - } - typeSet := make(map[string]bool) - for _, question := range payload.Questions { - if !validTypes[question.Type] { - return false - } - if strings.TrimSpace(question.Question) == "" || strings.TrimSpace(question.Answer) == "" { - return false - } - typeSet[question.Type] = true - switch question.Type { - case "single_choice", "multi_choice": - if len(question.Options) < 3 { - return false - } - case "true_false": - if len(question.Options) < 2 { - return false - } - } - } - // 至少2种不同题型 - if len(typeSet) < 2 { - return false - } - return true -} - -func validateNoteContent(content string) bool { - content = strings.TrimSpace(content) - if !strings.HasPrefix(content, "#") { - return false - } - lines := strings.Split(content, "\n") - bodyRunes := 0 - hasSection := false - for _, line := range lines[1:] { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "## ") { - hasSection = true - } - if line != "" && !strings.HasPrefix(line, "#") { - bodyRunes += len([]rune(line)) - } - } - return hasSection || bodyRunes >= 8 -} - -func stripSimpleHTML(content string) string { - var b strings.Builder - inTag := false - for _, r := range content { - switch r { - case '<': - inTag = true - case '>': - inTag = false - default: - if !inTag { - b.WriteRune(r) - } - } - } - return b.String() -} diff --git a/internal/service/importer_interface.go b/internal/service/importer_interface.go index 84d3c41..4e8dee0 100644 --- a/internal/service/importer_interface.go +++ b/internal/service/importer_interface.go @@ -1,26 +1,26 @@ -package service - -import ( - "YoudaoNoteLm/internal/model/entity" - "mime/multipart" -) - -// SearchResultItem 搜索结果项(用于导入时保留标题) -type SearchResultItem struct { - Title string // 标题 - URL string // URL -} - -// ImporterService 导入服务接口 -type ImporterService interface { - ImportFile(userID, notebookID uint, file *multipart.FileHeader) (*entity.Source, error) - // PreviewAudio 异步音频转写:上传文件后立即返回 previewID,后台执行 ASR 转写 - PreviewAudio(userID, notebookID uint, file *multipart.FileHeader) (previewID string, fileName string, err error) - // GetAudioPreviewStatus 查询音频预览状态(前端轮询用) - GetAudioPreviewStatus(userID uint, previewID string) (interface{}, error) - ConfirmAudio(userID uint, previewID string, editedContent *string) (*entity.Source, error) - // ImportSearchResults 批量导入搜索结果,返回任务 ID 和创建的 Source ID 列表 - ImportSearchResults(userID, notebookID uint, items []SearchResultItem) (taskID string, sourceIDs []uint, err error) - GetImportTask(taskID string) (interface{}, error) - DeleteImportTask(taskID string) error // 删除导入任务 -} +package service + +import ( + "YoudaoNoteLm/internal/model/entity" + "mime/multipart" +) + +// SearchResultItem 搜索结果项(用于导入时保留标题) +type SearchResultItem struct { + Title string // 标题 + URL string // URL +} + +// ImporterService 导入服务接口 +type ImporterService interface { + ImportFile(userID, notebookID uint, file *multipart.FileHeader) (*entity.Source, error) + // PreviewAudio 异步音频转写:上传文件后立即返回 previewID,后台执行 ASR 转写 + PreviewAudio(userID, notebookID uint, file *multipart.FileHeader) (previewID string, fileName string, err error) + // GetAudioPreviewStatus 查询音频预览状态(前端轮询用) + GetAudioPreviewStatus(userID uint, previewID string) (interface{}, error) + ConfirmAudio(userID uint, previewID string, editedContent *string) (*entity.Source, error) + // ImportSearchResults 批量导入搜索结果,返回任务 ID 和创建的 Source ID 列表 + ImportSearchResults(userID, notebookID uint, items []SearchResultItem) (taskID string, sourceIDs []uint, err error) + GetImportTask(taskID string) (interface{}, error) + DeleteImportTask(taskID string) error // 删除导入任务 +} diff --git a/internal/service/importer_service.go b/internal/service/importer_service.go index 46b9d4b..31797fd 100644 --- a/internal/service/importer_service.go +++ b/internal/service/importer_service.go @@ -1,1144 +1,1144 @@ -package service - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "mime/multipart" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "time" - - "YoudaoNoteLm/internal/llm" - "YoudaoNoteLm/internal/model/entity" - "YoudaoNoteLm/internal/rag" - "YoudaoNoteLm/internal/repository" - "YoudaoNoteLm/internal/service/external/asr" - externalMarkitdown "YoudaoNoteLm/internal/service/external/markitdown" - "YoudaoNoteLm/internal/service/external/storage" - "YoudaoNoteLm/pkg/cache" - bizerrors "YoudaoNoteLm/pkg/errors" - "YoudaoNoteLm/pkg/logger" - "YoudaoNoteLm/pkg/utils" - - "github.com/cloudwego/eino/components/model" - "github.com/cloudwego/eino/schema" - "github.com/google/uuid" - "go.uber.org/zap" -) - -var allowedFileTypes = map[string]bool{ - ".txt": true, ".md": true, ".docx": true, ".pdf": true, ".pptx": true, -} - -var allowedAudioTypes = map[string]bool{ - ".mp3": true, ".wav": true, -} - -const maxFileSize int64 = 30 << 20 // 30MB -const maxAudioSize int64 = 300 << 20 // 300MB - -type importerService struct { - configSvc ConfigService - markitdown externalMarkitdown.Client - storage storage.FileStorage - sourceRepo repository.SourceRepository - importCache *cache.ImportTaskCache - previewCache *cache.AudioPreviewCache - ingestionSvc rag.IngestionService - structurer MarkdownStructurer // LLM 结构化服务 - summaryCache *cache.SourceSummaryCache - cancelFuncs sync.Map // taskID -> context.CancelFunc,用于中止运行中的任务 -} - -// NewImporterService 创建导入服务 -func NewImporterService( - configSvc ConfigService, - markitdown externalMarkitdown.Client, - storage storage.FileStorage, - sourceRepo repository.SourceRepository, - importCache *cache.ImportTaskCache, - previewCache *cache.AudioPreviewCache, - ingestionSvc rag.IngestionService, - structurer MarkdownStructurer, - summaryCache *cache.SourceSummaryCache, -) ImporterService { - return &importerService{ - markitdown: markitdown, - configSvc: configSvc, - storage: storage, - sourceRepo: sourceRepo, - importCache: importCache, - previewCache: previewCache, - ingestionSvc: ingestionSvc, - structurer: structurer, - summaryCache: summaryCache, - } -} - -// ImportFile 文件上传导入(异步:立即创建 source,后台处理解析和入库) -func (s *importerService) ImportFile(userID, notebookID uint, file *multipart.FileHeader) (*entity.Source, error) { - ext := strings.ToLower(filepath.Ext(file.Filename)) - if !allowedFileTypes[ext] { - return nil, bizerrors.ErrUnsupportedFormat - } - if file.Size > maxFileSize { - return nil, bizerrors.ErrFileTooLarge - } - - logger.Info("开始文件导入", - zap.String("file", file.Filename), - zap.Int64("size", file.Size), - zap.Uint("user_id", userID), - ) - - // 上传到 MinIO 存储(必须同步,拿到 filePath) - filePath, err := s.storage.Upload(file) - if err != nil { - logger.Error("文件上传到存储服务失败", - zap.String("file", file.Filename), - zap.Int64("size", file.Size), - zap.Error(err), - ) - return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "文件上传失败", err) - } - - // 立即创建 source(status=processing),前端可以马上看到 - source := &entity.Source{ - UserID: userID, - NotebookID: notebookID, - Name: file.Filename, - Type: "file", - FilePath: filePath, - FileSize: file.Size, - MimeType: file.Header.Get("Content-Type"), - Status: "processing", - } - - if err := s.sourceRepo.Create(source); err != nil { - logger.Error("创建 Source 记录失败", - zap.String("file", file.Filename), - zap.Error(err), - ) - return nil, err - } - - logger.Info("Source 记录创建成功,后台开始处理", - zap.String("file", file.Filename), - zap.Uint("source_id", source.ID), - ) - - // 读取文件内容(后台 goroutine 需要,必须在 goroutine 外读取,避免 file 指针失效) - src, err := file.Open() - if err != nil { - s.sourceRepo.UpdateStatus(source.ID, "failed", "打开上传文件失败") - return source, nil - } - fileBytes, err := io.ReadAll(src) - src.Close() - if err != nil { - s.sourceRepo.UpdateStatus(source.ID, "failed", "读取上传文件失败") - return source, nil - } - - // 后台异步处理:MarkItDown → LLM 结构化 → 更新内容 → RAG 入库 - go s.processFileImport(source.ID, file.Filename, ext, filePath, file.Header.Get("Content-Type"), file.Size, userID, fileBytes) - - return source, nil -} - -// processFileImport 后台处理文件导入(解析、结构化、入库) -func (s *importerService) processFileImport(sourceID uint, fileName, ext, filePath, mimeType string, fileSize int64, userID uint, fileBytes []byte) { - totalStart := time.Now() - logger.Info("后台开始处理文件导入", - zap.String("file", fileName), - zap.Uint("source_id", sourceID), - zap.Int64("file_size", fileSize), - ) - - // 1. MarkItDown 转换 - stepStart := time.Now() - markdown, err := s.markitdown.ConvertReader(fileName, bytes.NewReader(fileBytes)) - if err != nil { - logger.Error("MarkItDown 转换失败", - zap.String("file", fileName), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - // 降级:对于文本文件,直接使用原始内容 - if ext == ".txt" || ext == ".md" { - markdown = string(fileBytes) - logger.Info("文本文件降级处理,使用原始内容", - zap.String("file", fileName), - zap.Int("content_len", len(markdown)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } else { - s.sourceRepo.UpdateStatus(sourceID, "failed", "文件解析失败") - return - } - } else { - logger.Info("MarkItDown 转换成功", - zap.String("file", fileName), - zap.Int("content_len", len(markdown)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } - - // 2. LLM 结构化 - stepStart = time.Now() - if s.structurer != nil { - result, err := s.structurer.Structure(context.Background(), userID, markdown, StructureMeta{ - Title: fileName, - SourceType: "file", - }) - if err != nil { - logger.Error("LLM 结构化失败,使用原始内容", - zap.String("file", fileName), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - } else if result.ActuallyCalled { - markdown = result.Content - logger.Info("LLM 结构化完成", - zap.String("file", fileName), - zap.Int("content_len", len(markdown)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } else { - logger.Warn("LLM 结构化被跳过(模型配置问题或 API Key 过期)", - zap.String("file", fileName), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } - } else { - logger.Warn("MarkdownStructurer 未配置,跳过结构化", zap.String("file", fileName)) - } - - // 3. 更新 source 内容和状态 - stepStart = time.Now() - if err := s.sourceRepo.UpdateContent(sourceID, markdown, "ready"); err != nil { - logger.Error("更新 Source 内容失败", - zap.String("file", fileName), - zap.Uint("source_id", sourceID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("保存失败: %v", err)) - return - } - - logger.Info("Source 内容更新成功", - zap.String("file", fileName), - zap.Uint("source_id", sourceID), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - // 4. RAG 入库 - stepStart = time.Now() - if s.ingestionSvc != nil { - if err := s.ingestionSvc.IngestSingle(context.Background(), sourceID); err != nil { - logger.Error("RAG 入库失败", - zap.String("file", fileName), - zap.Uint("source_id", sourceID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - // RAG 入库失败不影响 source 可见性,只记录日志 - return - } - logger.Info("RAG 入库成功", - zap.String("file", fileName), - zap.Uint("source_id", sourceID), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } - - // 5. 生成摘要(异步,不阻塞主流程) - go s.generateAndSaveSummary(sourceID, userID, markdown) - - logger.Info("文件导入完成", - zap.String("file", fileName), - zap.Uint("source_id", sourceID), - zap.Duration("total_elapsed", time.Since(totalStart)), - ) -} - -// PreviewAudio 异步音频转写:上传文件后立即返回 previewID,后台执行 ASR 转写 -func (s *importerService) PreviewAudio(userID, notebookID uint, file *multipart.FileHeader) (string, string, error) { - ext := strings.ToLower(filepath.Ext(file.Filename)) - if !allowedAudioTypes[ext] { - return "", "", bizerrors.ErrUnsupportedFormat - } - if file.Size > maxAudioSize { - return "", "", bizerrors.ErrFileTooLarge - } - - // 上传原始文件到 MinIO - filePath, err := s.storage.Upload(file) - if err != nil { - logger.Error("音频上传到存储服务失败", - zap.String("file", file.Filename), - zap.Int64("size", file.Size), - zap.Error(err), - ) - return "", "", bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "音频上传失败", err) - } - - previewID := uuid.New().String() - preview := &cache.AudioPreview{ - PreviewID: previewID, - UserID: userID, - NotebookID: notebookID, - FileName: file.Filename, - FilePath: filePath, - FileSize: file.Size, - Status: "pending", - ExpiresAt: time.Now().Add(30 * time.Minute).Unix(), - } - - ctx := context.Background() - if err := s.previewCache.Save(ctx, preview); err != nil { - return "", "", err - } - - // 后台异步执行 ASR 转写 - go s.doAudioTranscribe(previewID, userID, file, filePath, ext) - - return previewID, file.Filename, nil -} - -// doAudioTranscribe 后台执行音频转写,完成后更新缓存 -func (s *importerService) doAudioTranscribe(previewID string, userID uint, file *multipart.FileHeader, filePath, ext string) { - totalStart := time.Now() - ctx := context.Background() - - // 标记为处理中 - if err := s.previewCache.UpdateStatus(ctx, previewID, "processing"); err != nil { - logger.Error("更新预览状态为processing失败", zap.String("preview_id", previewID), zap.Error(err)) - return - } - - // 使用 ffmpeg 流式转换为 16kHz 单声道 WAV(内存占用低,支持各种格式) - asrFilePath := filePath - convertedPath, convertErr := s.convertAudioWithFFMPEG(filePath, ext) - if convertErr != nil { - logger.Warn("ffmpeg音频转换失败,使用原始文件", - zap.String("file", filePath), - zap.Error(convertErr), - ) - } else { - asrFilePath = convertedPath - logger.Info("音频已通过ffmpeg转换为16kHz单声道WAV", - zap.String("original", filePath), - zap.String("converted", asrFilePath), - ) - } - - // 获取 ASR 服务 - stepStart := time.Now() - asrSvc, err := s.getASR(userID) - if err != nil { - logger.Error("获取ASR服务失败", - zap.String("preview_id", previewID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - s.markPreviewFailed(ctx, previewID, "未配置 ASR 服务") - return - } - logger.Info("获取 ASR 服务完成", - zap.String("preview_id", previewID), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - // 执行转写 - stepStart = time.Now() - logger.Info("开始 ASR 转写", - zap.String("preview_id", previewID), - zap.String("asr_file", asrFilePath), - ) - text, err := asrSvc.Transcribe(asrFilePath) - if err != nil { - logger.Error("ASR转写失败", - zap.String("preview_id", previewID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - s.markPreviewFailed(ctx, previewID, fmt.Sprintf("音频转写失败: %v", err)) - return - } - logger.Info("ASR 转写完成", - zap.String("preview_id", previewID), - zap.Int("text_len", len(text)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - // 转写成功,更新缓存 - preview, err := s.previewCache.Get(ctx, previewID) - if err != nil || preview == nil { - logger.Error("转写完成但获取预览缓存失败", zap.String("preview_id", previewID), zap.Error(err)) - return - } - preview.TranscribedText = text - preview.Status = "ready" - if err := s.previewCache.Save(ctx, preview); err != nil { - logger.Error("保存转写结果失败", zap.String("preview_id", previewID), zap.Error(err)) - return - } - - logger.Info("音频转写流程完成", - zap.String("preview_id", previewID), - zap.Int("text_len", len(text)), - zap.Duration("total_elapsed", time.Since(totalStart)), - ) -} - -// markPreviewFailed 标记预览转写失败 -func (s *importerService) markPreviewFailed(ctx context.Context, previewID, errMsg string) { - preview, err := s.previewCache.Get(ctx, previewID) - if err != nil || preview == nil { - return - } - preview.Status = "failed" - preview.ErrorMsg = errMsg - if saveErr := s.previewCache.Save(ctx, preview); saveErr != nil { - logger.Error("保存预览失败状态出错", zap.String("preview_id", previewID), zap.Error(saveErr)) - } -} - -// GetAudioPreviewStatus 查询音频预览状态(前端轮询用) -func (s *importerService) GetAudioPreviewStatus(userID uint, previewID string) (interface{}, error) { - ctx := context.Background() - preview, err := s.previewCache.Get(ctx, previewID) - if err != nil { - return nil, bizerrors.ErrNotFound - } - if preview == nil { - return nil, bizerrors.ErrNotFound - } - if preview.UserID != userID { - return nil, bizerrors.ErrForbidden - } - return preview, nil -} - -// ConfirmAudio 确认音频导入 -func (s *importerService) ConfirmAudio(userID uint, previewID string, editedContent *string) (*entity.Source, error) { - totalStart := time.Now() - - ctx := context.Background() - preview, err := s.previewCache.Get(ctx, previewID) - if err != nil { - return nil, bizerrors.ErrNotFound - } - if preview == nil { - return nil, bizerrors.ErrNotFound - } - if preview.UserID != userID { - return nil, bizerrors.ErrForbidden - } - if time.Now().Unix() > preview.ExpiresAt { - return nil, bizerrors.ErrPreviewExpired - } - if preview.Status == "failed" { - return nil, bizerrors.New(bizerrors.CodeASTranscriptionFailed, preview.ErrorMsg) - } - if preview.Status != "ready" { - return nil, bizerrors.New(bizerrors.CodeBadRequest, "音频转写尚未完成,请稍后再试") - } - - logger.Info("开始确认音频导入", - zap.String("preview_id", previewID), - zap.String("file_name", preview.FileName), - zap.Uint("user_id", userID), - ) - - content := preview.TranscribedText - if editedContent != nil && *editedContent != "" { - content = *editedContent - logger.Info("使用用户编辑后的内容", - zap.String("preview_id", previewID), - zap.Int("content_len", len(content)), - ) - } else { - logger.Info("使用 ASR 转写结果", - zap.String("preview_id", previewID), - zap.Int("content_len", len(content)), - ) - } - - // LLM 结构化 - stepStart := time.Now() - if s.structurer != nil { - result, err := s.structurer.Structure(ctx, userID, content, StructureMeta{ - Title: preview.FileName, - SourceType: "audio", - }) - if err != nil { - logger.Error("LLM 结构化失败,使用原始内容", - zap.String("preview_id", previewID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - } else if result.ActuallyCalled && result.Content != content { - logger.Info("LLM 结构化成功,内容已优化", - zap.String("preview_id", previewID), - zap.Int("original_len", len(content)), - zap.Int("structured_len", len(result.Content)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - content = result.Content - } else if result.ActuallyCalled { - logger.Info("LLM 判断内容已有结构,无需结构化", - zap.String("preview_id", previewID), - zap.Int("content_len", len(content)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } else { - logger.Warn("LLM 结构化被跳过(模型配置问题或 API Key 过期)", - zap.String("preview_id", previewID), - zap.Int("content_len", len(content)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } - } else { - logger.Warn("MarkdownStructurer 未配置,跳过结构化", zap.String("preview_id", previewID)) - } - - // 创建 Source 记录 - stepStart = time.Now() - source := &entity.Source{ - UserID: userID, - NotebookID: preview.NotebookID, - Name: preview.FileName, - Type: "audio", - FilePath: preview.FilePath, - FileSize: preview.FileSize, - MarkdownContent: content, - Status: "ready", - } - - if err := s.sourceRepo.Create(source); err != nil { - logger.Error("创建 Source 记录失败", - zap.String("preview_id", previewID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - return nil, err - } - - logger.Info("Source 记录创建成功", - zap.String("preview_id", previewID), - zap.Uint("source_id", source.ID), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - // 同步触发 RAG 入库 - stepStart = time.Now() - if s.ingestionSvc != nil { - if err := s.ingestionSvc.IngestSingle(context.Background(), source.ID); err != nil { - logger.Error("RAG 入库失败", - zap.String("preview_id", previewID), - zap.Uint("source_id", source.ID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "RAG 入库失败", err) - } - logger.Info("RAG 入库成功", - zap.String("preview_id", previewID), - zap.Uint("source_id", source.ID), - zap.Duration("elapsed", time.Since(stepStart)), - ) - source.Vectorized = true - } - - // 生成摘要(异步,不阻塞主流程) - go s.generateAndSaveSummary(source.ID, userID, content) - - if err := s.previewCache.UpdateStatus(ctx, previewID, "confirmed"); err != nil { - logger.Warn("更新预览状态失败", zap.String("preview_id", previewID), zap.Error(err)) - } - - logger.Info("音频导入确认完成", - zap.String("preview_id", previewID), - zap.String("file_name", preview.FileName), - zap.Uint("source_id", source.ID), - zap.Duration("total_elapsed", time.Since(totalStart)), - ) - - return source, nil -} - -// convertAudioForASR 转换音频为 ASR 兼容格式 -// 如果已经是 16kHz 单声道则返回 nil(无需转换) -func (s *importerService) convertAudioForASR(file *multipart.FileHeader, ext string) ([]byte, error) { - // 读取文件内容 - src, err := file.Open() - if err != nil { - return nil, fmt.Errorf("打开音频文件失败: %w", err) - } - defer func(src multipart.File) { - err := src.Close() - if err != nil { - logger.Errorf("关闭文件失败:%s", err) - } - }(src) - - audioData, err := io.ReadAll(src) - if err != nil { - return nil, fmt.Errorf("读取音频文件失败: %w", err) - } - - // 转换为 16kHz 单声道 WAV - converted, err := utils.ConvertBytesToASRFormat(audioData, ext) - if err != nil { - return nil, fmt.Errorf("音频转换失败: %w", err) - } - - return converted, nil -} - -// convertAudioWithFFMPEG 使用 ffmpeg 流式转换音频为 16kHz 单声道 WAV -// 从 MinIO 下载 → ffmpeg 转换 → 上传回 MinIO,全程流式处理,内存占用低 -func (s *importerService) convertAudioWithFFMPEG(filePath, ext string) (string, error) { - // 1. 下载原始文件到临时文件 - srcData, err := s.storage.Download(filePath) - if err != nil { - return "", fmt.Errorf("下载原始音频失败: %w", err) - } - - tmpInput, err := os.CreateTemp("", "asr-input-*"+ext) - if err != nil { - return "", fmt.Errorf("创建临时输入文件失败: %w", err) - } - defer os.Remove(tmpInput.Name()) - defer tmpInput.Close() - - if _, err := tmpInput.Write(srcData); err != nil { - return "", fmt.Errorf("写入临时输入文件失败: %w", err) - } - tmpInput.Close() - - // 2. ffmpeg 转换为 16kHz 单声道 WAV - tmpOutput := tmpInput.Name() + "_16k.wav" - defer os.Remove(tmpOutput) - - cmd := exec.Command("ffmpeg", "-y", "-i", tmpInput.Name(), - "-ar", "16000", "-ac", "1", "-sample_fmt", "s16", - "-f", "wav", tmpOutput) - var stderr bytes.Buffer - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("ffmpeg转换失败: %w, stderr: %s", err, stderr.String()) - } - - // 3. 读取转换后的文件 - convertedData, err := os.ReadFile(tmpOutput) - if err != nil { - return "", fmt.Errorf("读取转换后文件失败: %w", err) - } - - // 4. 上传到 MinIO - convertedPath := filePath[:len(filePath)-len(filepath.Ext(filePath))] + "_16k.wav" - if err := s.storage.UploadBytes(convertedPath, convertedData, "audio/wav"); err != nil { - return "", fmt.Errorf("上传转换后音频失败: %w", err) - } - - return convertedPath, nil -} - -// ImportSearchResults 批量导入搜索结果 -// 为每个 URL 先创建 pending 状态的 Source 记录,然后异步处理 -// 返回创建的 Source ID 列表,前端可通过 Source 列表 API 查看每条的独立状态 -func (s *importerService) ImportSearchResults(userID, notebookID uint, items []SearchResultItem) (string, []uint, error) { - // 去重:同一个 URL 只创建一条记录(保留第一次出现的标题) - seen := make(map[string]string, len(items)) // url -> title - for _, item := range items { - if _, exists := seen[item.URL]; !exists { - seen[item.URL] = item.Title - } - } - - sourceIDs := make([]uint, 0, len(seen)) - - // 为每个 URL 创建 pending 状态的 Source - for url, title := range seen { - // 如果标题为空,使用 URL 作为标题 - name := title - if name == "" { - name = url - } - - source := &entity.Source{ - UserID: userID, - NotebookID: notebookID, - Name: name, - Type: "url", - OriginalURL: url, - Status: "pending", - } - if err := s.sourceRepo.Create(source); err != nil { - logger.Error("创建待导入Source失败", zap.String("url", url), zap.Error(err)) - continue - } - sourceIDs = append(sourceIDs, source.ID) - } - - if len(sourceIDs) == 0 { - return "", nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "创建导入记录失败", nil) - } - - // 创建可取消的 context,注册 cancel func 以便批量取消 - // 设置整体超时:每个 URL 最多 2 分钟,整体最多 10 分钟 - taskID := uuid.New().String() - maxTimeout := 10 * time.Minute - urlTimeout := time.Duration(len(seen)) * 2 * time.Minute - if urlTimeout > maxTimeout { - urlTimeout = maxTimeout - } - taskCtx, cancel := context.WithTimeout(context.Background(), urlTimeout) - s.cancelFuncs.Store(taskID, cancel) - - // 异步处理每个 Source - go s.processSources(taskCtx, taskID, sourceIDs) - - return taskID, sourceIDs, nil -} - -// processSources 异步处理 Source 列表(带并发控制,支持取消) -func (s *importerService) processSources(taskCtx context.Context, taskID string, sourceIDs []uint) { - // 任务结束后清理 cancel func - defer s.cancelFuncs.Delete(taskID) - - // 并发控制:最多同时处理 3 个 - concurrency := 3 - if len(sourceIDs) < concurrency { - concurrency = len(sourceIDs) - } - - idCh := make(chan uint, concurrency) - doneCh := make(chan struct{}, len(sourceIDs)) - - // 启动 worker - for i := 0; i < concurrency; i++ { - go func() { - for sourceID := range idCh { - if taskCtx.Err() != nil { - doneCh <- struct{}{} - continue - } - s.processSingleSource(taskCtx, sourceID) - doneCh <- struct{}{} - } - }() - } - - // 分发任务(支持取消中断分发) - go func() { - for _, sourceID := range sourceIDs { - if taskCtx.Err() != nil { - break - } - idCh <- sourceID - } - close(idCh) - }() - - // 等待所有任务完成 - for i := 0; i < len(sourceIDs); i++ { - <-doneCh - } - - // 将仍然处于 pending 状态的 Source 标记为 cancelled(被取消的任务) - if taskCtx.Err() != nil { - for _, sourceID := range sourceIDs { - src, err := s.sourceRepo.FindByID(sourceID) - if err != nil || src == nil { - continue - } - if src.Status == "pending" { - if err := s.sourceRepo.UpdateStatus(sourceID, "cancelled", "任务已取消"); err != nil { - logger.Warn("更新Source状态为cancelled失败", zap.Uint("source_id", sourceID), zap.Error(err)) - } - } - } - } -} - -// processSingleSource 处理单个 Source(支持取消) -func (s *importerService) processSingleSource(taskCtx context.Context, sourceID uint) { - totalStart := time.Now() - - // 处理前检查取消 - if taskCtx.Err() != nil { - return - } - - // 获取 Source 记录 - source, err := s.sourceRepo.FindByID(sourceID) - if err != nil || source == nil { - logger.Error("获取Source失败", zap.Uint("source_id", sourceID), zap.Error(err)) - return - } - - logger.Info("开始处理 URL 导入", - zap.Uint("source_id", sourceID), - zap.String("url", source.OriginalURL), - ) - - // 更新状态为 processing - if err := s.sourceRepo.UpdateStatus(sourceID, "processing", ""); err != nil { - logger.Warn("更新Source状态为processing失败", zap.Uint("source_id", sourceID), zap.Error(err)) - } - - // 转换 URL 内容 - stepStart := time.Now() - markdown, err := s.markitdown.ConvertFromURLWithContext(taskCtx, source.OriginalURL) - if err != nil { - // 如果是因为取消导致的错误 - if taskCtx.Err() != nil { - logger.Info("任务已取消,跳过Source处理", zap.Uint("source_id", sourceID)) - return - } - - // 处理结构化错误,返回用户友好的错误信息 - var userMsg string - var convertErr *externalMarkitdown.ConvertError - if errors.As(err, &convertErr) { - // 记录详细的技术错误信息到日志 - logger.Error("URL 转换失败", - zap.Uint("source_id", sourceID), - zap.String("url", source.OriginalURL), - zap.String("error_code", convertErr.Code), - zap.String("detail", convertErr.DetailMsg), - zap.Int("http_status", convertErr.HTTPStatus), - zap.Duration("elapsed", time.Since(stepStart)), - ) - // 使用用户友好的错误消息 - userMsg = convertErr.UserMsg - } else { - // 未知错误类型 - logger.Error("URL 转换失败", - zap.Uint("source_id", sourceID), - zap.String("url", source.OriginalURL), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - userMsg = "网页内容获取失败,请稍后重试" - } - - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", userMsg); updateErr != nil { - logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - - logger.Info("URL 转换成功", - zap.Uint("source_id", sourceID), - zap.String("url", source.OriginalURL), - zap.Int("content_len", len(markdown)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - // 转换完成后再检查一次 source 是否还存在(可能在转换期间被用户删除) - existing, _ := s.sourceRepo.FindByID(sourceID) - if existing == nil { - logger.Info("Source已被删除,丢弃转换结果", zap.Uint("source_id", sourceID)) - return - } - - // LLM 结构化 - stepStart = time.Now() - if s.structurer != nil { - result, err := s.structurer.Structure(taskCtx, source.UserID, markdown, StructureMeta{ - Title: source.Name, - SourceType: "url", - }) - if err != nil { - logger.Error("LLM 结构化失败,使用原始内容", - zap.Uint("source_id", sourceID), - zap.String("url", source.OriginalURL), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - } else if result.ActuallyCalled && result.Content != markdown { - logger.Info("LLM 结构化成功,内容已优化", - zap.Uint("source_id", sourceID), - zap.Int("original_len", len(markdown)), - zap.Int("structured_len", len(result.Content)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - markdown = result.Content - } else if result.ActuallyCalled { - logger.Info("LLM 判断内容已有结构,无需结构化", - zap.Uint("source_id", sourceID), - zap.Int("content_len", len(markdown)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } else { - logger.Warn("LLM 结构化被跳过(模型配置问题或 API Key 过期)", - zap.Uint("source_id", sourceID), - zap.Int("content_len", len(markdown)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } - } else { - logger.Warn("MarkdownStructurer 未配置,跳过结构化", zap.Uint("source_id", sourceID)) - } - - // 更新 Source 内容和状态为 ready - stepStart = time.Now() - source.MarkdownContent = markdown - source.Status = "ready" - if err := s.sourceRepo.Update(source); err != nil { - logger.Error("更新Source内容失败", zap.Uint("source_id", sourceID), zap.Duration("elapsed", time.Since(stepStart)), zap.Error(err)) - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("保存失败: %v", err)); updateErr != nil { - logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - - logger.Info("Source 记录更新成功", - zap.Uint("source_id", sourceID), - zap.String("url", source.OriginalURL), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - // 同步触发 RAG 入库 - stepStart = time.Now() - if s.ingestionSvc != nil { - if err := s.ingestionSvc.IngestSingle(taskCtx, sourceID); err != nil { - logger.Error("RAG 入库失败", - zap.Uint("source_id", sourceID), - zap.String("url", source.OriginalURL), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("RAG 入库失败: %v", err)); updateErr != nil { - logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - logger.Info("RAG 入库成功", - zap.Uint("source_id", sourceID), - zap.String("url", source.OriginalURL), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } - - // 生成摘要(异步,不阻塞主流程) - go s.generateAndSaveSummary(sourceID, source.UserID, markdown) - - logger.Info("URL 导入完成", - zap.Uint("source_id", sourceID), - zap.String("url", source.OriginalURL), - zap.Duration("total_elapsed", time.Since(totalStart)), - ) -} - -// GetImportTask 获取导入任务状态 -func (s *importerService) GetImportTask(taskID string) (interface{}, error) { - ctx := context.Background() - task, err := s.importCache.Get(ctx, taskID) - if err != nil { - return nil, bizerrors.ErrNotFound - } - if task == nil { - return nil, bizerrors.ErrNotFound - } - return task, nil -} - -// DeleteImportTask 删除/取消导入任务 -func (s *importerService) DeleteImportTask(taskID string) error { - ctx := context.Background() - - // 1. 尝试从 cancelFuncs 中取消正在运行的异步任务(新架构:Source-based 导入) - if cancel, ok := s.cancelFuncs.Load(taskID); ok { - cancel.(context.CancelFunc)() - s.cancelFuncs.Delete(taskID) - logger.Info("已发送取消信号给运行中的导入任务", zap.String("task_id", taskID)) - return nil - } - - // 2. 尝试从 importCache 中查找(旧架构:Redis-based 任务) - task, err := s.importCache.Get(ctx, taskID) - if err != nil { - return bizerrors.ErrNotFound - } - if task == nil { - return bizerrors.ErrNotFound - } - - // 如果任务正在运行中,标记为取消状态 - if task.Status == "running" { - task.Status = "cancelled" - if err := s.importCache.Save(ctx, task); err != nil { - logger.Warn("更新任务状态为取消失败", zap.String("task_id", taskID), zap.Error(err)) - } - } - - // 删除任务缓存 - return s.importCache.Delete(ctx, taskID) -} - -// getASR 获取 ASR 服务(从 ConfigService 动态加载) -func (s *importerService) getASR(userID uint) (asr.ASRService, error) { - if s.configSvc == nil { - return nil, fmt.Errorf("ConfigService 未初始化") - } - return s.configSvc.GetASRService(userID) -} - -// summarySystemPrompt 摘要生成的系统提示词 -const summarySystemPrompt = `你是一个资料摘要助手。请为以下文档内容生成一份简洁的摘要。 - -要求: -1. 摘要长度:200-400字 -2. 涵盖文档的核心主题、主要观点和关键信息 -3. 使用中文 -4. 保持客观,不添加个人评价 -5. 直接输出摘要内容,不要加任何前缀或解释` - -// generateAndSaveSummary 生成资料摘要并保存到 MySQL 和 Redis(importerService 的方法) -func (s *importerService) generateAndSaveSummary(sourceID uint, userID uint, content string) { - doGenerateAndSaveSummary(s.sourceRepo, s.configSvc, s.summaryCache, sourceID, userID, content) -} - -// fallbackSummaryLength 降级摘要的最大字符数 -const fallbackSummaryLength = 300 - -// doGenerateAndSaveSummary 生成资料摘要的包级别共享实现 -// LLM 失败时自动降级为截取内容前 N 个字符作为兜底摘要 -func doGenerateAndSaveSummary( - sourceRepo repository.SourceRepository, - configSvc ConfigService, - summaryCache *cache.SourceSummaryCache, - sourceID uint, userID uint, content string, -) { - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - startTime := time.Now() - - summary, usedFallback := tryGenerateWithLLM(ctx, configSvc, userID, content) - if usedFallback { - // LLM 失败,使用降级摘要 - summary = buildFallbackSummary(content) - logger.Warn("LLM 摘要生成失败,使用降级摘要", - zap.Uint("source_id", sourceID), - zap.Int("fallback_len", len(summary)), - ) - } - - if summary == "" { - logger.Warn("摘要生成失败且内容为空,跳过", - zap.Uint("source_id", sourceID), - ) - return - } - - // 保存到 MySQL - if err := sourceRepo.UpdateSummary(sourceID, summary); err != nil { - logger.Error("保存摘要到 MySQL 失败", - zap.Uint("source_id", sourceID), - zap.Error(err), - ) - return - } - - // 保存到 Redis - if summaryCache != nil { - if err := summaryCache.Set(ctx, sourceID, summary); err != nil { - logger.Warn("保存摘要到 Redis 失败", - zap.Uint("source_id", sourceID), - zap.Error(err), - ) - } - } - - logger.Info("资料摘要生成完成", - zap.Uint("source_id", sourceID), - zap.Int("summary_len", len(summary)), - zap.Bool("fallback", usedFallback), - zap.Duration("elapsed", time.Since(startTime)), - ) -} - -// tryGenerateWithLLM 尝试用 LLM 生成摘要,返回 (摘要内容, 是否需要降级) -func tryGenerateWithLLM(ctx context.Context, configSvc ConfigService, userID uint, content string) (string, bool) { - chatModel, err := getChatModelForSummary(ctx, configSvc, userID) - if err != nil || chatModel == nil { - return "", true - } - - userMsg := fmt.Sprintf("请为以下文档生成摘要:\n\n%s", content) - msg, err := chatModel.Generate(ctx, []*schema.Message{ - schema.SystemMessage(summarySystemPrompt), - schema.UserMessage(userMsg), - }, model.WithMaxTokens(1024)) - if err != nil { - return "", true - } - if msg == nil || strings.TrimSpace(msg.Content) == "" { - return "", true - } - - return strings.TrimSpace(msg.Content), false -} - -// buildFallbackSummary 从内容中提取降级摘要 、截取前 fallbackSummaryLength 个字符,尝试在句子边界截断 -func buildFallbackSummary(content string) string { - content = strings.TrimSpace(content) - if content == "" { - return "" - } - - runes := []rune(content) - if len(runes) <= fallbackSummaryLength { - return content - } - - // 截取前 N 个字符,尝试在句号、换行处断开 - truncated := runes[:fallbackSummaryLength] - cutPoints := []rune{'。', '\n', ';', '!', '?', '.', '!', '?'} - bestCut := fallbackSummaryLength - for i := fallbackSummaryLength - 1; i >= fallbackSummaryLength/2; i-- { - for _, cp := range cutPoints { - if truncated[i] == cp { - bestCut = i + 1 - break - } - } - if bestCut != fallbackSummaryLength { - break - } - } - - return string(runes[:bestCut]) + "..." -} - -// getChatModelForSummary 获取用于生成摘要的 ChatModel(包级别共享函数) -func getChatModelForSummary(ctx context.Context, configSvc ConfigService, userID uint) (model.ToolCallingChatModel, error) { - llmConfig, err := configSvc.GetUserLLMConfig(userID) - if err != nil { - return nil, fmt.Errorf("获取 LLM 配置失败: %w", err) - } - if llmConfig == nil || !llmConfig.Enabled { - return nil, nil - } - - chatModel, err := llm.NewChatModel(ctx, llmConfig) - if err != nil { - return nil, fmt.Errorf("创建 ChatModel 失败: %w", err) - } - return chatModel, nil -} +package service + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "mime/multipart" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "YoudaoNoteLm/internal/llm" + "YoudaoNoteLm/internal/model/entity" + "YoudaoNoteLm/internal/rag" + "YoudaoNoteLm/internal/repository" + "YoudaoNoteLm/internal/service/external/asr" + externalMarkitdown "YoudaoNoteLm/internal/service/external/markitdown" + "YoudaoNoteLm/internal/service/external/storage" + "YoudaoNoteLm/pkg/cache" + bizerrors "YoudaoNoteLm/pkg/errors" + "YoudaoNoteLm/pkg/logger" + "YoudaoNoteLm/pkg/utils" + + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + "github.com/google/uuid" + "go.uber.org/zap" +) + +var allowedFileTypes = map[string]bool{ + ".txt": true, ".md": true, ".docx": true, ".pdf": true, ".pptx": true, +} + +var allowedAudioTypes = map[string]bool{ + ".mp3": true, ".wav": true, +} + +const maxFileSize int64 = 30 << 20 // 30MB +const maxAudioSize int64 = 300 << 20 // 300MB + +type importerService struct { + configSvc ConfigService + markitdown externalMarkitdown.Client + storage storage.FileStorage + sourceRepo repository.SourceRepository + importCache *cache.ImportTaskCache + previewCache *cache.AudioPreviewCache + ingestionSvc rag.IngestionService + structurer MarkdownStructurer // LLM 结构化服务 + summaryCache *cache.SourceSummaryCache + cancelFuncs sync.Map // taskID -> context.CancelFunc,用于中止运行中的任务 +} + +// NewImporterService 创建导入服务 +func NewImporterService( + configSvc ConfigService, + markitdown externalMarkitdown.Client, + storage storage.FileStorage, + sourceRepo repository.SourceRepository, + importCache *cache.ImportTaskCache, + previewCache *cache.AudioPreviewCache, + ingestionSvc rag.IngestionService, + structurer MarkdownStructurer, + summaryCache *cache.SourceSummaryCache, +) ImporterService { + return &importerService{ + markitdown: markitdown, + configSvc: configSvc, + storage: storage, + sourceRepo: sourceRepo, + importCache: importCache, + previewCache: previewCache, + ingestionSvc: ingestionSvc, + structurer: structurer, + summaryCache: summaryCache, + } +} + +// ImportFile 文件上传导入(异步:立即创建 source,后台处理解析和入库) +func (s *importerService) ImportFile(userID, notebookID uint, file *multipart.FileHeader) (*entity.Source, error) { + ext := strings.ToLower(filepath.Ext(file.Filename)) + if !allowedFileTypes[ext] { + return nil, bizerrors.ErrUnsupportedFormat + } + if file.Size > maxFileSize { + return nil, bizerrors.ErrFileTooLarge + } + + logger.Info("开始文件导入", + zap.String("file", file.Filename), + zap.Int64("size", file.Size), + zap.Uint("user_id", userID), + ) + + // 上传到 MinIO 存储(必须同步,拿到 filePath) + filePath, err := s.storage.Upload(file) + if err != nil { + logger.Error("文件上传到存储服务失败", + zap.String("file", file.Filename), + zap.Int64("size", file.Size), + zap.Error(err), + ) + return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "文件上传失败", err) + } + + // 立即创建 source(status=processing),前端可以马上看到 + source := &entity.Source{ + UserID: userID, + NotebookID: notebookID, + Name: file.Filename, + Type: "file", + FilePath: filePath, + FileSize: file.Size, + MimeType: file.Header.Get("Content-Type"), + Status: "processing", + } + + if err := s.sourceRepo.Create(source); err != nil { + logger.Error("创建 Source 记录失败", + zap.String("file", file.Filename), + zap.Error(err), + ) + return nil, err + } + + logger.Info("Source 记录创建成功,后台开始处理", + zap.String("file", file.Filename), + zap.Uint("source_id", source.ID), + ) + + // 读取文件内容(后台 goroutine 需要,必须在 goroutine 外读取,避免 file 指针失效) + src, err := file.Open() + if err != nil { + s.sourceRepo.UpdateStatus(source.ID, "failed", "打开上传文件失败") + return source, nil + } + fileBytes, err := io.ReadAll(src) + src.Close() + if err != nil { + s.sourceRepo.UpdateStatus(source.ID, "failed", "读取上传文件失败") + return source, nil + } + + // 后台异步处理:MarkItDown → LLM 结构化 → 更新内容 → RAG 入库 + go s.processFileImport(source.ID, file.Filename, ext, filePath, file.Header.Get("Content-Type"), file.Size, userID, fileBytes) + + return source, nil +} + +// processFileImport 后台处理文件导入(解析、结构化、入库) +func (s *importerService) processFileImport(sourceID uint, fileName, ext, filePath, mimeType string, fileSize int64, userID uint, fileBytes []byte) { + totalStart := time.Now() + logger.Info("后台开始处理文件导入", + zap.String("file", fileName), + zap.Uint("source_id", sourceID), + zap.Int64("file_size", fileSize), + ) + + // 1. MarkItDown 转换 + stepStart := time.Now() + markdown, err := s.markitdown.ConvertReader(fileName, bytes.NewReader(fileBytes)) + if err != nil { + logger.Error("MarkItDown 转换失败", + zap.String("file", fileName), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + // 降级:对于文本文件,直接使用原始内容 + if ext == ".txt" || ext == ".md" { + markdown = string(fileBytes) + logger.Info("文本文件降级处理,使用原始内容", + zap.String("file", fileName), + zap.Int("content_len", len(markdown)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } else { + s.sourceRepo.UpdateStatus(sourceID, "failed", "文件解析失败") + return + } + } else { + logger.Info("MarkItDown 转换成功", + zap.String("file", fileName), + zap.Int("content_len", len(markdown)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } + + // 2. LLM 结构化 + stepStart = time.Now() + if s.structurer != nil { + result, err := s.structurer.Structure(context.Background(), userID, markdown, StructureMeta{ + Title: fileName, + SourceType: "file", + }) + if err != nil { + logger.Error("LLM 结构化失败,使用原始内容", + zap.String("file", fileName), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + } else if result.ActuallyCalled { + markdown = result.Content + logger.Info("LLM 结构化完成", + zap.String("file", fileName), + zap.Int("content_len", len(markdown)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } else { + logger.Warn("LLM 结构化被跳过(模型配置问题或 API Key 过期)", + zap.String("file", fileName), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } + } else { + logger.Warn("MarkdownStructurer 未配置,跳过结构化", zap.String("file", fileName)) + } + + // 3. 更新 source 内容和状态 + stepStart = time.Now() + if err := s.sourceRepo.UpdateContent(sourceID, markdown, "ready"); err != nil { + logger.Error("更新 Source 内容失败", + zap.String("file", fileName), + zap.Uint("source_id", sourceID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("保存失败: %v", err)) + return + } + + logger.Info("Source 内容更新成功", + zap.String("file", fileName), + zap.Uint("source_id", sourceID), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + // 4. RAG 入库 + stepStart = time.Now() + if s.ingestionSvc != nil { + if err := s.ingestionSvc.IngestSingle(context.Background(), sourceID); err != nil { + logger.Error("RAG 入库失败", + zap.String("file", fileName), + zap.Uint("source_id", sourceID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + // RAG 入库失败不影响 source 可见性,只记录日志 + return + } + logger.Info("RAG 入库成功", + zap.String("file", fileName), + zap.Uint("source_id", sourceID), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } + + // 5. 生成摘要(异步,不阻塞主流程) + go s.generateAndSaveSummary(sourceID, userID, markdown) + + logger.Info("文件导入完成", + zap.String("file", fileName), + zap.Uint("source_id", sourceID), + zap.Duration("total_elapsed", time.Since(totalStart)), + ) +} + +// PreviewAudio 异步音频转写:上传文件后立即返回 previewID,后台执行 ASR 转写 +func (s *importerService) PreviewAudio(userID, notebookID uint, file *multipart.FileHeader) (string, string, error) { + ext := strings.ToLower(filepath.Ext(file.Filename)) + if !allowedAudioTypes[ext] { + return "", "", bizerrors.ErrUnsupportedFormat + } + if file.Size > maxAudioSize { + return "", "", bizerrors.ErrFileTooLarge + } + + // 上传原始文件到 MinIO + filePath, err := s.storage.Upload(file) + if err != nil { + logger.Error("音频上传到存储服务失败", + zap.String("file", file.Filename), + zap.Int64("size", file.Size), + zap.Error(err), + ) + return "", "", bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "音频上传失败", err) + } + + previewID := uuid.New().String() + preview := &cache.AudioPreview{ + PreviewID: previewID, + UserID: userID, + NotebookID: notebookID, + FileName: file.Filename, + FilePath: filePath, + FileSize: file.Size, + Status: "pending", + ExpiresAt: time.Now().Add(30 * time.Minute).Unix(), + } + + ctx := context.Background() + if err := s.previewCache.Save(ctx, preview); err != nil { + return "", "", err + } + + // 后台异步执行 ASR 转写 + go s.doAudioTranscribe(previewID, userID, file, filePath, ext) + + return previewID, file.Filename, nil +} + +// doAudioTranscribe 后台执行音频转写,完成后更新缓存 +func (s *importerService) doAudioTranscribe(previewID string, userID uint, file *multipart.FileHeader, filePath, ext string) { + totalStart := time.Now() + ctx := context.Background() + + // 标记为处理中 + if err := s.previewCache.UpdateStatus(ctx, previewID, "processing"); err != nil { + logger.Error("更新预览状态为processing失败", zap.String("preview_id", previewID), zap.Error(err)) + return + } + + // 使用 ffmpeg 流式转换为 16kHz 单声道 WAV(内存占用低,支持各种格式) + asrFilePath := filePath + convertedPath, convertErr := s.convertAudioWithFFMPEG(filePath, ext) + if convertErr != nil { + logger.Warn("ffmpeg音频转换失败,使用原始文件", + zap.String("file", filePath), + zap.Error(convertErr), + ) + } else { + asrFilePath = convertedPath + logger.Info("音频已通过ffmpeg转换为16kHz单声道WAV", + zap.String("original", filePath), + zap.String("converted", asrFilePath), + ) + } + + // 获取 ASR 服务 + stepStart := time.Now() + asrSvc, err := s.getASR(userID) + if err != nil { + logger.Error("获取ASR服务失败", + zap.String("preview_id", previewID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + s.markPreviewFailed(ctx, previewID, "未配置 ASR 服务") + return + } + logger.Info("获取 ASR 服务完成", + zap.String("preview_id", previewID), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + // 执行转写 + stepStart = time.Now() + logger.Info("开始 ASR 转写", + zap.String("preview_id", previewID), + zap.String("asr_file", asrFilePath), + ) + text, err := asrSvc.Transcribe(asrFilePath) + if err != nil { + logger.Error("ASR转写失败", + zap.String("preview_id", previewID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + s.markPreviewFailed(ctx, previewID, fmt.Sprintf("音频转写失败: %v", err)) + return + } + logger.Info("ASR 转写完成", + zap.String("preview_id", previewID), + zap.Int("text_len", len(text)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + // 转写成功,更新缓存 + preview, err := s.previewCache.Get(ctx, previewID) + if err != nil || preview == nil { + logger.Error("转写完成但获取预览缓存失败", zap.String("preview_id", previewID), zap.Error(err)) + return + } + preview.TranscribedText = text + preview.Status = "ready" + if err := s.previewCache.Save(ctx, preview); err != nil { + logger.Error("保存转写结果失败", zap.String("preview_id", previewID), zap.Error(err)) + return + } + + logger.Info("音频转写流程完成", + zap.String("preview_id", previewID), + zap.Int("text_len", len(text)), + zap.Duration("total_elapsed", time.Since(totalStart)), + ) +} + +// markPreviewFailed 标记预览转写失败 +func (s *importerService) markPreviewFailed(ctx context.Context, previewID, errMsg string) { + preview, err := s.previewCache.Get(ctx, previewID) + if err != nil || preview == nil { + return + } + preview.Status = "failed" + preview.ErrorMsg = errMsg + if saveErr := s.previewCache.Save(ctx, preview); saveErr != nil { + logger.Error("保存预览失败状态出错", zap.String("preview_id", previewID), zap.Error(saveErr)) + } +} + +// GetAudioPreviewStatus 查询音频预览状态(前端轮询用) +func (s *importerService) GetAudioPreviewStatus(userID uint, previewID string) (interface{}, error) { + ctx := context.Background() + preview, err := s.previewCache.Get(ctx, previewID) + if err != nil { + return nil, bizerrors.ErrNotFound + } + if preview == nil { + return nil, bizerrors.ErrNotFound + } + if preview.UserID != userID { + return nil, bizerrors.ErrForbidden + } + return preview, nil +} + +// ConfirmAudio 确认音频导入 +func (s *importerService) ConfirmAudio(userID uint, previewID string, editedContent *string) (*entity.Source, error) { + totalStart := time.Now() + + ctx := context.Background() + preview, err := s.previewCache.Get(ctx, previewID) + if err != nil { + return nil, bizerrors.ErrNotFound + } + if preview == nil { + return nil, bizerrors.ErrNotFound + } + if preview.UserID != userID { + return nil, bizerrors.ErrForbidden + } + if time.Now().Unix() > preview.ExpiresAt { + return nil, bizerrors.ErrPreviewExpired + } + if preview.Status == "failed" { + return nil, bizerrors.New(bizerrors.CodeASTranscriptionFailed, preview.ErrorMsg) + } + if preview.Status != "ready" { + return nil, bizerrors.New(bizerrors.CodeBadRequest, "音频转写尚未完成,请稍后再试") + } + + logger.Info("开始确认音频导入", + zap.String("preview_id", previewID), + zap.String("file_name", preview.FileName), + zap.Uint("user_id", userID), + ) + + content := preview.TranscribedText + if editedContent != nil && *editedContent != "" { + content = *editedContent + logger.Info("使用用户编辑后的内容", + zap.String("preview_id", previewID), + zap.Int("content_len", len(content)), + ) + } else { + logger.Info("使用 ASR 转写结果", + zap.String("preview_id", previewID), + zap.Int("content_len", len(content)), + ) + } + + // LLM 结构化 + stepStart := time.Now() + if s.structurer != nil { + result, err := s.structurer.Structure(ctx, userID, content, StructureMeta{ + Title: preview.FileName, + SourceType: "audio", + }) + if err != nil { + logger.Error("LLM 结构化失败,使用原始内容", + zap.String("preview_id", previewID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + } else if result.ActuallyCalled && result.Content != content { + logger.Info("LLM 结构化成功,内容已优化", + zap.String("preview_id", previewID), + zap.Int("original_len", len(content)), + zap.Int("structured_len", len(result.Content)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + content = result.Content + } else if result.ActuallyCalled { + logger.Info("LLM 判断内容已有结构,无需结构化", + zap.String("preview_id", previewID), + zap.Int("content_len", len(content)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } else { + logger.Warn("LLM 结构化被跳过(模型配置问题或 API Key 过期)", + zap.String("preview_id", previewID), + zap.Int("content_len", len(content)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } + } else { + logger.Warn("MarkdownStructurer 未配置,跳过结构化", zap.String("preview_id", previewID)) + } + + // 创建 Source 记录 + stepStart = time.Now() + source := &entity.Source{ + UserID: userID, + NotebookID: preview.NotebookID, + Name: preview.FileName, + Type: "audio", + FilePath: preview.FilePath, + FileSize: preview.FileSize, + MarkdownContent: content, + Status: "ready", + } + + if err := s.sourceRepo.Create(source); err != nil { + logger.Error("创建 Source 记录失败", + zap.String("preview_id", previewID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + return nil, err + } + + logger.Info("Source 记录创建成功", + zap.String("preview_id", previewID), + zap.Uint("source_id", source.ID), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + // 同步触发 RAG 入库 + stepStart = time.Now() + if s.ingestionSvc != nil { + if err := s.ingestionSvc.IngestSingle(context.Background(), source.ID); err != nil { + logger.Error("RAG 入库失败", + zap.String("preview_id", previewID), + zap.Uint("source_id", source.ID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "RAG 入库失败", err) + } + logger.Info("RAG 入库成功", + zap.String("preview_id", previewID), + zap.Uint("source_id", source.ID), + zap.Duration("elapsed", time.Since(stepStart)), + ) + source.Vectorized = true + } + + // 生成摘要(异步,不阻塞主流程) + go s.generateAndSaveSummary(source.ID, userID, content) + + if err := s.previewCache.UpdateStatus(ctx, previewID, "confirmed"); err != nil { + logger.Warn("更新预览状态失败", zap.String("preview_id", previewID), zap.Error(err)) + } + + logger.Info("音频导入确认完成", + zap.String("preview_id", previewID), + zap.String("file_name", preview.FileName), + zap.Uint("source_id", source.ID), + zap.Duration("total_elapsed", time.Since(totalStart)), + ) + + return source, nil +} + +// convertAudioForASR 转换音频为 ASR 兼容格式 +// 如果已经是 16kHz 单声道则返回 nil(无需转换) +func (s *importerService) convertAudioForASR(file *multipart.FileHeader, ext string) ([]byte, error) { + // 读取文件内容 + src, err := file.Open() + if err != nil { + return nil, fmt.Errorf("打开音频文件失败: %w", err) + } + defer func(src multipart.File) { + err := src.Close() + if err != nil { + logger.Errorf("关闭文件失败:%s", err) + } + }(src) + + audioData, err := io.ReadAll(src) + if err != nil { + return nil, fmt.Errorf("读取音频文件失败: %w", err) + } + + // 转换为 16kHz 单声道 WAV + converted, err := utils.ConvertBytesToASRFormat(audioData, ext) + if err != nil { + return nil, fmt.Errorf("音频转换失败: %w", err) + } + + return converted, nil +} + +// convertAudioWithFFMPEG 使用 ffmpeg 流式转换音频为 16kHz 单声道 WAV +// 从 MinIO 下载 → ffmpeg 转换 → 上传回 MinIO,全程流式处理,内存占用低 +func (s *importerService) convertAudioWithFFMPEG(filePath, ext string) (string, error) { + // 1. 下载原始文件到临时文件 + srcData, err := s.storage.Download(filePath) + if err != nil { + return "", fmt.Errorf("下载原始音频失败: %w", err) + } + + tmpInput, err := os.CreateTemp("", "asr-input-*"+ext) + if err != nil { + return "", fmt.Errorf("创建临时输入文件失败: %w", err) + } + defer os.Remove(tmpInput.Name()) + defer tmpInput.Close() + + if _, err := tmpInput.Write(srcData); err != nil { + return "", fmt.Errorf("写入临时输入文件失败: %w", err) + } + tmpInput.Close() + + // 2. ffmpeg 转换为 16kHz 单声道 WAV + tmpOutput := tmpInput.Name() + "_16k.wav" + defer os.Remove(tmpOutput) + + cmd := exec.Command("ffmpeg", "-y", "-i", tmpInput.Name(), + "-ar", "16000", "-ac", "1", "-sample_fmt", "s16", + "-f", "wav", tmpOutput) + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("ffmpeg转换失败: %w, stderr: %s", err, stderr.String()) + } + + // 3. 读取转换后的文件 + convertedData, err := os.ReadFile(tmpOutput) + if err != nil { + return "", fmt.Errorf("读取转换后文件失败: %w", err) + } + + // 4. 上传到 MinIO + convertedPath := filePath[:len(filePath)-len(filepath.Ext(filePath))] + "_16k.wav" + if err := s.storage.UploadBytes(convertedPath, convertedData, "audio/wav"); err != nil { + return "", fmt.Errorf("上传转换后音频失败: %w", err) + } + + return convertedPath, nil +} + +// ImportSearchResults 批量导入搜索结果 +// 为每个 URL 先创建 pending 状态的 Source 记录,然后异步处理 +// 返回创建的 Source ID 列表,前端可通过 Source 列表 API 查看每条的独立状态 +func (s *importerService) ImportSearchResults(userID, notebookID uint, items []SearchResultItem) (string, []uint, error) { + // 去重:同一个 URL 只创建一条记录(保留第一次出现的标题) + seen := make(map[string]string, len(items)) // url -> title + for _, item := range items { + if _, exists := seen[item.URL]; !exists { + seen[item.URL] = item.Title + } + } + + sourceIDs := make([]uint, 0, len(seen)) + + // 为每个 URL 创建 pending 状态的 Source + for url, title := range seen { + // 如果标题为空,使用 URL 作为标题 + name := title + if name == "" { + name = url + } + + source := &entity.Source{ + UserID: userID, + NotebookID: notebookID, + Name: name, + Type: "url", + OriginalURL: url, + Status: "pending", + } + if err := s.sourceRepo.Create(source); err != nil { + logger.Error("创建待导入Source失败", zap.String("url", url), zap.Error(err)) + continue + } + sourceIDs = append(sourceIDs, source.ID) + } + + if len(sourceIDs) == 0 { + return "", nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "创建导入记录失败", nil) + } + + // 创建可取消的 context,注册 cancel func 以便批量取消 + // 设置整体超时:每个 URL 最多 2 分钟,整体最多 10 分钟 + taskID := uuid.New().String() + maxTimeout := 10 * time.Minute + urlTimeout := time.Duration(len(seen)) * 2 * time.Minute + if urlTimeout > maxTimeout { + urlTimeout = maxTimeout + } + taskCtx, cancel := context.WithTimeout(context.Background(), urlTimeout) + s.cancelFuncs.Store(taskID, cancel) + + // 异步处理每个 Source + go s.processSources(taskCtx, taskID, sourceIDs) + + return taskID, sourceIDs, nil +} + +// processSources 异步处理 Source 列表(带并发控制,支持取消) +func (s *importerService) processSources(taskCtx context.Context, taskID string, sourceIDs []uint) { + // 任务结束后清理 cancel func + defer s.cancelFuncs.Delete(taskID) + + // 并发控制:最多同时处理 3 个 + concurrency := 3 + if len(sourceIDs) < concurrency { + concurrency = len(sourceIDs) + } + + idCh := make(chan uint, concurrency) + doneCh := make(chan struct{}, len(sourceIDs)) + + // 启动 worker + for i := 0; i < concurrency; i++ { + go func() { + for sourceID := range idCh { + if taskCtx.Err() != nil { + doneCh <- struct{}{} + continue + } + s.processSingleSource(taskCtx, sourceID) + doneCh <- struct{}{} + } + }() + } + + // 分发任务(支持取消中断分发) + go func() { + for _, sourceID := range sourceIDs { + if taskCtx.Err() != nil { + break + } + idCh <- sourceID + } + close(idCh) + }() + + // 等待所有任务完成 + for i := 0; i < len(sourceIDs); i++ { + <-doneCh + } + + // 将仍然处于 pending 状态的 Source 标记为 cancelled(被取消的任务) + if taskCtx.Err() != nil { + for _, sourceID := range sourceIDs { + src, err := s.sourceRepo.FindByID(sourceID) + if err != nil || src == nil { + continue + } + if src.Status == "pending" { + if err := s.sourceRepo.UpdateStatus(sourceID, "cancelled", "任务已取消"); err != nil { + logger.Warn("更新Source状态为cancelled失败", zap.Uint("source_id", sourceID), zap.Error(err)) + } + } + } + } +} + +// processSingleSource 处理单个 Source(支持取消) +func (s *importerService) processSingleSource(taskCtx context.Context, sourceID uint) { + totalStart := time.Now() + + // 处理前检查取消 + if taskCtx.Err() != nil { + return + } + + // 获取 Source 记录 + source, err := s.sourceRepo.FindByID(sourceID) + if err != nil || source == nil { + logger.Error("获取Source失败", zap.Uint("source_id", sourceID), zap.Error(err)) + return + } + + logger.Info("开始处理 URL 导入", + zap.Uint("source_id", sourceID), + zap.String("url", source.OriginalURL), + ) + + // 更新状态为 processing + if err := s.sourceRepo.UpdateStatus(sourceID, "processing", ""); err != nil { + logger.Warn("更新Source状态为processing失败", zap.Uint("source_id", sourceID), zap.Error(err)) + } + + // 转换 URL 内容 + stepStart := time.Now() + markdown, err := s.markitdown.ConvertFromURLWithContext(taskCtx, source.OriginalURL) + if err != nil { + // 如果是因为取消导致的错误 + if taskCtx.Err() != nil { + logger.Info("任务已取消,跳过Source处理", zap.Uint("source_id", sourceID)) + return + } + + // 处理结构化错误,返回用户友好的错误信息 + var userMsg string + var convertErr *externalMarkitdown.ConvertError + if errors.As(err, &convertErr) { + // 记录详细的技术错误信息到日志 + logger.Error("URL 转换失败", + zap.Uint("source_id", sourceID), + zap.String("url", source.OriginalURL), + zap.String("error_code", convertErr.Code), + zap.String("detail", convertErr.DetailMsg), + zap.Int("http_status", convertErr.HTTPStatus), + zap.Duration("elapsed", time.Since(stepStart)), + ) + // 使用用户友好的错误消息 + userMsg = convertErr.UserMsg + } else { + // 未知错误类型 + logger.Error("URL 转换失败", + zap.Uint("source_id", sourceID), + zap.String("url", source.OriginalURL), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + userMsg = "无法获取该网页内容" + } + + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", userMsg); updateErr != nil { + logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + + logger.Info("URL 转换成功", + zap.Uint("source_id", sourceID), + zap.String("url", source.OriginalURL), + zap.Int("content_len", len(markdown)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + // 转换完成后再检查一次 source 是否还存在(可能在转换期间被用户删除) + existing, _ := s.sourceRepo.FindByID(sourceID) + if existing == nil { + logger.Info("Source已被删除,丢弃转换结果", zap.Uint("source_id", sourceID)) + return + } + + // LLM 结构化 + stepStart = time.Now() + if s.structurer != nil { + result, err := s.structurer.Structure(taskCtx, source.UserID, markdown, StructureMeta{ + Title: source.Name, + SourceType: "url", + }) + if err != nil { + logger.Error("LLM 结构化失败,使用原始内容", + zap.Uint("source_id", sourceID), + zap.String("url", source.OriginalURL), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + } else if result.ActuallyCalled && result.Content != markdown { + logger.Info("LLM 结构化成功,内容已优化", + zap.Uint("source_id", sourceID), + zap.Int("original_len", len(markdown)), + zap.Int("structured_len", len(result.Content)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + markdown = result.Content + } else if result.ActuallyCalled { + logger.Info("LLM 判断内容已有结构,无需结构化", + zap.Uint("source_id", sourceID), + zap.Int("content_len", len(markdown)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } else { + logger.Warn("LLM 结构化被跳过(模型配置问题或 API Key 过期)", + zap.Uint("source_id", sourceID), + zap.Int("content_len", len(markdown)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } + } else { + logger.Warn("MarkdownStructurer 未配置,跳过结构化", zap.Uint("source_id", sourceID)) + } + + // 更新 Source 内容和状态为 ready + stepStart = time.Now() + source.MarkdownContent = markdown + source.Status = "ready" + if err := s.sourceRepo.Update(source); err != nil { + logger.Error("更新Source内容失败", zap.Uint("source_id", sourceID), zap.Duration("elapsed", time.Since(stepStart)), zap.Error(err)) + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("保存失败: %v", err)); updateErr != nil { + logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + + logger.Info("Source 记录更新成功", + zap.Uint("source_id", sourceID), + zap.String("url", source.OriginalURL), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + // 同步触发 RAG 入库 + stepStart = time.Now() + if s.ingestionSvc != nil { + if err := s.ingestionSvc.IngestSingle(taskCtx, sourceID); err != nil { + logger.Error("RAG 入库失败", + zap.Uint("source_id", sourceID), + zap.String("url", source.OriginalURL), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("RAG 入库失败: %v", err)); updateErr != nil { + logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + logger.Info("RAG 入库成功", + zap.Uint("source_id", sourceID), + zap.String("url", source.OriginalURL), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } + + // 生成摘要(异步,不阻塞主流程) + go s.generateAndSaveSummary(sourceID, source.UserID, markdown) + + logger.Info("URL 导入完成", + zap.Uint("source_id", sourceID), + zap.String("url", source.OriginalURL), + zap.Duration("total_elapsed", time.Since(totalStart)), + ) +} + +// GetImportTask 获取导入任务状态 +func (s *importerService) GetImportTask(taskID string) (interface{}, error) { + ctx := context.Background() + task, err := s.importCache.Get(ctx, taskID) + if err != nil { + return nil, bizerrors.ErrNotFound + } + if task == nil { + return nil, bizerrors.ErrNotFound + } + return task, nil +} + +// DeleteImportTask 删除/取消导入任务 +func (s *importerService) DeleteImportTask(taskID string) error { + ctx := context.Background() + + // 1. 尝试从 cancelFuncs 中取消正在运行的异步任务(新架构:Source-based 导入) + if cancel, ok := s.cancelFuncs.Load(taskID); ok { + cancel.(context.CancelFunc)() + s.cancelFuncs.Delete(taskID) + logger.Info("已发送取消信号给运行中的导入任务", zap.String("task_id", taskID)) + return nil + } + + // 2. 尝试从 importCache 中查找(旧架构:Redis-based 任务) + task, err := s.importCache.Get(ctx, taskID) + if err != nil { + return bizerrors.ErrNotFound + } + if task == nil { + return bizerrors.ErrNotFound + } + + // 如果任务正在运行中,标记为取消状态 + if task.Status == "running" { + task.Status = "cancelled" + if err := s.importCache.Save(ctx, task); err != nil { + logger.Warn("更新任务状态为取消失败", zap.String("task_id", taskID), zap.Error(err)) + } + } + + // 删除任务缓存 + return s.importCache.Delete(ctx, taskID) +} + +// getASR 获取 ASR 服务(从 ConfigService 动态加载) +func (s *importerService) getASR(userID uint) (asr.ASRService, error) { + if s.configSvc == nil { + return nil, fmt.Errorf("ConfigService 未初始化") + } + return s.configSvc.GetASRService(userID) +} + +// summarySystemPrompt 摘要生成的系统提示词 +const summarySystemPrompt = `你是一个资料摘要助手。请为以下文档内容生成一份简洁的摘要。 + +要求: +1. 摘要长度:200-400字 +2. 涵盖文档的核心主题、主要观点和关键信息 +3. 使用中文 +4. 保持客观,不添加个人评价 +5. 直接输出摘要内容,不要加任何前缀或解释` + +// generateAndSaveSummary 生成资料摘要并保存到 MySQL 和 Redis(importerService 的方法) +func (s *importerService) generateAndSaveSummary(sourceID uint, userID uint, content string) { + doGenerateAndSaveSummary(s.sourceRepo, s.configSvc, s.summaryCache, sourceID, userID, content) +} + +// fallbackSummaryLength 降级摘要的最大字符数 +const fallbackSummaryLength = 300 + +// doGenerateAndSaveSummary 生成资料摘要的包级别共享实现 +// LLM 失败时自动降级为截取内容前 N 个字符作为兜底摘要 +func doGenerateAndSaveSummary( + sourceRepo repository.SourceRepository, + configSvc ConfigService, + summaryCache *cache.SourceSummaryCache, + sourceID uint, userID uint, content string, +) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + startTime := time.Now() + + summary, usedFallback := tryGenerateWithLLM(ctx, configSvc, userID, content) + if usedFallback { + // LLM 失败,使用降级摘要 + summary = buildFallbackSummary(content) + logger.Warn("LLM 摘要生成失败,使用降级摘要", + zap.Uint("source_id", sourceID), + zap.Int("fallback_len", len(summary)), + ) + } + + if summary == "" { + logger.Warn("摘要生成失败且内容为空,跳过", + zap.Uint("source_id", sourceID), + ) + return + } + + // 保存到 MySQL + if err := sourceRepo.UpdateSummary(sourceID, summary); err != nil { + logger.Error("保存摘要到 MySQL 失败", + zap.Uint("source_id", sourceID), + zap.Error(err), + ) + return + } + + // 保存到 Redis + if summaryCache != nil { + if err := summaryCache.Set(ctx, sourceID, summary); err != nil { + logger.Warn("保存摘要到 Redis 失败", + zap.Uint("source_id", sourceID), + zap.Error(err), + ) + } + } + + logger.Info("资料摘要生成完成", + zap.Uint("source_id", sourceID), + zap.Int("summary_len", len(summary)), + zap.Bool("fallback", usedFallback), + zap.Duration("elapsed", time.Since(startTime)), + ) +} + +// tryGenerateWithLLM 尝试用 LLM 生成摘要,返回 (摘要内容, 是否需要降级) +func tryGenerateWithLLM(ctx context.Context, configSvc ConfigService, userID uint, content string) (string, bool) { + chatModel, err := getChatModelForSummary(ctx, configSvc, userID) + if err != nil || chatModel == nil { + return "", true + } + + userMsg := fmt.Sprintf("请为以下文档生成摘要:\n\n%s", content) + msg, err := chatModel.Generate(ctx, []*schema.Message{ + schema.SystemMessage(summarySystemPrompt), + schema.UserMessage(userMsg), + }, model.WithMaxTokens(1024)) + if err != nil { + return "", true + } + if msg == nil || strings.TrimSpace(msg.Content) == "" { + return "", true + } + + return strings.TrimSpace(msg.Content), false +} + +// buildFallbackSummary 从内容中提取降级摘要 、截取前 fallbackSummaryLength 个字符,尝试在句子边界截断 +func buildFallbackSummary(content string) string { + content = strings.TrimSpace(content) + if content == "" { + return "" + } + + runes := []rune(content) + if len(runes) <= fallbackSummaryLength { + return content + } + + // 截取前 N 个字符,尝试在句号、换行处断开 + truncated := runes[:fallbackSummaryLength] + cutPoints := []rune{'。', '\n', ';', '!', '?', '.', '!', '?'} + bestCut := fallbackSummaryLength + for i := fallbackSummaryLength - 1; i >= fallbackSummaryLength/2; i-- { + for _, cp := range cutPoints { + if truncated[i] == cp { + bestCut = i + 1 + break + } + } + if bestCut != fallbackSummaryLength { + break + } + } + + return string(runes[:bestCut]) + "..." +} + +// getChatModelForSummary 获取用于生成摘要的 ChatModel(包级别共享函数) +func getChatModelForSummary(ctx context.Context, configSvc ConfigService, userID uint) (model.ToolCallingChatModel, error) { + llmConfig, err := configSvc.GetUserLLMConfig(userID) + if err != nil { + return nil, fmt.Errorf("获取 LLM 配置失败: %w", err) + } + if llmConfig == nil || !llmConfig.Enabled { + return nil, nil + } + + chatModel, err := llm.NewChatModel(ctx, llmConfig) + if err != nil { + return nil, fmt.Errorf("创建 ChatModel 失败: %w", err) + } + return chatModel, nil +} diff --git a/internal/service/markdown_structurer_interface.go b/internal/service/markdown_structurer_interface.go index 1c7e5c8..41b378e 100644 --- a/internal/service/markdown_structurer_interface.go +++ b/internal/service/markdown_structurer_interface.go @@ -1,23 +1,23 @@ -package service - -import "context" - -// StructureMeta 结构化元信息 -type StructureMeta struct { - Title string // 原始标题(如有) - SourceType string // "youdao" / "url" / "file" / "audio" -} - -// StructureResult 包含结构化结果和是否真正进行了 LLM 结构化 -type StructureResult struct { - Content string // 结构化后的内容(或原始内容) - ActuallyCalled bool // 是否真正调用了 LLM(false 表示跳过或失败降级) -} - -// MarkdownStructurer markdown 结构化服务接口 -type MarkdownStructurer interface { - // Structure 给 markdown 内容补充结构 - // - 已有结构(检测到 heading ≥ 2)→ 跳过 - // - 无结构 → 调用 LLM 补充标题/段落 - Structure(ctx context.Context, userID uint, content string, meta StructureMeta) (StructureResult, error) -} +package service + +import "context" + +// StructureMeta 结构化元信息 +type StructureMeta struct { + Title string // 原始标题(如有) + SourceType string // "youdao" / "url" / "file" / "audio" +} + +// StructureResult 包含结构化结果和是否真正进行了 LLM 结构化 +type StructureResult struct { + Content string // 结构化后的内容(或原始内容) + ActuallyCalled bool // 是否真正调用了 LLM(false 表示跳过或失败降级) +} + +// MarkdownStructurer markdown 结构化服务接口 +type MarkdownStructurer interface { + // Structure 给 markdown 内容补充结构 + // - 已有结构(检测到 heading ≥ 2)→ 跳过 + // - 无结构 → 调用 LLM 补充标题/段落 + Structure(ctx context.Context, userID uint, content string, meta StructureMeta) (StructureResult, error) +} diff --git a/internal/service/notebook_interface.go b/internal/service/notebook_interface.go index bf7b11b..3f67aca 100644 --- a/internal/service/notebook_interface.go +++ b/internal/service/notebook_interface.go @@ -1,18 +1,18 @@ -package service - -import ( - "YoudaoNoteLm/internal/model/dto/request" - "YoudaoNoteLm/internal/model/dto/response" -) - -// NotebookService 笔记本服务接口 -type NotebookService interface { - // Create 创建笔记本 - Create(userID uint, req *request.CreateNotebookRequest) (*response.NotebookResponse, error) - // List 查询用户的所有笔记本 - List(userID uint) ([]*response.NotebookResponse, error) - // Rename 重命名笔记本 - Rename(userID, notebookID uint, req *request.RenameNotebookRequest) error - // Delete 删除笔记本 - Delete(userID, notebookID uint) error -} +package service + +import ( + "YoudaoNoteLm/internal/model/dto/request" + "YoudaoNoteLm/internal/model/dto/response" +) + +// NotebookService 笔记本服务接口 +type NotebookService interface { + // Create 创建笔记本 + Create(userID uint, req *request.CreateNotebookRequest) (*response.NotebookResponse, error) + // List 查询用户的所有笔记本 + List(userID uint) ([]*response.NotebookResponse, error) + // Rename 重命名笔记本 + Rename(userID, notebookID uint, req *request.RenameNotebookRequest) error + // Delete 删除笔记本 + Delete(userID, notebookID uint) error +} diff --git a/internal/service/notebook_service.go b/internal/service/notebook_service.go index aeccfca..38e9270 100644 --- a/internal/service/notebook_service.go +++ b/internal/service/notebook_service.go @@ -1,270 +1,270 @@ -package service - -import ( - "context" - "time" - - "go.uber.org/zap" - - "YoudaoNoteLm/internal/model/dto/request" - "YoudaoNoteLm/internal/model/dto/response" - "YoudaoNoteLm/internal/model/entity" - "YoudaoNoteLm/internal/rag" - "YoudaoNoteLm/internal/repository" - "YoudaoNoteLm/pkg/cache" - bizerrors "YoudaoNoteLm/pkg/errors" - "YoudaoNoteLm/pkg/logger" -) - -// notebookService 笔记本服务实现 -type notebookService struct { - notebookRepo repository.NotebookRepository - sourceRepo repository.SourceRepository - conversationRepo repository.ConversationRepository - messageRepo repository.MessageRepository - ingestionSvc rag.IngestionService - chatCache *cache.ChatCache - summaryCache *cache.SourceSummaryCache -} - -// NewNotebookService 创建笔记本服务 -func NewNotebookService( - notebookRepo repository.NotebookRepository, - sourceRepo repository.SourceRepository, - conversationRepo repository.ConversationRepository, - messageRepo repository.MessageRepository, - ingestionSvc rag.IngestionService, - chatCache *cache.ChatCache, - summaryCache *cache.SourceSummaryCache, -) NotebookService { - return ¬ebookService{ - notebookRepo: notebookRepo, - sourceRepo: sourceRepo, - conversationRepo: conversationRepo, - messageRepo: messageRepo, - ingestionSvc: ingestionSvc, - chatCache: chatCache, - summaryCache: summaryCache, - } -} - -// Create 创建笔记本 -func (s *notebookService) Create(userID uint, req *request.CreateNotebookRequest) (*response.NotebookResponse, error) { - // 检查是否存在同名笔记本 - exists, err := s.notebookRepo.ExistsByName(userID, req.Name) - if err != nil { - return nil, err - } - if exists { - return nil, bizerrors.New(bizerrors.CodeConflict, "已存在同名笔记本") - } - - notebook := &entity.Notebook{ - UserID: userID, - Name: req.Name, - } - - if err := s.notebookRepo.Create(notebook); err != nil { - return nil, err - } - - return s.toResponse(notebook), nil -} - -// List 查询用户的所有笔记本 -func (s *notebookService) List(userID uint) ([]*response.NotebookResponse, error) { - notebooks, err := s.notebookRepo.ListByUserID(userID) - if err != nil { - return nil, err - } - - result := make([]*response.NotebookResponse, 0, len(notebooks)) - for _, nb := range notebooks { - result = append(result, s.toResponse(nb)) - } - return result, nil -} - -// Rename 重命名笔记本 -func (s *notebookService) Rename(userID, notebookID uint, req *request.RenameNotebookRequest) error { - notebook, err := s.notebookRepo.FindByID(notebookID) - if err != nil { - return err - } - if notebook == nil { - return bizerrors.ErrNotFound - } - - // 检查权限 - if notebook.UserID != userID { - return bizerrors.ErrForbidden - } - - // 检查是否存在同名笔记本(排除自身) - if notebook.Name != req.Name { - exists, err := s.notebookRepo.ExistsByName(userID, req.Name) - if err != nil { - return err - } - if exists { - return bizerrors.New(bizerrors.CodeConflict, "已存在同名笔记本") - } - } - - notebook.Name = req.Name - return s.notebookRepo.Update(notebook) -} - -// Delete 删除笔记本 -func (s *notebookService) Delete(userID, notebookID uint) error { - notebook, err := s.notebookRepo.FindByID(notebookID) - if err != nil { - return err - } - if notebook == nil { - return bizerrors.ErrNotFound - } - - // 检查权限 - if notebook.UserID != userID { - return bizerrors.ErrForbidden - } - - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) - defer cancel() - - // 删除该笔记本下所有 source 的向量和 parent_blocks - if s.ingestionSvc != nil && s.sourceRepo != nil { - s.deleteNotebookVectors(ctx, userID, notebookID) - } - - // 清理关联会话的 Redis 缓存和消息 - if s.conversationRepo != nil && s.chatCache != nil { - s.cleanupNotebookConversations(ctx, notebookID) - } - - // 清理该笔记本下所有 source 的摘要缓存 - if s.summaryCache != nil { - s.cleanupNotebookSourceSummaries(ctx, userID, notebookID) - } - - // 软删除该笔记本下所有 source(软删除不触发 CASCADE) - if err := s.sourceRepo.DeleteByNotebookID(notebookID); err != nil { - logger.Error("删除笔记本关联的 source 失败", - zap.Uint("notebook_id", notebookID), - zap.Error(err), - ) - } - - return s.notebookRepo.Delete(notebookID) -} - -// deleteNotebookVectors 删除笔记本下所有 source 的向量数据和 parent_blocks -func (s *notebookService) deleteNotebookVectors(ctx context.Context, userID, notebookID uint) { - // 查询该笔记本下所有 source - sources, _, err := s.sourceRepo.ListByNotebook(userID, notebookID, "", 0, 10000) - if err != nil { - logger.Error("查询笔记本关联的 source 失败", - zap.Uint("notebook_id", notebookID), - zap.Error(err), - ) - return - } - - // 逐个删除向量数据和 parent_blocks - for _, source := range sources { - if err := s.ingestionSvc.DeleteSource(ctx, userID, source.ID); err != nil { - logger.Error("删除笔记本关联的源数据失败", - zap.Uint("notebook_id", notebookID), - zap.Uint("source_id", source.ID), - zap.Error(err), - ) - } - } -} - -// cleanupNotebookConversations 清理笔记本下所有会话的 Redis 缓存和消息 -func (s *notebookService) cleanupNotebookConversations(ctx context.Context, notebookID uint) { - convs, err := s.conversationRepo.FindByNotebookID(notebookID) - if err != nil { - logger.Error("查询笔记本关联的会话失败", - zap.Uint("notebook_id", notebookID), - zap.Error(err), - ) - return - } - - for _, conv := range convs { - // 删除消息 - if s.messageRepo != nil { - if err := s.messageRepo.DeleteByConversationID(conv.ID); err != nil { - logger.Warn("删除会话消息失败", - zap.Uint("conversation_id", conv.ID), - zap.Error(err), - ) - } - } - // 清除 Redis 缓存(消息历史+摘要) - if err := s.chatCache.DeleteConversationCache(ctx, conv.ID); err != nil { - logger.Warn("清除会话缓存失败", - zap.Uint("conversation_id", conv.ID), - zap.Error(err), - ) - } - } - - // 删除会话本身 - if err := s.conversationRepo.DeleteByNotebookID(notebookID); err != nil { - logger.Error("删除笔记本关联的会话失败", - zap.Uint("notebook_id", notebookID), - zap.Error(err), - ) - } -} - -// cleanupNotebookSourceSummaries 清理笔记本下所有 source 的摘要缓存 -func (s *notebookService) cleanupNotebookSourceSummaries(ctx context.Context, userID, notebookID uint) { - // 查询该笔记本下所有 source - sources, _, err := s.sourceRepo.ListByNotebook(userID, notebookID, "", 0, 10000) - if err != nil { - logger.Error("查询笔记本关联的 source 失败(用于清理摘要缓存)", - zap.Uint("notebook_id", notebookID), - zap.Error(err), - ) - return - } - - if len(sources) == 0 { - return - } - - // 收集所有 sourceID - sourceIDs := make([]uint, len(sources)) - for i, source := range sources { - sourceIDs[i] = source.ID - } - - // 批量删除摘要缓存 - if err := s.summaryCache.BatchDelete(ctx, sourceIDs); err != nil { - logger.Warn("批量删除摘要缓存失败", - zap.Uint("notebook_id", notebookID), - zap.Int("count", len(sourceIDs)), - zap.Error(err), - ) - } else { - logger.Info("已清理笔记本关联的摘要缓存", - zap.Uint("notebook_id", notebookID), - zap.Int("count", len(sourceIDs)), - ) - } -} - -// toResponse 转换为响应 DTO -func (s *notebookService) toResponse(notebook *entity.Notebook) *response.NotebookResponse { - return &response.NotebookResponse{ - ID: notebook.ID, - Name: notebook.Name, - CreatedAt: notebook.CreatedAt, - UpdatedAt: notebook.UpdatedAt, - } -} +package service + +import ( + "context" + "time" + + "go.uber.org/zap" + + "YoudaoNoteLm/internal/model/dto/request" + "YoudaoNoteLm/internal/model/dto/response" + "YoudaoNoteLm/internal/model/entity" + "YoudaoNoteLm/internal/rag" + "YoudaoNoteLm/internal/repository" + "YoudaoNoteLm/pkg/cache" + bizerrors "YoudaoNoteLm/pkg/errors" + "YoudaoNoteLm/pkg/logger" +) + +// notebookService 笔记本服务实现 +type notebookService struct { + notebookRepo repository.NotebookRepository + sourceRepo repository.SourceRepository + conversationRepo repository.ConversationRepository + messageRepo repository.MessageRepository + ingestionSvc rag.IngestionService + chatCache *cache.ChatCache + summaryCache *cache.SourceSummaryCache +} + +// NewNotebookService 创建笔记本服务 +func NewNotebookService( + notebookRepo repository.NotebookRepository, + sourceRepo repository.SourceRepository, + conversationRepo repository.ConversationRepository, + messageRepo repository.MessageRepository, + ingestionSvc rag.IngestionService, + chatCache *cache.ChatCache, + summaryCache *cache.SourceSummaryCache, +) NotebookService { + return ¬ebookService{ + notebookRepo: notebookRepo, + sourceRepo: sourceRepo, + conversationRepo: conversationRepo, + messageRepo: messageRepo, + ingestionSvc: ingestionSvc, + chatCache: chatCache, + summaryCache: summaryCache, + } +} + +// Create 创建笔记本 +func (s *notebookService) Create(userID uint, req *request.CreateNotebookRequest) (*response.NotebookResponse, error) { + // 检查是否存在同名笔记本 + exists, err := s.notebookRepo.ExistsByName(userID, req.Name) + if err != nil { + return nil, err + } + if exists { + return nil, bizerrors.New(bizerrors.CodeConflict, "已存在同名笔记本") + } + + notebook := &entity.Notebook{ + UserID: userID, + Name: req.Name, + } + + if err := s.notebookRepo.Create(notebook); err != nil { + return nil, err + } + + return s.toResponse(notebook), nil +} + +// List 查询用户的所有笔记本 +func (s *notebookService) List(userID uint) ([]*response.NotebookResponse, error) { + notebooks, err := s.notebookRepo.ListByUserID(userID) + if err != nil { + return nil, err + } + + result := make([]*response.NotebookResponse, 0, len(notebooks)) + for _, nb := range notebooks { + result = append(result, s.toResponse(nb)) + } + return result, nil +} + +// Rename 重命名笔记本 +func (s *notebookService) Rename(userID, notebookID uint, req *request.RenameNotebookRequest) error { + notebook, err := s.notebookRepo.FindByID(notebookID) + if err != nil { + return err + } + if notebook == nil { + return bizerrors.ErrNotFound + } + + // 检查权限 + if notebook.UserID != userID { + return bizerrors.ErrForbidden + } + + // 检查是否存在同名笔记本(排除自身) + if notebook.Name != req.Name { + exists, err := s.notebookRepo.ExistsByName(userID, req.Name) + if err != nil { + return err + } + if exists { + return bizerrors.New(bizerrors.CodeConflict, "已存在同名笔记本") + } + } + + notebook.Name = req.Name + return s.notebookRepo.Update(notebook) +} + +// Delete 删除笔记本 +func (s *notebookService) Delete(userID, notebookID uint) error { + notebook, err := s.notebookRepo.FindByID(notebookID) + if err != nil { + return err + } + if notebook == nil { + return bizerrors.ErrNotFound + } + + // 检查权限 + if notebook.UserID != userID { + return bizerrors.ErrForbidden + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + // 删除该笔记本下所有 source 的向量和 parent_blocks + if s.ingestionSvc != nil && s.sourceRepo != nil { + s.deleteNotebookVectors(ctx, userID, notebookID) + } + + // 清理关联会话的 Redis 缓存和消息 + if s.conversationRepo != nil && s.chatCache != nil { + s.cleanupNotebookConversations(ctx, notebookID) + } + + // 清理该笔记本下所有 source 的摘要缓存 + if s.summaryCache != nil { + s.cleanupNotebookSourceSummaries(ctx, userID, notebookID) + } + + // 软删除该笔记本下所有 source(软删除不触发 CASCADE) + if err := s.sourceRepo.DeleteByNotebookID(notebookID); err != nil { + logger.Error("删除笔记本关联的 source 失败", + zap.Uint("notebook_id", notebookID), + zap.Error(err), + ) + } + + return s.notebookRepo.Delete(notebookID) +} + +// deleteNotebookVectors 删除笔记本下所有 source 的向量数据和 parent_blocks +func (s *notebookService) deleteNotebookVectors(ctx context.Context, userID, notebookID uint) { + // 查询该笔记本下所有 source + sources, _, err := s.sourceRepo.ListByNotebook(userID, notebookID, "", 0, 10000) + if err != nil { + logger.Error("查询笔记本关联的 source 失败", + zap.Uint("notebook_id", notebookID), + zap.Error(err), + ) + return + } + + // 逐个删除向量数据和 parent_blocks + for _, source := range sources { + if err := s.ingestionSvc.DeleteSource(ctx, userID, source.ID); err != nil { + logger.Error("删除笔记本关联的源数据失败", + zap.Uint("notebook_id", notebookID), + zap.Uint("source_id", source.ID), + zap.Error(err), + ) + } + } +} + +// cleanupNotebookConversations 清理笔记本下所有会话的 Redis 缓存和消息 +func (s *notebookService) cleanupNotebookConversations(ctx context.Context, notebookID uint) { + convs, err := s.conversationRepo.FindByNotebookID(notebookID) + if err != nil { + logger.Error("查询笔记本关联的会话失败", + zap.Uint("notebook_id", notebookID), + zap.Error(err), + ) + return + } + + for _, conv := range convs { + // 删除消息 + if s.messageRepo != nil { + if err := s.messageRepo.DeleteByConversationID(conv.ID); err != nil { + logger.Warn("删除会话消息失败", + zap.Uint("conversation_id", conv.ID), + zap.Error(err), + ) + } + } + // 清除 Redis 缓存(消息历史+摘要) + if err := s.chatCache.DeleteConversationCache(ctx, conv.ID); err != nil { + logger.Warn("清除会话缓存失败", + zap.Uint("conversation_id", conv.ID), + zap.Error(err), + ) + } + } + + // 删除会话本身 + if err := s.conversationRepo.DeleteByNotebookID(notebookID); err != nil { + logger.Error("删除笔记本关联的会话失败", + zap.Uint("notebook_id", notebookID), + zap.Error(err), + ) + } +} + +// cleanupNotebookSourceSummaries 清理笔记本下所有 source 的摘要缓存 +func (s *notebookService) cleanupNotebookSourceSummaries(ctx context.Context, userID, notebookID uint) { + // 查询该笔记本下所有 source + sources, _, err := s.sourceRepo.ListByNotebook(userID, notebookID, "", 0, 10000) + if err != nil { + logger.Error("查询笔记本关联的 source 失败(用于清理摘要缓存)", + zap.Uint("notebook_id", notebookID), + zap.Error(err), + ) + return + } + + if len(sources) == 0 { + return + } + + // 收集所有 sourceID + sourceIDs := make([]uint, len(sources)) + for i, source := range sources { + sourceIDs[i] = source.ID + } + + // 批量删除摘要缓存 + if err := s.summaryCache.BatchDelete(ctx, sourceIDs); err != nil { + logger.Warn("批量删除摘要缓存失败", + zap.Uint("notebook_id", notebookID), + zap.Int("count", len(sourceIDs)), + zap.Error(err), + ) + } else { + logger.Info("已清理笔记本关联的摘要缓存", + zap.Uint("notebook_id", notebookID), + zap.Int("count", len(sourceIDs)), + ) + } +} + +// toResponse 转换为响应 DTO +func (s *notebookService) toResponse(notebook *entity.Notebook) *response.NotebookResponse { + return &response.NotebookResponse{ + ID: notebook.ID, + Name: notebook.Name, + CreatedAt: notebook.CreatedAt, + UpdatedAt: notebook.UpdatedAt, + } +} diff --git a/internal/service/pptx_export_test.go b/internal/service/pptx_export_test.go deleted file mode 100644 index 3d10079..0000000 --- a/internal/service/pptx_export_test.go +++ /dev/null @@ -1,150 +0,0 @@ -package service - -import ( - "archive/zip" - "bytes" - "context" - "fmt" - "io" - "os" - "strings" - "testing" -) - -func TestExportPPTXNotCorrupted(t *testing.T) { - // 构建一个包含代码块的 PPT HTML - html := ` -
-

光合作用

-

植物通过光合作用将光能转化为化学能

-
-
-

代码示例

-

Go语言函数定义:

-
func main() {
-    fmt.Println("hello")
-}
-
` - - // 测试 Go 纯实现路径 (buildDynamicHTMLPPTX) - data, err := buildDynamicHTMLPPTX(html, "test-export") - if err != nil { - t.Fatalf("buildDynamicHTMLPPTX 失败: %v", err) - } - t.Logf("PPTX 大小: %d bytes", len(data)) - - // 验证 PPTX 结构 - if err := validatePPTXStructure(data, t); err != nil { - t.Fatalf("PPTX 结构验证失败: %v", err) - } - - // 保存到临时文件供手动验证 - tempDir, _ := os.MkdirTemp("", "pptx-validate-*") - path := tempDir + "/test_export.pptx" - os.WriteFile(path, data, 0644) - t.Logf("PPTX 保存到: %s", path) -} - -func TestExportPPTXWithDOMFallback(t *testing.T) { - // 测试 exportPPTWithDefaultEngine 路径 - // 当 Playwright 不可用时,应 fallback 到 buildDynamicHTMLPPTX - html := `

测试

内容

` - - data, err := exportPPTWithDefaultEngine(context.Background(), html, "test-dom") - if err != nil { - t.Fatalf("exportPPTWithDefaultEngine 失败: %v", err) - } - t.Logf("PPTX 大小: %d bytes", len(data)) - - // 验证 PK 签名 - if len(data) < 2 || data[0] != 0x50 || data[1] != 0x4b { - t.Fatalf("不是有效的 PPTX (ZIP) 文件") - } - - if err := validatePPTXStructure(data, t); err != nil { - t.Fatalf("PPTX 结构验证失败: %v", err) - } -} - -func validatePPTXStructure(data []byte, t *testing.T) error { - reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) - if err != nil { - return fmt.Errorf("ZIP 读取失败: %w", err) - } - - existing := make(map[string]bool) - for _, f := range reader.File { - existing[f.Name] = true - } - - // 检查必需文件 - required := []string{ - "[Content_Types].xml", - "ppt/presentation.xml", - "ppt/_rels/presentation.xml.rels", - "ppt/slideMasters/slideMaster1.xml", - } - for _, name := range required { - if !existing[name] { - return fmt.Errorf("缺少必需文件: %s", name) - } - } - - // 计算幻灯片数量 - slideCount := 0 - for _, f := range reader.File { - if strings.HasPrefix(f.Name, "ppt/slides/slide") && - strings.HasSuffix(f.Name, ".xml") && - !strings.Contains(f.Name, "_rels") { - slideCount++ - } - } - t.Logf("幻灯片数量: %d", slideCount) - - // 检查每个 slide 的 rels - for i := 1; i <= slideCount; i++ { - relsName := fmt.Sprintf("ppt/slides/_rels/slide%d.xml.rels", i) - if !existing[relsName] { - return fmt.Errorf("缺少 slide rels: %s", relsName) - } - } - - // 检查 Content_Types.xml - for _, f := range reader.File { - if f.Name == "[Content_Types].xml" { - rc, _ := f.Open() - content, _ := io.ReadAll(rc) - rc.Close() - ct := string(content) - for i := 1; i <= slideCount; i++ { - partName := fmt.Sprintf("/ppt/slides/slide%d.xml", i) - if !strings.Contains(ct, partName) { - t.Errorf("[Content_Types].xml 缺少 slide%d 的 Override", i) - } - } - } - } - - // 检查 presentation.xml 中的 sldId - for _, f := range reader.File { - if f.Name == "ppt/presentation.xml" { - rc, _ := f.Open() - content, _ := io.ReadAll(rc) - rc.Close() - pxml := string(content) - for i := 1; i <= slideCount; i++ { - sldId := fmt.Sprintf(`rId%d"`, i+2) - if !strings.Contains(pxml, sldId) { - t.Errorf("presentation.xml 缺少 slide%d 的 sldId (rId%d)", i, i+2) - } - } - } - } - - return nil -} diff --git a/internal/service/search_agent_interface.go b/internal/service/search_agent_interface.go index 9d94693..997fe48 100644 --- a/internal/service/search_agent_interface.go +++ b/internal/service/search_agent_interface.go @@ -1,50 +1,46 @@ -// internal/service/search_agent_interface.go -package service - -import ( - "context" - - "YoudaoNoteLm/internal/model/dto/response" -) - -// SearchAgentInterface 搜索 Agent 执行接口(由 agent/search.SearchAgent 实现) -type SearchAgentInterface interface { - // Execute 执行搜索任务(用户交互模式:只搜索不自动导入) - Execute(ctx context.Context, userID, notebookID uint, task string) (*SearchAgentResult, error) - // ExecuteStream 流式执行搜索任务,通过 channel 逐个推送事件,完成后关闭 channel - ExecuteStream(ctx context.Context, userID, notebookID uint, task string) <-chan *SearchAgentEvent - // ExecuteWithImport 执行搜索并自动导入任务(主Agent调用模式) - ExecuteWithImport(ctx context.Context, userID, notebookID uint, task string) (*SearchAgentResult, error) -} - -// SearchAgentResult Agent 执行结果(与 agent/search.AgentResult 对应) -type SearchAgentResult struct { - Content string `json:"content"` - SearchRounds int `json:"search_rounds"` -} - -// SearchAgentEvent Agent 流式执行事件 -type SearchAgentEvent struct { - Type string `json:"type"` // content, tool_call, search_round, error, done - Content string `json:"content,omitempty"` - Role string `json:"role,omitempty"` - ToolName string `json:"tool_name,omitempty"` - ToolArgs string `json:"tool_args,omitempty"` - SearchRounds int `json:"search_rounds,omitempty"` - Error string `json:"error,omitempty"` - ErrorCode int `json:"error_code,omitempty"` // 错误码,用于前端精确判断错误类型 -} - -// SearchAgentService 搜索 Agent 服务接口 -type SearchAgentService interface { - // Search 智能搜索:Agent 自主执行多轮搜索+分析(用户交互模式,不自动导入) - Search(userID, notebookID uint, query string) (*response.SearchResponse, error) - // SearchStream 智能搜索(流式):返回事件 channel,用于 SSE 推送 - SearchStream(userID, notebookID uint, query string) <-chan *SearchAgentEvent - // ImportFromURL URL 直接导入(返回任务 ID 和 Source ID) - ImportFromURL(userID, notebookID uint, url string) (taskID string, sourceID uint, err error) - // ImportSearchResults 批量导入搜索结果(带标题),返回任务 ID 和创建的 Source ID 列表 - ImportSearchResults(userID, notebookID uint, items []SearchResultItem) (taskID string, sourceIDs []uint, err error) - // SearchAndImport 搜索并自动导入:Agent 自主执行多轮搜索并自动导入结果(主Agent调用模式) - SearchAndImport(userID, notebookID uint, query string) (*response.SearchResponse, error) -} +// internal/service/search_agent_interface.go +package service + +import ( + "context" + + "YoudaoNoteLm/internal/model/dto/response" +) + +// SearchAgentInterface 搜索 Agent 执行接口(由 agent/search.SearchAgent 实现) +type SearchAgentInterface interface { + // Execute 执行搜索任务(用户交互模式:只搜索不自动导入) + Execute(ctx context.Context, userID, notebookID uint, task string) (*SearchAgentResult, error) + // ExecuteStream 流式执行搜索任务,通过 channel 逐个推送事件,完成后关闭 channel + ExecuteStream(ctx context.Context, userID, notebookID uint, task string) <-chan *SearchAgentEvent +} + +// SearchAgentResult Agent 执行结果(与 agent/search.AgentResult 对应) +type SearchAgentResult struct { + Content string `json:"content"` + SearchRounds int `json:"search_rounds"` +} + +// SearchAgentEvent Agent 流式执行事件 +type SearchAgentEvent struct { + Type string `json:"type"` // content, tool_call, search_round, error, done + Content string `json:"content,omitempty"` + Role string `json:"role,omitempty"` + ToolName string `json:"tool_name,omitempty"` + ToolArgs string `json:"tool_args,omitempty"` + SearchRounds int `json:"search_rounds,omitempty"` + Error string `json:"error,omitempty"` + ErrorCode int `json:"error_code,omitempty"` // 错误码,用于前端精确判断错误类型 +} + +// SearchAgentService 搜索 Agent 服务接口 +type SearchAgentService interface { + // Search 智能搜索:Agent 自主执行多轮搜索+分析(用户交互模式,不自动导入) + Search(ctx context.Context, userID, notebookID uint, query string) (*response.SearchResponse, error) + // SearchStream 智能搜索(流式):返回事件 channel,用于 SSE 推送 + SearchStream(ctx context.Context, userID, notebookID uint, query string) <-chan *SearchAgentEvent + // ImportFromURL URL 直接导入(返回任务 ID 和 Source ID) + ImportFromURL(userID, notebookID uint, url string) (taskID string, sourceID uint, err error) + // ImportSearchResults 批量导入搜索结果(带标题),返回任务 ID 和创建的 Source ID 列表 + ImportSearchResults(userID, notebookID uint, items []SearchResultItem) (taskID string, sourceIDs []uint, err error) +} diff --git a/internal/service/search_agent_service.go b/internal/service/search_agent_service.go index 83844f9..0564e36 100644 --- a/internal/service/search_agent_service.go +++ b/internal/service/search_agent_service.go @@ -1,197 +1,182 @@ -// internal/service/search_agent_service.go -package service - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - "YoudaoNoteLm/internal/model/dto/response" - "YoudaoNoteLm/pkg/logger" - - "go.uber.org/zap" -) - -type searchAgentService struct { - configService ConfigService - importer ImporterService - searchAgent SearchAgentInterface -} - -// NewSearchAgentService 创建搜索 Agent 服务 -func NewSearchAgentService( - configService ConfigService, - importer ImporterService, - searchAgent SearchAgentInterface, -) SearchAgentService { - return &searchAgentService{ - configService: configService, - importer: importer, - searchAgent: searchAgent, - } -} - -// Search 智能搜索 -func (s *searchAgentService) Search(userID, notebookID uint, query string) (*response.SearchResponse, error) { - // 执行 Agent - ctx := context.Background() - result, err := s.searchAgent.Execute(ctx, userID, notebookID, query) - if err != nil { - return nil, err - } - - // 解析 Agent 结果为 SearchResponse - return parseAgentResult(result.Content, result.SearchRounds) -} - -// SearchStream 智能搜索(流式):返回事件 channel,用于 SSE 推送 -func (s *searchAgentService) SearchStream(userID, notebookID uint, query string) <-chan *SearchAgentEvent { - ctx := context.Background() - return s.searchAgent.ExecuteStream(ctx, userID, notebookID, query) -} - -// ImportFromURL URL 直接导入(返回任务 ID 和 Source ID) -func (s *searchAgentService) ImportFromURL(userID, notebookID uint, url string) (string, uint, error) { - taskID, sourceIDs, err := s.importer.ImportSearchResults(userID, notebookID, []SearchResultItem{ - {URL: url}, - }) - if err != nil { - return "", 0, err - } - if len(sourceIDs) == 0 { - return "", 0, fmt.Errorf("导入失败:未创建Source记录") - } - - logger.Info("URL导入任务已创建", - zap.Uint("user_id", userID), - zap.String("url", url), - zap.String("task_id", taskID), - zap.Uint("source_id", sourceIDs[0]), - ) - - return taskID, sourceIDs[0], nil -} - -// ImportSearchResults 批量导入 -func (s *searchAgentService) ImportSearchResults(userID, notebookID uint, items []SearchResultItem) (string, []uint, error) { - return s.importer.ImportSearchResults(userID, notebookID, items) -} - -// SearchAndImport 搜索并自动导入(主Agent调用模式) -func (s *searchAgentService) SearchAndImport(userID, notebookID uint, query string) (*response.SearchResponse, error) { - // 执行 Agent(自动导入模式) - ctx := context.Background() - result, err := s.searchAgent.ExecuteWithImport(ctx, userID, notebookID, query) - if err != nil { - return nil, err - } - - // 解析 Agent 结果为 SearchResponse - return parseAgentResult(result.Content, result.SearchRounds) -} - -// parseAgentResult 解析 Agent 返回的内容为 SearchResponse -func parseAgentResult(content string, searchRounds int) (*response.SearchResponse, error) { - // 尝试从 Agent 回复中提取 JSON 代码块 - if jsonBlock := extractJSONBlock(content); jsonBlock != "" { - var result response.SearchResponse - if err := json.Unmarshal([]byte(jsonBlock), &result); err == nil { - result.SearchRounds = searchRounds - // summary 如果为空,用前面的文本 - if result.Summary == "" { - result.Summary = extractTextBeforeJSON(content) - } - return &result, nil - } - } - - // 尝试直接解析整个内容为 JSON - var result response.SearchResponse - if err := json.Unmarshal([]byte(content), &result); err == nil { - return &result, nil - } - - // 如果 Agent 没有返回结构化 JSON,则将整个内容作为 summary - result = response.SearchResponse{ - Results: []response.SearchResultItem{}, - Summary: content, - SearchRounds: searchRounds, - } - - // 尝试从文本中提取 URL 作为结果 - lines := strings.Split(content, "\n") - for _, line := range lines { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "http://") || strings.HasPrefix(line, "https://") { - result.Results = append(result.Results, response.SearchResultItem{ - URL: line, - }) - } - } - - // 如果没有提取到 URL,尝试从 Markdown 链接中提取 - if len(result.Results) == 0 { - for _, line := range lines { - if idx := strings.Index(line, "]("); idx != -1 { - start := idx + 2 - end := strings.Index(line[start:], ")") - if end != -1 { - url := line[start : start+end] - if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { - result.Results = append(result.Results, response.SearchResultItem{ - URL: url, - }) - } - } - } - } - } - - return &result, nil -} - -// extractJSONBlock 从文本中提取 ```json ... ``` 代码块 -func extractJSONBlock(content string) string { - startMarker := "```json" - endMarker := "```" - - startIdx := strings.Index(content, startMarker) - if startIdx == -1 { - // 尝试 ``` 开头 - startMarker = "```" - startIdx = strings.Index(content, startMarker) - if startIdx == -1 { - return "" - } - // 跳过 ```\n - startIdx += len(startMarker) - if startIdx < len(content) && content[startIdx] == '\n' { - startIdx++ - } - } else { - startIdx += len(startMarker) - if startIdx < len(content) && content[startIdx] == '\n' { - startIdx++ - } - } - - endIdx := strings.Index(content[startIdx:], endMarker) - if endIdx == -1 { - return "" - } - - return strings.TrimSpace(content[startIdx : startIdx+endIdx]) -} - -// extractTextBeforeJSON 提取 JSON 代码块之前的文本 -func extractTextBeforeJSON(content string) string { - idx := strings.Index(content, "```json") - if idx == -1 { - idx = strings.Index(content, "```") - } - if idx == -1 { - return content - } - return strings.TrimSpace(content[:idx]) -} +// internal/service/search_agent_service.go +package service + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "YoudaoNoteLm/internal/model/dto/response" + "YoudaoNoteLm/pkg/logger" + + "go.uber.org/zap" +) + +type searchAgentService struct { + configService ConfigService + importer ImporterService + searchAgent SearchAgentInterface +} + +// NewSearchAgentService 创建搜索 Agent 服务 +func NewSearchAgentService( + configService ConfigService, + importer ImporterService, + searchAgent SearchAgentInterface, +) SearchAgentService { + return &searchAgentService{ + configService: configService, + importer: importer, + searchAgent: searchAgent, + } +} + +// Search 智能搜索 +func (s *searchAgentService) Search(ctx context.Context, userID, notebookID uint, query string) (*response.SearchResponse, error) { + // 执行 Agent + result, err := s.searchAgent.Execute(ctx, userID, notebookID, query) + if err != nil { + return nil, err + } + + // 解析 Agent 结果为 SearchResponse + return parseAgentResult(result.Content, result.SearchRounds) +} + +// SearchStream 智能搜索(流式):返回事件 channel,用于 SSE 推送 +func (s *searchAgentService) SearchStream(ctx context.Context, userID, notebookID uint, query string) <-chan *SearchAgentEvent { + return s.searchAgent.ExecuteStream(ctx, userID, notebookID, query) +} + +// ImportFromURL URL 直接导入(返回任务 ID 和 Source ID) +func (s *searchAgentService) ImportFromURL(userID, notebookID uint, url string) (string, uint, error) { + taskID, sourceIDs, err := s.importer.ImportSearchResults(userID, notebookID, []SearchResultItem{ + {URL: url}, + }) + if err != nil { + return "", 0, err + } + if len(sourceIDs) == 0 { + return "", 0, fmt.Errorf("导入失败:未创建Source记录") + } + + logger.Info("URL导入任务已创建", + zap.Uint("user_id", userID), + zap.String("url", url), + zap.String("task_id", taskID), + zap.Uint("source_id", sourceIDs[0]), + ) + + return taskID, sourceIDs[0], nil +} + +// ImportSearchResults 批量导入 +func (s *searchAgentService) ImportSearchResults(userID, notebookID uint, items []SearchResultItem) (string, []uint, error) { + return s.importer.ImportSearchResults(userID, notebookID, items) +} + +// parseAgentResult 解析 Agent 返回的内容为 SearchResponse +func parseAgentResult(content string, searchRounds int) (*response.SearchResponse, error) { + // 尝试从 Agent 回复中提取 JSON 代码块 + if jsonBlock := extractJSONBlock(content); jsonBlock != "" { + var result response.SearchResponse + if err := json.Unmarshal([]byte(jsonBlock), &result); err == nil { + result.SearchRounds = searchRounds + // summary 如果为空,用前面的文本 + if result.Summary == "" { + result.Summary = extractTextBeforeJSON(content) + } + return &result, nil + } + } + + // 尝试直接解析整个内容为 JSON + var result response.SearchResponse + if err := json.Unmarshal([]byte(content), &result); err == nil { + return &result, nil + } + + // 如果 Agent 没有返回结构化 JSON,则将整个内容作为 summary + result = response.SearchResponse{ + Results: []response.SearchResultItem{}, + Summary: content, + SearchRounds: searchRounds, + } + + // 尝试从文本中提取 URL 作为结果 + lines := strings.Split(content, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "http://") || strings.HasPrefix(line, "https://") { + result.Results = append(result.Results, response.SearchResultItem{ + URL: line, + }) + } + } + + // 如果没有提取到 URL,尝试从 Markdown 链接中提取 + if len(result.Results) == 0 { + for _, line := range lines { + if idx := strings.Index(line, "]("); idx != -1 { + start := idx + 2 + end := strings.Index(line[start:], ")") + if end != -1 { + url := line[start : start+end] + if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { + result.Results = append(result.Results, response.SearchResultItem{ + URL: url, + }) + } + } + } + } + } + + return &result, nil +} + +// extractJSONBlock 从文本中提取 ```json ... ``` 代码块 +func extractJSONBlock(content string) string { + startMarker := "```json" + endMarker := "```" + + startIdx := strings.Index(content, startMarker) + if startIdx == -1 { + // 尝试 ``` 开头 + startMarker = "```" + startIdx = strings.Index(content, startMarker) + if startIdx == -1 { + return "" + } + // 跳过 ```\n + startIdx += len(startMarker) + if startIdx < len(content) && content[startIdx] == '\n' { + startIdx++ + } + } else { + startIdx += len(startMarker) + if startIdx < len(content) && content[startIdx] == '\n' { + startIdx++ + } + } + + endIdx := strings.Index(content[startIdx:], endMarker) + if endIdx == -1 { + return "" + } + + return strings.TrimSpace(content[startIdx : startIdx+endIdx]) +} + +// extractTextBeforeJSON 提取 JSON 代码块之前的文本 +func extractTextBeforeJSON(content string) string { + idx := strings.Index(content, "```json") + if idx == -1 { + idx = strings.Index(content, "```") + } + if idx == -1 { + return content + } + return strings.TrimSpace(content[:idx]) +} diff --git a/internal/service/search_interface.go b/internal/service/search_interface.go index a6abe98..941c543 100644 --- a/internal/service/search_interface.go +++ b/internal/service/search_interface.go @@ -1,93 +1,93 @@ -package service - -import ( - "context" - "time" -) - -// SearchScene 搜索使用场景。 -type SearchScene string - -const ( - SearchSceneGeneration SearchScene = "generation" - SearchSceneChat SearchScene = "chat" - SearchSceneImport SearchScene = "import" - SearchSceneSourcePreview SearchScene = "source_preview" -) - -// SearchRequest 统一搜索请求。 -type SearchRequest struct { - UserID uint `json:"user_id,omitempty"` - Scene SearchScene `json:"scene"` - Query string `json:"query"` - Freshness string `json:"freshness,omitempty"` - Count int `json:"count,omitempty"` - NeedSummary bool `json:"need_summary,omitempty"` - NeedContent bool `json:"need_content,omitempty"` - Language string `json:"language,omitempty"` - AllowedDomains []string `json:"allowed_domains,omitempty"` - BlockedDomains []string `json:"blocked_domains,omitempty"` - NotebookID uint `json:"notebook_id,omitempty"` - SourceID uint `json:"source_id,omitempty"` - TraceID string `json:"trace_id,omitempty"` - AllowDegrade bool `json:"allow_degrade,omitempty"` - SkipUserConfig bool `json:"-"` -} - -// SearchQuota 统一搜索额度信息。 -type SearchQuota struct { - DailyQuota *int `json:"daily_quota,omitempty"` - Used int `json:"used"` - Remaining *int `json:"remaining,omitempty"` - ResetAt *time.Time `json:"reset_at,omitempty"` -} - -// SearchResult 统一搜索结果。 -type SearchResult struct { - Title string `json:"title"` - Snippet string `json:"snippet,omitempty"` - URL string `json:"url"` - DisplayURL string `json:"display_url,omitempty"` - PublishedAt string `json:"published_at,omitempty"` - SiteName string `json:"site_name,omitempty"` - Score float64 `json:"score,omitempty"` - Content string `json:"content,omitempty"` - ProviderRawID string `json:"provider_raw_id,omitempty"` - Meta map[string]any `json:"meta,omitempty"` -} - -// SearchResponse 统一搜索响应。 -type SearchResponse struct { - Query string `json:"query"` - Provider string `json:"provider"` - Results []SearchResult `json:"results"` - Summary string `json:"summary,omitempty"` - Total int `json:"total"` - Cached bool `json:"cached"` - Quota *SearchQuota `json:"quota,omitempty"` - Meta map[string]any `json:"meta,omitempty"` -} - -// SearchImportRequest 搜索导入请求。 -type SearchImportRequest struct { - SearchRequest -} - -// SearchImportResponse 搜索导入预览结果。 -type SearchImportResponse struct { - Query string `json:"query"` - Provider string `json:"provider"` - Results []SearchResult `json:"results"` - URLs []string `json:"urls"` - Total int `json:"total"` - Cached bool `json:"cached"` - Quota *SearchQuota `json:"quota,omitempty"` - Meta map[string]any `json:"meta,omitempty"` -} - -// SearchService 统一搜索服务接口。 -type SearchService interface { - Search(ctx context.Context, req *SearchRequest) (*SearchResponse, error) - SearchAndSummarize(ctx context.Context, req *SearchRequest) (*SearchResponse, error) - SearchForImport(ctx context.Context, req *SearchImportRequest) (*SearchImportResponse, error) -} +package service + +import ( + "context" + "time" +) + +// SearchScene 搜索使用场景。 +type SearchScene string + +const ( + SearchSceneGeneration SearchScene = "generation" + SearchSceneChat SearchScene = "chat" + SearchSceneImport SearchScene = "import" + SearchSceneSourcePreview SearchScene = "source_preview" +) + +// SearchRequest 统一搜索请求。 +type SearchRequest struct { + UserID uint `json:"user_id,omitempty"` + Scene SearchScene `json:"scene"` + Query string `json:"query"` + Freshness string `json:"freshness,omitempty"` + Count int `json:"count,omitempty"` + NeedSummary bool `json:"need_summary,omitempty"` + NeedContent bool `json:"need_content,omitempty"` + Language string `json:"language,omitempty"` + AllowedDomains []string `json:"allowed_domains,omitempty"` + BlockedDomains []string `json:"blocked_domains,omitempty"` + NotebookID uint `json:"notebook_id,omitempty"` + SourceID uint `json:"source_id,omitempty"` + TraceID string `json:"trace_id,omitempty"` + AllowDegrade bool `json:"allow_degrade,omitempty"` + SkipUserConfig bool `json:"-"` +} + +// SearchQuota 统一搜索额度信息。 +type SearchQuota struct { + DailyQuota *int `json:"daily_quota,omitempty"` + Used int `json:"used"` + Remaining *int `json:"remaining,omitempty"` + ResetAt *time.Time `json:"reset_at,omitempty"` +} + +// SearchResult 统一搜索结果。 +type SearchResult struct { + Title string `json:"title"` + Snippet string `json:"snippet,omitempty"` + URL string `json:"url"` + DisplayURL string `json:"display_url,omitempty"` + PublishedAt string `json:"published_at,omitempty"` + SiteName string `json:"site_name,omitempty"` + Score float64 `json:"score,omitempty"` + Content string `json:"content,omitempty"` + ProviderRawID string `json:"provider_raw_id,omitempty"` + Meta map[string]any `json:"meta,omitempty"` +} + +// SearchResponse 统一搜索响应。 +type SearchResponse struct { + Query string `json:"query"` + Provider string `json:"provider"` + Results []SearchResult `json:"results"` + Summary string `json:"summary,omitempty"` + Total int `json:"total"` + Cached bool `json:"cached"` + Quota *SearchQuota `json:"quota,omitempty"` + Meta map[string]any `json:"meta,omitempty"` +} + +// SearchImportRequest 搜索导入请求。 +type SearchImportRequest struct { + SearchRequest +} + +// SearchImportResponse 搜索导入预览结果。 +type SearchImportResponse struct { + Query string `json:"query"` + Provider string `json:"provider"` + Results []SearchResult `json:"results"` + URLs []string `json:"urls"` + Total int `json:"total"` + Cached bool `json:"cached"` + Quota *SearchQuota `json:"quota,omitempty"` + Meta map[string]any `json:"meta,omitempty"` +} + +// SearchService 统一搜索服务接口。 +type SearchService interface { + Search(ctx context.Context, req *SearchRequest) (*SearchResponse, error) + SearchAndSummarize(ctx context.Context, req *SearchRequest) (*SearchResponse, error) + SearchForImport(ctx context.Context, req *SearchImportRequest) (*SearchImportResponse, error) +} diff --git a/internal/service/search_query_builder.go b/internal/service/search_query_builder.go index c60fe9d..0eac2a3 100644 --- a/internal/service/search_query_builder.go +++ b/internal/service/search_query_builder.go @@ -1,26 +1,26 @@ -package service - -import ( - "strings" - - bizerrors "YoudaoNoteLm/pkg/errors" -) - -// BuildSearchQuery 统一构建搜索 query。 -func BuildSearchQuery(req *SearchRequest) (string, error) { - if req == nil { - return "", bizerrors.New(bizerrors.CodeInvalidParam, "搜索请求不能为空") - } - - query := strings.Join(strings.Fields(strings.TrimSpace(req.Query)), " ") - if query == "" { - return "", bizerrors.New(bizerrors.CodeInvalidParam, "搜索 query 不能为空") - } - - switch req.Scene { - case SearchSceneGeneration, SearchSceneChat, SearchSceneImport, SearchSceneSourcePreview: - return query, nil - default: - return "", bizerrors.New(bizerrors.CodeInvalidParam, "无效的搜索场景") - } -} +package service + +import ( + "strings" + + bizerrors "YoudaoNoteLm/pkg/errors" +) + +// BuildSearchQuery 统一构建搜索 query。 +func BuildSearchQuery(req *SearchRequest) (string, error) { + if req == nil { + return "", bizerrors.New(bizerrors.CodeInvalidParam, "搜索请求不能为空") + } + + query := strings.Join(strings.Fields(strings.TrimSpace(req.Query)), " ") + if query == "" { + return "", bizerrors.New(bizerrors.CodeInvalidParam, "搜索 query 不能为空") + } + + switch req.Scene { + case SearchSceneGeneration, SearchSceneChat, SearchSceneImport, SearchSceneSourcePreview: + return query, nil + default: + return "", bizerrors.New(bizerrors.CodeInvalidParam, "无效的搜索场景") + } +} diff --git a/internal/service/search_result_normalizer.go b/internal/service/search_result_normalizer.go index eb0a625..fa40a92 100644 --- a/internal/service/search_result_normalizer.go +++ b/internal/service/search_result_normalizer.go @@ -1,139 +1,139 @@ -package service - -import ( - "net/url" - "sort" - "strings" - - "YoudaoNoteLm/internal/service/external" -) - -// NormalizeSearchResults 将 provider 结果清洗为统一结果。 -func NormalizeSearchResults(results []external.SearchProviderResult, needContent bool, allowedDomains, blockedDomains []string) []SearchResult { - allowed := toDomainSet(allowedDomains) - blocked := toDomainSet(blockedDomains) - seen := make(map[string]struct{}, len(results)) - normalized := make([]SearchResult, 0, len(results)) - - for _, item := range results { - rawURL := strings.TrimSpace(item.URL) - title := strings.TrimSpace(item.Title) - if rawURL == "" || title == "" { - continue - } - - host := hostFromURL(rawURL) - if len(allowed) > 0 && !matchDomainSet(host, allowed) { - continue - } - if matchDomainSet(host, blocked) { - continue - } - - key := normalizeURL(rawURL) - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - - content := "" - if needContent { - content = compactText(firstNonEmpty(item.Summary, item.Snippet), 1200) - } - - normalized = append(normalized, SearchResult{ - Title: compactText(title, 200), - Snippet: compactText(firstNonEmpty(item.Snippet, item.Summary), 500), - URL: rawURL, - DisplayURL: firstNonEmpty(strings.TrimSpace(item.DisplayURL), host), - PublishedAt: strings.TrimSpace(item.PublishedAt), - SiteName: firstNonEmpty(strings.TrimSpace(item.SiteName), host), - Score: item.Score, - Content: content, - ProviderRawID: strings.TrimSpace(item.ID), - Meta: item.Meta, - }) - } - - sort.SliceStable(normalized, func(i, j int) bool { - if normalized[i].Score == normalized[j].Score { - return normalized[i].PublishedAt > normalized[j].PublishedAt - } - return normalized[i].Score > normalized[j].Score - }) - - return normalized -} - -// BuildSearchSummary 按结果拼接简要摘要。 -func BuildSearchSummary(results []SearchResult) string { - if len(results) == 0 { - return "" - } - - parts := make([]string, 0, 3) - for i := 0; i < len(results) && i < 3; i++ { - text := strings.TrimSpace(firstNonEmpty(results[i].Content, results[i].Snippet)) - if text != "" { - parts = append(parts, text) - } - } - return strings.Join(parts, "\n") -} - -func toDomainSet(domains []string) map[string]struct{} { - if len(domains) == 0 { - return nil - } - result := make(map[string]struct{}, len(domains)) - for _, domain := range domains { - host := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(domain, "https://"), "http://"))) - host = strings.TrimPrefix(host, "www.") - host = strings.TrimSuffix(host, "/") - if host != "" { - result[host] = struct{}{} - } - } - return result -} - -func matchDomainSet(host string, domainSet map[string]struct{}) bool { - if len(domainSet) == 0 || host == "" { - return false - } - host = strings.ToLower(strings.TrimPrefix(host, "www.")) - for domain := range domainSet { - if host == domain || strings.HasSuffix(host, "."+domain) { - return true - } - } - return false -} - -func hostFromURL(raw string) string { - parsed, err := url.Parse(raw) - if err != nil { - return "" - } - return strings.ToLower(parsed.Hostname()) -} - -func normalizeURL(raw string) string { - parsed, err := url.Parse(strings.TrimSpace(raw)) - if err != nil { - return strings.TrimSpace(raw) - } - parsed.Fragment = "" - if host := strings.ToLower(parsed.Hostname()); host != "" { - parsed.Host = host - } - return parsed.String() -} - -func compactText(value string, maxLen int) string { - value = strings.Join(strings.Fields(strings.TrimSpace(value)), " ") - if maxLen > 0 && len(value) > maxLen { - return value[:maxLen] + "..." - } - return value -} +package service + +import ( + "net/url" + "sort" + "strings" + + "YoudaoNoteLm/internal/service/external" +) + +// NormalizeSearchResults 将 provider 结果清洗为统一结果。 +func NormalizeSearchResults(results []external.SearchProviderResult, needContent bool, allowedDomains, blockedDomains []string) []SearchResult { + allowed := toDomainSet(allowedDomains) + blocked := toDomainSet(blockedDomains) + seen := make(map[string]struct{}, len(results)) + normalized := make([]SearchResult, 0, len(results)) + + for _, item := range results { + rawURL := strings.TrimSpace(item.URL) + title := strings.TrimSpace(item.Title) + if rawURL == "" || title == "" { + continue + } + + host := hostFromURL(rawURL) + if len(allowed) > 0 && !matchDomainSet(host, allowed) { + continue + } + if matchDomainSet(host, blocked) { + continue + } + + key := normalizeURL(rawURL) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + + content := "" + if needContent { + content = compactText(firstNonEmpty(item.Summary, item.Snippet), 1200) + } + + normalized = append(normalized, SearchResult{ + Title: compactText(title, 200), + Snippet: compactText(firstNonEmpty(item.Snippet, item.Summary), 500), + URL: rawURL, + DisplayURL: firstNonEmpty(strings.TrimSpace(item.DisplayURL), host), + PublishedAt: strings.TrimSpace(item.PublishedAt), + SiteName: firstNonEmpty(strings.TrimSpace(item.SiteName), host), + Score: item.Score, + Content: content, + ProviderRawID: strings.TrimSpace(item.ID), + Meta: item.Meta, + }) + } + + sort.SliceStable(normalized, func(i, j int) bool { + if normalized[i].Score == normalized[j].Score { + return normalized[i].PublishedAt > normalized[j].PublishedAt + } + return normalized[i].Score > normalized[j].Score + }) + + return normalized +} + +// BuildSearchSummary 按结果拼接简要摘要。 +func BuildSearchSummary(results []SearchResult) string { + if len(results) == 0 { + return "" + } + + parts := make([]string, 0, 3) + for i := 0; i < len(results) && i < 3; i++ { + text := strings.TrimSpace(firstNonEmpty(results[i].Content, results[i].Snippet)) + if text != "" { + parts = append(parts, text) + } + } + return strings.Join(parts, "\n") +} + +func toDomainSet(domains []string) map[string]struct{} { + if len(domains) == 0 { + return nil + } + result := make(map[string]struct{}, len(domains)) + for _, domain := range domains { + host := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(domain, "https://"), "http://"))) + host = strings.TrimPrefix(host, "www.") + host = strings.TrimSuffix(host, "/") + if host != "" { + result[host] = struct{}{} + } + } + return result +} + +func matchDomainSet(host string, domainSet map[string]struct{}) bool { + if len(domainSet) == 0 || host == "" { + return false + } + host = strings.ToLower(strings.TrimPrefix(host, "www.")) + for domain := range domainSet { + if host == domain || strings.HasSuffix(host, "."+domain) { + return true + } + } + return false +} + +func hostFromURL(raw string) string { + parsed, err := url.Parse(raw) + if err != nil { + return "" + } + return strings.ToLower(parsed.Hostname()) +} + +func normalizeURL(raw string) string { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return strings.TrimSpace(raw) + } + parsed.Fragment = "" + if host := strings.ToLower(parsed.Hostname()); host != "" { + parsed.Host = host + } + return parsed.String() +} + +func compactText(value string, maxLen int) string { + value = strings.Join(strings.Fields(strings.TrimSpace(value)), " ") + if maxLen > 0 && len(value) > maxLen { + return value[:maxLen] + "..." + } + return value +} diff --git a/internal/service/search_service.go b/internal/service/search_service.go index 001fc60..1607451 100644 --- a/internal/service/search_service.go +++ b/internal/service/search_service.go @@ -1,395 +1,395 @@ -package service - -import ( - "context" - "crypto/sha1" - "encoding/hex" - "fmt" - "strings" - "time" - - "YoudaoNoteLm/internal/model/entity" - "YoudaoNoteLm/internal/repository" - "YoudaoNoteLm/internal/service/external" - "YoudaoNoteLm/pkg/cache" - "YoudaoNoteLm/pkg/config" - bizerrors "YoudaoNoteLm/pkg/errors" - "YoudaoNoteLm/pkg/logger" - - "go.uber.org/zap" -) - -type searchService struct { - client external.WebSearchClient - userConfigRepo repository.UserConfigRepository - cache *cache.Cache - bochaConfig config.BochaConfig -} - -type cachedSearchResponse struct { - Response *SearchResponse `json:"response"` -} - -// NewSearchService 创建统一搜索服务。 -func NewSearchService( - client external.WebSearchClient, - userConfigRepo repository.UserConfigRepository, - cacheClient *cache.Cache, - bochaConfig config.BochaConfig, -) SearchService { - return &searchService{ - client: client, - userConfigRepo: userConfigRepo, - cache: cacheClient, - bochaConfig: bochaConfig, - } -} - -func (s *searchService) Search(ctx context.Context, req *SearchRequest) (*SearchResponse, error) { - resolved, query, err := s.prepareRequest(req) - if err != nil { - return nil, err - } - - queryHash := hashQuery(query) - cacheKey := s.buildCacheKey(resolved, query) - if resp, ok := s.getCachedResponse(ctx, cacheKey); ok { - resp.Cached = true - if resp.Meta == nil { - resp.Meta = map[string]any{} - } - resp.Meta["query_hash"] = queryHash - resp.Meta["degraded"] = false - return resp, nil - } - - userCfg, providerCfg, quota, err := s.resolveProviderConfig(resolved.UserID, resolved.SkipUserConfig) - if err != nil { - return nil, err - } - - providerResp, err := s.client.Search(ctx, providerCfg, &external.SearchProviderRequest{ - Query: query, - Freshness: resolved.Freshness, - Count: resolved.Count, - NeedSummary: resolved.NeedSummary, - NeedContent: resolved.NeedContent, - Language: resolved.Language, - AllowedDomains: resolved.AllowedDomains, - BlockedDomains: resolved.BlockedDomains, - TraceID: resolved.TraceID, - }) - if err != nil { - if degraded := s.tryDegrade(resolved, err, queryHash); degraded != nil { - return degraded, nil - } - return nil, err - } - if len(providerResp.Results) == 0 { - return nil, bizerrors.ErrSearchProviderEmptyResult - } - - results := NormalizeSearchResults(providerResp.Results, resolved.NeedContent, resolved.AllowedDomains, resolved.BlockedDomains) - if len(results) == 0 { - return nil, bizerrors.ErrSearchNormalizedEmptyResult - } - - summary := "" - if resolved.NeedSummary { - summary = firstNonEmpty(providerResp.Summary, BuildSearchSummary(results)) - } - - response := &SearchResponse{ - Query: query, - Provider: providerResp.Provider, - Results: results, - Summary: summary, - Total: len(results), - Cached: false, - Quota: quota, - Meta: map[string]any{ - "query_hash": queryHash, - "provider_total": providerResp.Total, - "scene": resolved.Scene, - "degraded": false, - "normalized_query": query, - }, - } - - if err := s.consumeQuota(userCfg, quota); err != nil { - logger.Warn("consume search quota failed", zap.Uint("user_id", resolved.UserID), zap.Error(err)) - } - if err := s.setCachedResponse(ctx, cacheKey, response); err != nil { - logger.Warn("cache search response failed", zap.String("cache_key", cacheKey), zap.Error(err)) - } - - logger.Info("search completed", - zap.Uint("user_id", resolved.UserID), - zap.String("scene", string(resolved.Scene)), - zap.String("provider", response.Provider), - zap.String("query_hash", queryHash), - zap.Int("result_count", len(results)), - ) - - return response, nil -} - -func (s *searchService) SearchAndSummarize(ctx context.Context, req *SearchRequest) (*SearchResponse, error) { - cloned := cloneSearchRequest(req) - cloned.NeedSummary = true - return s.Search(ctx, cloned) -} - -func (s *searchService) SearchForImport(ctx context.Context, req *SearchImportRequest) (*SearchImportResponse, error) { - if req == nil { - return nil, bizerrors.New(bizerrors.CodeInvalidParam, "搜索导入请求不能为空") - } - cloned := cloneSearchRequest(&req.SearchRequest) - if cloned.Scene == "" { - cloned.Scene = SearchSceneImport - } - resp, err := s.Search(ctx, cloned) - if err != nil { - return nil, err - } - - urls := make([]string, 0, len(resp.Results)) - for _, result := range resp.Results { - urls = append(urls, result.URL) - } - - return &SearchImportResponse{ - Query: resp.Query, - Provider: resp.Provider, - Results: resp.Results, - URLs: urls, - Total: resp.Total, - Cached: resp.Cached, - Quota: resp.Quota, - Meta: resp.Meta, - }, nil -} - -func (s *searchService) prepareRequest(req *SearchRequest) (*SearchRequest, string, error) { - cloned := cloneSearchRequest(req) - if cloned.Count <= 0 { - cloned.Count = max(1, s.bochaConfig.DefaultCount) - } - maxCount := s.bochaConfig.MaxCount - if maxCount <= 0 { - maxCount = 10 - } - if cloned.Count > maxCount { - cloned.Count = maxCount - } - if strings.TrimSpace(cloned.Freshness) == "" { - cloned.Freshness = "noLimit" - } - - query, err := BuildSearchQuery(cloned) - if err != nil { - return nil, "", err - } - return cloned, query, nil -} - -func (s *searchService) resolveProviderConfig(userID uint, skipUserConfig bool) (*entity.UserConfig, external.SearchProviderConfig, *SearchQuota, error) { - providerCfg := external.SearchProviderConfig{ - BaseURL: strings.TrimSpace(s.bochaConfig.BaseURL), - APIKey: strings.TrimSpace(s.bochaConfig.APIKey), - Timeout: time.Duration(max(1, s.bochaConfig.TimeoutSeconds)) * time.Second, - } - - if skipUserConfig { - if providerCfg.APIKey == "" || providerCfg.BaseURL == "" { - return nil, external.SearchProviderConfig{}, nil, bizerrors.ErrSearchProviderNotConfigured - } - return nil, providerCfg, nil, nil - } - - var quota *SearchQuota - var userCfg *entity.UserConfig - if userID == 0 || s.userConfigRepo == nil { - if providerCfg.APIKey == "" || providerCfg.BaseURL == "" { - return nil, external.SearchProviderConfig{}, nil, bizerrors.ErrSearchProviderNotConfigured - } - return nil, providerCfg, nil, nil - } - - cfg, err := s.userConfigRepo.FindByUserAndType(userID, "search") - if err != nil { - return nil, external.SearchProviderConfig{}, nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "读取用户搜索配置失败", err) - } - userCfg = cfg - if userCfg != nil { - if !userCfg.Enabled { - return nil, external.SearchProviderConfig{}, nil, bizerrors.New(bizerrors.CodeSearchProviderNotConfigured, "当前用户未启用联网搜索") - } - if provider := strings.TrimSpace(strings.ToLower(userCfg.Provider)); provider != "" && provider != "bocha" { - return nil, external.SearchProviderConfig{}, nil, bizerrors.New(bizerrors.CodeSearchProviderNotConfigured, "当前仅支持 bocha 搜索 provider") - } - if value := strings.TrimSpace(userCfg.APIURL); value != "" { - providerCfg.BaseURL = value - } - if value := strings.TrimSpace(userCfg.APIKey); value != "" { - providerCfg.APIKey = value - } - if err := s.resetQuotaIfNeeded(userCfg); err != nil { - return nil, external.SearchProviderConfig{}, nil, err - } - quota = buildSearchQuota(userCfg) - if userCfg.DailyQuota != nil && userCfg.QuotaUsed >= *userCfg.DailyQuota { - return nil, external.SearchProviderConfig{}, quota, bizerrors.ErrSearchQuotaExhausted - } - } - - if providerCfg.APIKey == "" || providerCfg.BaseURL == "" { - return nil, external.SearchProviderConfig{}, quota, bizerrors.ErrSearchProviderNotConfigured - } - return userCfg, providerCfg, quota, nil -} - -func (s *searchService) resetQuotaIfNeeded(cfg *entity.UserConfig) error { - if cfg == nil || cfg.QuotaResetAt == nil || s.userConfigRepo == nil { - return nil - } - now := time.Now() - if cfg.QuotaResetAt.After(now) { - return nil - } - next := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, now.Location()) - cfg.QuotaUsed = 0 - cfg.QuotaResetAt = &next - return s.userConfigRepo.Update(cfg) -} - -func (s *searchService) consumeQuota(cfg *entity.UserConfig, quota *SearchQuota) error { - if cfg == nil || cfg.DailyQuota == nil || s.userConfigRepo == nil { - return nil - } - cfg.QuotaUsed++ - if cfg.QuotaResetAt == nil { - next := time.Now().Add(24 * time.Hour) - cfg.QuotaResetAt = &next - } - if err := s.userConfigRepo.Update(cfg); err != nil { - return err - } - if quota != nil { - quota.Used = cfg.QuotaUsed - if quota.DailyQuota != nil { - remaining := *quota.DailyQuota - quota.Used - if remaining < 0 { - remaining = 0 - } - quota.Remaining = &remaining - } - quota.ResetAt = cfg.QuotaResetAt - } - return nil -} - -func buildSearchQuota(cfg *entity.UserConfig) *SearchQuota { - if cfg == nil { - return nil - } - quota := &SearchQuota{ - DailyQuota: cfg.DailyQuota, - Used: cfg.QuotaUsed, - ResetAt: cfg.QuotaResetAt, - } - if cfg.DailyQuota != nil { - remaining := *cfg.DailyQuota - cfg.QuotaUsed - if remaining < 0 { - remaining = 0 - } - quota.Remaining = &remaining - } - return quota -} - -func (s *searchService) tryDegrade(req *SearchRequest, err error, queryHash string) *SearchResponse { - if req == nil || !req.AllowDegrade { - return nil - } - if req.Scene != SearchSceneGeneration && req.Scene != SearchSceneChat { - return nil - } - - logger.Warn("search degraded to local-only", - zap.Uint("user_id", req.UserID), - zap.String("scene", string(req.Scene)), - zap.String("query_hash", queryHash), - zap.Error(err), - ) - - return &SearchResponse{ - Query: req.Query, - Provider: "bocha", - Results: []SearchResult{}, - Total: 0, - Cached: false, - Meta: map[string]any{ - "degraded": true, - "query_hash": queryHash, - "reason": err.Error(), - }, - } -} - -func (s *searchService) buildCacheKey(req *SearchRequest, query string) string { - return fmt.Sprintf( - "search:bocha:%s:%s:%s:%d:%t:%t", - req.Scene, - hashQuery(query), - req.Freshness, - req.Count, - req.NeedSummary, - req.NeedContent, - ) -} - -func (s *searchService) getCachedResponse(ctx context.Context, key string) (*SearchResponse, bool) { - if s.cache == nil || strings.TrimSpace(key) == "" { - return nil, false - } - - var payload cachedSearchResponse - if err := s.cache.Get(ctx, key, &payload); err != nil || payload.Response == nil { - return nil, false - } - return payload.Response, true -} - -func (s *searchService) setCachedResponse(ctx context.Context, key string, resp *SearchResponse) error { - if s.cache == nil || resp == nil || strings.TrimSpace(key) == "" { - return nil - } - ttlSeconds := s.bochaConfig.CacheTTLSeconds - if ttlSeconds <= 0 { - ttlSeconds = 300 - } - return s.cache.Set(ctx, key, &cachedSearchResponse{Response: resp}, time.Duration(ttlSeconds)*time.Second) -} - -func cloneSearchRequest(req *SearchRequest) *SearchRequest { - if req == nil { - return &SearchRequest{} - } - cloned := *req - cloned.AllowedDomains = append([]string(nil), req.AllowedDomains...) - cloned.BlockedDomains = append([]string(nil), req.BlockedDomains...) - return &cloned -} - -func hashQuery(query string) string { - sum := sha1.Sum([]byte(strings.TrimSpace(query))) - return hex.EncodeToString(sum[:]) -} - -func max(a, b int) int { - if a > b { - return a - } - return b -} +package service + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "fmt" + "strings" + "time" + + "YoudaoNoteLm/internal/model/entity" + "YoudaoNoteLm/internal/repository" + "YoudaoNoteLm/internal/service/external" + "YoudaoNoteLm/pkg/cache" + "YoudaoNoteLm/pkg/config" + bizerrors "YoudaoNoteLm/pkg/errors" + "YoudaoNoteLm/pkg/logger" + + "go.uber.org/zap" +) + +type searchService struct { + client external.WebSearchClient + userConfigRepo repository.UserConfigRepository + cache *cache.Cache + bochaConfig config.BochaConfig +} + +type cachedSearchResponse struct { + Response *SearchResponse `json:"response"` +} + +// NewSearchService 创建统一搜索服务。 +func NewSearchService( + client external.WebSearchClient, + userConfigRepo repository.UserConfigRepository, + cacheClient *cache.Cache, + bochaConfig config.BochaConfig, +) SearchService { + return &searchService{ + client: client, + userConfigRepo: userConfigRepo, + cache: cacheClient, + bochaConfig: bochaConfig, + } +} + +func (s *searchService) Search(ctx context.Context, req *SearchRequest) (*SearchResponse, error) { + resolved, query, err := s.prepareRequest(req) + if err != nil { + return nil, err + } + + queryHash := hashQuery(query) + cacheKey := s.buildCacheKey(resolved, query) + if resp, ok := s.getCachedResponse(ctx, cacheKey); ok { + resp.Cached = true + if resp.Meta == nil { + resp.Meta = map[string]any{} + } + resp.Meta["query_hash"] = queryHash + resp.Meta["degraded"] = false + return resp, nil + } + + userCfg, providerCfg, quota, err := s.resolveProviderConfig(resolved.UserID, resolved.SkipUserConfig) + if err != nil { + return nil, err + } + + providerResp, err := s.client.Search(ctx, providerCfg, &external.SearchProviderRequest{ + Query: query, + Freshness: resolved.Freshness, + Count: resolved.Count, + NeedSummary: resolved.NeedSummary, + NeedContent: resolved.NeedContent, + Language: resolved.Language, + AllowedDomains: resolved.AllowedDomains, + BlockedDomains: resolved.BlockedDomains, + TraceID: resolved.TraceID, + }) + if err != nil { + if degraded := s.tryDegrade(resolved, err, queryHash); degraded != nil { + return degraded, nil + } + return nil, err + } + if len(providerResp.Results) == 0 { + return nil, bizerrors.ErrSearchProviderEmptyResult + } + + results := NormalizeSearchResults(providerResp.Results, resolved.NeedContent, resolved.AllowedDomains, resolved.BlockedDomains) + if len(results) == 0 { + return nil, bizerrors.ErrSearchNormalizedEmptyResult + } + + summary := "" + if resolved.NeedSummary { + summary = firstNonEmpty(providerResp.Summary, BuildSearchSummary(results)) + } + + response := &SearchResponse{ + Query: query, + Provider: providerResp.Provider, + Results: results, + Summary: summary, + Total: len(results), + Cached: false, + Quota: quota, + Meta: map[string]any{ + "query_hash": queryHash, + "provider_total": providerResp.Total, + "scene": resolved.Scene, + "degraded": false, + "normalized_query": query, + }, + } + + if err := s.consumeQuota(userCfg, quota); err != nil { + logger.Warn("consume search quota failed", zap.Uint("user_id", resolved.UserID), zap.Error(err)) + } + if err := s.setCachedResponse(ctx, cacheKey, response); err != nil { + logger.Warn("cache search response failed", zap.String("cache_key", cacheKey), zap.Error(err)) + } + + logger.Info("search completed", + zap.Uint("user_id", resolved.UserID), + zap.String("scene", string(resolved.Scene)), + zap.String("provider", response.Provider), + zap.String("query_hash", queryHash), + zap.Int("result_count", len(results)), + ) + + return response, nil +} + +func (s *searchService) SearchAndSummarize(ctx context.Context, req *SearchRequest) (*SearchResponse, error) { + cloned := cloneSearchRequest(req) + cloned.NeedSummary = true + return s.Search(ctx, cloned) +} + +func (s *searchService) SearchForImport(ctx context.Context, req *SearchImportRequest) (*SearchImportResponse, error) { + if req == nil { + return nil, bizerrors.New(bizerrors.CodeInvalidParam, "搜索导入请求不能为空") + } + cloned := cloneSearchRequest(&req.SearchRequest) + if cloned.Scene == "" { + cloned.Scene = SearchSceneImport + } + resp, err := s.Search(ctx, cloned) + if err != nil { + return nil, err + } + + urls := make([]string, 0, len(resp.Results)) + for _, result := range resp.Results { + urls = append(urls, result.URL) + } + + return &SearchImportResponse{ + Query: resp.Query, + Provider: resp.Provider, + Results: resp.Results, + URLs: urls, + Total: resp.Total, + Cached: resp.Cached, + Quota: resp.Quota, + Meta: resp.Meta, + }, nil +} + +func (s *searchService) prepareRequest(req *SearchRequest) (*SearchRequest, string, error) { + cloned := cloneSearchRequest(req) + if cloned.Count <= 0 { + cloned.Count = max(1, s.bochaConfig.DefaultCount) + } + maxCount := s.bochaConfig.MaxCount + if maxCount <= 0 { + maxCount = 10 + } + if cloned.Count > maxCount { + cloned.Count = maxCount + } + if strings.TrimSpace(cloned.Freshness) == "" { + cloned.Freshness = "noLimit" + } + + query, err := BuildSearchQuery(cloned) + if err != nil { + return nil, "", err + } + return cloned, query, nil +} + +func (s *searchService) resolveProviderConfig(userID uint, skipUserConfig bool) (*entity.UserConfig, external.SearchProviderConfig, *SearchQuota, error) { + providerCfg := external.SearchProviderConfig{ + BaseURL: strings.TrimSpace(s.bochaConfig.BaseURL), + APIKey: strings.TrimSpace(s.bochaConfig.APIKey), + Timeout: time.Duration(max(1, s.bochaConfig.TimeoutSeconds)) * time.Second, + } + + if skipUserConfig { + if providerCfg.APIKey == "" || providerCfg.BaseURL == "" { + return nil, external.SearchProviderConfig{}, nil, bizerrors.ErrSearchProviderNotConfigured + } + return nil, providerCfg, nil, nil + } + + var quota *SearchQuota + var userCfg *entity.UserConfig + if userID == 0 || s.userConfigRepo == nil { + if providerCfg.APIKey == "" || providerCfg.BaseURL == "" { + return nil, external.SearchProviderConfig{}, nil, bizerrors.ErrSearchProviderNotConfigured + } + return nil, providerCfg, nil, nil + } + + cfg, err := s.userConfigRepo.FindByUserAndType(userID, "search") + if err != nil { + return nil, external.SearchProviderConfig{}, nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "读取用户搜索配置失败", err) + } + userCfg = cfg + if userCfg != nil { + if !userCfg.Enabled { + return nil, external.SearchProviderConfig{}, nil, bizerrors.New(bizerrors.CodeSearchProviderNotConfigured, "当前用户未启用联网搜索") + } + if provider := strings.TrimSpace(strings.ToLower(userCfg.Provider)); provider != "" && provider != "bocha" { + return nil, external.SearchProviderConfig{}, nil, bizerrors.New(bizerrors.CodeSearchProviderNotConfigured, "当前仅支持 bocha 搜索 provider") + } + if value := strings.TrimSpace(userCfg.APIURL); value != "" { + providerCfg.BaseURL = value + } + if value := strings.TrimSpace(userCfg.APIKey); value != "" { + providerCfg.APIKey = value + } + if err := s.resetQuotaIfNeeded(userCfg); err != nil { + return nil, external.SearchProviderConfig{}, nil, err + } + quota = buildSearchQuota(userCfg) + if userCfg.DailyQuota != nil && userCfg.QuotaUsed >= *userCfg.DailyQuota { + return nil, external.SearchProviderConfig{}, quota, bizerrors.ErrSearchQuotaExhausted + } + } + + if providerCfg.APIKey == "" || providerCfg.BaseURL == "" { + return nil, external.SearchProviderConfig{}, quota, bizerrors.ErrSearchProviderNotConfigured + } + return userCfg, providerCfg, quota, nil +} + +func (s *searchService) resetQuotaIfNeeded(cfg *entity.UserConfig) error { + if cfg == nil || cfg.QuotaResetAt == nil || s.userConfigRepo == nil { + return nil + } + now := time.Now() + if cfg.QuotaResetAt.After(now) { + return nil + } + next := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, now.Location()) + cfg.QuotaUsed = 0 + cfg.QuotaResetAt = &next + return s.userConfigRepo.Update(cfg) +} + +func (s *searchService) consumeQuota(cfg *entity.UserConfig, quota *SearchQuota) error { + if cfg == nil || cfg.DailyQuota == nil || s.userConfigRepo == nil { + return nil + } + cfg.QuotaUsed++ + if cfg.QuotaResetAt == nil { + next := time.Now().Add(24 * time.Hour) + cfg.QuotaResetAt = &next + } + if err := s.userConfigRepo.Update(cfg); err != nil { + return err + } + if quota != nil { + quota.Used = cfg.QuotaUsed + if quota.DailyQuota != nil { + remaining := *quota.DailyQuota - quota.Used + if remaining < 0 { + remaining = 0 + } + quota.Remaining = &remaining + } + quota.ResetAt = cfg.QuotaResetAt + } + return nil +} + +func buildSearchQuota(cfg *entity.UserConfig) *SearchQuota { + if cfg == nil { + return nil + } + quota := &SearchQuota{ + DailyQuota: cfg.DailyQuota, + Used: cfg.QuotaUsed, + ResetAt: cfg.QuotaResetAt, + } + if cfg.DailyQuota != nil { + remaining := *cfg.DailyQuota - cfg.QuotaUsed + if remaining < 0 { + remaining = 0 + } + quota.Remaining = &remaining + } + return quota +} + +func (s *searchService) tryDegrade(req *SearchRequest, err error, queryHash string) *SearchResponse { + if req == nil || !req.AllowDegrade { + return nil + } + if req.Scene != SearchSceneGeneration && req.Scene != SearchSceneChat { + return nil + } + + logger.Warn("search degraded to local-only", + zap.Uint("user_id", req.UserID), + zap.String("scene", string(req.Scene)), + zap.String("query_hash", queryHash), + zap.Error(err), + ) + + return &SearchResponse{ + Query: req.Query, + Provider: "bocha", + Results: []SearchResult{}, + Total: 0, + Cached: false, + Meta: map[string]any{ + "degraded": true, + "query_hash": queryHash, + "reason": err.Error(), + }, + } +} + +func (s *searchService) buildCacheKey(req *SearchRequest, query string) string { + return fmt.Sprintf( + "search:bocha:%s:%s:%s:%d:%t:%t", + req.Scene, + hashQuery(query), + req.Freshness, + req.Count, + req.NeedSummary, + req.NeedContent, + ) +} + +func (s *searchService) getCachedResponse(ctx context.Context, key string) (*SearchResponse, bool) { + if s.cache == nil || strings.TrimSpace(key) == "" { + return nil, false + } + + var payload cachedSearchResponse + if err := s.cache.Get(ctx, key, &payload); err != nil || payload.Response == nil { + return nil, false + } + return payload.Response, true +} + +func (s *searchService) setCachedResponse(ctx context.Context, key string, resp *SearchResponse) error { + if s.cache == nil || resp == nil || strings.TrimSpace(key) == "" { + return nil + } + ttlSeconds := s.bochaConfig.CacheTTLSeconds + if ttlSeconds <= 0 { + ttlSeconds = 300 + } + return s.cache.Set(ctx, key, &cachedSearchResponse{Response: resp}, time.Duration(ttlSeconds)*time.Second) +} + +func cloneSearchRequest(req *SearchRequest) *SearchRequest { + if req == nil { + return &SearchRequest{} + } + cloned := *req + cloned.AllowedDomains = append([]string(nil), req.AllowedDomains...) + cloned.BlockedDomains = append([]string(nil), req.BlockedDomains...) + return &cloned +} + +func hashQuery(query string) string { + sum := sha1.Sum([]byte(strings.TrimSpace(query))) + return hex.EncodeToString(sum[:]) +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/internal/service/source_interface.go b/internal/service/source_interface.go index 268de03..833dd4e 100644 --- a/internal/service/source_interface.go +++ b/internal/service/source_interface.go @@ -1,27 +1,27 @@ -package service - -import ( - "YoudaoNoteLm/internal/model/dto/response" - "YoudaoNoteLm/internal/model/entity" -) - -// SourceService 资料来源服务接口 -type SourceService interface { - List(userID, notebookID uint, keyword string, page, size int) ([]*response.SourceResponse, int64, error) - GetByID(id uint) (*entity.Source, error) - Rename(id uint, name string) error - Delete(id uint) error - BatchDelete(ids []uint) error - DeleteFailed(userID, notebookID uint) (int64, error) - GetContent(id uint) (string, error) - GetOriginalContent(id uint) (content string, contentType string, err error) - GetDownloadURL(id uint) (string, error) - // ReimportAll 重新导入用户所有未向量化的资料 - ReimportAll(userID uint) (int, error) - // ReimportSelected 重新导入指定的未向量化资料 - ReimportSelected(sourceIDs []uint) (int, error) - // CreateFromNote 将笔记内容保存为来源 - CreateFromNote(userID, notebookID uint, title, content string) (*response.SourceResponse, error) - // DeleteByNoteAndNotebook 根据笔记标题和笔记本ID删除来源 - DeleteByNoteAndNotebook(userID, notebookID uint, title string) error -} +package service + +import ( + "YoudaoNoteLm/internal/model/dto/response" + "YoudaoNoteLm/internal/model/entity" +) + +// SourceService 资料来源服务接口 +type SourceService interface { + List(userID, notebookID uint, keyword string, page, size int) ([]*response.SourceResponse, int64, error) + GetByID(id uint) (*entity.Source, error) + Rename(id uint, name string) error + Delete(id uint) error + BatchDelete(ids []uint) error + DeleteFailed(userID, notebookID uint) (int64, error) + GetContent(id uint) (string, error) + GetOriginalContent(id uint) (content string, contentType string, err error) + GetDownloadURL(id uint) (string, error) + // ReimportAll 重新导入用户所有未向量化的资料 + ReimportAll(userID uint) (int, error) + // ReimportSelected 重新导入指定的未向量化资料 + ReimportSelected(sourceIDs []uint) (int, error) + // CreateFromNote 将笔记内容保存为来源 + CreateFromNote(userID, notebookID uint, title, content string) (*response.SourceResponse, error) + // DeleteByNoteAndNotebook 根据笔记标题和笔记本ID删除来源 + DeleteByNoteAndNotebook(userID, notebookID uint, title string) error +} diff --git a/internal/service/source_service.go b/internal/service/source_service.go index 25a7263..356edbb 100644 --- a/internal/service/source_service.go +++ b/internal/service/source_service.go @@ -1,385 +1,385 @@ -package service - -import ( - "YoudaoNoteLm/internal/rag" - "YoudaoNoteLm/internal/service/external/storage" - "context" - "time" - - "YoudaoNoteLm/internal/model/dto/response" - "YoudaoNoteLm/internal/model/entity" - "YoudaoNoteLm/internal/repository" - "YoudaoNoteLm/pkg/cache" - bizerrors "YoudaoNoteLm/pkg/errors" - "YoudaoNoteLm/pkg/logger" - "go.uber.org/zap" -) - -type sourceService struct { - sourceRepo repository.SourceRepository - storage storage.FileStorage - ingestionSvc rag.IngestionService - summaryCache *cache.SourceSummaryCache -} - -func NewSourceService(sourceRepo repository.SourceRepository, storage storage.FileStorage, ingestionSvc rag.IngestionService, summaryCache *cache.SourceSummaryCache) SourceService { - return &sourceService{sourceRepo: sourceRepo, storage: storage, ingestionSvc: ingestionSvc, summaryCache: summaryCache} -} - -func (s *sourceService) List(userID, notebookID uint, keyword string, page, size int) ([]*response.SourceResponse, int64, error) { - if page < 1 { - page = 1 - } - if size < 1 { - size = 10 - } - if size > 100 { - size = 100 - } - - offset := (page - 1) * size - sources, total, err := s.sourceRepo.ListByNotebook(userID, notebookID, keyword, offset, size) - if err != nil { - return nil, 0, err - } - - list := make([]*response.SourceResponse, 0, len(sources)) - for _, src := range sources { - list = append(list, toSourceResponse(src)) - } - - return list, total, nil -} - -func (s *sourceService) GetByID(id uint) (*entity.Source, error) { - source, err := s.sourceRepo.FindByID(id) - if err != nil { - return nil, err - } - if source == nil { - return nil, bizerrors.ErrNotFound - } - return source, nil -} - -func (s *sourceService) Rename(id uint, name string) error { - source, err := s.GetByID(id) - if err != nil { - return err - } - source.Name = name - return s.sourceRepo.Update(source) -} - -func (s *sourceService) Delete(id uint) error { - source, err := s.GetByID(id) - if err != nil { - return err - } - - // 删除 Milvus 中的向量数据 - if s.ingestionSvc != nil && source.Vectorized { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if err := s.ingestionSvc.DeleteSource(ctx, source.UserID, id); err != nil { - logger.Error("删除源向量数据失败", - zap.Uint("source_id", id), - zap.Error(err), - ) - // 向量删除失败不阻塞主流程,记录日志继续 - } - } - - // 删除摘要缓存 - if s.summaryCache != nil { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := s.summaryCache.Delete(ctx, id); err != nil { - logger.Warn("删除摘要缓存失败", - zap.Uint("source_id", id), - zap.Error(err), - ) - } - } - - return s.sourceRepo.Delete(id) -} - -func (s *sourceService) BatchDelete(ids []uint) error { - // 批量删除前,先删除每个 source 的向量数据 - if s.ingestionSvc != nil && len(ids) > 0 { - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - for _, id := range ids { - source, err := s.sourceRepo.FindByID(id) - if err != nil || source == nil { - continue - } - if source.Vectorized { - if err := s.ingestionSvc.DeleteSource(ctx, source.UserID, id); err != nil { - logger.Error("批量删除时删除源向量数据失败", - zap.Uint("source_id", id), - zap.Error(err), - ) - } - } - } - } - - // 批量删除摘要缓存 - if s.summaryCache != nil && len(ids) > 0 { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := s.summaryCache.BatchDelete(ctx, ids); err != nil { - logger.Warn("批量删除摘要缓存失败", - zap.Uints("source_ids", ids), - zap.Error(err), - ) - } - } - - return s.sourceRepo.BatchDelete(ids) -} - -func (s *sourceService) DeleteFailed(userID, notebookID uint) (int64, error) { - return s.sourceRepo.DeleteFailedByNotebook(userID, notebookID) -} - -func (s *sourceService) GetContent(id uint) (string, error) { - source, err := s.GetByID(id) - if err != nil { - return "", err - } - return source.MarkdownContent, nil -} - -func (s *sourceService) GetOriginalContent(id uint) (string, string, error) { - source, err := s.GetByID(id) - if err != nil { - return "", "", err - } - - switch source.Type { - case "file": - // 对于文件类型,返回 Markdown 内容作为原内容展示 - // 原始文件通过 GetDownloadURL 提供下载 - return source.MarkdownContent, source.MimeType, nil - case "url": - return source.OriginalURL, "url", nil - case "audio": - return source.MarkdownContent, "audio_transcript", nil - case "note", "youdao": - return source.MarkdownContent, "raw_markdown", nil - default: - return "", "", bizerrors.New(bizerrors.CodeBadRequest, "该类型不支持查看原格式") - } -} - -func (s *sourceService) GetDownloadURL(id uint) (string, error) { - source, err := s.GetByID(id) - if err != nil { - return "", err - } - if source.FilePath == "" { - return "", bizerrors.New(bizerrors.CodeBadRequest, "该来源没有可下载的文件") - } - - // 如果 storage 是 MinIO,生成预签名 URL - if minioStorage, ok := s.storage.(interface { - GetPresignedURL(filePath string, expiry time.Duration) (string, error) - }); ok { - url, err := minioStorage.GetPresignedURL(source.FilePath, 10*time.Minute) - if err != nil { - return "", bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "生成下载链接失败", err) - } - return url, nil - } - - return "", bizerrors.New(bizerrors.CodeInternalServiceError, "存储服务不支持生成下载链接") -} - -func toSourceResponse(src *entity.Source) *response.SourceResponse { - return &response.SourceResponse{ - ID: src.ID, - NotebookID: src.NotebookID, - Name: src.Name, - Type: src.Type, - OriginalURL: src.OriginalURL, - FilePath: src.FilePath, - FileSize: src.FileSize, - MimeType: src.MimeType, - Status: src.Status, - ErrorMessage: src.ErrorMessage, - Vectorized: src.Vectorized, - CreatedAt: src.CreatedAt, - UpdatedAt: src.UpdatedAt, - } -} - -// ReimportAll 重新导入用户所有未向量化的资料 -func (s *sourceService) ReimportAll(userID uint) (int, error) { - if s.ingestionSvc == nil { - return 0, bizerrors.New(bizerrors.CodeInternalServiceError, "向量入库服务未初始化") - } - - // 获取用户所有未向量化的资料 - sources, err := s.sourceRepo.FindUnvectorizedByUserID(userID) - if err != nil { - return 0, err - } - - if len(sources) == 0 { - return 0, nil - } - - // 收集所有 source ID - sourceIDs := make([]uint, 0, len(sources)) - for _, src := range sources { - sourceIDs = append(sourceIDs, src.ID) - } - - // 批量入库 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - // 即使部分失败也继续,返回成功导入的数量 - ingestErr := s.ingestionSvc.Ingest(ctx, sourceIDs) - - // 统计实际成功导入的数量(vectorized=true 的数量) - successCount := 0 - for _, src := range sources { - updated, err := s.sourceRepo.FindByID(src.ID) - if err == nil && updated != nil && updated.Vectorized { - successCount++ - } - } - - if ingestErr != nil { - logger.Warn("批量重新入库部分失败", - zap.Uint("user_id", userID), - zap.Int("total", len(sourceIDs)), - zap.Int("success", successCount), - zap.Error(ingestErr), - ) - } - - logger.Info("批量重新入库完成", - zap.Uint("user_id", userID), - zap.Int("total", len(sourceIDs)), - zap.Int("success", successCount), - ) - - return successCount, nil -} - -// ReimportSelected 重新导入指定的未向量化资料 -func (s *sourceService) ReimportSelected(sourceIDs []uint) (int, error) { - if s.ingestionSvc == nil { - return 0, bizerrors.New(bizerrors.CodeInternalServiceError, "向量入库服务未初始化") - } - - if len(sourceIDs) == 0 { - return 0, nil - } - - // 验证所有 source 都存在且未向量化 - for _, id := range sourceIDs { - source, err := s.sourceRepo.FindByID(id) - if err != nil { - return 0, err - } - if source == nil { - return 0, bizerrors.New(bizerrors.CodeNotFound, "资料不存在") - } - if source.Vectorized { - return 0, bizerrors.New(bizerrors.CodeBadRequest, "资料已入库,无需重复导入") - } - } - - // 批量入库 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - ingestErr := s.ingestionSvc.Ingest(ctx, sourceIDs) - - // 统计实际成功导入的数量 - successCount := 0 - for _, id := range sourceIDs { - updated, err := s.sourceRepo.FindByID(id) - if err == nil && updated != nil && updated.Vectorized { - successCount++ - } - } - - if ingestErr != nil { - logger.Warn("批量重新入库部分失败", - zap.Int("total", len(sourceIDs)), - zap.Int("success", successCount), - zap.Error(ingestErr), - ) - } - - logger.Info("批量重新入库完成", - zap.Int("total", len(sourceIDs)), - zap.Int("success", successCount), - ) - - return successCount, nil -} - -// CreateFromNote 将笔记内容保存为来源 -func (s *sourceService) CreateFromNote(userID, notebookID uint, title, content string) (*response.SourceResponse, error) { - if title == "" { - return nil, bizerrors.New(bizerrors.CodeBadRequest, "标题不能为空") - } - if content == "" { - return nil, bizerrors.New(bizerrors.CodeBadRequest, "内容不能为空") - } - - source := &entity.Source{ - UserID: userID, - NotebookID: notebookID, - Name: title, - Type: "note", - MarkdownContent: content, - Status: "pending", - Vectorized: false, - } - - if err := s.sourceRepo.Create(source); err != nil { - return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "创建来源失败", err) - } - - // 异步进行向量化入库 - if s.ingestionSvc != nil { - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - if err := s.ingestionSvc.Ingest(ctx, []uint{source.ID}); err != nil { - logger.Warn("笔记来源向量化入库失败", - zap.Uint("source_id", source.ID), - zap.Error(err), - ) - } - }() - } - - return toSourceResponse(source), nil -} - -// DeleteByNoteAndNotebook 根据笔记标题和笔记本ID删除来源 -func (s *sourceService) DeleteByNoteAndNotebook(userID, notebookID uint, title string) error { - sources, _, err := s.sourceRepo.ListByNotebook(userID, notebookID, title, 0, 100) - if err != nil { - return err - } - - for _, src := range sources { - if src.Name == title && src.Type == "note" { - return s.Delete(src.ID) - } - } - - return bizerrors.New(bizerrors.CodeNotFound, "未找到对应的笔记来源") -} +package service + +import ( + "YoudaoNoteLm/internal/rag" + "YoudaoNoteLm/internal/service/external/storage" + "context" + "time" + + "YoudaoNoteLm/internal/model/dto/response" + "YoudaoNoteLm/internal/model/entity" + "YoudaoNoteLm/internal/repository" + "YoudaoNoteLm/pkg/cache" + bizerrors "YoudaoNoteLm/pkg/errors" + "YoudaoNoteLm/pkg/logger" + "go.uber.org/zap" +) + +type sourceService struct { + sourceRepo repository.SourceRepository + storage storage.FileStorage + ingestionSvc rag.IngestionService + summaryCache *cache.SourceSummaryCache +} + +func NewSourceService(sourceRepo repository.SourceRepository, storage storage.FileStorage, ingestionSvc rag.IngestionService, summaryCache *cache.SourceSummaryCache) SourceService { + return &sourceService{sourceRepo: sourceRepo, storage: storage, ingestionSvc: ingestionSvc, summaryCache: summaryCache} +} + +func (s *sourceService) List(userID, notebookID uint, keyword string, page, size int) ([]*response.SourceResponse, int64, error) { + if page < 1 { + page = 1 + } + if size < 1 { + size = 10 + } + if size > 100 { + size = 100 + } + + offset := (page - 1) * size + sources, total, err := s.sourceRepo.ListByNotebook(userID, notebookID, keyword, offset, size) + if err != nil { + return nil, 0, err + } + + list := make([]*response.SourceResponse, 0, len(sources)) + for _, src := range sources { + list = append(list, toSourceResponse(src)) + } + + return list, total, nil +} + +func (s *sourceService) GetByID(id uint) (*entity.Source, error) { + source, err := s.sourceRepo.FindByID(id) + if err != nil { + return nil, err + } + if source == nil { + return nil, bizerrors.ErrNotFound + } + return source, nil +} + +func (s *sourceService) Rename(id uint, name string) error { + source, err := s.GetByID(id) + if err != nil { + return err + } + source.Name = name + return s.sourceRepo.Update(source) +} + +func (s *sourceService) Delete(id uint) error { + source, err := s.GetByID(id) + if err != nil { + return err + } + + // 删除 Milvus 中的向量数据 + if s.ingestionSvc != nil && source.Vectorized { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := s.ingestionSvc.DeleteSource(ctx, source.UserID, id); err != nil { + logger.Error("删除源向量数据失败", + zap.Uint("source_id", id), + zap.Error(err), + ) + // 向量删除失败不阻塞主流程,记录日志继续 + } + } + + // 删除摘要缓存 + if s.summaryCache != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.summaryCache.Delete(ctx, id); err != nil { + logger.Warn("删除摘要缓存失败", + zap.Uint("source_id", id), + zap.Error(err), + ) + } + } + + return s.sourceRepo.Delete(id) +} + +func (s *sourceService) BatchDelete(ids []uint) error { + // 批量删除前,先删除每个 source 的向量数据 + if s.ingestionSvc != nil && len(ids) > 0 { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + for _, id := range ids { + source, err := s.sourceRepo.FindByID(id) + if err != nil || source == nil { + continue + } + if source.Vectorized { + if err := s.ingestionSvc.DeleteSource(ctx, source.UserID, id); err != nil { + logger.Error("批量删除时删除源向量数据失败", + zap.Uint("source_id", id), + zap.Error(err), + ) + } + } + } + } + + // 批量删除摘要缓存 + if s.summaryCache != nil && len(ids) > 0 { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.summaryCache.BatchDelete(ctx, ids); err != nil { + logger.Warn("批量删除摘要缓存失败", + zap.Uints("source_ids", ids), + zap.Error(err), + ) + } + } + + return s.sourceRepo.BatchDelete(ids) +} + +func (s *sourceService) DeleteFailed(userID, notebookID uint) (int64, error) { + return s.sourceRepo.DeleteFailedByNotebook(userID, notebookID) +} + +func (s *sourceService) GetContent(id uint) (string, error) { + source, err := s.GetByID(id) + if err != nil { + return "", err + } + return source.MarkdownContent, nil +} + +func (s *sourceService) GetOriginalContent(id uint) (string, string, error) { + source, err := s.GetByID(id) + if err != nil { + return "", "", err + } + + switch source.Type { + case "file": + // 对于文件类型,返回 Markdown 内容作为原内容展示 + // 原始文件通过 GetDownloadURL 提供下载 + return source.MarkdownContent, source.MimeType, nil + case "url": + return source.OriginalURL, "url", nil + case "audio": + return source.MarkdownContent, "audio_transcript", nil + case "note", "youdao": + return source.MarkdownContent, "raw_markdown", nil + default: + return "", "", bizerrors.New(bizerrors.CodeBadRequest, "该类型不支持查看原格式") + } +} + +func (s *sourceService) GetDownloadURL(id uint) (string, error) { + source, err := s.GetByID(id) + if err != nil { + return "", err + } + if source.FilePath == "" { + return "", bizerrors.New(bizerrors.CodeBadRequest, "该来源没有可下载的文件") + } + + // 如果 storage 是 MinIO,生成预签名 URL + if minioStorage, ok := s.storage.(interface { + GetPresignedURL(filePath string, expiry time.Duration) (string, error) + }); ok { + url, err := minioStorage.GetPresignedURL(source.FilePath, 10*time.Minute) + if err != nil { + return "", bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "生成下载链接失败", err) + } + return url, nil + } + + return "", bizerrors.New(bizerrors.CodeInternalServiceError, "存储服务不支持生成下载链接") +} + +func toSourceResponse(src *entity.Source) *response.SourceResponse { + return &response.SourceResponse{ + ID: src.ID, + NotebookID: src.NotebookID, + Name: src.Name, + Type: src.Type, + OriginalURL: src.OriginalURL, + FilePath: src.FilePath, + FileSize: src.FileSize, + MimeType: src.MimeType, + Status: src.Status, + ErrorMessage: src.ErrorMessage, + Vectorized: src.Vectorized, + CreatedAt: src.CreatedAt, + UpdatedAt: src.UpdatedAt, + } +} + +// ReimportAll 重新导入用户所有未向量化的资料 +func (s *sourceService) ReimportAll(userID uint) (int, error) { + if s.ingestionSvc == nil { + return 0, bizerrors.New(bizerrors.CodeInternalServiceError, "向量入库服务未初始化") + } + + // 获取用户所有未向量化的资料 + sources, err := s.sourceRepo.FindUnvectorizedByUserID(userID) + if err != nil { + return 0, err + } + + if len(sources) == 0 { + return 0, nil + } + + // 收集所有 source ID + sourceIDs := make([]uint, 0, len(sources)) + for _, src := range sources { + sourceIDs = append(sourceIDs, src.ID) + } + + // 批量入库 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // 即使部分失败也继续,返回成功导入的数量 + ingestErr := s.ingestionSvc.Ingest(ctx, sourceIDs) + + // 统计实际成功导入的数量(vectorized=true 的数量) + successCount := 0 + for _, src := range sources { + updated, err := s.sourceRepo.FindByID(src.ID) + if err == nil && updated != nil && updated.Vectorized { + successCount++ + } + } + + if ingestErr != nil { + logger.Warn("批量重新入库部分失败", + zap.Uint("user_id", userID), + zap.Int("total", len(sourceIDs)), + zap.Int("success", successCount), + zap.Error(ingestErr), + ) + } + + logger.Info("批量重新入库完成", + zap.Uint("user_id", userID), + zap.Int("total", len(sourceIDs)), + zap.Int("success", successCount), + ) + + return successCount, nil +} + +// ReimportSelected 重新导入指定的未向量化资料 +func (s *sourceService) ReimportSelected(sourceIDs []uint) (int, error) { + if s.ingestionSvc == nil { + return 0, bizerrors.New(bizerrors.CodeInternalServiceError, "向量入库服务未初始化") + } + + if len(sourceIDs) == 0 { + return 0, nil + } + + // 验证所有 source 都存在且未向量化 + for _, id := range sourceIDs { + source, err := s.sourceRepo.FindByID(id) + if err != nil { + return 0, err + } + if source == nil { + return 0, bizerrors.New(bizerrors.CodeNotFound, "资料不存在") + } + if source.Vectorized { + return 0, bizerrors.New(bizerrors.CodeBadRequest, "资料已入库,无需重复导入") + } + } + + // 批量入库 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + ingestErr := s.ingestionSvc.Ingest(ctx, sourceIDs) + + // 统计实际成功导入的数量 + successCount := 0 + for _, id := range sourceIDs { + updated, err := s.sourceRepo.FindByID(id) + if err == nil && updated != nil && updated.Vectorized { + successCount++ + } + } + + if ingestErr != nil { + logger.Warn("批量重新入库部分失败", + zap.Int("total", len(sourceIDs)), + zap.Int("success", successCount), + zap.Error(ingestErr), + ) + } + + logger.Info("批量重新入库完成", + zap.Int("total", len(sourceIDs)), + zap.Int("success", successCount), + ) + + return successCount, nil +} + +// CreateFromNote 将笔记内容保存为来源 +func (s *sourceService) CreateFromNote(userID, notebookID uint, title, content string) (*response.SourceResponse, error) { + if title == "" { + return nil, bizerrors.New(bizerrors.CodeBadRequest, "标题不能为空") + } + if content == "" { + return nil, bizerrors.New(bizerrors.CodeBadRequest, "内容不能为空") + } + + source := &entity.Source{ + UserID: userID, + NotebookID: notebookID, + Name: title, + Type: "note", + MarkdownContent: content, + Status: "pending", + Vectorized: false, + } + + if err := s.sourceRepo.Create(source); err != nil { + return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "创建来源失败", err) + } + + // 异步进行向量化入库 + if s.ingestionSvc != nil { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + if err := s.ingestionSvc.Ingest(ctx, []uint{source.ID}); err != nil { + logger.Warn("笔记来源向量化入库失败", + zap.Uint("source_id", source.ID), + zap.Error(err), + ) + } + }() + } + + return toSourceResponse(source), nil +} + +// DeleteByNoteAndNotebook 根据笔记标题和笔记本ID删除来源 +func (s *sourceService) DeleteByNoteAndNotebook(userID, notebookID uint, title string) error { + sources, _, err := s.sourceRepo.ListByNotebook(userID, notebookID, title, 0, 100) + if err != nil { + return err + } + + for _, src := range sources { + if src.Name == title && src.Type == "note" { + return s.Delete(src.ID) + } + } + + return bizerrors.New(bizerrors.CodeNotFound, "未找到对应的笔记来源") +} diff --git a/internal/service/testdata/photosynthesis_dynamic_ppt_sample.html b/internal/service/testdata/photosynthesis_dynamic_ppt_sample.html deleted file mode 100644 index 42c32b0..0000000 --- a/internal/service/testdata/photosynthesis_dynamic_ppt_sample.html +++ /dev/null @@ -1,394 +0,0 @@ - - - - - - 光合作用 · 学习课件 - - - -
- -
-

光合作用

-

从光能到化学能的生命引擎

-
    -
  • 基于笔记整理的学习材料
  • -
  • 核心内容:光反应 · 卡尔文循环 · 影响因素
  • -
  • 适用对象:中学生物 / 大学通识 · 自学复习
  • -
-
绿色植物 · 藻类 · 光合细菌
-
- -
-

目录

-
-
背景与目标
-
概念框架
-
⚙️机制与流程
-
案例与应用
-
⚠️易错辨析
-
总结复盘
-
-
- 素材来源: 本地笔记 · 光反应、卡尔文循环、影响因素;联网参考补充实验背景(希尔反应、鲁宾-卡门、阿尔农)。 -
-
- -
-

背景与目标

-
    -
  • 光合作用是地球生命能量流动的起点——将光能转化为化学能,驱动碳循环与氧气生成
  • -
  • 绿色植物、藻类和光合细菌利用叶绿体(或类似结构)完成这一过程
  • -
  • 学习目标① 理解光反应与卡尔文循环的分工协作关系
  • -
  • 学习目标② 掌握光照强度、CO₂浓度、温度对光合效率的影响机制
  • -
  • 学习目标③ 能够联系农业生产、温室调控和生态碳汇等实际应用
  • -
-
- 光合作用总反应式:6CO₂ + 12H₂O → C₆H₁₂O₆ + 6O₂ + 6H₂O(光能驱动) -
-
- 补充: 1937年希尔反应证明水的光解与CO₂还原可分开进行;1941年鲁宾-卡门用¹⁸O标记确认氧气全部来源于水。 -
-
- -
-

概念框架 · 光合作用三大模块

-
-
-

☀️ 光反应

-

类囊体膜上发生,依赖光照。
叶绿素吸收光能 → 水光解 → 产生 ATP + NADPH + O₂。

-
-
-

卡尔文循环

-

叶绿体基质中进行,不直接需光。
利用 ATP 和 NADPH 固定 CO₂,最终合成糖类(如葡萄糖)。

-
-
-

影响因素

-

光照强度、CO₂浓度、温度 三者协同调控光合速率,存在短板效应。

-
-
-
    -
  • 光反应 ↔ 卡尔文循环:光反应为暗反应提供 ATP(能量)和 NADPH(还原力);暗反应消耗光反应产物,推动循环持续
  • -
  • 三者相互联系,缺一不可——光反应受阻则暗反应无能量来源;暗反应受阻则光反应产物积累,导致反馈抑制
  • -
-
- 本地笔记依据: 光反应将水分解并产生 ATP 和 NADPH;卡尔文循环固定 CO₂ 合成糖类;三大因子影响效率。 -
-
- -
-

⚙️ 机制与流程 · 光反应详解

-
    -
  • ① 光能的吸收与传递:叶绿素(主要吸收红光和蓝紫光)捕获光子,激发电子进入高能态
  • -
  • ② 水的光解:2H₂O → 4H⁺ + 4e⁻ + O₂↑(氧气释放;电子用于补充叶绿素失去的电子)
  • -
  • ③ 电子传递与光合磷酸化:高能电子经电子传递链 → 驱动 H⁺ 跨膜运输 → 化学渗透合成 ATP
  • -
  • ④ NADPH 形成:电子最终传递给 NADP⁺,与 H⁺ 结合生成 NADPH(还原力)
  • -
-
- 关键物质:ATP(能量货币)和 NADPH(还原剂)在光反应中生成,直接供给卡尔文循环。 -
-
-

卡尔文循环(C3 循环)

-
    -
  • ① 羧化阶段:CO₂ + RuBP(五碳化合物)→ 2 × 3-磷酸甘油酸(3-PGA)——由 Rubisco 催化
  • -
  • ② 还原阶段:3-PGA 在 ATP 和 NADPH 作用下还原为 3-磷酸甘油醛(G3P)
  • -
  • ③ 再生阶段:大部分 G3P 用于再生 RuBP,少部分合成葡萄糖等有机物
  • -
-
- 实验支撑: 1954年阿尔农发现光照下叶绿体可合成 ATP;1957年证实该过程与水的光解相伴随。光反应与 CO₂ 固定可分离(希尔反应)。 -
-
-
- -
-

⚡ 光反应与卡尔文循环的能量耦联

-
    -
  • ATP 和 NADPH 是连接两个阶段的“能量货币”与“还原力载体”
  • -
  • 光反应产生的 ATP 和 NADPH 在卡尔文循环中被消耗,转化为 ADP、Pi 和 NADP⁺,重新返回光反应
  • -
  • 这种循环耦联保证了光合作用的持续进行——光反应为“暗反应”提供燃料,暗反应为光反应提供再生底物
  • -
  • 氧气全部来源于水(希尔反应 + 鲁宾-卡门同位素标记实验证实),与 CO₂ 中的氧无关
  • -
-
-

记忆要点

-

光反应:类囊体 · 需光 · 产 ATP、NADPH、O₂
- 卡尔文循环:基质 · 不需光(但依赖光产物) · 耗 ATP、NADPH · 固碳成糖

-
-
- -
-

案例与应用 · 光合作用在真实世界

-
    -
  • 农业增产:合理密植(充分利用光照)、大棚补充 CO₂(气肥)、延长光照时间或补光
  • -
  • 温室调控:控制温度在植物最适光合温度(如 25~30℃),避免高温导致的光抑制和 Rubisco 失活
  • -
  • 植物工厂:使用 LED 灯精准调节光质(红光 + 蓝光组合)和光周期,最大化能量转化效率
  • -
  • 碳达峰与碳中和:利用造林、草地恢复等提升光合作用固碳能力,发展林业碳汇抵消碳排放
  • -
-
- 补充解释: “光抑制”指过强光照导致光反应系统受损,实际光合速率反而下降;适当遮阴或调控光质可缓解。 -
-
- 光合作用是全球碳循环的核心——每年约固定 2×10¹¹ 吨碳,维持大气 O₂/CO₂ 平衡。 -
-
- -
-

⚠️ 易错辨析 · 常见误区澄清

-
    -
  • ❌ “暗反应”= 黑暗中反应? 卡尔文循环虽不直接需光,但依赖光反应产物(ATP、NADPH)。无光时产物耗尽,循环停止;“暗反应”应理解为“不直接依赖光”
  • -
  • ❌ 光合作用释放的氧气来自 CO₂? 同位素示踪(鲁宾-卡门,¹⁸O 标记 H₂O 和 CO₂)明确证明:氧气全部来源于水,而非 CO₂
  • -
  • ❌ 叶绿体只进行“暗反应”? 叶绿体是完整场所:光反应在类囊体,暗反应在基质,两者相互协作,缺一不可
  • -
  • ❌ ATP 和 NADPH 只是能量载体? NADPH 还提供还原力,直接参与 CO₂ 还原为糖类的过程;既是能量也是电子供体
  • -
  • ❌ 光照越强光合速率越快? 存在光饱和点;超过最适光强后,光抑制导致效率下降,甚至损伤光合系统
  • -
-
- 小结: 以上误区在考试和实际理解中高频出现,牢记实验证据和概念边界可有效避免。 -
-
- -
-

总结复盘 · 光合作用核心线索

-
    -
  • 总反应式:6CO₂ + 12H₂O → C₆H₁₂O₆ + 6O₂ + 6H₂O(光能驱动,叶绿体)
  • -
  • 光反应:类囊体膜 · 光 → ATP + NADPH + O₂(水的光解)
  • -
  • 卡尔文循环:基质 · CO₂ → 糖类(消耗 ATP 和 NADPH)
  • -
  • 耦联纽带:ATP 和 NADPH 在两个阶段间循环使用
  • -
  • 三大影响因子:光照强度、CO₂浓度、温度 —— 存在短板效应,实际光合速率受最低因子限制
  • -
  • 应用延伸:农业增产、温室调控、植物工厂、碳汇工程
  • -
-
- 理解光合作用是生态学、农学、环境科学和气候行动的共通基础 —— 从分子到全球,生命的光合引擎永不停歇。 -
-
- ✅ 基于本地笔记 · 联网补充实验背景 · 学习课件 v1.0 -
-
- -
- - diff --git a/internal/service/token_blacklist_interface.go b/internal/service/token_blacklist_interface.go index 0e8df72..27e58b3 100644 --- a/internal/service/token_blacklist_interface.go +++ b/internal/service/token_blacklist_interface.go @@ -1,11 +1,25 @@ -package service - -import "context" - -// TokenBlacklistService Token 黑名单服务接口 -type TokenBlacklistService interface { - // RevokeToken 将 token 加入黑名单,TTL = token 剩余过期时间 - RevokeToken(ctx context.Context, tokenString string) error - // IsRevoked 检查 token 是否已被撤销 - IsRevoked(ctx context.Context, jti string) (bool, error) -} +package service + +import ( + "context" + "time" +) + +// TokenBlacklistService Token 黑名单服务接口 +type TokenBlacklistService interface { + // RevokeToken 将单个 token 加入黑名单,TTL = token 剩余过期时间 + RevokeToken(ctx context.Context, tokenString string) error + // IsRevoked 检查 token 是否已被撤销 + IsRevoked(ctx context.Context, jti string) (bool, error) + + // AddUserToken 将 jti 加入用户 token 集合,并续期集合 TTL + // 用于登录/refresh 时登记新会话,配合 RevokeUserTokens 实现用户级批量吊销 + AddUserToken(ctx context.Context, userID uint, jti string, ttl time.Duration) error + // RemoveUserToken 从用户 token 集合移除 jti + // 用于 refresh 换发新 access token 后清理旧 jti,避免集合膨胀(可选调用) + RemoveUserToken(ctx context.Context, userID uint, jti string) error + // RevokeUserTokens 拉黑该用户集合中所有 token,并删除集合,返回拉黑数量 + // tokenTTL 应取 refresh token 的有效期(24h),保证 refresh token 也被拉黑到过期; + // access token 15m 自然过期,多拉黑无害 + RevokeUserTokens(ctx context.Context, userID uint, tokenTTL time.Duration) (int, error) +} diff --git a/internal/service/token_blacklist_service.go b/internal/service/token_blacklist_service.go index 1644e30..2e58659 100644 --- a/internal/service/token_blacklist_service.go +++ b/internal/service/token_blacklist_service.go @@ -1,70 +1,121 @@ -package service - -import ( - "YoudaoNoteLm/pkg/jwt" - "context" - "fmt" - "time" - - "github.com/redis/go-redis/v9" -) - -const ( - // tokenBlacklistKeyPrefix 单个 token 黑名单 key 前缀 - tokenBlacklistKeyPrefix = "token:blacklist:" - // userTokenKeyPrefix 用户 token 集合 key 前缀(用于批量吊销) - userTokenKeyPrefix = "token:user:" -) - -// tokenBlacklistService Token 黑名单服务实现 -type tokenBlacklistService struct { - redis *redis.Client -} - -// NewTokenBlacklistService 创建 Token 黑名单服务 -func NewTokenBlacklistService(redisClient *redis.Client) TokenBlacklistService { - return &tokenBlacklistService{ - redis: redisClient, - } -} - -// blacklistKey 黑名单 key: token:blacklist:{jti} -func (s *tokenBlacklistService) blacklistKey(jti string) string { - return fmt.Sprintf("%s%s", tokenBlacklistKeyPrefix, jti) -} - -// RevokeToken 将 token 加入黑名单 -func (s *tokenBlacklistService) RevokeToken(ctx context.Context, tokenString string) error { - // 解析 token 获取 JTI 和过期时间(不做有效性校验,过期的 token 也需要记录) - parser := jwt.GetParser() - claims := &jwt.CustomClaims{} - _, _, err := parser.ParseUnverified(tokenString, claims) - if err != nil { - return fmt.Errorf("解析 token 失败: %w", err) - } - - if claims.ID == "" { - return fmt.Errorf("token 缺少 JTI") - } - - // 计算 token 剩余过期时间作为黑名单 TTL - ttl := time.Until(claims.ExpiresAt.Time) - if ttl <= 0 { - // token 已过期,无需加入黑名单 - return nil - } - - // 存入 Redis,过期后自动清除 - key := s.blacklistKey(claims.ID) - return s.redis.Set(ctx, key, "1", ttl).Err() -} - -// IsRevoked 检查 token 是否已被撤销 -func (s *tokenBlacklistService) IsRevoked(ctx context.Context, jti string) (bool, error) { - key := s.blacklistKey(jti) - exists, err := s.redis.Exists(ctx, key).Result() - if err != nil { - return false, fmt.Errorf("查询黑名单失败: %w", err) - } - return exists > 0, nil -} +package service + +import ( + "YoudaoNoteLm/pkg/jwt" + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + // tokenBlacklistKeyPrefix 单个 token 黑名单 key 前缀 + tokenBlacklistKeyPrefix = "token:blacklist:" + // userTokenKeyPrefix 用户 token 集合 key 前缀(用于批量吊销) + userTokenKeyPrefix = "token:user:" +) + +// tokenBlacklistService Token 黑名单服务实现 +type tokenBlacklistService struct { + redis *redis.Client +} + +// NewTokenBlacklistService 创建 Token 黑名单服务 +func NewTokenBlacklistService(redisClient *redis.Client) TokenBlacklistService { + return &tokenBlacklistService{ + redis: redisClient, + } +} + +// blacklistKey 黑名单 key: token:blacklist:{jti} +func (s *tokenBlacklistService) blacklistKey(jti string) string { + return fmt.Sprintf("%s%s", tokenBlacklistKeyPrefix, jti) +} + +// RevokeToken 将 token 加入黑名单 +func (s *tokenBlacklistService) RevokeToken(ctx context.Context, tokenString string) error { + // 解析 token 获取 JTI 和过期时间(不做有效性校验,过期的 token 也需要记录) + parser := jwt.GetParser() + claims := &jwt.CustomClaims{} + _, _, err := parser.ParseUnverified(tokenString, claims) + if err != nil { + return fmt.Errorf("解析 token 失败: %w", err) + } + + if claims.ID == "" { + return fmt.Errorf("token 缺少 JTI") + } + + // 计算 token 剩余过期时间作为黑名单 TTL + ttl := time.Until(claims.ExpiresAt.Time) + if ttl <= 0 { + // token 已过期,无需加入黑名单 + return nil + } + + // 存入 Redis,过期后自动清除 + key := s.blacklistKey(claims.ID) + return s.redis.Set(ctx, key, "1", ttl).Err() +} + +// IsRevoked 检查 token 是否已被撤销 +func (s *tokenBlacklistService) IsRevoked(ctx context.Context, jti string) (bool, error) { + key := s.blacklistKey(jti) + exists, err := s.redis.Exists(ctx, key).Result() + if err != nil { + return false, fmt.Errorf("查询黑名单失败: %w", err) + } + return exists > 0, nil +} + +// userTokenKey 用户 token 集合 key: token:user:{userID} +func (s *tokenBlacklistService) userTokenKey(userID uint) string { + return fmt.Sprintf("%s%d", userTokenKeyPrefix, userID) +} + +// AddUserToken 将 jti 加入用户 token 集合,并续期集合 TTL +// Redis Set 不支持元素级 TTL,每次 SAdd 后整体 Expire 续期 +func (s *tokenBlacklistService) AddUserToken(ctx context.Context, userID uint, jti string, ttl time.Duration) error { + key := s.userTokenKey(userID) + pipe := s.redis.Pipeline() + pipe.SAdd(ctx, key, jti) + pipe.Expire(ctx, key, ttl) + if _, err := pipe.Exec(ctx); err != nil { + return fmt.Errorf("添加用户 token 到集合失败: %w", err) + } + return nil +} + +// RemoveUserToken 从用户 token 集合移除 jti(refresh 换发新 token 后清理旧 jti) +func (s *tokenBlacklistService) RemoveUserToken(ctx context.Context, userID uint, jti string) error { + key := s.userTokenKey(userID) + if err := s.redis.SRem(ctx, key, jti).Err(); err != nil { + return fmt.Errorf("从用户 token 集合移除失败: %w", err) + } + return nil +} + +// RevokeUserTokens 拉黑该用户集合中所有 token,并删除集合 +// tokenTTL 应取 refresh token 有效期,保证 refresh token 被拉黑到过期 +func (s *tokenBlacklistService) RevokeUserTokens(ctx context.Context, userID uint, tokenTTL time.Duration) (int, error) { + key := s.userTokenKey(userID) + jtis, err := s.redis.SMembers(ctx, key).Result() + if err != nil { + return 0, fmt.Errorf("查询用户 token 集合失败: %w", err) + } + if len(jtis) == 0 { + return 0, nil + } + + // 批量拉黑 + 删除集合,Pipeline 减少往返 + pipe := s.redis.Pipeline() + for _, jti := range jtis { + pipe.Set(ctx, s.blacklistKey(jti), "1", tokenTTL) + } + pipe.Del(ctx, key) + if _, err := pipe.Exec(ctx); err != nil { + return 0, fmt.Errorf("批量拉黑用户 token 失败: %w", err) + } + return len(jtis), nil +} diff --git a/internal/service/user_config_interface.go b/internal/service/user_config_interface.go index d16415a..977270b 100644 --- a/internal/service/user_config_interface.go +++ b/internal/service/user_config_interface.go @@ -1,36 +1,42 @@ -package service - -import "YoudaoNoteLm/internal/model/entity" - -// UserConfigService 用户配置服务接口 -type UserConfigService interface { - // LLM 配置(独立表 user_llm_config,支持多条) - ListLLMConfigs(userID uint) ([]*entity.UserLLMConfig, error) - CreateLLMConfig(userID uint, config *entity.UserLLMConfig) error - UpdateLLMConfig(userID uint, id uint, config *entity.UserLLMConfig) error - DeleteLLMConfig(userID uint, id uint) error - - // 搜索配置 - ListSearchConfigs(userID uint) ([]*entity.UserConfig, error) - CreateSearchConfig(userID uint, config *entity.UserConfig) error - UpdateSearchConfig(id uint, config *entity.UserConfig) error - DeleteSearchConfig(id uint) error - - // ASR 配置 - ListASRConfigs(userID uint) ([]*entity.UserConfig, error) - CreateASRConfig(userID uint, config *entity.UserConfig) error - UpdateASRConfig(id uint, config *entity.UserConfig) error - DeleteASRConfig(id uint) error - - // Embedding 配置 - ListEmbeddingConfigs(userID uint) ([]*entity.UserConfig, error) - CreateEmbeddingConfig(userID uint, config *entity.UserConfig) error - UpdateEmbeddingConfig(id uint, config *entity.UserConfig) error - DeleteEmbeddingConfig(id uint) error - - // 获取当前生效的配置(用户配置 > 系统配置 > 默认值) - GetActiveConfig(userID uint, configType string) (*entity.UserConfig, error) - - // TestConfig 测试配置连通性(保存前验证) - TestConfig(configType string, config *entity.UserConfig) *HealthCheckResult -} +package service + +import "YoudaoNoteLm/internal/model/entity" + +// UserConfigService 用户配置服务接口 +type UserConfigService interface { + // LLM 配置(独立表 user_llm_config,支持多条) + ListLLMConfigs(userID uint) ([]*entity.UserLLMConfig, error) + CreateLLMConfig(userID uint, config *entity.UserLLMConfig) error + UpdateLLMConfig(userID uint, id uint, config *entity.UserLLMConfig) error + DeleteLLMConfig(userID uint, id uint) error + + // 搜索配置 + ListSearchConfigs(userID uint) ([]*entity.UserConfig, error) + CreateSearchConfig(userID uint, config *entity.UserConfig) error + UpdateSearchConfig(id uint, config *entity.UserConfig) error + DeleteSearchConfig(id uint) error + + // ASR 配置 + ListASRConfigs(userID uint) ([]*entity.UserConfig, error) + CreateASRConfig(userID uint, config *entity.UserConfig) error + UpdateASRConfig(id uint, config *entity.UserConfig) error + DeleteASRConfig(id uint) error + + // Embedding 配置 + ListEmbeddingConfigs(userID uint) ([]*entity.UserConfig, error) + CreateEmbeddingConfig(userID uint, config *entity.UserConfig) error + UpdateEmbeddingConfig(id uint, config *entity.UserConfig) error + DeleteEmbeddingConfig(id uint) error + + // Reranker 配置 + ListRerankerConfigs(userID uint) ([]*entity.UserConfig, error) + CreateRerankerConfig(userID uint, config *entity.UserConfig) error + UpdateRerankerConfig(id uint, config *entity.UserConfig) error + DeleteRerankerConfig(id uint) error + + // 获取当前生效的配置(用户配置 > 系统配置 > 默认值) + GetActiveConfig(userID uint, configType string) (*entity.UserConfig, error) + + // TestConfig 测试配置连通性(保存前验证) + TestConfig(configType string, config *entity.UserConfig) *HealthCheckResult +} diff --git a/internal/service/user_config_service.go b/internal/service/user_config_service.go index 1ebec76..e578e6e 100644 --- a/internal/service/user_config_service.go +++ b/internal/service/user_config_service.go @@ -1,553 +1,657 @@ -package service - -import ( - "encoding/json" - - "YoudaoNoteLm/internal/model/entity" - "YoudaoNoteLm/internal/repository" - bizerrors "YoudaoNoteLm/pkg/errors" - "YoudaoNoteLm/pkg/logger" - "YoudaoNoteLm/pkg/utils" - - "go.uber.org/zap" - "gorm.io/gorm" -) - -type userConfigService struct { - configRepo repository.UserConfigRepository - llmConfigRepo repository.UserLLMConfigRepository - configSvc ConfigService // 配置路由服务,用于获取系统配置 - healthChk *ConfigHealthChecker // 配置健康检查器 - encryptionKey []byte // API Key 加密密钥 -} - -func NewUserConfigService(configRepo repository.UserConfigRepository, llmConfigRepo repository.UserLLMConfigRepository, configSvc ConfigService, encryptionKey string) UserConfigService { - return &userConfigService{ - configRepo: configRepo, - llmConfigRepo: llmConfigRepo, - configSvc: configSvc, - healthChk: NewConfigHealthChecker(), - encryptionKey: []byte(encryptionKey), - } -} - -// ===== LLM Config ===== - -func (s *userConfigService) ListLLMConfigs(userID uint) ([]*entity.UserLLMConfig, error) { - configs, err := s.llmConfigRepo.FindByUserID(userID) - if err != nil { - return nil, err - } - // 解密 API Key - for _, config := range configs { - if config.APIKey != "" { - decrypted, err := utils.Decrypt(config.APIKey, s.encryptionKey) - if err != nil { - // 解密失败,可能数据未加密或使用了不同的密钥,保留原值 - logger.Debug("解密 LLM API Key 失败(可能未加密)", zap.Uint("config_id", config.ID), zap.Error(err)) - } else { - config.APIKey = decrypted - } - } - } - return configs, nil -} - -func (s *userConfigService) CreateLLMConfig(userID uint, config *entity.UserLLMConfig) error { - config.UserID = userID - // 加密 API Key - if config.APIKey != "" { - encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Error("加密 LLM API Key 失败", zap.Error(err)) - return err - } - config.APIKey = encrypted - } - return s.llmConfigRepo.Create(config) -} - -func (s *userConfigService) UpdateLLMConfig(userID uint, id uint, config *entity.UserLLMConfig) error { - existing, err := s.llmConfigRepo.FindByID(id) - if err != nil { - logger.Error("查找配置失败", zap.Uint("id", id), zap.Error(err)) - return err - } - if existing == nil { - return bizerrors.ErrNotFound - } - if existing.UserID != userID { - return bizerrors.New(bizerrors.CodeForbidden, "无权操作此配置") - } - config.ID = id - config.UserID = existing.UserID - config.CreatedAt = existing.CreatedAt - config.UpdatedAt = existing.UpdatedAt - // 加密 API Key - if config.APIKey != "" { - encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Error("加密 LLM API Key 失败", zap.Error(err)) - return err - } - config.APIKey = encrypted - } - logger.Info("更新LLM配置", zap.Uint("id", id), zap.Any("config", config)) - return s.llmConfigRepo.Update(config) -} - -func (s *userConfigService) DeleteLLMConfig(userID uint, id uint) error { - existing, err := s.llmConfigRepo.FindByID(id) - if err != nil { - return err - } - if existing == nil { - return bizerrors.ErrNotFound - } - if existing.UserID != userID { - return bizerrors.New(bizerrors.CodeForbidden, "无权操作此配置") - } - return s.llmConfigRepo.Delete(id) -} - -// ===== Search Config ===== - -func (s *userConfigService) ListSearchConfigs(userID uint) ([]*entity.UserConfig, error) { - config, err := s.configRepo.FindByUserAndType(userID, "search") - if err != nil { - return nil, err - } - if config == nil { - return []*entity.UserConfig{}, nil - } - // 解密 API Key - if config.APIKey != "" { - decrypted, err := utils.Decrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Debug("解密 Search API Key 失败(可能未加密)", zap.Uint("config_id", config.ID), zap.Error(err)) - } else { - config.APIKey = decrypted - } - } - return []*entity.UserConfig{config}, nil -} - -func (s *userConfigService) CreateSearchConfig(userID uint, config *entity.UserConfig) error { - config.UserID = userID - config.ConfigType = "search" - if config.ExtraConfig == "" { - config.ExtraConfig = "{}" - } - - // 加密 API Key - if config.APIKey != "" { - encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Error("加密 Search API Key 失败", zap.Error(err)) - return err - } - config.APIKey = encrypted - } - - // 检查是否已经存在相同类型的配置(包括已删除的记录) - existing, err := s.configRepo.FindByUserAndTypeIncludingDeleted(userID, "search") - if err != nil { - return err - } - - if existing != nil { - // 如果存在已删除的记录,则更新它并恢复为未删除状态 - config.ID = existing.ID - config.CreatedAt = existing.CreatedAt - config.UpdatedAt = existing.UpdatedAt - config.DeletedAt = gorm.DeletedAt{} // 恢复为未删除状态 - return s.configRepo.Update(config) - } - - return s.configRepo.Create(config) -} - -func (s *userConfigService) UpdateSearchConfig(id uint, config *entity.UserConfig) error { - existing, err := s.configRepo.FindByID(id) - if err != nil { - return err - } - if existing == nil { - return bizerrors.ErrNotFound - } - config.ID = id - config.UserID = existing.UserID - config.ConfigType = "search" - config.CreatedAt = existing.CreatedAt - config.UpdatedAt = existing.UpdatedAt - if config.ExtraConfig == "" { - config.ExtraConfig = "{}" - } - // 加密 API Key - if config.APIKey != "" { - encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Error("加密 Search API Key 失败", zap.Error(err)) - return err - } - config.APIKey = encrypted - } - if err := s.configRepo.Update(config); err != nil { - return err - } - s.configSvc.ClearUserConfigCache(existing.UserID, "search") - return nil -} - -func (s *userConfigService) DeleteSearchConfig(id uint) error { - existing, err := s.configRepo.FindByID(id) - if err != nil { - return err - } - if existing == nil { - return bizerrors.ErrNotFound - } - if err := s.configRepo.Delete(id); err != nil { - return err - } - s.configSvc.ClearUserConfigCache(existing.UserID, "search") - return nil -} - -// ===== ASR Config ===== - -func (s *userConfigService) ListASRConfigs(userID uint) ([]*entity.UserConfig, error) { - config, err := s.configRepo.FindByUserAndType(userID, "asr") - if err != nil { - return nil, err - } - if config == nil { - return []*entity.UserConfig{}, nil - } - // 解密 API Key - if config.APIKey != "" { - decrypted, err := utils.Decrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Debug("解密 ASR API Key 失败(可能未加密)", zap.Uint("config_id", config.ID), zap.Error(err)) - } else { - config.APIKey = decrypted - } - } - return []*entity.UserConfig{config}, nil -} - -func (s *userConfigService) CreateASRConfig(userID uint, config *entity.UserConfig) error { - config.UserID = userID - config.ConfigType = "asr" - if config.ExtraConfig == "" { - config.ExtraConfig = "{}" - } - - // 加密 API Key - if config.APIKey != "" { - encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Error("加密 ASR API Key 失败", zap.Error(err)) - return err - } - config.APIKey = encrypted - } - - // 检查是否已经存在相同类型的配置(包括已删除的记录) - existing, err := s.configRepo.FindByUserAndTypeIncludingDeleted(userID, "asr") - if err != nil { - return err - } - - if existing != nil { - // 如果存在已删除的记录,则更新它并恢复为未删除状态 - config.ID = existing.ID - config.CreatedAt = existing.CreatedAt - config.UpdatedAt = existing.UpdatedAt - config.DeletedAt = gorm.DeletedAt{} // 恢复为未删除状态 - return s.configRepo.Update(config) - } - - return s.configRepo.Create(config) -} - -func (s *userConfigService) UpdateASRConfig(id uint, config *entity.UserConfig) error { - existing, err := s.configRepo.FindByID(id) - if err != nil { - return err - } - if existing == nil { - return bizerrors.ErrNotFound - } - config.ID = id - config.UserID = existing.UserID - config.ConfigType = "asr" - config.CreatedAt = existing.CreatedAt - config.UpdatedAt = existing.UpdatedAt - if config.ExtraConfig == "" { - config.ExtraConfig = "{}" - } - // 加密 API Key - if config.APIKey != "" { - encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Error("加密 ASR API Key 失败", zap.Error(err)) - return err - } - config.APIKey = encrypted - } - if err := s.configRepo.Update(config); err != nil { - return err - } - s.configSvc.ClearUserConfigCache(existing.UserID, "asr") - return nil -} - -func (s *userConfigService) DeleteASRConfig(id uint) error { - existing, err := s.configRepo.FindByID(id) - if err != nil { - return err - } - if existing == nil { - return bizerrors.ErrNotFound - } - if err := s.configRepo.Delete(id); err != nil { - return err - } - s.configSvc.ClearUserConfigCache(existing.UserID, "asr") - return nil -} - -// ===== Embedding Config ===== - -func (s *userConfigService) ListEmbeddingConfigs(userID uint) ([]*entity.UserConfig, error) { - config, err := s.configRepo.FindByUserAndType(userID, "embedding") - if err != nil { - return nil, err - } - if config == nil { - return []*entity.UserConfig{}, nil - } - // 解密 API Key - if config.APIKey != "" { - decrypted, err := utils.Decrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Debug("解密 Embedding API Key 失败(可能未加密)", zap.Uint("config_id", config.ID), zap.Error(err)) - } else { - config.APIKey = decrypted - } - } - return []*entity.UserConfig{config}, nil -} - -func (s *userConfigService) CreateEmbeddingConfig(userID uint, config *entity.UserConfig) error { - config.UserID = userID - config.ConfigType = "embedding" - if config.ExtraConfig == "" { - config.ExtraConfig = "{}" - } - - // 加密 API Key - if config.APIKey != "" { - encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Error("加密 Embedding API Key 失败", zap.Error(err)) - return err - } - config.APIKey = encrypted - } - - // 检查是否已经存在相同类型的配置(包括已删除的记录) - existing, err := s.configRepo.FindByUserAndTypeIncludingDeleted(userID, "embedding") - if err != nil { - return err - } - - if existing != nil { - // 如果存在已删除的记录,则更新它并恢复为未删除状态 - config.ID = existing.ID - config.CreatedAt = existing.CreatedAt - config.UpdatedAt = existing.UpdatedAt - config.DeletedAt = gorm.DeletedAt{} // 恢复为未删除状态 - return s.configRepo.Update(config) - } - - return s.configRepo.Create(config) -} - -func (s *userConfigService) UpdateEmbeddingConfig(id uint, config *entity.UserConfig) error { - existing, err := s.configRepo.FindByID(id) - if err != nil { - return err - } - if existing == nil { - return bizerrors.ErrNotFound - } - config.ID = id - config.UserID = existing.UserID - config.ConfigType = "embedding" - config.CreatedAt = existing.CreatedAt - config.UpdatedAt = existing.UpdatedAt - if config.ExtraConfig == "" { - config.ExtraConfig = "{}" - } - // 加密 API Key - if config.APIKey != "" { - encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) - if err != nil { - logger.Error("加密 Embedding API Key 失败", zap.Error(err)) - return err - } - config.APIKey = encrypted - } - if err := s.configRepo.Update(config); err != nil { - return err - } - s.configSvc.ClearUserConfigCache(existing.UserID, "embedding") - return nil -} - -func (s *userConfigService) DeleteEmbeddingConfig(id uint) error { - existing, err := s.configRepo.FindByID(id) - if err != nil { - return err - } - if existing == nil { - return bizerrors.ErrNotFound - } - if err := s.configRepo.Delete(id); err != nil { - return err - } - s.configSvc.ClearUserConfigCache(existing.UserID, "embedding") - return nil -} - -// GetActiveConfig 获取当前生效的配置(用户配置 > 系统配置) -func (s *userConfigService) GetActiveConfig(userID uint, configType string) (*entity.UserConfig, error) { - // LLM 配置存储在独立的 user_llm_config 表,需要特殊处理 - if configType == "llm" { - return s.getActiveLLMConfig(userID) - } - - // 1. 优先返回用户配置(必须启用) - userCfg, err := s.configRepo.FindByUserAndType(userID, configType) - if err == nil && userCfg != nil && userCfg.Enabled { - userCfg.Source = "user" - // 解密 API Key - if userCfg.APIKey != "" { - decrypted, err := utils.Decrypt(userCfg.APIKey, s.encryptionKey) - if err != nil { - logger.Debug("解密用户配置 API Key 失败(可能未加密)", zap.Uint("user_id", userID), zap.Error(err)) - } else { - userCfg.APIKey = decrypted - } - } - return userCfg, nil - } - - // 2. 降级到系统配置 - sysCfg, err := s.configSvc.GetSysConfig(configType) - if err == nil && sysCfg != nil { - // 解析系统配置的 JSON 值 - var params map[string]interface{} - if jsonErr := json.Unmarshal([]byte(sysCfg.ConfigValue), ¶ms); jsonErr == nil { - getStr := func(key string) string { - if v, ok := params[key].(string); ok { - return v - } - return "" - } - - return &entity.UserConfig{ - ConfigType: configType, - Name: getStr("name"), - Provider: getStr("provider"), - APIURL: getStr("api_url"), - APIKey: getStr("api_key"), - Model: getStr("model"), - Enabled: sysCfg.Enabled, - Source: "system", - }, nil - } - - // 如果解析失败,尝试作为纯 URL 处理 - return &entity.UserConfig{ - ConfigType: configType, - Name: sysCfg.ConfigKey, - Provider: sysCfg.ConfigKey, - APIURL: sysCfg.ConfigValue, - Enabled: sysCfg.Enabled, - Source: "system", - }, nil - } - - // 3. 没有配置 - return nil, nil -} - -// getActiveLLMConfig 获取当前生效的 LLM 配置(LLM 存储在 user_llm_config 表) -func (s *userConfigService) getActiveLLMConfig(userID uint) (*entity.UserConfig, error) { - // 1. 优先返回用户 LLM 配置(第一个启用的) - llmCfg, err := s.llmConfigRepo.FindDefaultByUserID(userID) - if err == nil && llmCfg != nil && llmCfg.Enabled { - apiKey := llmCfg.APIKey - if apiKey != "" { - decrypted, err := utils.Decrypt(apiKey, s.encryptionKey) - if err != nil { - logger.Debug("解密 LLM API Key 失败(可能未加密)", zap.Uint("user_id", userID), zap.Error(err)) - } else { - apiKey = decrypted - } - } - return &entity.UserConfig{ - ConfigType: "llm", - Name: llmCfg.Name, - Provider: llmCfg.Provider, - APIURL: llmCfg.APIURL, - APIKey: apiKey, - Model: llmCfg.Model, - Enabled: llmCfg.Enabled, - Source: "user", - }, nil - } - - // 2. 降级到系统配置 - sysCfg, err := s.configSvc.GetSysConfig("llm") - if err == nil && sysCfg != nil { - var params map[string]interface{} - if jsonErr := json.Unmarshal([]byte(sysCfg.ConfigValue), ¶ms); jsonErr == nil { - getStr := func(key string) string { - if v, ok := params[key].(string); ok { - return v - } - return "" - } - return &entity.UserConfig{ - ConfigType: "llm", - Name: getStr("name"), - Provider: getStr("provider"), - APIURL: getStr("api_url"), - APIKey: getStr("api_key"), - Model: getStr("model"), - Enabled: sysCfg.Enabled, - Source: "system", - }, nil - } - return &entity.UserConfig{ - ConfigType: "llm", - Name: sysCfg.ConfigKey, - Provider: sysCfg.ConfigKey, - APIURL: sysCfg.ConfigValue, - Enabled: sysCfg.Enabled, - Source: "system", - }, nil - } - - // 3. 没有配置 - return nil, nil -} - -// TestConfig 测试配置连通性(保存前验证) -func (s *userConfigService) TestConfig(configType string, config *entity.UserConfig) *HealthCheckResult { - return s.healthChk.TestConfig(configType, config) -} +package service + +import ( + "encoding/json" + + "YoudaoNoteLm/internal/model/entity" + "YoudaoNoteLm/internal/repository" + bizerrors "YoudaoNoteLm/pkg/errors" + "YoudaoNoteLm/pkg/logger" + "YoudaoNoteLm/pkg/utils" + + "go.uber.org/zap" + "gorm.io/gorm" +) + +type userConfigService struct { + configRepo repository.UserConfigRepository + llmConfigRepo repository.UserLLMConfigRepository + configSvc ConfigService // 配置路由服务,用于获取系统配置 + healthChk *ConfigHealthChecker // 配置健康检查器 + encryptionKey []byte // API Key 加密密钥 +} + +func NewUserConfigService(configRepo repository.UserConfigRepository, llmConfigRepo repository.UserLLMConfigRepository, configSvc ConfigService, encryptionKey string) UserConfigService { + return &userConfigService{ + configRepo: configRepo, + llmConfigRepo: llmConfigRepo, + configSvc: configSvc, + healthChk: NewConfigHealthChecker(), + encryptionKey: []byte(encryptionKey), + } +} + +// ===== LLM Config ===== + +func (s *userConfigService) ListLLMConfigs(userID uint) ([]*entity.UserLLMConfig, error) { + configs, err := s.llmConfigRepo.FindByUserID(userID) + if err != nil { + return nil, err + } + // 解密 API Key + for _, config := range configs { + if config.APIKey != "" { + decrypted, err := utils.Decrypt(config.APIKey, s.encryptionKey) + if err != nil { + // 解密失败,可能数据未加密或使用了不同的密钥,保留原值 + logger.Debug("解密 LLM API Key 失败(可能未加密)", zap.Uint("config_id", config.ID), zap.Error(err)) + } else { + config.APIKey = decrypted + } + } + } + return configs, nil +} + +func (s *userConfigService) CreateLLMConfig(userID uint, config *entity.UserLLMConfig) error { + config.UserID = userID + // 加密 API Key + if config.APIKey != "" { + encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Error("加密 LLM API Key 失败", zap.Error(err)) + return err + } + config.APIKey = encrypted + } + return s.llmConfigRepo.Create(config) +} + +func (s *userConfigService) UpdateLLMConfig(userID uint, id uint, config *entity.UserLLMConfig) error { + existing, err := s.llmConfigRepo.FindByID(id) + if err != nil { + logger.Error("查找配置失败", zap.Uint("id", id), zap.Error(err)) + return err + } + if existing == nil { + return bizerrors.ErrNotFound + } + if existing.UserID != userID { + return bizerrors.New(bizerrors.CodeForbidden, "无权操作此配置") + } + config.ID = id + config.UserID = existing.UserID + config.CreatedAt = existing.CreatedAt + config.UpdatedAt = existing.UpdatedAt + // 加密 API Key + if config.APIKey != "" { + encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Error("加密 LLM API Key 失败", zap.Error(err)) + return err + } + config.APIKey = encrypted + } + logger.Info("更新LLM配置", zap.Uint("id", id), zap.Any("config", config)) + return s.llmConfigRepo.Update(config) +} + +func (s *userConfigService) DeleteLLMConfig(userID uint, id uint) error { + existing, err := s.llmConfigRepo.FindByID(id) + if err != nil { + return err + } + if existing == nil { + return bizerrors.ErrNotFound + } + if existing.UserID != userID { + return bizerrors.New(bizerrors.CodeForbidden, "无权操作此配置") + } + return s.llmConfigRepo.Delete(id) +} + +// ===== Search Config ===== + +func (s *userConfigService) ListSearchConfigs(userID uint) ([]*entity.UserConfig, error) { + config, err := s.configRepo.FindByUserAndType(userID, "search") + if err != nil { + return nil, err + } + if config == nil { + return []*entity.UserConfig{}, nil + } + // 解密 API Key + if config.APIKey != "" { + decrypted, err := utils.Decrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Debug("解密 Search API Key 失败(可能未加密)", zap.Uint("config_id", config.ID), zap.Error(err)) + } else { + config.APIKey = decrypted + } + } + return []*entity.UserConfig{config}, nil +} + +func (s *userConfigService) CreateSearchConfig(userID uint, config *entity.UserConfig) error { + config.UserID = userID + config.ConfigType = "search" + if config.ExtraConfig == "" { + config.ExtraConfig = "{}" + } + + // 加密 API Key + if config.APIKey != "" { + encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Error("加密 Search API Key 失败", zap.Error(err)) + return err + } + config.APIKey = encrypted + } + + // 检查是否已经存在相同类型的配置(包括已删除的记录) + existing, err := s.configRepo.FindByUserAndTypeIncludingDeleted(userID, "search") + if err != nil { + return err + } + + if existing != nil { + // 如果存在已删除的记录,则更新它并恢复为未删除状态 + config.ID = existing.ID + config.CreatedAt = existing.CreatedAt + config.UpdatedAt = existing.UpdatedAt + config.DeletedAt = gorm.DeletedAt{} // 恢复为未删除状态 + return s.configRepo.Update(config) + } + + return s.configRepo.Create(config) +} + +func (s *userConfigService) UpdateSearchConfig(id uint, config *entity.UserConfig) error { + existing, err := s.configRepo.FindByID(id) + if err != nil { + return err + } + if existing == nil { + return bizerrors.ErrNotFound + } + config.ID = id + config.UserID = existing.UserID + config.ConfigType = "search" + config.CreatedAt = existing.CreatedAt + config.UpdatedAt = existing.UpdatedAt + if config.ExtraConfig == "" { + config.ExtraConfig = "{}" + } + // 加密 API Key + if config.APIKey != "" { + encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Error("加密 Search API Key 失败", zap.Error(err)) + return err + } + config.APIKey = encrypted + } + if err := s.configRepo.Update(config); err != nil { + return err + } + s.configSvc.ClearUserConfigCache(existing.UserID, "search") + return nil +} + +func (s *userConfigService) DeleteSearchConfig(id uint) error { + existing, err := s.configRepo.FindByID(id) + if err != nil { + return err + } + if existing == nil { + return bizerrors.ErrNotFound + } + if err := s.configRepo.Delete(id); err != nil { + return err + } + s.configSvc.ClearUserConfigCache(existing.UserID, "search") + return nil +} + +// ===== ASR Config ===== + +func (s *userConfigService) ListASRConfigs(userID uint) ([]*entity.UserConfig, error) { + config, err := s.configRepo.FindByUserAndType(userID, "asr") + if err != nil { + return nil, err + } + if config == nil { + return []*entity.UserConfig{}, nil + } + // 解密 API Key + if config.APIKey != "" { + decrypted, err := utils.Decrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Debug("解密 ASR API Key 失败(可能未加密)", zap.Uint("config_id", config.ID), zap.Error(err)) + } else { + config.APIKey = decrypted + } + } + return []*entity.UserConfig{config}, nil +} + +func (s *userConfigService) CreateASRConfig(userID uint, config *entity.UserConfig) error { + config.UserID = userID + config.ConfigType = "asr" + if config.ExtraConfig == "" { + config.ExtraConfig = "{}" + } + + // 加密 API Key + if config.APIKey != "" { + encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Error("加密 ASR API Key 失败", zap.Error(err)) + return err + } + config.APIKey = encrypted + } + + // 检查是否已经存在相同类型的配置(包括已删除的记录) + existing, err := s.configRepo.FindByUserAndTypeIncludingDeleted(userID, "asr") + if err != nil { + return err + } + + if existing != nil { + // 如果存在已删除的记录,则更新它并恢复为未删除状态 + config.ID = existing.ID + config.CreatedAt = existing.CreatedAt + config.UpdatedAt = existing.UpdatedAt + config.DeletedAt = gorm.DeletedAt{} // 恢复为未删除状态 + return s.configRepo.Update(config) + } + + return s.configRepo.Create(config) +} + +func (s *userConfigService) UpdateASRConfig(id uint, config *entity.UserConfig) error { + existing, err := s.configRepo.FindByID(id) + if err != nil { + return err + } + if existing == nil { + return bizerrors.ErrNotFound + } + config.ID = id + config.UserID = existing.UserID + config.ConfigType = "asr" + config.CreatedAt = existing.CreatedAt + config.UpdatedAt = existing.UpdatedAt + if config.ExtraConfig == "" { + config.ExtraConfig = "{}" + } + // 加密 API Key + if config.APIKey != "" { + encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Error("加密 ASR API Key 失败", zap.Error(err)) + return err + } + config.APIKey = encrypted + } + if err := s.configRepo.Update(config); err != nil { + return err + } + s.configSvc.ClearUserConfigCache(existing.UserID, "asr") + return nil +} + +func (s *userConfigService) DeleteASRConfig(id uint) error { + existing, err := s.configRepo.FindByID(id) + if err != nil { + return err + } + if existing == nil { + return bizerrors.ErrNotFound + } + if err := s.configRepo.Delete(id); err != nil { + return err + } + s.configSvc.ClearUserConfigCache(existing.UserID, "asr") + return nil +} + +// ===== Embedding Config ===== + +func (s *userConfigService) ListEmbeddingConfigs(userID uint) ([]*entity.UserConfig, error) { + config, err := s.configRepo.FindByUserAndType(userID, "embedding") + if err != nil { + return nil, err + } + if config == nil { + return []*entity.UserConfig{}, nil + } + // 解密 API Key + if config.APIKey != "" { + decrypted, err := utils.Decrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Debug("解密 Embedding API Key 失败(可能未加密)", zap.Uint("config_id", config.ID), zap.Error(err)) + } else { + config.APIKey = decrypted + } + } + return []*entity.UserConfig{config}, nil +} + +func (s *userConfigService) CreateEmbeddingConfig(userID uint, config *entity.UserConfig) error { + config.UserID = userID + config.ConfigType = "embedding" + if config.ExtraConfig == "" { + config.ExtraConfig = "{}" + } + + // 加密 API Key + if config.APIKey != "" { + encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Error("加密 Embedding API Key 失败", zap.Error(err)) + return err + } + config.APIKey = encrypted + } + + // 检查是否已经存在相同类型的配置(包括已删除的记录) + existing, err := s.configRepo.FindByUserAndTypeIncludingDeleted(userID, "embedding") + if err != nil { + return err + } + + if existing != nil { + // 如果存在已删除的记录,则更新它并恢复为未删除状态 + config.ID = existing.ID + config.CreatedAt = existing.CreatedAt + config.UpdatedAt = existing.UpdatedAt + config.DeletedAt = gorm.DeletedAt{} // 恢复为未删除状态 + return s.configRepo.Update(config) + } + + return s.configRepo.Create(config) +} + +func (s *userConfigService) UpdateEmbeddingConfig(id uint, config *entity.UserConfig) error { + existing, err := s.configRepo.FindByID(id) + if err != nil { + return err + } + if existing == nil { + return bizerrors.ErrNotFound + } + config.ID = id + config.UserID = existing.UserID + config.ConfigType = "embedding" + config.CreatedAt = existing.CreatedAt + config.UpdatedAt = existing.UpdatedAt + if config.ExtraConfig == "" { + config.ExtraConfig = "{}" + } + // 加密 API Key + if config.APIKey != "" { + encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Error("加密 Embedding API Key 失败", zap.Error(err)) + return err + } + config.APIKey = encrypted + } + if err := s.configRepo.Update(config); err != nil { + return err + } + s.configSvc.ClearUserConfigCache(existing.UserID, "embedding") + return nil +} + +func (s *userConfigService) DeleteEmbeddingConfig(id uint) error { + existing, err := s.configRepo.FindByID(id) + if err != nil { + return err + } + if existing == nil { + return bizerrors.ErrNotFound + } + if err := s.configRepo.Delete(id); err != nil { + return err + } + s.configSvc.ClearUserConfigCache(existing.UserID, "embedding") + return nil +} + +// ===== Reranker Config ===== + +func (s *userConfigService) ListRerankerConfigs(userID uint) ([]*entity.UserConfig, error) { + config, err := s.configRepo.FindByUserAndType(userID, "reranker") + if err != nil { + return nil, err + } + if config == nil { + return []*entity.UserConfig{}, nil + } + // 解密 API Key + if config.APIKey != "" { + decrypted, err := utils.Decrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Debug("解密 Reranker API Key 失败(可能未加密)", zap.Uint("config_id", config.ID), zap.Error(err)) + } else { + config.APIKey = decrypted + } + } + return []*entity.UserConfig{config}, nil +} + +func (s *userConfigService) CreateRerankerConfig(userID uint, config *entity.UserConfig) error { + config.UserID = userID + config.ConfigType = "reranker" + if config.ExtraConfig == "" { + config.ExtraConfig = "{}" + } + + // 加密 API Key + if config.APIKey != "" { + encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Error("加密 Reranker API Key 失败", zap.Error(err)) + return err + } + config.APIKey = encrypted + } + + // 检查是否已经存在相同类型的配置(包括已删除的记录) + existing, err := s.configRepo.FindByUserAndTypeIncludingDeleted(userID, "reranker") + if err != nil { + return err + } + + if existing != nil { + // 如果存在已删除的记录,则更新它并恢复为未删除状态 + config.ID = existing.ID + config.CreatedAt = existing.CreatedAt + config.UpdatedAt = existing.UpdatedAt + config.DeletedAt = gorm.DeletedAt{} // 恢复为未删除状态 + return s.configRepo.Update(config) + } + + return s.configRepo.Create(config) +} + +func (s *userConfigService) UpdateRerankerConfig(id uint, config *entity.UserConfig) error { + existing, err := s.configRepo.FindByID(id) + if err != nil { + return err + } + if existing == nil { + return bizerrors.ErrNotFound + } + config.ID = id + config.UserID = existing.UserID + config.ConfigType = "reranker" + config.CreatedAt = existing.CreatedAt + config.UpdatedAt = existing.UpdatedAt + if config.ExtraConfig == "" { + config.ExtraConfig = "{}" + } + // 加密 API Key + if config.APIKey != "" { + encrypted, err := utils.Encrypt(config.APIKey, s.encryptionKey) + if err != nil { + logger.Error("加密 Reranker API Key 失败", zap.Error(err)) + return err + } + config.APIKey = encrypted + } + if err := s.configRepo.Update(config); err != nil { + return err + } + s.configSvc.ClearUserConfigCache(existing.UserID, "reranker") + return nil +} + +func (s *userConfigService) DeleteRerankerConfig(id uint) error { + existing, err := s.configRepo.FindByID(id) + if err != nil { + return err + } + if existing == nil { + return bizerrors.ErrNotFound + } + if err := s.configRepo.Delete(id); err != nil { + return err + } + s.configSvc.ClearUserConfigCache(existing.UserID, "reranker") + return nil +} + +// GetActiveConfig 获取当前生效的配置(用户配置 > 系统配置) +func (s *userConfigService) GetActiveConfig(userID uint, configType string) (*entity.UserConfig, error) { + // LLM 配置存储在独立的 user_llm_config 表,需要特殊处理 + if configType == "llm" { + return s.getActiveLLMConfig(userID) + } + + // 1. 优先返回用户配置(必须启用) + userCfg, err := s.configRepo.FindByUserAndType(userID, configType) + if err == nil && userCfg != nil && userCfg.Enabled { + userCfg.Source = "user" + // 解密 API Key + if userCfg.APIKey != "" { + decrypted, err := utils.Decrypt(userCfg.APIKey, s.encryptionKey) + if err != nil { + logger.Debug("解密用户配置 API Key 失败(可能未加密)", zap.Uint("user_id", userID), zap.Error(err)) + } else { + userCfg.APIKey = decrypted + } + } + return userCfg, nil + } + + // 2. 降级到系统配置 + sysCfg, err := s.configSvc.GetSysConfig(configType) + if err == nil && sysCfg != nil { + // 解析系统配置的 JSON 值 + var params map[string]interface{} + if jsonErr := json.Unmarshal([]byte(sysCfg.ConfigValue), ¶ms); jsonErr == nil { + getStr := func(key string) string { + if v, ok := params[key].(string); ok { + return v + } + return "" + } + + return &entity.UserConfig{ + ConfigType: configType, + Name: getStr("name"), + Provider: getStr("provider"), + APIURL: getStr("api_url"), + APIKey: getStr("api_key"), + Model: getStr("model"), + Enabled: sysCfg.Enabled, + Source: "system", + }, nil + } + + // 如果解析失败,尝试作为纯 URL 处理 + return &entity.UserConfig{ + ConfigType: configType, + Name: sysCfg.ConfigKey, + Provider: sysCfg.ConfigKey, + APIURL: sysCfg.ConfigValue, + Enabled: sysCfg.Enabled, + Source: "system", + }, nil + } + + // 3. 没有配置 + return nil, nil +} + +// getActiveLLMConfig 获取当前生效的 LLM 配置(LLM 存储在 user_llm_config 表) +func (s *userConfigService) getActiveLLMConfig(userID uint) (*entity.UserConfig, error) { + // 1. 优先返回用户 LLM 配置(第一个启用的) + llmCfg, err := s.llmConfigRepo.FindDefaultByUserID(userID) + if err == nil && llmCfg != nil && llmCfg.Enabled { + apiKey := llmCfg.APIKey + if apiKey != "" { + decrypted, err := utils.Decrypt(apiKey, s.encryptionKey) + if err != nil { + logger.Debug("解密 LLM API Key 失败(可能未加密)", zap.Uint("user_id", userID), zap.Error(err)) + } else { + apiKey = decrypted + } + } + return &entity.UserConfig{ + ConfigType: "llm", + Name: llmCfg.Name, + Provider: llmCfg.Provider, + APIURL: llmCfg.APIURL, + APIKey: apiKey, + Model: llmCfg.Model, + Enabled: llmCfg.Enabled, + Source: "user", + }, nil + } + + // 2. 降级到系统配置 + sysCfg, err := s.configSvc.GetSysConfig("llm") + if err == nil && sysCfg != nil { + var params map[string]interface{} + if jsonErr := json.Unmarshal([]byte(sysCfg.ConfigValue), ¶ms); jsonErr == nil { + getStr := func(key string) string { + if v, ok := params[key].(string); ok { + return v + } + return "" + } + return &entity.UserConfig{ + ConfigType: "llm", + Name: getStr("name"), + Provider: getStr("provider"), + APIURL: getStr("api_url"), + APIKey: getStr("api_key"), + Model: getStr("model"), + Enabled: sysCfg.Enabled, + Source: "system", + }, nil + } + return &entity.UserConfig{ + ConfigType: "llm", + Name: sysCfg.ConfigKey, + Provider: sysCfg.ConfigKey, + APIURL: sysCfg.ConfigValue, + Enabled: sysCfg.Enabled, + Source: "system", + }, nil + } + + // 3. 没有配置 + return nil, nil +} + +// TestConfig 测试配置连通性(保存前验证) +func (s *userConfigService) TestConfig(configType string, config *entity.UserConfig) *HealthCheckResult { + return s.healthChk.TestConfig(configType, config) +} diff --git a/internal/service/user_interface.go b/internal/service/user_interface.go index 6f4bcaf..35174c6 100644 --- a/internal/service/user_interface.go +++ b/internal/service/user_interface.go @@ -1,32 +1,32 @@ -package service - -import ( - "YoudaoNoteLm/internal/model/dto/request" - dto "YoudaoNoteLm/internal/model/dto/response" - "YoudaoNoteLm/internal/model/entity" - "YoudaoNoteLm/pkg/response" - "context" - "mime/multipart" -) - -// UserService 用户服务接口 -type UserService interface { - // Register 用户注册(邮箱+验证码) - Register(ctx context.Context, req *request.RegisterRequest) error - // GetUserByID 根据 ID 获取用户 - GetUserByID(id uint) (*entity.User, error) - // UpdateUser 更新用户信息 - UpdateUser(id uint, req *request.UpdateUserRequest) error - // UpdateUsername 修改用户名 - UpdateUsername(id uint, req *request.UpdateUsernameRequest) error - // UploadAvatar 上传头像 - UploadAvatar(id uint, file *multipart.FileHeader) (string, error) - // ChangePassword 修改密码 - ChangePassword(id uint, req *request.ChangePasswordRequest) error - // DeleteAccount 注销用户(硬删除) - DeleteAccount(id uint, req *request.DeleteAccountRequest) error - // GetUserResponse 获取用户响应 - GetUserResponse(user *entity.User) *dto.UserResponse - // ListUsers 分页获取用户列表 - ListUsers(req *request.UserListRequest) (*response.PageResponse, error) -} +package service + +import ( + "YoudaoNoteLm/internal/model/dto/request" + dto "YoudaoNoteLm/internal/model/dto/response" + "YoudaoNoteLm/internal/model/entity" + "YoudaoNoteLm/pkg/response" + "context" + "mime/multipart" +) + +// UserService 用户服务接口 +type UserService interface { + // Register 用户注册(邮箱+验证码) + Register(ctx context.Context, req *request.RegisterRequest) error + // GetUserByID 根据 ID 获取用户 + GetUserByID(id uint) (*entity.User, error) + // UpdateUser 更新用户信息 + UpdateUser(id uint, req *request.UpdateUserRequest) error + // UpdateUsername 修改用户名 + UpdateUsername(id uint, req *request.UpdateUsernameRequest) error + // UploadAvatar 上传头像 + UploadAvatar(id uint, file *multipart.FileHeader) (string, error) + // ChangePassword 修改密码 + ChangePassword(id uint, req *request.ChangePasswordRequest) error + // DeleteAccount 注销用户(硬删除) + DeleteAccount(id uint, req *request.DeleteAccountRequest) error + // GetUserResponse 获取用户响应 + GetUserResponse(user *entity.User) *dto.UserResponse + // ListUsers 分页获取用户列表 + ListUsers(req *request.UserListRequest) (*response.PageResponse, error) +} diff --git a/internal/service/utils.go b/internal/service/utils.go index aef22df..b305fe8 100644 --- a/internal/service/utils.go +++ b/internal/service/utils.go @@ -1,13 +1,13 @@ -package service - -import "strings" - -// firstNonEmpty 返回第一个非空字符串 -func firstNonEmpty(values ...string) string { - for _, value := range values { - if strings.TrimSpace(value) != "" { - return value - } - } - return "" -} +package service + +import "strings" + +// firstNonEmpty 返回第一个非空字符串 +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} diff --git a/internal/service/verify_code_interface.go b/internal/service/verify_code_interface.go index f83ba36..3353588 100644 --- a/internal/service/verify_code_interface.go +++ b/internal/service/verify_code_interface.go @@ -1,13 +1,13 @@ -package service - -import "context" - -// VerifyCodeService 验证码服务接口 -type VerifyCodeService interface { - // Generate 生成验证码并存储到 Redis,返回验证码和是否在冷却中 - Generate(ctx context.Context, email string, codeType string) (string, error) - // Verify 校验验证码 - Verify(ctx context.Context, email string, codeType string, code string) error - // GetCooldownRemaining 获取剩余冷却秒数,0 表示不在冷却中 - GetCooldownRemaining(ctx context.Context, email string, codeType string) (int, error) -} +package service + +import "context" + +// VerifyCodeService 验证码服务接口 +type VerifyCodeService interface { + // Generate 生成验证码并存储到 Redis,返回验证码和是否在冷却中 + Generate(ctx context.Context, email string, codeType string) (string, error) + // Verify 校验验证码 + Verify(ctx context.Context, email string, codeType string, code string) error + // GetCooldownRemaining 获取剩余冷却秒数,0 表示不在冷却中 + GetCooldownRemaining(ctx context.Context, email string, codeType string) (int, error) +} diff --git a/internal/service/verify_code_service.go b/internal/service/verify_code_service.go index 367296a..387703a 100644 --- a/internal/service/verify_code_service.go +++ b/internal/service/verify_code_service.go @@ -1,151 +1,151 @@ -package service - -import ( - bizerrors "YoudaoNoteLm/pkg/errors" - "YoudaoNoteLm/pkg/logger" - "YoudaoNoteLm/pkg/utils" - "context" - "fmt" - "time" - - "github.com/redis/go-redis/v9" - "go.uber.org/zap" -) - -const ( - verifyCodeTTL = 5 * time.Minute // 验证码有效期 5 分钟 - verifyCodeCooldown = 60 * time.Second // 发送冷却 60 秒 - verifyCodeMaxRetry = 5 // 最大验证次数 -) - -// verifyCodeService 验证码服务实现 -type verifyCodeService struct { - redis *redis.Client - emailSvc EmailService -} - -// NewVerifyCodeService 创建验证码服务 -func NewVerifyCodeService(redisClient *redis.Client, emailSvc EmailService) VerifyCodeService { - return &verifyCodeService{ - redis: redisClient, - emailSvc: emailSvc, - } -} - -// codeKey 验证码存储 key: verify_code:{type}:{email} -func (s *verifyCodeService) codeKey(email, codeType string) string { - return fmt.Sprintf("verify_code:%s:%s", codeType, email) -} - -// cooldownKey 冷却 key: verify_code_cooldown:{type}:{email} -func (s *verifyCodeService) cooldownKey(email, codeType string) string { - return fmt.Sprintf("verify_code_cooldown:%s:%s", codeType, email) -} - -// retryKey 重试次数 key: verify_code_retry:{type}:{email} -func (s *verifyCodeService) retryKey(email, codeType string) string { - return fmt.Sprintf("verify_code_retry:%s:%s", codeType, email) -} - -// Generate 生成验证码并存储到 Redis,同时发送邮件 -func (s *verifyCodeService) Generate(ctx context.Context, email string, codeType string) (string, error) { - // 检查冷却时间 - remaining, err := s.GetCooldownRemaining(ctx, email, codeType) - if err != nil { - return "", err - } - if remaining > 0 { - return "", bizerrors.ErrVerifyCodeTooFrequent - } - - // 生成 6 位数字验证码 - code, err := utils.GenerateRandomString(6, utils.Numeric) - if err != nil { - logger.Error("生成验证码失败", zap.Error(err)) - return "", fmt.Errorf("生成验证码失败: %w", err) - } - - // 存储验证码到 Redis,5 分钟过期 - if err := s.redis.Set(ctx, s.codeKey(email, codeType), code, verifyCodeTTL).Err(); err != nil { - logger.Error("存储验证码失败", zap.Error(err)) - return "", fmt.Errorf("存储验证码失败: %w", err) - } - - // 重置重试次数 - if err := s.redis.Del(ctx, s.retryKey(email, codeType)).Err(); err != nil { - logger.Warn("清除重试次数失败", zap.String("email", email), zap.Error(err)) - } - - // 设置冷却时间 - if err := s.redis.Set(ctx, s.cooldownKey(email, codeType), 1, verifyCodeCooldown).Err(); err != nil { - logger.Warn("设置冷却时间失败", zap.String("email", email), zap.Error(err)) - } - - // 发送邮件 - if err := s.emailSvc.SendVerifyCode(email, code); err != nil { - logger.Error("发送验证码邮件失败", zap.String("email", email), zap.Error(err)) - // 邮件发送失败不影响验证码已生成,返回错误让用户重试 - return "", fmt.Errorf("发送验证码邮件失败: %w", err) - } - - return code, nil -} - -// Verify 校验验证码 -func (s *verifyCodeService) Verify(ctx context.Context, email string, codeType string, code string) error { - // 检查重试次数 - retryCount, err := s.redis.Get(ctx, s.retryKey(email, codeType)).Int() - if err != nil && err != redis.Nil { - return fmt.Errorf("查询重试次数失败: %w", err) - } - if retryCount >= verifyCodeMaxRetry { - // 清除验证码 - if err := s.redis.Del(ctx, s.codeKey(email, codeType)).Err(); err != nil { - logger.Warn("清除验证码失败", zap.String("email", email), zap.Error(err)) - } - return bizerrors.ErrVerifyCodeLocked - } - - // 获取存储的验证码 - storedCode, err := s.redis.Get(ctx, s.codeKey(email, codeType)).Result() - if err == redis.Nil { - return bizerrors.ErrVerifyCodeExpired - } - if err != nil { - return fmt.Errorf("查询验证码失败: %w", err) - } - - // 校验验证码 - if storedCode != code { - // 增加重试次数 - if err := s.redis.Incr(ctx, s.retryKey(email, codeType)).Err(); err != nil { - logger.Warn("增加重试次数失败", zap.String("email", email), zap.Error(err)) - } - if err := s.redis.Expire(ctx, s.retryKey(email, codeType), verifyCodeTTL).Err(); err != nil { - logger.Warn("设置重试次数过期时间失败", zap.String("email", email), zap.Error(err)) - } - return bizerrors.ErrVerifyCodeInvalid - } - - // 验证成功,清除验证码和重试次数 - if err := s.redis.Del(ctx, s.codeKey(email, codeType)).Err(); err != nil { - logger.Warn("清除验证码失败", zap.String("email", email), zap.Error(err)) - } - if err := s.redis.Del(ctx, s.retryKey(email, codeType)).Err(); err != nil { - logger.Warn("清除重试次数失败", zap.String("email", email), zap.Error(err)) - } - - return nil -} - -// GetCooldownRemaining 获取剩余冷却秒数 -func (s *verifyCodeService) GetCooldownRemaining(ctx context.Context, email string, codeType string) (int, error) { - ttl, err := s.redis.TTL(ctx, s.cooldownKey(email, codeType)).Result() - if err != nil { - return 0, fmt.Errorf("查询冷却时间失败: %w", err) - } - if ttl <= 0 { - return 0, nil - } - return int(ttl.Seconds()), nil -} +package service + +import ( + bizerrors "YoudaoNoteLm/pkg/errors" + "YoudaoNoteLm/pkg/logger" + "YoudaoNoteLm/pkg/utils" + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" + "go.uber.org/zap" +) + +const ( + verifyCodeTTL = 5 * time.Minute // 验证码有效期 5 分钟 + verifyCodeCooldown = 60 * time.Second // 发送冷却 60 秒 + verifyCodeMaxRetry = 5 // 最大验证次数 +) + +// verifyCodeService 验证码服务实现 +type verifyCodeService struct { + redis *redis.Client + emailSvc EmailService +} + +// NewVerifyCodeService 创建验证码服务 +func NewVerifyCodeService(redisClient *redis.Client, emailSvc EmailService) VerifyCodeService { + return &verifyCodeService{ + redis: redisClient, + emailSvc: emailSvc, + } +} + +// codeKey 验证码存储 key: verify_code:{type}:{email} +func (s *verifyCodeService) codeKey(email, codeType string) string { + return fmt.Sprintf("verify_code:%s:%s", codeType, email) +} + +// cooldownKey 冷却 key: verify_code_cooldown:{type}:{email} +func (s *verifyCodeService) cooldownKey(email, codeType string) string { + return fmt.Sprintf("verify_code_cooldown:%s:%s", codeType, email) +} + +// retryKey 重试次数 key: verify_code_retry:{type}:{email} +func (s *verifyCodeService) retryKey(email, codeType string) string { + return fmt.Sprintf("verify_code_retry:%s:%s", codeType, email) +} + +// Generate 生成验证码并存储到 Redis,同时发送邮件 +func (s *verifyCodeService) Generate(ctx context.Context, email string, codeType string) (string, error) { + // 检查冷却时间 + remaining, err := s.GetCooldownRemaining(ctx, email, codeType) + if err != nil { + return "", err + } + if remaining > 0 { + return "", bizerrors.ErrVerifyCodeTooFrequent + } + + // 生成 6 位数字验证码 + code, err := utils.GenerateRandomString(6, utils.Numeric) + if err != nil { + logger.Error("生成验证码失败", zap.Error(err)) + return "", fmt.Errorf("生成验证码失败: %w", err) + } + + // 存储验证码到 Redis,5 分钟过期 + if err := s.redis.Set(ctx, s.codeKey(email, codeType), code, verifyCodeTTL).Err(); err != nil { + logger.Error("存储验证码失败", zap.Error(err)) + return "", fmt.Errorf("存储验证码失败: %w", err) + } + + // 重置重试次数 + if err := s.redis.Del(ctx, s.retryKey(email, codeType)).Err(); err != nil { + logger.Warn("清除重试次数失败", zap.String("email", email), zap.Error(err)) + } + + // 设置冷却时间 + if err := s.redis.Set(ctx, s.cooldownKey(email, codeType), 1, verifyCodeCooldown).Err(); err != nil { + logger.Warn("设置冷却时间失败", zap.String("email", email), zap.Error(err)) + } + + // 发送邮件 + if err := s.emailSvc.SendVerifyCode(email, code); err != nil { + logger.Error("发送验证码邮件失败", zap.String("email", email), zap.Error(err)) + // 邮件发送失败不影响验证码已生成,返回错误让用户重试 + return "", fmt.Errorf("发送验证码邮件失败: %w", err) + } + + return code, nil +} + +// Verify 校验验证码 +func (s *verifyCodeService) Verify(ctx context.Context, email string, codeType string, code string) error { + // 检查重试次数 + retryCount, err := s.redis.Get(ctx, s.retryKey(email, codeType)).Int() + if err != nil && err != redis.Nil { + return fmt.Errorf("查询重试次数失败: %w", err) + } + if retryCount >= verifyCodeMaxRetry { + // 清除验证码 + if err := s.redis.Del(ctx, s.codeKey(email, codeType)).Err(); err != nil { + logger.Warn("清除验证码失败", zap.String("email", email), zap.Error(err)) + } + return bizerrors.ErrVerifyCodeLocked + } + + // 获取存储的验证码 + storedCode, err := s.redis.Get(ctx, s.codeKey(email, codeType)).Result() + if err == redis.Nil { + return bizerrors.ErrVerifyCodeExpired + } + if err != nil { + return fmt.Errorf("查询验证码失败: %w", err) + } + + // 校验验证码 + if storedCode != code { + // 增加重试次数 + if err := s.redis.Incr(ctx, s.retryKey(email, codeType)).Err(); err != nil { + logger.Warn("增加重试次数失败", zap.String("email", email), zap.Error(err)) + } + if err := s.redis.Expire(ctx, s.retryKey(email, codeType), verifyCodeTTL).Err(); err != nil { + logger.Warn("设置重试次数过期时间失败", zap.String("email", email), zap.Error(err)) + } + return bizerrors.ErrVerifyCodeInvalid + } + + // 验证成功,清除验证码和重试次数 + if err := s.redis.Del(ctx, s.codeKey(email, codeType)).Err(); err != nil { + logger.Warn("清除验证码失败", zap.String("email", email), zap.Error(err)) + } + if err := s.redis.Del(ctx, s.retryKey(email, codeType)).Err(); err != nil { + logger.Warn("清除重试次数失败", zap.String("email", email), zap.Error(err)) + } + + return nil +} + +// GetCooldownRemaining 获取剩余冷却秒数 +func (s *verifyCodeService) GetCooldownRemaining(ctx context.Context, email string, codeType string) (int, error) { + ttl, err := s.redis.TTL(ctx, s.cooldownKey(email, codeType)).Result() + if err != nil { + return 0, fmt.Errorf("查询冷却时间失败: %w", err) + } + if ttl <= 0 { + return 0, nil + } + return int(ttl.Seconds()), nil +} diff --git a/internal/service/youdao_service.go b/internal/service/youdao_service.go index 515d2c6..d6ecf79 100644 --- a/internal/service/youdao_service.go +++ b/internal/service/youdao_service.go @@ -1,666 +1,697 @@ -package service - -import ( - "context" - "fmt" - "strings" - "sync" - "time" - - "YoudaoNoteLm/internal/model/entity" - "YoudaoNoteLm/internal/rag" - "YoudaoNoteLm/internal/repository" - externalYoudao "YoudaoNoteLm/internal/service/external/youdao" - "YoudaoNoteLm/pkg/cache" - "YoudaoNoteLm/pkg/logger" - - "github.com/google/uuid" - "go.uber.org/zap" -) - -type youdaoService struct { - cli externalYoudao.CLI - bindingRepo repository.YoudaoBindingRepository - sourceRepo repository.SourceRepository - ingestionSvc rag.IngestionService - structurer MarkdownStructurer // LLM 结构化服务 - configSvc ConfigService // 用于获取用户 LLM 配置(摘要生成) - summaryCache *cache.SourceSummaryCache - cancelFuncs sync.Map // taskID -> context.CancelFunc - cookiesPath string // youdaonote cookies 文件路径(用于 .note 格式转换) -} - -// NewYoudaoService 创建有道云笔记服务 -func NewYoudaoService( - cli externalYoudao.CLI, - bindingRepo repository.YoudaoBindingRepository, - sourceRepo repository.SourceRepository, - ingestionSvc rag.IngestionService, - cookiesPath string, - structurer MarkdownStructurer, - configSvc ConfigService, - summaryCache *cache.SourceSummaryCache, -) YoudaoService { - return &youdaoService{ - cli: cli, - bindingRepo: bindingRepo, - sourceRepo: sourceRepo, - ingestionSvc: ingestionSvc, - cookiesPath: cookiesPath, - structurer: structurer, - configSvc: configSvc, - summaryCache: summaryCache, - } -} - -// getAPIKey 获取用户的有道 API Key(内部辅助方法) -func (s *youdaoService) getAPIKey(userID uint) (string, error) { - binding, err := s.bindingRepo.FindByUserID(userID) - if err != nil { - return "", fmt.Errorf("查询绑定信息失败: %w", err) - } - if binding == nil || binding.Status != "active" { - return "", fmt.Errorf("请先绑定有道云笔记账号") - } - return binding.APIKey, nil -} - -// generateAndSaveSummary 生成资料摘要并保存到 MySQL 和 Redis -func (s *youdaoService) generateAndSaveSummary(sourceID uint, userID uint, content string) { - doGenerateAndSaveSummary(s.sourceRepo, s.configSvc, s.summaryCache, sourceID, userID, content) -} - -// Bind 绑定有道 API Key -func (s *youdaoService) Bind(userID uint, apiKey string) error { - // 1. 检查 CLI 是否可用 - if err := s.cli.CheckAvailable(); err != nil { - return fmt.Errorf("youdaonote CLI 不可用: %w", err) - } - - // 2. 验证 Key 有效性(调用 list 测试) - _, err := s.cli.List(apiKey, "") - if err != nil { - return fmt.Errorf("API Key 验证失败(CLI 返回错误: %w),请检查 Key 是否正确或网络是否正常", err) - } - - // 3. 使用 Upsert 原子操作,避免并发冲突 - binding := &entity.YoudaoBinding{ - UserID: userID, - APIKey: apiKey, - Status: "active", - } - return s.bindingRepo.Upsert(binding) -} - -// Unbind 解绑有道账号 -func (s *youdaoService) Unbind(userID uint) error { - return s.bindingRepo.Delete(userID) -} - -// GetBinding 获取绑定信息 -func (s *youdaoService) GetBinding(userID uint) (*entity.YoudaoBinding, error) { - return s.bindingRepo.FindByUserID(userID) -} - -// ListNotes 浏览有道云笔记目录 -func (s *youdaoService) ListNotes(userID uint, folderID string) ([]externalYoudao.NoteItem, error) { - apiKey, err := s.getAPIKey(userID) - if err != nil { - return nil, err - } - - items, err := s.cli.List(apiKey, folderID) - if err != nil { - return nil, fmt.Errorf("获取笔记列表失败: %w", err) - } - - return items, nil -} - -// ImportNote 导入单篇有道云笔记到本系统 -func (s *youdaoService) ImportNote(userID uint, notebookID uint, fileID string) (*entity.Source, error) { - totalStart := time.Now() - - apiKey, err := s.getAPIKey(userID) - if err != nil { - return nil, err - } - - logger.Info("开始导入有道笔记", - zap.Uint("user_id", userID), - zap.String("file_id", fileID), - ) - - // 1. 读取笔记内容 - stepStart := time.Now() - readResult, err := s.cli.Read(apiKey, fileID) - if err != nil { - logger.Error("读取有道笔记内容失败", - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - return nil, fmt.Errorf("读取笔记内容失败: %w", err) - } - - logger.Info("有道笔记内容读取成功", - zap.String("file_id", fileID), - zap.String("format", readResult.RawFormat), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - content := strings.TrimSpace(readResult.Content) - - // .note 格式必须转换为 Markdown(向量化要求 Markdown 格式) - if readResult.RawFormat == "note" { - // 空笔记无需转换,直接返回空内容,由调用方处理 - if content == "" && s.cookiesPath == "" { - return nil, fmt.Errorf("笔记内容为空") - } - if s.cookiesPath == "" { - return nil, fmt.Errorf("笔记为 .note 格式,但未配置 cookies 文件路径,无法转换") - } - logger.Info("笔记为 .note 格式,开始转换为 Markdown", zap.String("file_id", fileID)) - convertStart := time.Now() - convertedContent, convertErr := s.cli.ConvertNote(fileID, s.cookiesPath) - if convertErr != nil { - logger.Error(".note 格式转换失败", - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(convertStart)), - zap.Error(convertErr), - ) - return nil, fmt.Errorf(".note 格式转换失败: %w", convertErr) - } - if strings.TrimSpace(convertedContent) == "" { - return nil, fmt.Errorf(".note 格式转换后内容为空") - } - content = convertedContent - logger.Info(".note 格式转换成功", - zap.String("file_id", fileID), - zap.Int("content_len", len(content)), - zap.Duration("elapsed", time.Since(convertStart)), - ) - } else if content == "" && s.cookiesPath != "" { - // 非 .note 格式但内容为空,尝试转换(可能是格式识别错误) - logger.Info("内容为空,尝试使用 youdaonote-pull 转换", zap.String("file_id", fileID)) - convertStart := time.Now() - convertedContent, convertErr := s.cli.ConvertNote(fileID, s.cookiesPath) - if convertErr != nil { - logger.Warn("youdaonote-pull 转换失败", zap.String("file_id", fileID), zap.Duration("elapsed", time.Since(convertStart)), zap.Error(convertErr)) - } else if strings.TrimSpace(convertedContent) != "" { - content = convertedContent - logger.Info("youdaonote-pull 转换成功", zap.String("file_id", fileID), zap.Duration("elapsed", time.Since(convertStart))) - } - } - - // 检查内容是否为空 - if content == "" { - return nil, fmt.Errorf("笔记内容为空或格式不支持") - } - - // 2. 通过 list 获取笔记名称 - stepStart = time.Now() - noteName := fileID // 降级使用 fileID - items, listErr := s.cli.List(apiKey, "") - if listErr == nil { - for _, item := range items { - if item.ID == fileID { - noteName = item.Name - break - } - } - } - logger.Info("获取笔记名称完成", - zap.String("file_id", fileID), - zap.String("note_name", noteName), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - // LLM 结构化 - stepStart = time.Now() - if s.structurer != nil { - result, err := s.structurer.Structure(context.Background(), userID, content, StructureMeta{ - Title: noteName, - SourceType: "youdao", - }) - if err != nil { - logger.Error("LLM 结构化失败,使用原始内容", - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - } else if result.ActuallyCalled && result.Content != content { - logger.Info("LLM 结构化成功,内容已优化", - zap.String("file_id", fileID), - zap.Int("original_len", len(content)), - zap.Int("structured_len", len(result.Content)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - content = result.Content - } else if result.ActuallyCalled { - logger.Info("LLM 判断内容已有结构,无需结构化", - zap.String("file_id", fileID), - zap.Int("content_len", len(content)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } else { - logger.Warn("LLM 结构化被跳过(模型配置问题或 API Key 过期)", - zap.String("file_id", fileID), - zap.Int("content_len", len(content)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } - } else { - logger.Warn("MarkdownStructurer 未配置,跳过结构化", zap.String("file_id", fileID)) - } - - // 3. 创建 Source 记录 - stepStart = time.Now() - source := &entity.Source{ - UserID: userID, - NotebookID: notebookID, - Name: noteName, - Type: "youdao", - ExternalID: fileID, - MarkdownContent: content, - Status: "ready", - } - - if err := s.sourceRepo.Create(source); err != nil { - logger.Error("创建 Source 记录失败", - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - return nil, fmt.Errorf("创建 Source 记录失败: %w", err) - } - - logger.Info("Source 记录创建成功", - zap.String("file_id", fileID), - zap.Uint("source_id", source.ID), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - // 4. 同步触发 RAG 入库 - stepStart = time.Now() - if s.ingestionSvc != nil { - if err := s.ingestionSvc.IngestSingle(context.Background(), source.ID); err != nil { - logger.Error("RAG 入库失败", - zap.String("file_id", fileID), - zap.Uint("source_id", source.ID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - return nil, fmt.Errorf("RAG 入库失败: %w", err) - } - logger.Info("RAG 入库成功", - zap.String("file_id", fileID), - zap.Uint("source_id", source.ID), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } - - // 5. 生成摘要(异步,不阻塞主流程) - go s.generateAndSaveSummary(source.ID, userID, content) - - logger.Info("有道笔记导入完成", - zap.Uint("user_id", userID), - zap.String("file_id", fileID), - zap.String("name", noteName), - zap.Uint("source_id", source.ID), - zap.Duration("total_elapsed", time.Since(totalStart)), - ) - - return source, nil -} - -// ImportNotesBatch 批量导入有道云笔记 -func (s *youdaoService) ImportNotesBatch(userID uint, notebookID uint, fileIDs []string, fileNames map[string]string) (string, []uint, error) { - apiKey, err := s.getAPIKey(userID) - if err != nil { - return "", nil, err - } - - // 去重 - seen := make(map[string]struct{}, len(fileIDs)) - uniqueIDs := make([]string, 0, len(fileIDs)) - for _, id := range fileIDs { - if _, exists := seen[id]; exists { - continue - } - seen[id] = struct{}{} - uniqueIDs = append(uniqueIDs, id) - } - - sourceIDs := make([]uint, 0, len(uniqueIDs)) - - // 为每个 fileID 创建 pending 状态的 Source - for _, fileID := range uniqueIDs { - // 优先使用前端传递的笔记标题,降级使用 fileID - noteName := fileID - if name, ok := fileNames[fileID]; ok && name != "" { - noteName = name - } - - source := &entity.Source{ - UserID: userID, - NotebookID: notebookID, - Name: noteName, - Type: "youdao", - ExternalID: fileID, - Status: "pending", - } - if err := s.sourceRepo.Create(source); err != nil { - logger.Error("创建待导入有道笔记Source失败", zap.String("file_id", fileID), zap.Error(err)) - continue - } - sourceIDs = append(sourceIDs, source.ID) - } - - if len(sourceIDs) == 0 { - return "", nil, fmt.Errorf("创建导入记录失败") - } - - // 创建可取消的 context - taskID := uuid.New().String() - taskCtx, cancel := context.WithCancel(context.Background()) - s.cancelFuncs.Store(taskID, cancel) - - // 异步处理 - go s.processBatch(taskCtx, taskID, apiKey, sourceIDs, uniqueIDs) - - return taskID, sourceIDs, nil -} - -// processBatch 批量处理有道笔记导入 -func (s *youdaoService) processBatch(taskCtx context.Context, taskID string, apiKey string, sourceIDs []uint, fileIDs []string) { - defer s.cancelFuncs.Delete(taskID) - - concurrency := 3 - if len(fileIDs) < concurrency { - concurrency = len(fileIDs) - } - - type task struct { - sourceID uint - fileID string - } - - taskCh := make(chan task, concurrency) - doneCh := make(chan struct{}, len(fileIDs)) - - // 启动 worker - for i := 0; i < concurrency; i++ { - go func() { - for t := range taskCh { - if taskCtx.Err() != nil { - doneCh <- struct{}{} - continue - } - s.processSingleNote(taskCtx, apiKey, t.sourceID, t.fileID) - doneCh <- struct{}{} - } - }() - } - - // 分发任务 - go func() { - for i, fileID := range fileIDs { - if taskCtx.Err() != nil { - break - } - taskCh <- task{sourceID: sourceIDs[i], fileID: fileID} - } - close(taskCh) - }() - - // 等待完成 - for i := 0; i < len(fileIDs); i++ { - <-doneCh - } - - // 处理被取消的 pending 任务 - if taskCtx.Err() != nil { - for _, sourceID := range sourceIDs { - src, err := s.sourceRepo.FindByID(sourceID) - if err != nil || src == nil { - continue - } - if src.Status == "pending" { - if err := s.sourceRepo.UpdateStatus(sourceID, "cancelled", "任务已取消"); err != nil { - logger.Warn("更新Source状态为cancelled失败", zap.Uint("source_id", sourceID), zap.Error(err)) - } - } - } - } -} - -// processSingleNote 处理单篇有道笔记导入 -func (s *youdaoService) processSingleNote(taskCtx context.Context, apiKey string, sourceID uint, fileID string) { - totalStart := time.Now() - - if taskCtx.Err() != nil { - return - } - - logger.Info("开始处理有道笔记导入", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - ) - - // 更新状态为 processing - if err := s.sourceRepo.UpdateStatus(sourceID, "processing", ""); err != nil { - logger.Warn("更新Source状态为processing失败", zap.Uint("source_id", sourceID), zap.Error(err)) - } - - // 读取笔记内容 - stepStart := time.Now() - readResult, err := s.cli.Read(apiKey, fileID) - if err != nil { - if taskCtx.Err() != nil { - return - } - logger.Error("读取有道笔记内容失败", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("读取失败: %v", err)); updateErr != nil { - logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - - logger.Info("有道笔记内容读取成功", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - zap.String("format", readResult.RawFormat), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - content := strings.TrimSpace(readResult.Content) - - // .note 格式必须转换为 Markdown(向量化要求 Markdown 格式) - if readResult.RawFormat == "note" { - // 空笔记无需转换,跳过入库 - if content == "" && s.cookiesPath == "" { - logger.Info("笔记内容为空,跳过入库", zap.String("file_id", fileID)) - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "ready", ""); updateErr != nil { - logger.Warn("更新Source状态失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - if s.cookiesPath == "" { - logger.Error("笔记为 .note 格式,但未配置 cookies 文件路径", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - ) - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", "笔记为 .note 格式,但未配置 cookies 文件路径"); updateErr != nil { - logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - logger.Info("笔记为 .note 格式,开始转换为 Markdown", zap.String("file_id", fileID)) - convertStart := time.Now() - convertedContent, convertErr := s.cli.ConvertNote(fileID, s.cookiesPath) - if convertErr != nil { - logger.Error(".note 格式转换失败", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(convertStart)), - zap.Error(convertErr), - ) - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf(".note 格式转换失败: %v", convertErr)); updateErr != nil { - logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - if strings.TrimSpace(convertedContent) == "" { - logger.Error(".note 格式转换后内容为空", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(convertStart)), - ) - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", ".note 格式转换后内容为空"); updateErr != nil { - logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - content = convertedContent - logger.Info(".note 格式转换成功", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - zap.Int("content_len", len(content)), - zap.Duration("elapsed", time.Since(convertStart)), - ) - } else if content == "" && s.cookiesPath != "" { - // 非 .note 格式但内容为空,尝试转换(可能是格式识别错误) - logger.Info("内容为空,尝试使用 youdaonote-pull 转换", zap.String("file_id", fileID)) - convertStart := time.Now() - convertedContent, convertErr := s.cli.ConvertNote(fileID, s.cookiesPath) - if convertErr != nil { - logger.Warn("youdaonote-pull 转换失败", zap.String("file_id", fileID), zap.Duration("elapsed", time.Since(convertStart)), zap.Error(convertErr)) - } else if strings.TrimSpace(convertedContent) != "" { - content = convertedContent - logger.Info("youdaonote-pull 转换成功", zap.String("file_id", fileID), zap.Duration("elapsed", time.Since(convertStart))) - } - } - - // 检查内容是否为空 - if content == "" { - if taskCtx.Err() != nil { - return - } - logger.Error("笔记内容为空或格式不支持", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - ) - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", "笔记内容为空或格式不支持"); updateErr != nil { - logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - - // 检查 Source 是否还存在 - existing, err := s.sourceRepo.FindByID(sourceID) - if err != nil { - logger.Warn("查询Source失败", zap.Uint("source_id", sourceID), zap.Error(err)) - return - } - if existing == nil { - return - } - - // LLM 结构化 - stepStart = time.Now() - if s.structurer != nil { - result, err := s.structurer.Structure(taskCtx, existing.UserID, content, StructureMeta{ - Title: existing.Name, - SourceType: "youdao", - }) - if err != nil { - logger.Error("LLM 结构化失败,使用原始内容", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - } else if result.ActuallyCalled && result.Content != content { - logger.Info("LLM 结构化成功,内容已优化", - zap.Uint("source_id", sourceID), - zap.Int("original_len", len(content)), - zap.Int("structured_len", len(result.Content)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - content = result.Content - } else if result.ActuallyCalled { - logger.Info("LLM 判断内容已有结构,无需结构化", - zap.Uint("source_id", sourceID), - zap.Int("content_len", len(content)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } else { - logger.Warn("LLM 结构化被跳过(模型配置问题或 API Key 过期)", - zap.Uint("source_id", sourceID), - zap.Int("content_len", len(content)), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } - } else { - logger.Warn("MarkdownStructurer 未配置,跳过结构化", zap.Uint("source_id", sourceID)) - } - - // 更新内容和状态 - stepStart = time.Now() - existing.MarkdownContent = content - existing.Status = "ready" - if err := s.sourceRepo.Update(existing); err != nil { - logger.Error("更新 Source 内容失败", - zap.Uint("source_id", sourceID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("保存失败: %v", err)); updateErr != nil { - logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - - logger.Info("Source 记录更新成功", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(stepStart)), - ) - - // 同步触发 RAG 入库 - stepStart = time.Now() - if s.ingestionSvc != nil { - if err := s.ingestionSvc.IngestSingle(context.Background(), sourceID); err != nil { - logger.Error("RAG 入库失败", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(stepStart)), - zap.Error(err), - ) - if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("RAG 入库失败: %v", err)); updateErr != nil { - logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) - } - return - } - logger.Info("RAG 入库成功", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - zap.Duration("elapsed", time.Since(stepStart)), - ) - } - - // 生成摘要(异步,不阻塞主流程) - go s.generateAndSaveSummary(sourceID, existing.UserID, content) - - logger.Info("有道笔记导入完成", - zap.Uint("source_id", sourceID), - zap.String("file_id", fileID), - zap.Duration("total_elapsed", time.Since(totalStart)), - ) -} +package service + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "YoudaoNoteLm/internal/model/entity" + "YoudaoNoteLm/internal/rag" + "YoudaoNoteLm/internal/repository" + externalYoudao "YoudaoNoteLm/internal/service/external/youdao" + "YoudaoNoteLm/pkg/cache" + bizerrors "YoudaoNoteLm/pkg/errors" + "YoudaoNoteLm/pkg/logger" + + "github.com/google/uuid" + "go.uber.org/zap" +) + +type youdaoService struct { + cli externalYoudao.CLI + bindingRepo repository.YoudaoBindingRepository + sourceRepo repository.SourceRepository + ingestionSvc rag.IngestionService + structurer MarkdownStructurer // LLM 结构化服务 + configSvc ConfigService // 用于获取用户 LLM 配置(摘要生成) + summaryCache *cache.SourceSummaryCache + cancelFuncs sync.Map // taskID -> context.CancelFunc + cookiesPath string // youdaonote cookies 文件路径(用于 .note 格式转换) +} + +// NewYoudaoService 创建有道云笔记服务 +func NewYoudaoService( + cli externalYoudao.CLI, + bindingRepo repository.YoudaoBindingRepository, + sourceRepo repository.SourceRepository, + ingestionSvc rag.IngestionService, + cookiesPath string, + structurer MarkdownStructurer, + configSvc ConfigService, + summaryCache *cache.SourceSummaryCache, +) YoudaoService { + return &youdaoService{ + cli: cli, + bindingRepo: bindingRepo, + sourceRepo: sourceRepo, + ingestionSvc: ingestionSvc, + cookiesPath: cookiesPath, + structurer: structurer, + configSvc: configSvc, + summaryCache: summaryCache, + } +} + +// getAPIKey 获取用户的有道 API Key(内部辅助方法) +func (s *youdaoService) getAPIKey(userID uint) (string, error) { + binding, err := s.bindingRepo.FindByUserID(userID) + if err != nil { + return "", fmt.Errorf("查询绑定信息失败: %w", err) + } + if binding == nil || binding.Status != "active" { + return "", fmt.Errorf("请先绑定有道云笔记账号") + } + return binding.APIKey, nil +} + +// mapYoudaoAuthError 将有道 CLI 的认证失败错误(HTTP 401 / API Key 无效或过期) +// 映射为用户友好的业务错误 BizError,使前端看到的是 "有道云笔记 API Key 无效或已过期,请重新绑定" +// 而不是原始的 "CLI 执行失败: SSE error: Non-200 status code (401)"。 +// 非认证错误返回 nil,由调用方按原逻辑包装上下文信息。 +func mapYoudaoAuthError(err error) error { + if err != nil && errors.Is(err, externalYoudao.ErrAuthFailed) { + return bizerrors.NewWithErr(bizerrors.CodeInvalidYoudaoAPIKey, + "有道云笔记 API Key 无效或已过期,请重新绑定", err) + } + return nil +} + +// generateAndSaveSummary 生成资料摘要并保存到 MySQL 和 Redis +func (s *youdaoService) generateAndSaveSummary(sourceID uint, userID uint, content string) { + doGenerateAndSaveSummary(s.sourceRepo, s.configSvc, s.summaryCache, sourceID, userID, content) +} + +// Bind 绑定有道 API Key +func (s *youdaoService) Bind(userID uint, apiKey string) error { + // 1. 检查 CLI 是否可用 + if err := s.cli.CheckAvailable(); err != nil { + return fmt.Errorf("youdaonote CLI 不可用: %w", err) + } + + // 2. 验证 Key 有效性(调用 list 测试) + _, err := s.cli.List(apiKey, "") + if err != nil { + if friendly := mapYoudaoAuthError(err); friendly != nil { + return friendly + } + return fmt.Errorf("API Key 验证失败(CLI 返回错误: %w),请检查 Key 是否正确或网络是否正常", err) + } + + // 3. 使用 Upsert 原子操作,避免并发冲突 + binding := &entity.YoudaoBinding{ + UserID: userID, + APIKey: apiKey, + Status: "active", + } + return s.bindingRepo.Upsert(binding) +} + +// Unbind 解绑有道账号 +func (s *youdaoService) Unbind(userID uint) error { + return s.bindingRepo.Delete(userID) +} + +// GetBinding 获取绑定信息 +func (s *youdaoService) GetBinding(userID uint) (*entity.YoudaoBinding, error) { + return s.bindingRepo.FindByUserID(userID) +} + +// ListNotes 浏览有道云笔记目录 +func (s *youdaoService) ListNotes(userID uint, folderID string) ([]externalYoudao.NoteItem, error) { + apiKey, err := s.getAPIKey(userID) + if err != nil { + return nil, err + } + + items, err := s.cli.List(apiKey, folderID) + if err != nil { + if friendly := mapYoudaoAuthError(err); friendly != nil { + return nil, friendly + } + return nil, fmt.Errorf("获取笔记列表失败: %w", err) + } + + return items, nil +} + +// ImportNote 导入单篇有道云笔记到本系统 +func (s *youdaoService) ImportNote(userID uint, notebookID uint, fileID string) (*entity.Source, error) { + totalStart := time.Now() + + apiKey, err := s.getAPIKey(userID) + if err != nil { + return nil, err + } + + logger.Info("开始导入有道笔记", + zap.Uint("user_id", userID), + zap.String("file_id", fileID), + ) + + // 1. 读取笔记内容 + stepStart := time.Now() + readResult, err := s.cli.Read(apiKey, fileID) + if err != nil { + logger.Error("读取有道笔记内容失败", + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + if friendly := mapYoudaoAuthError(err); friendly != nil { + return nil, friendly + } + return nil, fmt.Errorf("读取笔记内容失败: %w", err) + } + + logger.Info("有道笔记内容读取成功", + zap.String("file_id", fileID), + zap.String("format", readResult.RawFormat), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + content := strings.TrimSpace(readResult.Content) + + // .note 格式必须转换为 Markdown(向量化要求 Markdown 格式) + if readResult.RawFormat == "note" { + // 空笔记无需转换,直接返回空内容,由调用方处理 + if content == "" && s.cookiesPath == "" { + return nil, fmt.Errorf("笔记内容为空") + } + if s.cookiesPath == "" { + return nil, fmt.Errorf("笔记为 .note 格式,但未配置 cookies 文件路径,无法转换") + } + logger.Info("笔记为 .note 格式,开始转换为 Markdown", zap.String("file_id", fileID)) + convertStart := time.Now() + convertedContent, convertErr := s.cli.ConvertNote(fileID, s.cookiesPath) + if convertErr != nil { + logger.Error(".note 格式转换失败", + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(convertStart)), + zap.Error(convertErr), + ) + return nil, fmt.Errorf(".note 格式转换失败: %w", convertErr) + } + if strings.TrimSpace(convertedContent) == "" { + return nil, fmt.Errorf(".note 格式转换后内容为空") + } + content = convertedContent + logger.Info(".note 格式转换成功", + zap.String("file_id", fileID), + zap.Int("content_len", len(content)), + zap.Duration("elapsed", time.Since(convertStart)), + ) + } else if content == "" && s.cookiesPath != "" { + // 非 .note 格式但内容为空,尝试转换(可能是格式识别错误) + logger.Info("内容为空,尝试使用 youdaonote-pull 转换", zap.String("file_id", fileID)) + convertStart := time.Now() + convertedContent, convertErr := s.cli.ConvertNote(fileID, s.cookiesPath) + if convertErr != nil { + logger.Warn("youdaonote-pull 转换失败", zap.String("file_id", fileID), zap.Duration("elapsed", time.Since(convertStart)), zap.Error(convertErr)) + } else if strings.TrimSpace(convertedContent) != "" { + content = convertedContent + logger.Info("youdaonote-pull 转换成功", zap.String("file_id", fileID), zap.Duration("elapsed", time.Since(convertStart))) + } + } + + // 检查内容是否为空 + if content == "" { + return nil, fmt.Errorf("笔记内容为空或格式不支持") + } + + // 2. 通过 list 获取笔记名称 + stepStart = time.Now() + noteName := fileID // 降级使用 fileID + items, listErr := s.cli.List(apiKey, "") + if listErr == nil { + for _, item := range items { + if item.ID == fileID { + noteName = item.Name + break + } + } + } + logger.Info("获取笔记名称完成", + zap.String("file_id", fileID), + zap.String("note_name", noteName), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + // LLM 结构化 + stepStart = time.Now() + if s.structurer != nil { + result, err := s.structurer.Structure(context.Background(), userID, content, StructureMeta{ + Title: noteName, + SourceType: "youdao", + }) + if err != nil { + logger.Error("LLM 结构化失败,使用原始内容", + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + } else if result.ActuallyCalled && result.Content != content { + logger.Info("LLM 结构化成功,内容已优化", + zap.String("file_id", fileID), + zap.Int("original_len", len(content)), + zap.Int("structured_len", len(result.Content)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + content = result.Content + } else if result.ActuallyCalled { + logger.Info("LLM 判断内容已有结构,无需结构化", + zap.String("file_id", fileID), + zap.Int("content_len", len(content)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } else { + logger.Warn("LLM 结构化被跳过(模型配置问题或 API Key 过期)", + zap.String("file_id", fileID), + zap.Int("content_len", len(content)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } + } else { + logger.Warn("MarkdownStructurer 未配置,跳过结构化", zap.String("file_id", fileID)) + } + + // 3. 创建 Source 记录 + stepStart = time.Now() + source := &entity.Source{ + UserID: userID, + NotebookID: notebookID, + Name: noteName, + Type: "youdao", + ExternalID: fileID, + MarkdownContent: content, + Status: "ready", + } + + if err := s.sourceRepo.Create(source); err != nil { + logger.Error("创建 Source 记录失败", + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + return nil, fmt.Errorf("创建 Source 记录失败: %w", err) + } + + logger.Info("Source 记录创建成功", + zap.String("file_id", fileID), + zap.Uint("source_id", source.ID), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + // 4. 同步触发 RAG 入库 + stepStart = time.Now() + if s.ingestionSvc != nil { + if err := s.ingestionSvc.IngestSingle(context.Background(), source.ID); err != nil { + logger.Error("RAG 入库失败", + zap.String("file_id", fileID), + zap.Uint("source_id", source.ID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + return nil, fmt.Errorf("RAG 入库失败: %w", err) + } + logger.Info("RAG 入库成功", + zap.String("file_id", fileID), + zap.Uint("source_id", source.ID), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } + + // 5. 生成摘要(异步,不阻塞主流程) + go s.generateAndSaveSummary(source.ID, userID, content) + + logger.Info("有道笔记导入完成", + zap.Uint("user_id", userID), + zap.String("file_id", fileID), + zap.String("name", noteName), + zap.Uint("source_id", source.ID), + zap.Duration("total_elapsed", time.Since(totalStart)), + ) + + return source, nil +} + +// ImportNotesBatch 批量导入有道云笔记 +func (s *youdaoService) ImportNotesBatch(userID uint, notebookID uint, fileIDs []string, fileNames map[string]string) (string, []uint, error) { + apiKey, err := s.getAPIKey(userID) + if err != nil { + return "", nil, err + } + + // 去重 + seen := make(map[string]struct{}, len(fileIDs)) + uniqueIDs := make([]string, 0, len(fileIDs)) + for _, id := range fileIDs { + if _, exists := seen[id]; exists { + continue + } + seen[id] = struct{}{} + uniqueIDs = append(uniqueIDs, id) + } + + sourceIDs := make([]uint, 0, len(uniqueIDs)) + + // 为每个 fileID 创建 pending 状态的 Source + for _, fileID := range uniqueIDs { + // 优先使用前端传递的笔记标题,降级使用 fileID + noteName := fileID + if name, ok := fileNames[fileID]; ok && name != "" { + noteName = name + } + + source := &entity.Source{ + UserID: userID, + NotebookID: notebookID, + Name: noteName, + Type: "youdao", + ExternalID: fileID, + Status: "pending", + } + if err := s.sourceRepo.Create(source); err != nil { + logger.Error("创建待导入有道笔记Source失败", zap.String("file_id", fileID), zap.Error(err)) + continue + } + sourceIDs = append(sourceIDs, source.ID) + } + + if len(sourceIDs) == 0 { + return "", nil, fmt.Errorf("创建导入记录失败") + } + + // 创建可取消的 context + taskID := uuid.New().String() + taskCtx, cancel := context.WithCancel(context.Background()) + s.cancelFuncs.Store(taskID, cancel) + + // 异步处理 + go s.processBatch(taskCtx, taskID, apiKey, sourceIDs, uniqueIDs) + + return taskID, sourceIDs, nil +} + +// processBatch 批量处理有道笔记导入 +func (s *youdaoService) processBatch(taskCtx context.Context, taskID string, apiKey string, sourceIDs []uint, fileIDs []string) { + defer s.cancelFuncs.Delete(taskID) + + concurrency := 3 + if len(fileIDs) < concurrency { + concurrency = len(fileIDs) + } + + type task struct { + sourceID uint + fileID string + } + + taskCh := make(chan task, concurrency) + doneCh := make(chan struct{}, len(fileIDs)) + + // 启动 worker + for i := 0; i < concurrency; i++ { + go func() { + for t := range taskCh { + if taskCtx.Err() != nil { + doneCh <- struct{}{} + continue + } + s.processSingleNote(taskCtx, apiKey, t.sourceID, t.fileID) + doneCh <- struct{}{} + } + }() + } + + // 分发任务 + go func() { + for i, fileID := range fileIDs { + if taskCtx.Err() != nil { + break + } + taskCh <- task{sourceID: sourceIDs[i], fileID: fileID} + } + close(taskCh) + }() + + // 等待完成 + for i := 0; i < len(fileIDs); i++ { + <-doneCh + } + + // 处理被取消的 pending 任务 + if taskCtx.Err() != nil { + for _, sourceID := range sourceIDs { + src, err := s.sourceRepo.FindByID(sourceID) + if err != nil || src == nil { + continue + } + if src.Status == "pending" { + if err := s.sourceRepo.UpdateStatus(sourceID, "cancelled", "任务已取消"); err != nil { + logger.Warn("更新Source状态为cancelled失败", zap.Uint("source_id", sourceID), zap.Error(err)) + } + } + } + } +} + +// processSingleNote 处理单篇有道笔记导入 +func (s *youdaoService) processSingleNote(taskCtx context.Context, apiKey string, sourceID uint, fileID string) { + totalStart := time.Now() + + if taskCtx.Err() != nil { + return + } + + logger.Info("开始处理有道笔记导入", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + ) + + // 更新状态为 processing + if err := s.sourceRepo.UpdateStatus(sourceID, "processing", ""); err != nil { + logger.Warn("更新Source状态为processing失败", zap.Uint("source_id", sourceID), zap.Error(err)) + } + + // 读取笔记内容 + stepStart := time.Now() + readResult, err := s.cli.Read(apiKey, fileID) + if err != nil { + if taskCtx.Err() != nil { + return + } + logger.Error("读取有道笔记内容失败", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + failMsg := fmt.Sprintf("读取失败: %v", err) + if friendly := mapYoudaoAuthError(err); friendly != nil { + if bizErr, ok := friendly.(*bizerrors.BizError); ok { + failMsg = bizErr.Message + } else { + failMsg = friendly.Error() + } + } + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", failMsg); updateErr != nil { + logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + + logger.Info("有道笔记内容读取成功", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + zap.String("format", readResult.RawFormat), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + content := strings.TrimSpace(readResult.Content) + + // .note 格式必须转换为 Markdown(向量化要求 Markdown 格式) + if readResult.RawFormat == "note" { + // 空笔记无需转换,跳过入库 + if content == "" && s.cookiesPath == "" { + logger.Info("笔记内容为空,跳过入库", zap.String("file_id", fileID)) + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "ready", ""); updateErr != nil { + logger.Warn("更新Source状态失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + if s.cookiesPath == "" { + logger.Error("笔记为 .note 格式,但未配置 cookies 文件路径", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + ) + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", "笔记为 .note 格式,但未配置 cookies 文件路径"); updateErr != nil { + logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + logger.Info("笔记为 .note 格式,开始转换为 Markdown", zap.String("file_id", fileID)) + convertStart := time.Now() + convertedContent, convertErr := s.cli.ConvertNote(fileID, s.cookiesPath) + if convertErr != nil { + logger.Error(".note 格式转换失败", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(convertStart)), + zap.Error(convertErr), + ) + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf(".note 格式转换失败: %v", convertErr)); updateErr != nil { + logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + if strings.TrimSpace(convertedContent) == "" { + logger.Error(".note 格式转换后内容为空", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(convertStart)), + ) + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", ".note 格式转换后内容为空"); updateErr != nil { + logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + content = convertedContent + logger.Info(".note 格式转换成功", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + zap.Int("content_len", len(content)), + zap.Duration("elapsed", time.Since(convertStart)), + ) + } else if content == "" && s.cookiesPath != "" { + // 非 .note 格式但内容为空,尝试转换(可能是格式识别错误) + logger.Info("内容为空,尝试使用 youdaonote-pull 转换", zap.String("file_id", fileID)) + convertStart := time.Now() + convertedContent, convertErr := s.cli.ConvertNote(fileID, s.cookiesPath) + if convertErr != nil { + logger.Warn("youdaonote-pull 转换失败", zap.String("file_id", fileID), zap.Duration("elapsed", time.Since(convertStart)), zap.Error(convertErr)) + } else if strings.TrimSpace(convertedContent) != "" { + content = convertedContent + logger.Info("youdaonote-pull 转换成功", zap.String("file_id", fileID), zap.Duration("elapsed", time.Since(convertStart))) + } + } + + // 检查内容是否为空 + if content == "" { + if taskCtx.Err() != nil { + return + } + logger.Error("笔记内容为空或格式不支持", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + ) + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", "笔记内容为空或格式不支持"); updateErr != nil { + logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + + // 检查 Source 是否还存在 + existing, err := s.sourceRepo.FindByID(sourceID) + if err != nil { + logger.Warn("查询Source失败", zap.Uint("source_id", sourceID), zap.Error(err)) + return + } + if existing == nil { + return + } + + // LLM 结构化 + stepStart = time.Now() + if s.structurer != nil { + result, err := s.structurer.Structure(taskCtx, existing.UserID, content, StructureMeta{ + Title: existing.Name, + SourceType: "youdao", + }) + if err != nil { + logger.Error("LLM 结构化失败,使用原始内容", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + } else if result.ActuallyCalled && result.Content != content { + logger.Info("LLM 结构化成功,内容已优化", + zap.Uint("source_id", sourceID), + zap.Int("original_len", len(content)), + zap.Int("structured_len", len(result.Content)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + content = result.Content + } else if result.ActuallyCalled { + logger.Info("LLM 判断内容已有结构,无需结构化", + zap.Uint("source_id", sourceID), + zap.Int("content_len", len(content)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } else { + logger.Warn("LLM 结构化被跳过(模型配置问题或 API Key 过期)", + zap.Uint("source_id", sourceID), + zap.Int("content_len", len(content)), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } + } else { + logger.Warn("MarkdownStructurer 未配置,跳过结构化", zap.Uint("source_id", sourceID)) + } + + // 更新内容和状态 + stepStart = time.Now() + existing.MarkdownContent = content + existing.Status = "ready" + if err := s.sourceRepo.Update(existing); err != nil { + logger.Error("更新 Source 内容失败", + zap.Uint("source_id", sourceID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("保存失败: %v", err)); updateErr != nil { + logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + + logger.Info("Source 记录更新成功", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(stepStart)), + ) + + // 同步触发 RAG 入库 + stepStart = time.Now() + if s.ingestionSvc != nil { + if err := s.ingestionSvc.IngestSingle(context.Background(), sourceID); err != nil { + logger.Error("RAG 入库失败", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(stepStart)), + zap.Error(err), + ) + if updateErr := s.sourceRepo.UpdateStatus(sourceID, "failed", fmt.Sprintf("RAG 入库失败: %v", err)); updateErr != nil { + logger.Warn("更新Source状态为failed失败", zap.Uint("source_id", sourceID), zap.Error(updateErr)) + } + return + } + logger.Info("RAG 入库成功", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + zap.Duration("elapsed", time.Since(stepStart)), + ) + } + + // 生成摘要(异步,不阻塞主流程) + go s.generateAndSaveSummary(sourceID, existing.UserID, content) + + logger.Info("有道笔记导入完成", + zap.Uint("source_id", sourceID), + zap.String("file_id", fileID), + zap.Duration("total_elapsed", time.Since(totalStart)), + ) +} diff --git a/internal/service/youdao_service_interface.go b/internal/service/youdao_service_interface.go index 871b06e..2510bdd 100644 --- a/internal/service/youdao_service_interface.go +++ b/internal/service/youdao_service_interface.go @@ -1,23 +1,23 @@ -package service - -import ( - "YoudaoNoteLm/internal/model/entity" - externalYoudao "YoudaoNoteLm/internal/service/external/youdao" -) - -// YoudaoService 有道云笔记服务接口 - -// YoudaoService 有道云笔记服务接口 -type YoudaoService interface { - // 绑定管理 - Bind(userID uint, apiKey string) error - Unbind(userID uint) error - GetBinding(userID uint) (*entity.YoudaoBinding, error) - - // 浏览 - ListNotes(userID uint, folderID string) ([]externalYoudao.NoteItem, error) - - // 导入 - ImportNote(userID uint, notebookID uint, fileID string) (*entity.Source, error) - ImportNotesBatch(userID uint, notebookID uint, fileIDs []string, fileNames map[string]string) (taskID string, sourceIDs []uint, err error) -} +package service + +import ( + "YoudaoNoteLm/internal/model/entity" + externalYoudao "YoudaoNoteLm/internal/service/external/youdao" +) + +// YoudaoService 有道云笔记服务接口 + +// YoudaoService 有道云笔记服务接口 +type YoudaoService interface { + // 绑定管理 + Bind(userID uint, apiKey string) error + Unbind(userID uint) error + GetBinding(userID uint) (*entity.YoudaoBinding, error) + + // 浏览 + ListNotes(userID uint, folderID string) ([]externalYoudao.NoteItem, error) + + // 导入 + ImportNote(userID uint, notebookID uint, fileID string) (*entity.Source, error) + ImportNotesBatch(userID uint, notebookID uint, fileIDs []string, fileNames map[string]string) (taskID string, sourceIDs []uint, err error) +} diff --git a/markitdown_service/main.py b/markitdown_service/main.py index 4f12761..d703ce1 100644 --- a/markitdown_service/main.py +++ b/markitdown_service/main.py @@ -62,6 +62,12 @@ def fetch_webpage(url: str) -> tuple[bytes, str]: return b"", f"网络请求失败: {str(e)}" +@app.get("/health") +async def health(): + """健康检查端点""" + return {"status": "ok"} + + @app.post("/convert") async def convert(file: UploadFile = File(...)): """文件转 Markdown""" diff --git a/nginx.conf b/nginx.conf index 76d7b20..d0c3140 100644 --- a/nginx.conf +++ b/nginx.conf @@ -25,7 +25,10 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; # 超时配置 proxy_connect_timeout 300s; diff --git a/pkg/cache/audio_preview.go b/pkg/cache/audio_preview.go index aeadc77..ff08f77 100644 --- a/pkg/cache/audio_preview.go +++ b/pkg/cache/audio_preview.go @@ -1,74 +1,74 @@ -package cache - -import ( - "context" - "fmt" - "time" -) - -// AudioPreview 音频预览缓存结构 -type AudioPreview struct { - PreviewID string `json:"preview_id"` // 预览ID(UUID) - UserID uint `json:"user_id"` // 所属用户 - NotebookID uint `json:"notebook_id"` // 所属笔记本 - FileName string `json:"file_name"` // 文件名 - FilePath string `json:"file_path"` // 对象存储路径 - FileSize int64 `json:"file_size"` // 文件大小(字节) - TranscribedText string `json:"transcribed_text"` // ASR转写文本(转写完成后填充) - Status string `json:"status"` // 状态: pending/processing/ready/failed - ErrorMsg string `json:"error_msg"` // 失败原因 - ExpiresAt int64 `json:"expires_at"` // 过期时间戳 -} - -// AudioPreviewCache 音频预览缓存操作 -type AudioPreviewCache struct { - cache *Cache -} - -// NewAudioPreviewCache 创建音频预览缓存 -func NewAudioPreviewCache(cache *Cache) *AudioPreviewCache { - return &AudioPreviewCache{cache: cache} -} - -// key 前缀 -const audioPreviewPrefix = "audio:preview:" - -// Save 保存音频预览(按过期时间设置TTL) -func (c *AudioPreviewCache) Save(ctx context.Context, preview *AudioPreview) error { - key := fmt.Sprintf("%s%s", audioPreviewPrefix, preview.PreviewID) - expireAt := time.Unix(preview.ExpiresAt, 0) - return c.cache.SetWithExpire(ctx, key, preview, expireAt) -} - -// Get 获取音频预览 -func (c *AudioPreviewCache) Get(ctx context.Context, previewID string) (*AudioPreview, error) { - key := fmt.Sprintf("%s%s", audioPreviewPrefix, previewID) - var preview AudioPreview - err := c.cache.Get(ctx, key, &preview) - if err != nil { - return nil, err - } - return &preview, nil -} - -// UpdateStatus 更新预览状态 -func (c *AudioPreviewCache) UpdateStatus(ctx context.Context, previewID string, status string) error { - preview, err := c.Get(ctx, previewID) - if err != nil { - return err - } - preview.Status = status - return c.Save(ctx, preview) -} - -// Delete 删除音频预览 -func (c *AudioPreviewCache) Delete(ctx context.Context, previewID string) error { - key := fmt.Sprintf("%s%s", audioPreviewPrefix, previewID) - return c.cache.Delete(ctx, key) -} - -// Exists 检查预览是否存在 -func (c *AudioPreviewCache) Exists(ctx context.Context, previewID string) (bool, error) { - key := fmt.Sprintf("%s%s", audioPreviewPrefix, previewID) - return c.cache.Exists(ctx, key) -} +package cache + +import ( + "context" + "fmt" + "time" +) + +// AudioPreview 音频预览缓存结构 +type AudioPreview struct { + PreviewID string `json:"preview_id"` // 预览ID(UUID) + UserID uint `json:"user_id"` // 所属用户 + NotebookID uint `json:"notebook_id"` // 所属笔记本 + FileName string `json:"file_name"` // 文件名 + FilePath string `json:"file_path"` // 对象存储路径 + FileSize int64 `json:"file_size"` // 文件大小(字节) + TranscribedText string `json:"transcribed_text"` // ASR转写文本(转写完成后填充) + Status string `json:"status"` // 状态: pending/processing/ready/failed + ErrorMsg string `json:"error_msg"` // 失败原因 + ExpiresAt int64 `json:"expires_at"` // 过期时间戳 +} + +// AudioPreviewCache 音频预览缓存操作 +type AudioPreviewCache struct { + cache *Cache +} + +// NewAudioPreviewCache 创建音频预览缓存 +func NewAudioPreviewCache(cache *Cache) *AudioPreviewCache { + return &AudioPreviewCache{cache: cache} +} + +// key 前缀 +const audioPreviewPrefix = "audio:preview:" + +// Save 保存音频预览(按过期时间设置TTL) +func (c *AudioPreviewCache) Save(ctx context.Context, preview *AudioPreview) error { + key := fmt.Sprintf("%s%s", audioPreviewPrefix, preview.PreviewID) + expireAt := time.Unix(preview.ExpiresAt, 0) + return c.cache.SetWithExpire(ctx, key, preview, expireAt) +} + +// Get 获取音频预览 +func (c *AudioPreviewCache) Get(ctx context.Context, previewID string) (*AudioPreview, error) { + key := fmt.Sprintf("%s%s", audioPreviewPrefix, previewID) + var preview AudioPreview + err := c.cache.Get(ctx, key, &preview) + if err != nil { + return nil, err + } + return &preview, nil +} + +// UpdateStatus 更新预览状态 +func (c *AudioPreviewCache) UpdateStatus(ctx context.Context, previewID string, status string) error { + preview, err := c.Get(ctx, previewID) + if err != nil { + return err + } + preview.Status = status + return c.Save(ctx, preview) +} + +// Delete 删除音频预览 +func (c *AudioPreviewCache) Delete(ctx context.Context, previewID string) error { + key := fmt.Sprintf("%s%s", audioPreviewPrefix, previewID) + return c.cache.Delete(ctx, key) +} + +// Exists 检查预览是否存在 +func (c *AudioPreviewCache) Exists(ctx context.Context, previewID string) (bool, error) { + key := fmt.Sprintf("%s%s", audioPreviewPrefix, previewID) + return c.cache.Exists(ctx, key) +} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 261bb82..627ebde 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -1,86 +1,86 @@ -package cache - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/redis/go-redis/v9" -) - -// Cache Redis缓存操作封装 -type Cache struct { - client *redis.Client -} - -// New 创建缓存实例 -func New(client *redis.Client) *Cache { - return &Cache{client: client} -} - -// Set 设置缓存 -func (c *Cache) Set(ctx context.Context, key string, value interface{}, expiration time.Duration) error { - data, err := json.Marshal(value) - if err != nil { - return fmt.Errorf("序列化失败: %w", err) - } - return c.client.Set(ctx, key, data, expiration).Err() -} - -// Get 获取缓存 -func (c *Cache) Get(ctx context.Context, key string, dest interface{}) error { - data, err := c.client.Get(ctx, key).Bytes() - if err != nil { - return err - } - return json.Unmarshal(data, dest) -} - -// Delete 删除缓存 -func (c *Cache) Delete(ctx context.Context, keys ...string) error { - return c.client.Del(ctx, keys...).Err() -} - -// Exists 检查key是否存在 -func (c *Cache) Exists(ctx context.Context, key string) (bool, error) { - result, err := c.client.Exists(ctx, key).Result() - return result > 0, err -} - -// SetHash 设置Hash缓存 -func (c *Cache) SetHash(ctx context.Context, key string, field string, value interface{}) error { - data, err := json.Marshal(value) - if err != nil { - return fmt.Errorf("序列化失败: %w", err) - } - return c.client.HSet(ctx, key, field, data).Err() -} - -// GetHash 获取Hash字段 -func (c *Cache) GetHash(ctx context.Context, key string, field string, dest interface{}) error { - data, err := c.client.HGet(ctx, key, field).Bytes() - if err != nil { - return err - } - return json.Unmarshal(data, dest) -} - -// GetAllHash 获取Hash所有字段 -func (c *Cache) GetAllHash(ctx context.Context, key string) (map[string]string, error) { - return c.client.HGetAll(ctx, key).Result() -} - -// DeleteHashField 删除Hash字段 -func (c *Cache) DeleteHashField(ctx context.Context, key string, fields ...string) error { - return c.client.HDel(ctx, key, fields...).Err() -} - -// SetWithExpire 设置带过期时间的缓存(以key为维度) -func (c *Cache) SetWithExpire(ctx context.Context, key string, value interface{}, expireAt time.Time) error { - ttl := time.Until(expireAt) - if ttl <= 0 { - return nil // 已过期,不存储 - } - return c.Set(ctx, key, value, ttl) -} +package cache + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +// Cache Redis缓存操作封装 +type Cache struct { + client *redis.Client +} + +// New 创建缓存实例 +func New(client *redis.Client) *Cache { + return &Cache{client: client} +} + +// Set 设置缓存 +func (c *Cache) Set(ctx context.Context, key string, value interface{}, expiration time.Duration) error { + data, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("序列化失败: %w", err) + } + return c.client.Set(ctx, key, data, expiration).Err() +} + +// Get 获取缓存 +func (c *Cache) Get(ctx context.Context, key string, dest interface{}) error { + data, err := c.client.Get(ctx, key).Bytes() + if err != nil { + return err + } + return json.Unmarshal(data, dest) +} + +// Delete 删除缓存 +func (c *Cache) Delete(ctx context.Context, keys ...string) error { + return c.client.Del(ctx, keys...).Err() +} + +// Exists 检查key是否存在 +func (c *Cache) Exists(ctx context.Context, key string) (bool, error) { + result, err := c.client.Exists(ctx, key).Result() + return result > 0, err +} + +// SetHash 设置Hash缓存 +func (c *Cache) SetHash(ctx context.Context, key string, field string, value interface{}) error { + data, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("序列化失败: %w", err) + } + return c.client.HSet(ctx, key, field, data).Err() +} + +// GetHash 获取Hash字段 +func (c *Cache) GetHash(ctx context.Context, key string, field string, dest interface{}) error { + data, err := c.client.HGet(ctx, key, field).Bytes() + if err != nil { + return err + } + return json.Unmarshal(data, dest) +} + +// GetAllHash 获取Hash所有字段 +func (c *Cache) GetAllHash(ctx context.Context, key string) (map[string]string, error) { + return c.client.HGetAll(ctx, key).Result() +} + +// DeleteHashField 删除Hash字段 +func (c *Cache) DeleteHashField(ctx context.Context, key string, fields ...string) error { + return c.client.HDel(ctx, key, fields...).Err() +} + +// SetWithExpire 设置带过期时间的缓存(以key为维度) +func (c *Cache) SetWithExpire(ctx context.Context, key string, value interface{}, expireAt time.Time) error { + ttl := time.Until(expireAt) + if ttl <= 0 { + return nil // 已过期,不存储 + } + return c.Set(ctx, key, value, ttl) +} diff --git a/pkg/cache/chat_cache.go b/pkg/cache/chat_cache.go index 6e5c704..b45ad98 100644 --- a/pkg/cache/chat_cache.go +++ b/pkg/cache/chat_cache.go @@ -1,130 +1,130 @@ -package cache - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/google/uuid" - "github.com/redis/go-redis/v9" -) - -const ( - recentMessagesKey = "chat:%d:recent_messages" - summaryKey = "chat:%d:summary" - lockKey = "chat:%d:lock" - - messagesTTL = 7 * 24 * time.Hour - summaryTTL = 7 * 24 * time.Hour - lockTTL = 120 * time.Second -) - -// MessagePair 消息对 -type MessagePair struct { - User string `json:"user"` - Assistant string `json:"assistant"` -} - -// ChatCache 对话缓存 -type ChatCache struct { - rdb *redis.Client -} - -// NewChatCache 创建对话缓存 -func NewChatCache(rdb *redis.Client) *ChatCache { - return &ChatCache{rdb: rdb} -} - -// GetRecentMessages 获取最近消息 -func (c *ChatCache) GetRecentMessages(ctx context.Context, conversationID uint) ([]MessagePair, error) { - key := fmt.Sprintf(recentMessagesKey, conversationID) - data, err := c.rdb.LRange(ctx, key, 0, -1).Result() - if err != nil { - return nil, err - } - - var pairs []MessagePair - for _, item := range data { - var pair MessagePair - if err := json.Unmarshal([]byte(item), &pair); err != nil { - continue - } - pairs = append(pairs, pair) - } - return pairs, nil -} - -// AddMessage 添加消息对到缓存 -func (c *ChatCache) AddMessage(ctx context.Context, conversationID uint, userMsg, assistantMsg string) error { - key := fmt.Sprintf(recentMessagesKey, conversationID) - - pair := MessagePair{ - User: userMsg, - Assistant: assistantMsg, - } - data, err := json.Marshal(pair) - if err != nil { - return err - } - - pipe := c.rdb.Pipeline() - pipe.RPush(ctx, key, string(data)) - // 每个 list 元素就是一个 MessagePair(一轮对话),保留最近 10 轮 - pipe.LTrim(ctx, key, -10, -1) - pipe.Expire(ctx, key, messagesTTL) - _, err = pipe.Exec(ctx) - return err -} - -// GetSummary 获取对话摘要 -func (c *ChatCache) GetSummary(ctx context.Context, conversationID uint) (string, bool, error) { - key := fmt.Sprintf(summaryKey, conversationID) - val, err := c.rdb.Get(ctx, key).Result() - if err == redis.Nil { - return "", false, nil - } - if err != nil { - return "", false, err - } - return val, true, nil -} - -// SetSummary 设置对话摘要 -func (c *ChatCache) SetSummary(ctx context.Context, conversationID uint, summary string) error { - key := fmt.Sprintf(summaryKey, conversationID) - return c.rdb.Set(ctx, key, summary, summaryTTL).Err() -} - -// AcquireLock 获取并发锁,返回 lockValue 用于释放 -func (c *ChatCache) AcquireLock(ctx context.Context, conversationID uint) (string, bool, error) { - key := fmt.Sprintf(lockKey, conversationID) - lockValue := uuid.New().String() - ok, err := c.rdb.SetNX(ctx, key, lockValue, lockTTL).Result() - if err != nil { - return "", false, err - } - return lockValue, ok, nil -} - -var releaseLockScript = redis.NewScript(` - if redis.call("get", KEYS[1]) == ARGV[1] then - return redis.call("del", KEYS[1]) - end - return 0 -`) - -// ReleaseLock 释放并发锁 -func (c *ChatCache) ReleaseLock(ctx context.Context, conversationID uint, lockValue string) error { - key := fmt.Sprintf(lockKey, conversationID) - return releaseLockScript.Run(ctx, c.rdb, []string{key}, lockValue).Err() -} - -// DeleteConversationCache 删除对话的所有缓存(消息+摘要) -func (c *ChatCache) DeleteConversationCache(ctx context.Context, conversationID uint) error { - keys := []string{ - fmt.Sprintf(recentMessagesKey, conversationID), - fmt.Sprintf(summaryKey, conversationID), - } - return c.rdb.Del(ctx, keys...).Err() -} +package cache + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" +) + +const ( + recentMessagesKey = "chat:%d:recent_messages" + summaryKey = "chat:%d:summary" + lockKey = "chat:%d:lock" + + messagesTTL = 7 * 24 * time.Hour + summaryTTL = 7 * 24 * time.Hour + lockTTL = 120 * time.Second +) + +// MessagePair 消息对 +type MessagePair struct { + User string `json:"user"` + Assistant string `json:"assistant"` +} + +// ChatCache 对话缓存 +type ChatCache struct { + rdb *redis.Client +} + +// NewChatCache 创建对话缓存 +func NewChatCache(rdb *redis.Client) *ChatCache { + return &ChatCache{rdb: rdb} +} + +// GetRecentMessages 获取最近消息 +func (c *ChatCache) GetRecentMessages(ctx context.Context, conversationID uint) ([]MessagePair, error) { + key := fmt.Sprintf(recentMessagesKey, conversationID) + data, err := c.rdb.LRange(ctx, key, 0, -1).Result() + if err != nil { + return nil, err + } + + var pairs []MessagePair + for _, item := range data { + var pair MessagePair + if err := json.Unmarshal([]byte(item), &pair); err != nil { + continue + } + pairs = append(pairs, pair) + } + return pairs, nil +} + +// AddMessage 添加消息对到缓存 +func (c *ChatCache) AddMessage(ctx context.Context, conversationID uint, userMsg, assistantMsg string) error { + key := fmt.Sprintf(recentMessagesKey, conversationID) + + pair := MessagePair{ + User: userMsg, + Assistant: assistantMsg, + } + data, err := json.Marshal(pair) + if err != nil { + return err + } + + pipe := c.rdb.Pipeline() + pipe.RPush(ctx, key, string(data)) + // 每个 list 元素就是一个 MessagePair(一轮对话),保留最近 10 轮 + pipe.LTrim(ctx, key, -10, -1) + pipe.Expire(ctx, key, messagesTTL) + _, err = pipe.Exec(ctx) + return err +} + +// GetSummary 获取对话摘要 +func (c *ChatCache) GetSummary(ctx context.Context, conversationID uint) (string, bool, error) { + key := fmt.Sprintf(summaryKey, conversationID) + val, err := c.rdb.Get(ctx, key).Result() + if err == redis.Nil { + return "", false, nil + } + if err != nil { + return "", false, err + } + return val, true, nil +} + +// SetSummary 设置对话摘要 +func (c *ChatCache) SetSummary(ctx context.Context, conversationID uint, summary string) error { + key := fmt.Sprintf(summaryKey, conversationID) + return c.rdb.Set(ctx, key, summary, summaryTTL).Err() +} + +// AcquireLock 获取并发锁,返回 lockValue 用于释放 +func (c *ChatCache) AcquireLock(ctx context.Context, conversationID uint) (string, bool, error) { + key := fmt.Sprintf(lockKey, conversationID) + lockValue := uuid.New().String() + ok, err := c.rdb.SetNX(ctx, key, lockValue, lockTTL).Result() + if err != nil { + return "", false, err + } + return lockValue, ok, nil +} + +var releaseLockScript = redis.NewScript(` + if redis.call("get", KEYS[1]) == ARGV[1] then + return redis.call("del", KEYS[1]) + end + return 0 +`) + +// ReleaseLock 释放并发锁 +func (c *ChatCache) ReleaseLock(ctx context.Context, conversationID uint, lockValue string) error { + key := fmt.Sprintf(lockKey, conversationID) + return releaseLockScript.Run(ctx, c.rdb, []string{key}, lockValue).Err() +} + +// DeleteConversationCache 删除对话的所有缓存(消息+摘要) +func (c *ChatCache) DeleteConversationCache(ctx context.Context, conversationID uint) error { + keys := []string{ + fmt.Sprintf(recentMessagesKey, conversationID), + fmt.Sprintf(summaryKey, conversationID), + } + return c.rdb.Del(ctx, keys...).Err() +} diff --git a/pkg/cache/generation_memory.go b/pkg/cache/generation_memory.go index 3048bcb..8d2b46e 100644 --- a/pkg/cache/generation_memory.go +++ b/pkg/cache/generation_memory.go @@ -1,80 +1,80 @@ -package cache - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/redis/go-redis/v9" -) - -const ( - generationMemoryKeyFormat = "generation:memory:%d:%d:%s" - generationMemoryMaxEntries = 10 - generationMemoryTTL = 7 * 24 * time.Hour -) - -type GenerationMemoryCacheEntry struct { - Prompt string `json:"prompt"` - InputSummary string `json:"input_summary"` - OutputSummary string `json:"output_summary"` - CreatedAt time.Time `json:"created_at"` -} - -type GenerationMemoryCache struct { - rdb *redis.Client -} - -func NewGenerationMemoryCache(rdb *redis.Client) *GenerationMemoryCache { - return &GenerationMemoryCache{rdb: rdb} -} - -func (c *GenerationMemoryCache) GetRecent(ctx context.Context, userID, notebookID uint, typ string, limit int) ([]GenerationMemoryCacheEntry, error) { - if c == nil || c.rdb == nil { - return []GenerationMemoryCacheEntry{}, nil - } - limit = normalizeGenerationMemoryLimit(limit) - key := generationMemoryKey(userID, notebookID, typ) - values, err := c.rdb.LRange(ctx, key, 0, int64(limit-1)).Result() - if err != nil { - return nil, err - } - entries := make([]GenerationMemoryCacheEntry, 0, len(values)) - for _, value := range values { - var entry GenerationMemoryCacheEntry - if err := json.Unmarshal([]byte(value), &entry); err != nil { - continue - } - entries = append(entries, entry) - } - return entries, nil -} - -func (c *GenerationMemoryCache) Add(ctx context.Context, userID, notebookID uint, typ string, entry GenerationMemoryCacheEntry) error { - if c == nil || c.rdb == nil { - return nil - } - data, err := json.Marshal(entry) - if err != nil { - return err - } - key := generationMemoryKey(userID, notebookID, typ) - pipe := c.rdb.Pipeline() - pipe.LPush(ctx, key, string(data)) - pipe.LTrim(ctx, key, 0, generationMemoryMaxEntries-1) - pipe.Expire(ctx, key, generationMemoryTTL) - _, err = pipe.Exec(ctx) - return err -} - -func generationMemoryKey(userID, notebookID uint, typ string) string { - return fmt.Sprintf(generationMemoryKeyFormat, userID, notebookID, typ) -} - -func normalizeGenerationMemoryLimit(limit int) int { - if limit <= 0 || limit > generationMemoryMaxEntries { - return generationMemoryMaxEntries - } - return limit -} +package cache + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + generationMemoryKeyFormat = "generation:memory:%d:%d:%s" + generationMemoryMaxEntries = 10 + generationMemoryTTL = 7 * 24 * time.Hour +) + +type GenerationMemoryCacheEntry struct { + Prompt string `json:"prompt"` + InputSummary string `json:"input_summary"` + OutputSummary string `json:"output_summary"` + CreatedAt time.Time `json:"created_at"` +} + +type GenerationMemoryCache struct { + rdb *redis.Client +} + +func NewGenerationMemoryCache(rdb *redis.Client) *GenerationMemoryCache { + return &GenerationMemoryCache{rdb: rdb} +} + +func (c *GenerationMemoryCache) GetRecent(ctx context.Context, userID, notebookID uint, typ string, limit int) ([]GenerationMemoryCacheEntry, error) { + if c == nil || c.rdb == nil { + return []GenerationMemoryCacheEntry{}, nil + } + limit = normalizeGenerationMemoryLimit(limit) + key := generationMemoryKey(userID, notebookID, typ) + values, err := c.rdb.LRange(ctx, key, 0, int64(limit-1)).Result() + if err != nil { + return nil, err + } + entries := make([]GenerationMemoryCacheEntry, 0, len(values)) + for _, value := range values { + var entry GenerationMemoryCacheEntry + if err := json.Unmarshal([]byte(value), &entry); err != nil { + continue + } + entries = append(entries, entry) + } + return entries, nil +} + +func (c *GenerationMemoryCache) Add(ctx context.Context, userID, notebookID uint, typ string, entry GenerationMemoryCacheEntry) error { + if c == nil || c.rdb == nil { + return nil + } + data, err := json.Marshal(entry) + if err != nil { + return err + } + key := generationMemoryKey(userID, notebookID, typ) + pipe := c.rdb.Pipeline() + pipe.LPush(ctx, key, string(data)) + pipe.LTrim(ctx, key, 0, generationMemoryMaxEntries-1) + pipe.Expire(ctx, key, generationMemoryTTL) + _, err = pipe.Exec(ctx) + return err +} + +func generationMemoryKey(userID, notebookID uint, typ string) string { + return fmt.Sprintf(generationMemoryKeyFormat, userID, notebookID, typ) +} + +func normalizeGenerationMemoryLimit(limit int) int { + if limit <= 0 || limit > generationMemoryMaxEntries { + return generationMemoryMaxEntries + } + return limit +} diff --git a/pkg/cache/generation_task.go b/pkg/cache/generation_task.go new file mode 100644 index 0000000..9d5918c --- /dev/null +++ b/pkg/cache/generation_task.go @@ -0,0 +1,129 @@ +package cache + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + generationTaskPrefix = "generation:task:" + generationTaskRequestPrefix = "generation:task:request:" + generationTaskUserPrefix = "generation:task:user:" + // generationTaskQueueKey 队列基于 Redis Set 实现,SADD 入队、SPOP 出队。 + // Set 天然去重,SPOP 原子弹出;不保证严格 FIFO,但生成任务串行处理,顺序无关紧要。 + generationTaskQueueKey = "generation:task:queue" + generationTaskDefaultTTL = 24 * time.Hour + generationTaskDefaultSize = 100 +) + +type GenerationTaskCache struct { + cache *Cache +} + +func NewGenerationTaskCache(cache *Cache) *GenerationTaskCache { + return &GenerationTaskCache{cache: cache} +} + +func (c *GenerationTaskCache) Save(ctx context.Context, taskID string, userID uint, sequence int64, task interface{}) error { + key := fmt.Sprintf("%s%s", generationTaskPrefix, taskID) + score := float64(sequence) + if sequence <= 0 { + score = float64(time.Now().UnixNano()) + } + pipe := c.cache.client.TxPipeline() + data, err := marshalCacheValue(task) + if err != nil { + return err + } + pipe.Set(ctx, key, data, generationTaskDefaultTTL) + if userID != 0 { + userKey := generationTaskUserKey(userID) + pipe.ZAdd(ctx, userKey, redis.Z{Score: score, Member: taskID}) + pipe.Expire(ctx, userKey, generationTaskDefaultTTL) + } + _, err = pipe.Exec(ctx) + return err +} + +func (c *GenerationTaskCache) Get(ctx context.Context, taskID string, dest interface{}) error { + key := fmt.Sprintf("%s%s", generationTaskPrefix, taskID) + return c.cache.Get(ctx, key, dest) +} + +func (c *GenerationTaskCache) ListUserTaskIDs(ctx context.Context, userID uint, limit int) ([]string, error) { + if limit <= 0 { + limit = generationTaskDefaultSize + } + return c.cache.client.ZRange(ctx, generationTaskUserKey(userID), 0, int64(limit-1)).Result() +} + +// Delete 按 taskID 删除任务:先读取任务拿到 userID,再用 pipeline 同时删 task 数据和 +// user sorted set 中的 member。任务不存在或已删除均返回 nil(幂等)。 +func (c *GenerationTaskCache) Delete(ctx context.Context, taskID string) error { + if taskID == "" { + return nil + } + key := fmt.Sprintf("%s%s", generationTaskPrefix, taskID) + + // 先读取任务以拿到 user_id,便于从 user sorted set 中移除。 + // 读不到也继续删除 task key 本身,保证幂等。 + var payload struct { + UserID uint `json:"user_id"` + } + _ = c.cache.Get(ctx, key, &payload) + + pipe := c.cache.client.TxPipeline() + pipe.Del(ctx, key) + pipe.Del(ctx, fmt.Sprintf("%s%s", generationTaskRequestPrefix, taskID)) + if payload.UserID != 0 { + pipe.ZRem(ctx, generationTaskUserKey(payload.UserID), taskID) + } + _, err := pipe.Exec(ctx) + return err +} + +// Enqueue 将任务 ID 投递到 Redis Set 队列,并缓存请求体。 +// 使用 SADD 入队,Set 结构天然去重,重复投递同一 taskID 不会产生重复消费。 +func (c *GenerationTaskCache) Enqueue(ctx context.Context, taskID string, req interface{}) error { + reqKey := fmt.Sprintf("%s%s", generationTaskRequestPrefix, taskID) + data, err := marshalCacheValue(req) + if err != nil { + return err + } + pipe := c.cache.client.TxPipeline() + pipe.Set(ctx, reqKey, data, generationTaskDefaultTTL) + pipe.SAdd(ctx, generationTaskQueueKey, taskID) + _, err = pipe.Exec(ctx) + return err +} + +// Dequeue 从 Redis Set 队列原子弹出一个 taskID 并读取其请求体。 +// 队列为空时返回 redis.Nil 错误,调用方应轮询重试。 +func (c *GenerationTaskCache) Dequeue(ctx context.Context, dest interface{}) (string, error) { + taskID, err := c.cache.client.SPop(ctx, generationTaskQueueKey).Result() + if err != nil { + return "", err + } + reqKey := fmt.Sprintf("%s%s", generationTaskRequestPrefix, taskID) + if err := c.cache.Get(ctx, reqKey, dest); err != nil { + return taskID, err + } + _ = c.cache.Delete(ctx, reqKey) + return taskID, nil +} + +func generationTaskUserKey(userID uint) string { + return fmt.Sprintf("%s%d", generationTaskUserPrefix, userID) +} + +func marshalCacheValue(value interface{}) ([]byte, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("序列化失败: %w", err) + } + return data, nil +} diff --git a/pkg/cache/import_task.go b/pkg/cache/import_task.go index 7f37d3d..19feace 100644 --- a/pkg/cache/import_task.go +++ b/pkg/cache/import_task.go @@ -1,76 +1,76 @@ -package cache - -import ( - "context" - "fmt" - "time" -) - -// ImportTask 导入任务缓存结构 -type ImportTask struct { - TaskID string `json:"task_id"` // 任务ID - UserID uint `json:"user_id"` // 所属用户 - NotebookID uint `json:"notebook_id"` // 所属笔记本 - TaskType string `json:"task_type"` // 任务类型: batch_file/batch_url/youdao - TotalCount int `json:"total_count"` // 总数 - ProcessedCount int `json:"processed_count"` // 已处理数 - SuccessCount int `json:"success_count"` // 成功数 - FailCount int `json:"fail_count"` // 失败数 - Status string `json:"status"` // 状态: pending/running/completed/failed/partial_failed/cancelled - ErrorDetail string `json:"error_detail"` // 错误详情(JSON) - CreatedAt int64 `json:"created_at"` // 创建时间戳 -} - -// ImportTaskCache 导入任务缓存操作 -type ImportTaskCache struct { - cache *Cache -} - -// NewImportTaskCache 创建导入任务缓存 -func NewImportTaskCache(cache *Cache) *ImportTaskCache { - return &ImportTaskCache{cache: cache} -} - -// key 前缀 -const importTaskPrefix = "import:task:" - -// Save 保存导入任务(默认24小时过期) -func (c *ImportTaskCache) Save(ctx context.Context, task *ImportTask) error { - key := fmt.Sprintf("%s%s", importTaskPrefix, task.TaskID) - return c.cache.Set(ctx, key, task, 24*time.Hour) -} - -// Get 获取导入任务 -func (c *ImportTaskCache) Get(ctx context.Context, taskID string) (*ImportTask, error) { - key := fmt.Sprintf("%s%s", importTaskPrefix, taskID) - var task ImportTask - err := c.cache.Get(ctx, key, &task) - if err != nil { - return nil, err - } - return &task, nil -} - -// UpdateStatus 更新任务状态 -func (c *ImportTaskCache) UpdateStatus(ctx context.Context, taskID string, status string, successCount, failCount int) error { - task, err := c.Get(ctx, taskID) - if err != nil { - return err - } - task.Status = status - task.SuccessCount = successCount - task.FailCount = failCount - return c.Save(ctx, task) -} - -// Delete 删除导入任务 -func (c *ImportTaskCache) Delete(ctx context.Context, taskID string) error { - key := fmt.Sprintf("%s%s", importTaskPrefix, taskID) - return c.cache.Delete(ctx, key) -} - -// Exists 检查任务是否存在 -func (c *ImportTaskCache) Exists(ctx context.Context, taskID string) (bool, error) { - key := fmt.Sprintf("%s%s", importTaskPrefix, taskID) - return c.cache.Exists(ctx, key) -} +package cache + +import ( + "context" + "fmt" + "time" +) + +// ImportTask 导入任务缓存结构 +type ImportTask struct { + TaskID string `json:"task_id"` // 任务ID + UserID uint `json:"user_id"` // 所属用户 + NotebookID uint `json:"notebook_id"` // 所属笔记本 + TaskType string `json:"task_type"` // 任务类型: batch_file/batch_url/youdao + TotalCount int `json:"total_count"` // 总数 + ProcessedCount int `json:"processed_count"` // 已处理数 + SuccessCount int `json:"success_count"` // 成功数 + FailCount int `json:"fail_count"` // 失败数 + Status string `json:"status"` // 状态: pending/running/completed/failed/partial_failed/cancelled + ErrorDetail string `json:"error_detail"` // 错误详情(JSON) + CreatedAt int64 `json:"created_at"` // 创建时间戳 +} + +// ImportTaskCache 导入任务缓存操作 +type ImportTaskCache struct { + cache *Cache +} + +// NewImportTaskCache 创建导入任务缓存 +func NewImportTaskCache(cache *Cache) *ImportTaskCache { + return &ImportTaskCache{cache: cache} +} + +// key 前缀 +const importTaskPrefix = "import:task:" + +// Save 保存导入任务(默认24小时过期) +func (c *ImportTaskCache) Save(ctx context.Context, task *ImportTask) error { + key := fmt.Sprintf("%s%s", importTaskPrefix, task.TaskID) + return c.cache.Set(ctx, key, task, 24*time.Hour) +} + +// Get 获取导入任务 +func (c *ImportTaskCache) Get(ctx context.Context, taskID string) (*ImportTask, error) { + key := fmt.Sprintf("%s%s", importTaskPrefix, taskID) + var task ImportTask + err := c.cache.Get(ctx, key, &task) + if err != nil { + return nil, err + } + return &task, nil +} + +// UpdateStatus 更新任务状态 +func (c *ImportTaskCache) UpdateStatus(ctx context.Context, taskID string, status string, successCount, failCount int) error { + task, err := c.Get(ctx, taskID) + if err != nil { + return err + } + task.Status = status + task.SuccessCount = successCount + task.FailCount = failCount + return c.Save(ctx, task) +} + +// Delete 删除导入任务 +func (c *ImportTaskCache) Delete(ctx context.Context, taskID string) error { + key := fmt.Sprintf("%s%s", importTaskPrefix, taskID) + return c.cache.Delete(ctx, key) +} + +// Exists 检查任务是否存在 +func (c *ImportTaskCache) Exists(ctx context.Context, taskID string) (bool, error) { + key := fmt.Sprintf("%s%s", importTaskPrefix, taskID) + return c.cache.Exists(ctx, key) +} diff --git a/pkg/cache/source_summary_cache.go b/pkg/cache/source_summary_cache.go index b527c4e..36f88d1 100644 --- a/pkg/cache/source_summary_cache.go +++ b/pkg/cache/source_summary_cache.go @@ -1,65 +1,65 @@ -package cache - -import ( - "context" - "fmt" - "time" - - "github.com/redis/go-redis/v9" -) - -const ( - sourceSummaryKey = "source:%d:summary" - sourceSummaryTTL = 30 * 24 * time.Hour // 30天 -) - -// SourceSummaryCache 资料摘要缓存 -type SourceSummaryCache struct { - rdb *redis.Client -} - -// NewSourceSummaryCache 创建资料摘要缓存 -func NewSourceSummaryCache(rdb *redis.Client) *SourceSummaryCache { - return &SourceSummaryCache{rdb: rdb} -} - -// Get 获取资料摘要 -func (c *SourceSummaryCache) Get(ctx context.Context, sourceID uint) (string, bool, error) { - key := formatSourceSummaryKey(sourceID) - val, err := c.rdb.Get(ctx, key).Result() - if err == redis.Nil { - return "", false, nil - } - if err != nil { - return "", false, err - } - return val, true, nil -} - -// Set 设置资料摘要 -func (c *SourceSummaryCache) Set(ctx context.Context, sourceID uint, summary string) error { - key := formatSourceSummaryKey(sourceID) - return c.rdb.Set(ctx, key, summary, sourceSummaryTTL).Err() -} - -// Delete 删除资料摘要 -func (c *SourceSummaryCache) Delete(ctx context.Context, sourceID uint) error { - key := formatSourceSummaryKey(sourceID) - return c.rdb.Del(ctx, key).Err() -} - -// BatchDelete 批量删除资料摘要 -func (c *SourceSummaryCache) BatchDelete(ctx context.Context, sourceIDs []uint) error { - if len(sourceIDs) == 0 { - return nil - } - keys := make([]string, len(sourceIDs)) - for i, id := range sourceIDs { - keys[i] = formatSourceSummaryKey(id) - } - return c.rdb.Del(ctx, keys...).Err() -} - -func formatSourceSummaryKey(sourceID uint) string { - return fmt.Sprintf(sourceSummaryKey, sourceID) -} +package cache + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + sourceSummaryKey = "source:%d:summary" + sourceSummaryTTL = 30 * 24 * time.Hour // 30天 +) + +// SourceSummaryCache 资料摘要缓存 +type SourceSummaryCache struct { + rdb *redis.Client +} + +// NewSourceSummaryCache 创建资料摘要缓存 +func NewSourceSummaryCache(rdb *redis.Client) *SourceSummaryCache { + return &SourceSummaryCache{rdb: rdb} +} + +// Get 获取资料摘要 +func (c *SourceSummaryCache) Get(ctx context.Context, sourceID uint) (string, bool, error) { + key := formatSourceSummaryKey(sourceID) + val, err := c.rdb.Get(ctx, key).Result() + if err == redis.Nil { + return "", false, nil + } + if err != nil { + return "", false, err + } + return val, true, nil +} + +// Set 设置资料摘要 +func (c *SourceSummaryCache) Set(ctx context.Context, sourceID uint, summary string) error { + key := formatSourceSummaryKey(sourceID) + return c.rdb.Set(ctx, key, summary, sourceSummaryTTL).Err() +} + +// Delete 删除资料摘要 +func (c *SourceSummaryCache) Delete(ctx context.Context, sourceID uint) error { + key := formatSourceSummaryKey(sourceID) + return c.rdb.Del(ctx, key).Err() +} + +// BatchDelete 批量删除资料摘要 +func (c *SourceSummaryCache) BatchDelete(ctx context.Context, sourceIDs []uint) error { + if len(sourceIDs) == 0 { + return nil + } + keys := make([]string, len(sourceIDs)) + for i, id := range sourceIDs { + keys[i] = formatSourceSummaryKey(id) + } + return c.rdb.Del(ctx, keys...).Err() +} + +func formatSourceSummaryKey(sourceID uint) string { + return fmt.Sprintf(sourceSummaryKey, sourceID) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index af1f7fd..eb9e0d1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,313 +1,325 @@ -package config - -import ( - "fmt" - "time" -) - -// Validate 校验所有必填配置项,缺失或无效时返回 error。 -func (c *Config) Validate() error { - // App - if c.App.Name == "" { - return fmt.Errorf("app.name 不能为空") - } - if c.App.Port == 0 { - return fmt.Errorf("app.port 不能为空") - } - - // MySQL - if c.Database.MySQL.Host == "" { - return fmt.Errorf("database.mysql.host 不能为空") - } - if c.Database.MySQL.Port == 0 { - return fmt.Errorf("database.mysql.port 不能为空") - } - if c.Database.MySQL.Username == "" { - return fmt.Errorf("database.mysql.username 不能为空") - } - if c.Database.MySQL.Database == "" { - return fmt.Errorf("database.mysql.database 不能为空") - } - - // Redis - if c.Database.Redis.Host == "" { - return fmt.Errorf("database.redis.host 不能为空") - } - if c.Database.Redis.Port == 0 { - return fmt.Errorf("database.redis.port 不能为空") - } - - // JWT - if c.JWT.Secret == "" { - return fmt.Errorf("jwt.secret 不能为空") - } - - // Log - if c.Log.Filename == "" { - return fmt.Errorf("log.filename 不能为空") - } - - // Email - if c.Email.Host == "" { - return fmt.Errorf("email.host 不能为空") - } - if c.Email.Port == 0 { - return fmt.Errorf("email.port 不能为空") - } - if c.Email.Username == "" { - return fmt.Errorf("email.username 不能为空") - } - if c.Email.Password == "" { - return fmt.Errorf("email.password 不能为空") - } - - // Milvus - if c.Milvus.Host == "" { - return fmt.Errorf("milvus.host 不能为空") - } - if c.Milvus.Port == 0 { - return fmt.Errorf("milvus.port 不能为空") - } - - // External - MarkItDown - if c.External.MarkItDown.URL == "" { - return fmt.Errorf("external.markitdown.url 不能为空") - } - - // External - MinIO - if c.External.MinIO.Endpoint == "" { - return fmt.Errorf("external.minio.endpoint 不能为空") - } - if c.External.MinIO.AccessKey == "" { - return fmt.Errorf("external.minio.access_key 不能为空") - } - if c.External.MinIO.SecretKey == "" { - return fmt.Errorf("external.minio.secret_key 不能为空") - } - if c.External.MinIO.Bucket == "" { - return fmt.Errorf("external.minio.bucket 不能为空") - } - - // Security - if c.Security.EncryptionKey == "" { - return fmt.Errorf("security.encryption_key 不能为空") - } - if len(c.Security.EncryptionKey) != 32 { - return fmt.Errorf("security.encryption_key 必须为32字节,当前%d字节", len(c.Security.EncryptionKey)) - } - - return nil -} - -// Config 应用配置结构体 -type Config struct { - App AppConfig `mapstructure:"app"` - Database DatabaseConfig `mapstructure:"database"` - JWT JWTConfig `mapstructure:"jwt"` - Log LogConfig `mapstructure:"log"` - CORS CORSConfig `mapstructure:"cors"` - Email EmailConfig `mapstructure:"email"` - External ExternalConfig `mapstructure:"external"` - Milvus MilvusConfig `mapstructure:"milvus"` - Security SecurityConfig `mapstructure:"security"` -} - -// MilvusConfig Milvus 向量数据库配置 -type MilvusConfig struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` -} - -// SecurityConfig 安全配置 -type SecurityConfig struct { - EncryptionKey string `mapstructure:"encryption_key"` // API Key 加密密钥,32字节 -} - -// GetAddress 返回 host:port 格式的地址 -func (c *MilvusConfig) GetAddress() string { - return fmt.Sprintf("%s:%d", c.Host, c.Port) -} - -// ExternalConfig 外部服务配置 -type ExternalConfig struct { - MarkItDown MarkItDownConfig `mapstructure:"markitdown"` - ASR ASRConfig `mapstructure:"asr"` - MinIO MinIOConfig `mapstructure:"minio"` - Youdao YoudaoConfig `mapstructure:"youdao"` - Bocha BochaConfig `mapstructure:"bocha"` -} - -// BochaConfig 博查联网搜索配置 -type BochaConfig struct { - BaseURL string `mapstructure:"base_url"` - Endpoint string `mapstructure:"endpoint"` - APIKey string `mapstructure:"api_key"` - TimeoutSeconds int `mapstructure:"timeout_seconds"` - DefaultCount int `mapstructure:"default_count"` - Summary bool `mapstructure:"summary"` - CacheTTLSeconds int `mapstructure:"cache_ttl_seconds"` - MaxCount int `mapstructure:"max_count"` -} - -// YoudaoConfig 有道云笔记 CLI 配置 -type YoudaoConfig struct { - CLIPath string `mapstructure:"cli_path"` // CLI 路径,默认 "youdaonote"(在 PATH 中) - ConverterScriptPath string `mapstructure:"converter_script_path"` // youdaonote-pull 转换脚本路径(可选,用于 .note 格式转换) - CookiesPath string `mapstructure:"cookies_path"` // youdaonote cookies 文件路径(可选,用于 .note 格式转换) -} - -// MarkItDownConfig 文档转换服务配置 -type MarkItDownConfig struct { - URL string `mapstructure:"url"` -} - -// ASRConfig ASR 语音转文本配置 -type ASRConfig struct { - Provider string `mapstructure:"provider"` - Params map[string]interface{} `mapstructure:"params"` -} - -// GetString 获取参数中的字符串值 -func (c *ASRConfig) GetString(key string) string { - if c == nil { - return "" - } - if v, ok := c.Params[key]; ok { - if s, ok := v.(string); ok { - return s - } - } - return "" -} - -// GetInt 获取参数中的整数值 -func (c *ASRConfig) GetInt(key string) int { - if c == nil { - return 0 - } - if v, ok := c.Params[key]; ok { - switch n := v.(type) { - case int: - return n - case int64: - return int(n) - case float64: - return int(n) - } - } - return 0 -} - -// MinIOConfig MinIO 对象存储配置 -type MinIOConfig struct { - Endpoint string `mapstructure:"endpoint"` - PublicEndpoint string `mapstructure:"public_endpoint"` - AccessKey string `mapstructure:"access_key"` - SecretKey string `mapstructure:"secret_key"` - Bucket string `mapstructure:"bucket"` -} - -// AppConfig 应用配置 -type AppConfig struct { - Name string `mapstructure:"name"` - Version string `mapstructure:"version"` - Mode string `mapstructure:"mode"` // debug, release, test - Port int `mapstructure:"port"` -} - -// DatabaseConfig 数据库配置 -type DatabaseConfig struct { - MySQL MySQLConfig `mapstructure:"mysql"` - Redis RedisConfig `mapstructure:"redis"` -} - -// MySQLConfig MySQL 配置 -type MySQLConfig struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - Database string `mapstructure:"database"` - MaxIdleConns int `mapstructure:"max_idle_conns"` - MaxOpenConns int `mapstructure:"max_open_conns"` -} - -// RedisConfig Redis 配置 -type RedisConfig struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - Password string `mapstructure:"password"` - DB int `mapstructure:"db"` - PoolSize int `mapstructure:"pool_size"` -} - -// JWTConfig JWT 配置 -type JWTConfig struct { - Secret string `mapstructure:"secret"` - ExpireHours time.Duration `mapstructure:"expire_hours"` - AccessTokenExp string `mapstructure:"access_token_exp"` - RefreshTokenExp string `mapstructure:"refresh_token_exp"` - Issuer string `mapstructure:"issuer"` -} - -// GetAccessTokenExp 获取 Access Token 过期时间 -func (c *JWTConfig) GetAccessTokenExp() time.Duration { - if c.AccessTokenExp != "" { - d, err := time.ParseDuration(c.AccessTokenExp) - if err == nil { - return d - } - } - // 默认 15 分钟 - return 15 * time.Minute -} - -// GetRefreshTokenExp 获取 Refresh Token 过期时间 -func (c *JWTConfig) GetRefreshTokenExp() time.Duration { - if c.RefreshTokenExp != "" { - d, err := time.ParseDuration(c.RefreshTokenExp) - if err == nil { - return d - } - } - // 默认 7 天 - return 7 * 24 * time.Hour -} - -// GetIssuer 获取签发者 -func (c *JWTConfig) GetIssuer() string { - if c.Issuer != "" { - return c.Issuer - } - return "youdaonotelm" -} - -// LogConfig 日志配置 -type LogConfig struct { - Level string `mapstructure:"level"` // debug, info, warn, error - Filename string `mapstructure:"filename"` // 日志文件路径 - MaxSize int `mapstructure:"max_size"` // 单个日志文件最大大小(MB) - MaxBackups int `mapstructure:"max_backups"` // 保留的旧日志文件数量 - MaxAge int `mapstructure:"max_age"` // 保留旧日志文件的最大天数 - Compress bool `mapstructure:"compress"` // 是否压缩 -} - -// CORSConfig CORS 配置 -type CORSConfig struct { - Enabled bool `mapstructure:"enabled"` - AllowOrigins []string `mapstructure:"allow_origins"` - AllowMethods []string `mapstructure:"allow_methods"` - AllowHeaders []string `mapstructure:"allow_headers"` - ExposeHeaders []string `mapstructure:"expose_headers"` - AllowCredentials bool `mapstructure:"allow_credentials"` - MaxAge int `mapstructure:"max_age"` -} - -// EmailConfig 邮箱配置 -type EmailConfig struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - From string `mapstructure:"from"` // 发件人地址,默认使用 Username -} +package config + +import ( + "fmt" + "time" +) + +// Validate 校验所有必填配置项,缺失或无效时返回 error。 +func (c *Config) Validate() error { + // App + if c.App.Name == "" { + return fmt.Errorf("app.name 不能为空") + } + if c.App.Port == 0 { + return fmt.Errorf("app.port 不能为空") + } + + // MySQL + if c.Database.MySQL.Host == "" { + return fmt.Errorf("database.mysql.host 不能为空") + } + if c.Database.MySQL.Port == 0 { + return fmt.Errorf("database.mysql.port 不能为空") + } + if c.Database.MySQL.Username == "" { + return fmt.Errorf("database.mysql.username 不能为空") + } + if c.Database.MySQL.Database == "" { + return fmt.Errorf("database.mysql.database 不能为空") + } + + // Redis + if c.Database.Redis.Host == "" { + return fmt.Errorf("database.redis.host 不能为空") + } + if c.Database.Redis.Port == 0 { + return fmt.Errorf("database.redis.port 不能为空") + } + + // JWT + if c.JWT.Secret == "" { + return fmt.Errorf("jwt.secret 不能为空") + } + + // Log + if c.Log.Filename == "" { + return fmt.Errorf("log.filename 不能为空") + } + + // Email + if c.Email.Host == "" { + return fmt.Errorf("email.host 不能为空") + } + if c.Email.Port == 0 { + return fmt.Errorf("email.port 不能为空") + } + if c.Email.Username == "" { + return fmt.Errorf("email.username 不能为空") + } + if c.Email.Password == "" { + return fmt.Errorf("email.password 不能为空") + } + + // Milvus + if c.Milvus.Host == "" { + return fmt.Errorf("milvus.host 不能为空") + } + if c.Milvus.Port == 0 { + return fmt.Errorf("milvus.port 不能为空") + } + + // External - MarkItDown + if c.External.MarkItDown.URL == "" { + return fmt.Errorf("external.markitdown.url 不能为空") + } + + // External - MinIO + if c.External.MinIO.Endpoint == "" { + return fmt.Errorf("external.minio.endpoint 不能为空") + } + if c.External.MinIO.AccessKey == "" { + return fmt.Errorf("external.minio.access_key 不能为空") + } + if c.External.MinIO.SecretKey == "" { + return fmt.Errorf("external.minio.secret_key 不能为空") + } + if c.External.MinIO.Bucket == "" { + return fmt.Errorf("external.minio.bucket 不能为空") + } + + // Security + if c.Security.EncryptionKey == "" { + return fmt.Errorf("security.encryption_key 不能为空") + } + if len(c.Security.EncryptionKey) != 32 { + return fmt.Errorf("security.encryption_key 必须为32字节,当前%d字节", len(c.Security.EncryptionKey)) + } + + return nil +} + +// Config 应用配置结构体 +type Config struct { + App AppConfig `mapstructure:"app"` + Database DatabaseConfig `mapstructure:"database"` + JWT JWTConfig `mapstructure:"jwt"` + Log LogConfig `mapstructure:"log"` + CORS CORSConfig `mapstructure:"cors"` + Email EmailConfig `mapstructure:"email"` + External ExternalConfig `mapstructure:"external"` + Milvus MilvusConfig `mapstructure:"milvus"` + Security SecurityConfig `mapstructure:"security"` + Agent AgentConfig `mapstructure:"agent"` +} + +// AgentConfig 搜索 Agent 运行参数(零值时使用代码内默认值,无需在 yaml 配置) +type AgentConfig struct { + MaxSearchRounds int `mapstructure:"max_search_rounds"` // 最大搜索轮数,默认 2 + MaxIterations int `mapstructure:"max_iterations"` // ReAct 最大迭代数,默认 4 + ExecuteTimeout time.Duration `mapstructure:"execute_timeout"` // 搜索超时,默认 3min + ExecuteWithImportTimeout time.Duration `mapstructure:"execute_with_import_timeout"` // 含导入的搜索超时,默认 5min + MaxConcurrent int `mapstructure:"max_concurrent"` // per-user 最大并发,默认 1 + CancelTimeout time.Duration `mapstructure:"cancel_timeout"` // 中断后等待安全点超时,默认 5s + MainAgentEnabled bool `mapstructure:"main_agent_enabled"` // 主从协同开关,默认 false(关闭时行为等价于现有) +} + +// MilvusConfig Milvus 向量数据库配置 +type MilvusConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` +} + +// SecurityConfig 安全配置 +type SecurityConfig struct { + EncryptionKey string `mapstructure:"encryption_key"` // API Key 加密密钥,32字节 +} + +// GetAddress 返回 host:port 格式的地址 +func (c *MilvusConfig) GetAddress() string { + return fmt.Sprintf("%s:%d", c.Host, c.Port) +} + +// ExternalConfig 外部服务配置 +type ExternalConfig struct { + MarkItDown MarkItDownConfig `mapstructure:"markitdown"` + ASR ASRConfig `mapstructure:"asr"` + MinIO MinIOConfig `mapstructure:"minio"` + Youdao YoudaoConfig `mapstructure:"youdao"` + Bocha BochaConfig `mapstructure:"bocha"` +} + +// BochaConfig 博查联网搜索配置 +type BochaConfig struct { + BaseURL string `mapstructure:"base_url"` + Endpoint string `mapstructure:"endpoint"` + APIKey string `mapstructure:"api_key"` + TimeoutSeconds int `mapstructure:"timeout_seconds"` + DefaultCount int `mapstructure:"default_count"` + Summary bool `mapstructure:"summary"` + CacheTTLSeconds int `mapstructure:"cache_ttl_seconds"` + MaxCount int `mapstructure:"max_count"` +} + +// YoudaoConfig 有道云笔记 CLI 配置 +type YoudaoConfig struct { + CLIPath string `mapstructure:"cli_path"` // CLI 路径,默认 "youdaonote"(在 PATH 中) + ConverterScriptPath string `mapstructure:"converter_script_path"` // youdaonote-pull 转换脚本路径(可选,用于 .note 格式转换) + CookiesPath string `mapstructure:"cookies_path"` // youdaonote cookies 文件路径(可选,用于 .note 格式转换) +} + +// MarkItDownConfig 文档转换服务配置 +type MarkItDownConfig struct { + URL string `mapstructure:"url"` +} + +// ASRConfig ASR 语音转文本配置 +type ASRConfig struct { + Provider string `mapstructure:"provider"` + Params map[string]interface{} `mapstructure:"params"` +} + +// GetString 获取参数中的字符串值 +func (c *ASRConfig) GetString(key string) string { + if c == nil { + return "" + } + if v, ok := c.Params[key]; ok { + if s, ok := v.(string); ok { + return s + } + } + return "" +} + +// GetInt 获取参数中的整数值 +func (c *ASRConfig) GetInt(key string) int { + if c == nil { + return 0 + } + if v, ok := c.Params[key]; ok { + switch n := v.(type) { + case int: + return n + case int64: + return int(n) + case float64: + return int(n) + } + } + return 0 +} + +// MinIOConfig MinIO 对象存储配置 +type MinIOConfig struct { + Endpoint string `mapstructure:"endpoint"` + PublicEndpoint string `mapstructure:"public_endpoint"` + AccessKey string `mapstructure:"access_key"` + SecretKey string `mapstructure:"secret_key"` + Bucket string `mapstructure:"bucket"` +} + +// AppConfig 应用配置 +type AppConfig struct { + Name string `mapstructure:"name"` + Version string `mapstructure:"version"` + Mode string `mapstructure:"mode"` // debug, release, test + Port int `mapstructure:"port"` +} + +// DatabaseConfig 数据库配置 +type DatabaseConfig struct { + MySQL MySQLConfig `mapstructure:"mysql"` + Redis RedisConfig `mapstructure:"redis"` +} + +// MySQLConfig MySQL 配置 +type MySQLConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` + MaxIdleConns int `mapstructure:"max_idle_conns"` + MaxOpenConns int `mapstructure:"max_open_conns"` +} + +// RedisConfig Redis 配置 +type RedisConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Password string `mapstructure:"password"` + DB int `mapstructure:"db"` + PoolSize int `mapstructure:"pool_size"` +} + +// JWTConfig JWT 配置 +type JWTConfig struct { + Secret string `mapstructure:"secret"` + ExpireHours time.Duration `mapstructure:"expire_hours"` + AccessTokenExp string `mapstructure:"access_token_exp"` + RefreshTokenExp string `mapstructure:"refresh_token_exp"` + Issuer string `mapstructure:"issuer"` +} + +// GetAccessTokenExp 获取 Access Token 过期时间 +func (c *JWTConfig) GetAccessTokenExp() time.Duration { + if c.AccessTokenExp != "" { + d, err := time.ParseDuration(c.AccessTokenExp) + if err == nil { + return d + } + } + // 默认 15 分钟 + return 15 * time.Minute +} + +// GetRefreshTokenExp 获取 Refresh Token 过期时间 +func (c *JWTConfig) GetRefreshTokenExp() time.Duration { + if c.RefreshTokenExp != "" { + d, err := time.ParseDuration(c.RefreshTokenExp) + if err == nil { + return d + } + } + // 默认 7 天 + return 7 * 24 * time.Hour +} + +// GetIssuer 获取签发者 +func (c *JWTConfig) GetIssuer() string { + if c.Issuer != "" { + return c.Issuer + } + return "youdaonotelm" +} + +// LogConfig 日志配置 +type LogConfig struct { + Level string `mapstructure:"level"` // debug, info, warn, error + Filename string `mapstructure:"filename"` // 日志文件路径 + MaxSize int `mapstructure:"max_size"` // 单个日志文件最大大小(MB) + MaxBackups int `mapstructure:"max_backups"` // 保留的旧日志文件数量 + MaxAge int `mapstructure:"max_age"` // 保留旧日志文件的最大天数 + Compress bool `mapstructure:"compress"` // 是否压缩 +} + +// CORSConfig CORS 配置 +type CORSConfig struct { + Enabled bool `mapstructure:"enabled"` + AllowOrigins []string `mapstructure:"allow_origins"` + AllowMethods []string `mapstructure:"allow_methods"` + AllowHeaders []string `mapstructure:"allow_headers"` + ExposeHeaders []string `mapstructure:"expose_headers"` + AllowCredentials bool `mapstructure:"allow_credentials"` + MaxAge int `mapstructure:"max_age"` +} + +// EmailConfig 邮箱配置 +type EmailConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + From string `mapstructure:"from"` // 发件人地址,默认使用 Username +} diff --git a/pkg/database/mysql.go b/pkg/database/mysql.go index 67804a3..4426e03 100644 --- a/pkg/database/mysql.go +++ b/pkg/database/mysql.go @@ -1,96 +1,96 @@ -package database - -import ( - "YoudaoNoteLm/pkg/config" - "YoudaoNoteLm/pkg/logger" - "fmt" - "time" - - "go.uber.org/zap" - "gorm.io/driver/mysql" - "gorm.io/gorm" - gormlogger "gorm.io/gorm/logger" -) - -var mysqlDB *gorm.DB - -// InitMySQL 初始化 MySQL 连接 -func InitMySQL(cfg *config.MySQLConfig) (*gorm.DB, error) { - // 构建 DSN - dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", - cfg.Username, - cfg.Password, - cfg.Host, - cfg.Port, - cfg.Database, - ) - - // GORM 配置 - gormConfig := &gorm.Config{ - // 禁用外键约束(应用层手动管理级联删除,不依赖数据库外键) - DisableForeignKeyConstraintWhenMigrating: true, - // 跳过默认事务 - SkipDefaultTransaction: true, - } - - // 根据应用模式设置日志级别 - if config.Get().App.Mode == "debug" { - gormConfig.Logger = gormlogger.Default.LogMode(gormlogger.Info) - } else { - gormConfig.Logger = gormlogger.Default.LogMode(gormlogger.Silent) - } - - // 连接数据库 - db, err := gorm.Open(mysql.Open(dsn), gormConfig) - if err != nil { - logger.Error("MySQL 连接失败", zap.Error(err)) - return nil, fmt.Errorf("MySQL 连接失败: %w", err) - } - - // 获取底层 sql.DB - sqlDB, err := db.DB() - if err != nil { - logger.Error("获取 SQL DB 失败", zap.Error(err)) - return nil, fmt.Errorf("获取 SQL DB 失败: %w", err) - } - - // 设置连接池 - sqlDB.SetMaxIdleConns(cfg.MaxIdleConns) - sqlDB.SetMaxOpenConns(cfg.MaxOpenConns) - sqlDB.SetConnMaxLifetime(time.Hour) - - // 测试连接 - if err := sqlDB.Ping(); err != nil { - logger.Error("MySQL Ping 失败", zap.Error(err)) - return nil, fmt.Errorf("MySQL Ping 失败: %w", err) - } - - mysqlDB = db - logger.Info("MySQL 连接成功", - zap.String("host", cfg.Host), - zap.Int("port", cfg.Port), - zap.String("database", cfg.Database), - ) - - return db, nil -} - -// GetMySQL 获取 MySQL 实例 -func GetMySQL() *gorm.DB { - if mysqlDB == nil { - panic("MySQL 未初始化") - } - return mysqlDB -} - -// CloseMySQL 关闭 MySQL 连接 -func CloseMySQL() error { - if mysqlDB != nil { - sqlDB, err := mysqlDB.DB() - if err != nil { - return err - } - return sqlDB.Close() - } - return nil -} +package database + +import ( + "YoudaoNoteLm/pkg/config" + "YoudaoNoteLm/pkg/logger" + "fmt" + "time" + + "go.uber.org/zap" + "gorm.io/driver/mysql" + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" +) + +var mysqlDB *gorm.DB + +// InitMySQL 初始化 MySQL 连接 +func InitMySQL(cfg *config.MySQLConfig) (*gorm.DB, error) { + // 构建 DSN + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", + cfg.Username, + cfg.Password, + cfg.Host, + cfg.Port, + cfg.Database, + ) + + // GORM 配置 + gormConfig := &gorm.Config{ + // 禁用外键约束(应用层手动管理级联删除,不依赖数据库外键) + DisableForeignKeyConstraintWhenMigrating: true, + // 跳过默认事务 + SkipDefaultTransaction: true, + } + + // 根据应用模式设置日志级别 + if config.Get().App.Mode == "debug" { + gormConfig.Logger = gormlogger.Default.LogMode(gormlogger.Info) + } else { + gormConfig.Logger = gormlogger.Default.LogMode(gormlogger.Silent) + } + + // 连接数据库 + db, err := gorm.Open(mysql.Open(dsn), gormConfig) + if err != nil { + logger.Error("MySQL 连接失败", zap.Error(err)) + return nil, fmt.Errorf("MySQL 连接失败: %w", err) + } + + // 获取底层 sql.DB + sqlDB, err := db.DB() + if err != nil { + logger.Error("获取 SQL DB 失败", zap.Error(err)) + return nil, fmt.Errorf("获取 SQL DB 失败: %w", err) + } + + // 设置连接池 + sqlDB.SetMaxIdleConns(cfg.MaxIdleConns) + sqlDB.SetMaxOpenConns(cfg.MaxOpenConns) + sqlDB.SetConnMaxLifetime(time.Hour) + + // 测试连接 + if err := sqlDB.Ping(); err != nil { + logger.Error("MySQL Ping 失败", zap.Error(err)) + return nil, fmt.Errorf("MySQL Ping 失败: %w", err) + } + + mysqlDB = db + logger.Info("MySQL 连接成功", + zap.String("host", cfg.Host), + zap.Int("port", cfg.Port), + zap.String("database", cfg.Database), + ) + + return db, nil +} + +// GetMySQL 获取 MySQL 实例 +func GetMySQL() *gorm.DB { + if mysqlDB == nil { + panic("MySQL 未初始化") + } + return mysqlDB +} + +// CloseMySQL 关闭 MySQL 连接 +func CloseMySQL() error { + if mysqlDB != nil { + sqlDB, err := mysqlDB.DB() + if err != nil { + return err + } + return sqlDB.Close() + } + return nil +} diff --git a/pkg/database/redis.go b/pkg/database/redis.go index 84bb14b..ef3e8e3 100644 --- a/pkg/database/redis.go +++ b/pkg/database/redis.go @@ -1,57 +1,57 @@ -package database - -import ( - "YoudaoNoteLm/pkg/config" - "YoudaoNoteLm/pkg/logger" - "context" - "fmt" - "time" - - "github.com/redis/go-redis/v9" - "go.uber.org/zap" -) - -var redisClient *redis.Client - -// InitRedis 初始化 Redis 连接 -func InitRedis(cfg *config.RedisConfig) (*redis.Client, error) { - client := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), - Password: cfg.Password, - DB: cfg.DB, - PoolSize: cfg.PoolSize, - }) - - // 测试连接 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - if err := client.Ping(ctx).Err(); err != nil { - logger.Error("Redis 连接失败", zap.Error(err)) - return nil, fmt.Errorf("Redis 连接失败: %w", err) - } - - redisClient = client - logger.Info("Redis 连接成功", - zap.String("addr", fmt.Sprintf("%s:%d", cfg.Host, cfg.Port)), - zap.Int("db", cfg.DB), - ) - - return client, nil -} - -// GetRedis 获取 Redis 实例 -func GetRedis() *redis.Client { - if redisClient == nil { - panic("Redis 未初始化") - } - return redisClient -} - -// CloseRedis 关闭 Redis 连接 -func CloseRedis() error { - if redisClient != nil { - return redisClient.Close() - } - return nil -} +package database + +import ( + "YoudaoNoteLm/pkg/config" + "YoudaoNoteLm/pkg/logger" + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" + "go.uber.org/zap" +) + +var redisClient *redis.Client + +// InitRedis 初始化 Redis 连接 +func InitRedis(cfg *config.RedisConfig) (*redis.Client, error) { + client := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), + Password: cfg.Password, + DB: cfg.DB, + PoolSize: cfg.PoolSize, + }) + + // 测试连接 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := client.Ping(ctx).Err(); err != nil { + logger.Error("Redis 连接失败", zap.Error(err)) + return nil, fmt.Errorf("Redis 连接失败: %w", err) + } + + redisClient = client + logger.Info("Redis 连接成功", + zap.String("addr", fmt.Sprintf("%s:%d", cfg.Host, cfg.Port)), + zap.Int("db", cfg.DB), + ) + + return client, nil +} + +// GetRedis 获取 Redis 实例 +func GetRedis() *redis.Client { + if redisClient == nil { + panic("Redis 未初始化") + } + return redisClient +} + +// CloseRedis 关闭 Redis 连接 +func CloseRedis() error { + if redisClient != nil { + return redisClient.Close() + } + return nil +} diff --git a/pkg/eino/web_search_runner.go b/pkg/eino/web_search_runner.go index dd0fb6f..5cc0baa 100644 --- a/pkg/eino/web_search_runner.go +++ b/pkg/eino/web_search_runner.go @@ -1,75 +1,75 @@ -package eino - -import ( - "context" - "encoding/json" - "fmt" - - "YoudaoNoteLm/internal/service" - bizerrors "YoudaoNoteLm/pkg/errors" - - "github.com/bytedance/sonic" - "github.com/cloudwego/eino/components/tool" - "github.com/cloudwego/eino/compose" - "github.com/cloudwego/eino/schema" -) - -// WebSearchRunner 通过 Eino ToolsNode 调用统一搜索工具。 -type WebSearchRunner struct { - toolNode *compose.ToolsNode -} - -// NewWebSearchRunner 创建基于 Eino 的搜索执行器。 -func NewWebSearchRunner(ctx context.Context, searchService service.SearchService) (*WebSearchRunner, error) { - searchTool, err := NewWebSearchTool(searchService) - if err != nil { - return nil, fmt.Errorf("create web search tool failed: %w", err) - } - - toolNode, err := compose.NewToolNode(ctx, &compose.ToolsNodeConfig{ - Tools: []tool.BaseTool{searchTool}, - }) - if err != nil { - return nil, fmt.Errorf("create tool node failed: %w", err) - } - - return &WebSearchRunner{toolNode: toolNode}, nil -} - -// Search 通过 Eino ToolsNode 执行联网搜索。 -func (r *WebSearchRunner) Search(ctx context.Context, req *service.SearchRequest) (*service.SearchResponse, error) { - if r == nil || r.toolNode == nil { - return nil, bizerrors.New(bizerrors.CodeInternalServiceError, "搜索执行器未初始化") - } - if req == nil { - return nil, bizerrors.New(bizerrors.CodeInvalidParam, "搜索请求不能为空") - } - - arguments, err := json.Marshal(req) - if err != nil { - return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "序列化搜索参数失败", err) - } - - messages, err := r.toolNode.Invoke(ctx, schema.AssistantMessage("", []schema.ToolCall{ - { - ID: "web-search-call-1", - Type: "function", - Function: schema.FunctionCall{ - Name: webSearchToolName, - Arguments: string(arguments), - }, - }, - })) - if err != nil { - return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "Eino 搜索编排执行失败", err) - } - if len(messages) == 0 { - return nil, bizerrors.New(bizerrors.CodeInternalServiceError, "Eino 搜索编排未返回结果") - } - - var resp service.SearchResponse - if err := sonic.UnmarshalString(messages[0].Content, &resp); err != nil { - return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "解析 Eino 搜索结果失败", err) - } - return &resp, nil -} +package eino + +import ( + "context" + "encoding/json" + "fmt" + + "YoudaoNoteLm/internal/service" + bizerrors "YoudaoNoteLm/pkg/errors" + + "github.com/bytedance/sonic" + "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/schema" +) + +// WebSearchRunner 通过 Eino ToolsNode 调用统一搜索工具。 +type WebSearchRunner struct { + toolNode *compose.ToolsNode +} + +// NewWebSearchRunner 创建基于 Eino 的搜索执行器。 +func NewWebSearchRunner(ctx context.Context, searchService service.SearchService) (*WebSearchRunner, error) { + searchTool, err := NewWebSearchTool(searchService) + if err != nil { + return nil, fmt.Errorf("create web search tool failed: %w", err) + } + + toolNode, err := compose.NewToolNode(ctx, &compose.ToolsNodeConfig{ + Tools: []tool.BaseTool{searchTool}, + }) + if err != nil { + return nil, fmt.Errorf("create tool node failed: %w", err) + } + + return &WebSearchRunner{toolNode: toolNode}, nil +} + +// Search 通过 Eino ToolsNode 执行联网搜索。 +func (r *WebSearchRunner) Search(ctx context.Context, req *service.SearchRequest) (*service.SearchResponse, error) { + if r == nil || r.toolNode == nil { + return nil, bizerrors.New(bizerrors.CodeInternalServiceError, "搜索执行器未初始化") + } + if req == nil { + return nil, bizerrors.New(bizerrors.CodeInvalidParam, "搜索请求不能为空") + } + + arguments, err := json.Marshal(req) + if err != nil { + return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "序列化搜索参数失败", err) + } + + messages, err := r.toolNode.Invoke(ctx, schema.AssistantMessage("", []schema.ToolCall{ + { + ID: "web-search-call-1", + Type: "function", + Function: schema.FunctionCall{ + Name: webSearchToolName, + Arguments: string(arguments), + }, + }, + })) + if err != nil { + return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "Eino 搜索编排执行失败", err) + } + if len(messages) == 0 { + return nil, bizerrors.New(bizerrors.CodeInternalServiceError, "Eino 搜索编排未返回结果") + } + + var resp service.SearchResponse + if err := sonic.UnmarshalString(messages[0].Content, &resp); err != nil { + return nil, bizerrors.NewWithErr(bizerrors.CodeInternalServiceError, "解析 Eino 搜索结果失败", err) + } + return &resp, nil +} diff --git a/pkg/eino/web_search_tool.go b/pkg/eino/web_search_tool.go index 5eab954..6c1a610 100644 --- a/pkg/eino/web_search_tool.go +++ b/pkg/eino/web_search_tool.go @@ -1,23 +1,23 @@ -package eino - -import ( - "context" - - "YoudaoNoteLm/internal/service" - - "github.com/cloudwego/eino/components/tool" - toolutils "github.com/cloudwego/eino/components/tool/utils" -) - -const webSearchToolName = "web_search" - -// NewWebSearchTool 将统一搜索服务封装为 Eino Tool。 -func NewWebSearchTool(searchService service.SearchService) (tool.InvokableTool, error) { - return toolutils.InferTool[service.SearchRequest, *service.SearchResponse]( - webSearchToolName, - "通过项目内基于 Bocha 的统一搜索服务执行联网搜索。", - func(ctx context.Context, input service.SearchRequest) (*service.SearchResponse, error) { - return searchService.Search(ctx, &input) - }, - ) -} +package eino + +import ( + "context" + + "YoudaoNoteLm/internal/service" + + "github.com/cloudwego/eino/components/tool" + toolutils "github.com/cloudwego/eino/components/tool/utils" +) + +const webSearchToolName = "web_search" + +// NewWebSearchTool 将统一搜索服务封装为 Eino Tool。 +func NewWebSearchTool(searchService service.SearchService) (tool.InvokableTool, error) { + return toolutils.InferTool[service.SearchRequest, *service.SearchResponse]( + webSearchToolName, + "通过项目内基于 Bocha 的统一搜索服务执行联网搜索。", + func(ctx context.Context, input service.SearchRequest) (*service.SearchResponse, error) { + return searchService.Search(ctx, &input) + }, + ) +} diff --git a/pkg/errors/code.go b/pkg/errors/code.go index c6f1355..246814c 100644 --- a/pkg/errors/code.go +++ b/pkg/errors/code.go @@ -1,139 +1,149 @@ -package errors - -// 错误码定义 -const ( - // 成功 - CodeSuccess = 0 - - // 通用错误 4xx - CodeBadRequest = 400 - CodeUnauthorized = 401 - CodeForbidden = 403 - CodeNotFound = 404 - CodeMethodNotAllowed = 405 - CodeRequestTimeout = 408 - CodeConflict = 409 - - // 服务器错误 5xx - CodeInternalError = 500 - CodeNotImplemented = 501 - CodeServiceUnavailable = 503 - - // 业务错误 1xxx - CodeUserNotFound = 1001 - CodeUserAlreadyExists = 1002 - CodeInvalidCredentials = 1003 - CodeUserDisabled = 1004 - CodeInvalidToken = 1005 - CodeTokenExpired = 1006 - CodeUserLocked = 1007 - - // 验证码错误 11xx - CodeVerifyCodeExpired = 1101 - CodeVerifyCodeInvalid = 1102 - CodeVerifyCodeLocked = 1103 - CodeVerifyCodeTooFrequent = 1104 - - // 参数错误 2xxx - CodeInvalidParam = 2001 - CodeMissingParam = 2002 - CodeParamFormatError = 2003 - - // 资源错误 3xxx - CodeResourceNotFound = 3001 - CodeResourceAlreadyExists = 3002 - CodeResourceLocked = 3003 - - // 导入模块错误 4xxxx - CodeUnsupportedFormat = 40001 - CodeFileTooLarge = 40002 - CodeFileParseFailed = 40003 - CodeWebScrapeFailed = 40004 - CodeASTranscriptionFailed = 40005 - CodeSearchQuotaExhausted = 40006 - CodeInvalidYoudaoAPIKey = 40007 - CodeDuplicateImport = 40008 - CodePreviewExpired = 40009 - CodeSearchProviderNotConfigured = 40010 - CodeSearchInvalidAPIKey = 40011 - CodeSearchRequestTimeout = 40012 - CodeSearchProviderUnavailable = 40013 - CodeSearchInvalidResponse = 40014 - CodeSearchProviderEmptyResult = 40015 - CodeSearchNormalizedEmptyResult = 40016 - - // LLM / Agent 错误码 4002x - CodeLLMNotConfigured = 40020 - CodeLLMCallFailed = 40021 - CodeLLMResponseInvalid = 40022 - CodeSearchAgentTimeout = 40023 - - // 服务器错误 5xxxx - CodeInternalServiceError = 50001 - - // 配置健康检查错误 6xxxx - CodeConfigTestFailed = 60001 // 配置连通性测试失败 - CodeConfigTestTimeout = 60002 // 配置连通性测试超时 - CodeConfigTestInvalid = 60003 // 配置参数无效 -) - -// 错误码默认消息 -var codeMessages = map[int]string{ - CodeSuccess: "成功", - CodeBadRequest: "请求参数错误", - CodeUnauthorized: "未授权", - CodeForbidden: "禁止访问", - CodeNotFound: "资源不存在", - CodeMethodNotAllowed: "方法不允许", - CodeRequestTimeout: "请求超时", - CodeConflict: "资源冲突", - CodeInternalError: "服务器内部错误", - CodeNotImplemented: "功能未实现", - CodeServiceUnavailable: "服务不可用", - CodeUserNotFound: "用户不存在", - CodeUserAlreadyExists: "用户已存在", - CodeInvalidCredentials: "邮箱或密码错误", - CodeUserDisabled: "用户已被禁用", - CodeUserLocked: "账户已被锁定,请15分钟后重试", - CodeInvalidToken: "无效的令牌", - CodeTokenExpired: "令牌已过期", - CodeVerifyCodeExpired: "验证码已过期,请重新获取", - CodeVerifyCodeInvalid: "验证码错误", - CodeVerifyCodeLocked: "验证码输入错误次数过多,请重新获取", - CodeVerifyCodeTooFrequent: "验证码发送过于频繁,请60秒后重试", - CodeInvalidParam: "参数错误", - CodeMissingParam: "缺少必要参数", - CodeParamFormatError: "参数格式错误", - CodeResourceNotFound: "资源不存在", - CodeResourceAlreadyExists: "资源已存在", - CodeResourceLocked: "资源已被锁定", - - CodeUnsupportedFormat: "不支持的文件格式", - CodeFileTooLarge: "文件大小超限", - CodeFileParseFailed: "文件解析失败", - CodeWebScrapeFailed: "网页抓取失败", - CodeASTranscriptionFailed: "音频转写失败", - CodeSearchQuotaExhausted: "搜索API配额耗尽", - CodeInvalidYoudaoAPIKey: "有道API Key无效", - CodeDuplicateImport: "重复导入", - CodePreviewExpired: "预览已过期", - CodeLLMNotConfigured: "请先在设置中配置 LLM 服务", - CodeLLMCallFailed: "LLM 服务调用失败", - CodeLLMResponseInvalid: "LLM 返回结果格式异常", - CodeSearchAgentTimeout: "搜索 Agent 执行超时", - CodeInternalServiceError: "内部服务错误", - CodeConfigTestFailed: "配置连通性测试失败", - CodeConfigTestTimeout: "配置连通性测试超时", - CodeConfigTestInvalid: "配置参数无效", - CodeSearchInvalidResponse: "搜索 Provider 返回结构异常", - CodeSearchProviderEmptyResult: "搜索未返回结果", - CodeSearchNormalizedEmptyResult: "搜索结果清洗后为空", -} - -// GetMessage 获取错误码消息 -func GetMessage(code int) string { - if msg, ok := codeMessages[code]; ok { - return msg - } - return "未知错误" -} +package errors + +// 错误码定义 +const ( + // 成功 + CodeSuccess = 0 + + // 通用错误 4xx + CodeBadRequest = 400 + CodeUnauthorized = 401 + CodeForbidden = 403 + CodeNotFound = 404 + CodeMethodNotAllowed = 405 + CodeRequestTimeout = 408 + CodeConflict = 409 + + // 服务器错误 5xx + CodeInternalError = 500 + CodeNotImplemented = 501 + CodeServiceUnavailable = 503 + + // 业务错误 1xxx + CodeUserNotFound = 1001 + CodeUserAlreadyExists = 1002 + CodeInvalidCredentials = 1003 + CodeUserDisabled = 1004 + CodeInvalidToken = 1005 + CodeTokenExpired = 1006 + CodeUserLocked = 1007 + + // 验证码错误 11xx + CodeVerifyCodeExpired = 1101 + CodeVerifyCodeInvalid = 1102 + CodeVerifyCodeLocked = 1103 + CodeVerifyCodeTooFrequent = 1104 + + // 参数错误 2xxx + CodeInvalidParam = 2001 + CodeMissingParam = 2002 + CodeParamFormatError = 2003 + + // 资源错误 3xxx + CodeResourceNotFound = 3001 + CodeResourceAlreadyExists = 3002 + CodeResourceLocked = 3003 + + // 导入模块错误 4xxxx + CodeUnsupportedFormat = 40001 + CodeFileTooLarge = 40002 + CodeFileParseFailed = 40003 + CodeWebScrapeFailed = 40004 + CodeASTranscriptionFailed = 40005 + CodeSearchQuotaExhausted = 40006 + CodeInvalidYoudaoAPIKey = 40007 + CodeDuplicateImport = 40008 + CodePreviewExpired = 40009 + CodeSearchProviderNotConfigured = 40010 + CodeSearchInvalidAPIKey = 40011 + CodeSearchRequestTimeout = 40012 + CodeSearchProviderUnavailable = 40013 + CodeSearchInvalidResponse = 40014 + CodeSearchProviderEmptyResult = 40015 + CodeSearchNormalizedEmptyResult = 40016 + + // LLM / Agent 错误码 4002x + CodeLLMNotConfigured = 40020 + CodeLLMCallFailed = 40021 + CodeLLMResponseInvalid = 40022 + CodeSearchAgentTimeout = 40023 + + // 其他服务未配置错误码 4003x + CodeEmbeddingNotConfigured = 40030 + CodeASRNotConfigured = 40031 + + // 服务器错误 5xxxx + CodeInternalServiceError = 50001 + + // 配置健康检查错误 6xxxx + CodeConfigTestFailed = 60001 // 配置连通性测试失败 + CodeConfigTestTimeout = 60002 // 配置连通性测试超时 + CodeConfigTestInvalid = 60003 // 配置参数无效 +) + +// 错误码默认消息 +var codeMessages = map[int]string{ + CodeSuccess: "成功", + CodeBadRequest: "请求参数错误", + CodeUnauthorized: "未授权", + CodeForbidden: "禁止访问", + CodeNotFound: "资源不存在", + CodeMethodNotAllowed: "方法不允许", + CodeRequestTimeout: "请求超时", + CodeConflict: "资源冲突", + CodeInternalError: "服务器内部错误", + CodeNotImplemented: "功能未实现", + CodeServiceUnavailable: "服务不可用", + CodeUserNotFound: "用户不存在", + CodeUserAlreadyExists: "用户已存在", + CodeInvalidCredentials: "邮箱或密码错误", + CodeUserDisabled: "用户已被禁用", + CodeUserLocked: "账户已被锁定,请15分钟后重试", + CodeInvalidToken: "无效的令牌", + CodeTokenExpired: "令牌已过期", + CodeVerifyCodeExpired: "验证码已过期,请重新获取", + CodeVerifyCodeInvalid: "验证码错误", + CodeVerifyCodeLocked: "验证码输入错误次数过多,请重新获取", + CodeVerifyCodeTooFrequent: "验证码发送过于频繁,请60秒后重试", + CodeInvalidParam: "参数错误", + CodeMissingParam: "缺少必要参数", + CodeParamFormatError: "参数格式错误", + CodeResourceNotFound: "资源不存在", + CodeResourceAlreadyExists: "资源已存在", + CodeResourceLocked: "资源已被锁定", + + CodeUnsupportedFormat: "不支持的文件格式", + CodeFileTooLarge: "文件大小超限", + CodeFileParseFailed: "文件解析失败", + CodeWebScrapeFailed: "网页抓取失败", + CodeASTranscriptionFailed: "音频转写失败", + CodeSearchQuotaExhausted: "搜索API配额耗尽", + CodeInvalidYoudaoAPIKey: "有道API Key无效", + CodeDuplicateImport: "重复导入", + CodePreviewExpired: "预览已过期", + CodeSearchProviderNotConfigured: "请先在设置中配置搜索引擎服务", + CodeSearchInvalidAPIKey: "搜索引擎 API Key 无效", + CodeSearchRequestTimeout: "搜索请求超时", + CodeSearchProviderUnavailable: "搜索引擎服务暂不可用", + CodeLLMNotConfigured: "请先在设置中配置 LLM 服务", + CodeLLMCallFailed: "LLM 服务调用失败", + CodeLLMResponseInvalid: "LLM 返回结果格式异常", + CodeSearchAgentTimeout: "搜索 Agent 执行超时", + CodeEmbeddingNotConfigured: "请先在设置中配置 Embedding 服务", + CodeASRNotConfigured: "请先在设置中配置 ASR 语音识别服务", + CodeInternalServiceError: "内部服务错误", + CodeConfigTestFailed: "配置连通性测试失败", + CodeConfigTestTimeout: "配置连通性测试超时", + CodeConfigTestInvalid: "配置参数无效", + CodeSearchInvalidResponse: "搜索 Provider 返回结构异常", + CodeSearchProviderEmptyResult: "搜索未返回结果", + CodeSearchNormalizedEmptyResult: "搜索结果清洗后为空", +} + +// GetMessage 获取错误码消息 +func GetMessage(code int) string { + if msg, ok := codeMessages[code]; ok { + return msg + } + return "未知错误" +} diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 00a3fab..188cb33 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -1,92 +1,96 @@ -package errors - -import "fmt" - -// BizError 业务错误 -type BizError struct { - Code int - Message string - Err error -} - -// Error 实现 error 接口 -func (e *BizError) Error() string { - if e.Err != nil { - return fmt.Sprintf("[%d] %s: %v", e.Code, e.Message, e.Err) - } - return fmt.Sprintf("[%d] %s", e.Code, e.Message) -} - -// New 创建新的业务错误 -func New(code int, message string) *BizError { - return &BizError{ - Code: code, - Message: message, - } -} - -// NewWithErr 创建带原始错误的业务错误 -func NewWithErr(code int, message string, err error) *BizError { - return &BizError{ - Code: code, - Message: message, - Err: err, - } -} - -// NewDefault 创建使用默认消息的业务错误 -func NewDefault(code int) *BizError { - return &BizError{ - Code: code, - Message: GetMessage(code), - } -} - -// 预定义常用错误 -var ( - ErrBadRequest = NewDefault(CodeBadRequest) - ErrUnauthorized = NewDefault(CodeUnauthorized) - ErrForbidden = NewDefault(CodeForbidden) - ErrNotFound = NewDefault(CodeNotFound) - ErrInternalError = NewDefault(CodeInternalError) - ErrUserNotFound = NewDefault(CodeUserNotFound) - ErrUserAlreadyExists = NewDefault(CodeUserAlreadyExists) - ErrUserDisabled = NewDefault(CodeUserDisabled) - ErrUserLocked = NewDefault(CodeUserLocked) - ErrInvalidCredentials = NewDefault(CodeInvalidCredentials) - ErrInvalidToken = NewDefault(CodeInvalidToken) - ErrTokenExpired = NewDefault(CodeTokenExpired) - ErrInvalidParam = NewDefault(CodeInvalidParam) - ErrMissingParam = NewDefault(CodeMissingParam) - - // 验证码相关错误 - ErrVerifyCodeExpired = NewDefault(CodeVerifyCodeExpired) - ErrVerifyCodeInvalid = NewDefault(CodeVerifyCodeInvalid) - ErrVerifyCodeLocked = NewDefault(CodeVerifyCodeLocked) - ErrVerifyCodeTooFrequent = NewDefault(CodeVerifyCodeTooFrequent) - - ErrUnsupportedFormat = NewDefault(CodeUnsupportedFormat) - ErrFileTooLarge = NewDefault(CodeFileTooLarge) - ErrFileParseFailed = NewDefault(CodeFileParseFailed) - ErrWebScrapeFailed = NewDefault(CodeWebScrapeFailed) - ErrASTranscriptionFailed = NewDefault(CodeASTranscriptionFailed) - ErrSearchQuotaExhausted = NewDefault(CodeSearchQuotaExhausted) - ErrSearchProviderNotConfigured = NewDefault(CodeSearchProviderNotConfigured) - ErrSearchInvalidAPIKey = NewDefault(CodeSearchInvalidAPIKey) - ErrSearchInvalidResponse = NewDefault(CodeSearchInvalidResponse) - ErrInvalidYoudaoAPIKey = NewDefault(CodeInvalidYoudaoAPIKey) - ErrDuplicateImport = NewDefault(CodeDuplicateImport) - ErrPreviewExpired = NewDefault(CodePreviewExpired) - ErrSearchRequestTimeout = NewDefault(CodeSearchRequestTimeout) - ErrSearchProviderUnavailable = NewDefault(CodeSearchProviderUnavailable) - ErrSearchProviderEmptyResult = NewDefault(CodeSearchProviderEmptyResult) - ErrSearchNormalizedEmptyResult = NewDefault(CodeSearchNormalizedEmptyResult) - - // 搜索 Agent 相关错误 - ErrLLMNotConfigured = NewDefault(CodeLLMNotConfigured) - ErrLLMCallFailed = NewDefault(CodeLLMCallFailed) - ErrLLMResponseInvalid = NewDefault(CodeLLMResponseInvalid) - ErrSearchAgentTimeout = NewDefault(CodeSearchAgentTimeout) - - ErrInternalServiceError = NewDefault(CodeInternalServiceError) -) +package errors + +import "fmt" + +// BizError 业务错误 +type BizError struct { + Code int + Message string + Err error +} + +// Error 实现 error 接口 +func (e *BizError) Error() string { + if e.Err != nil { + return fmt.Sprintf("[%d] %s: %v", e.Code, e.Message, e.Err) + } + return fmt.Sprintf("[%d] %s", e.Code, e.Message) +} + +// New 创建新的业务错误 +func New(code int, message string) *BizError { + return &BizError{ + Code: code, + Message: message, + } +} + +// NewWithErr 创建带原始错误的业务错误 +func NewWithErr(code int, message string, err error) *BizError { + return &BizError{ + Code: code, + Message: message, + Err: err, + } +} + +// NewDefault 创建使用默认消息的业务错误 +func NewDefault(code int) *BizError { + return &BizError{ + Code: code, + Message: GetMessage(code), + } +} + +// 预定义常用错误 +var ( + ErrBadRequest = NewDefault(CodeBadRequest) + ErrUnauthorized = NewDefault(CodeUnauthorized) + ErrForbidden = NewDefault(CodeForbidden) + ErrNotFound = NewDefault(CodeNotFound) + ErrInternalError = NewDefault(CodeInternalError) + ErrUserNotFound = NewDefault(CodeUserNotFound) + ErrUserAlreadyExists = NewDefault(CodeUserAlreadyExists) + ErrUserDisabled = NewDefault(CodeUserDisabled) + ErrUserLocked = NewDefault(CodeUserLocked) + ErrInvalidCredentials = NewDefault(CodeInvalidCredentials) + ErrInvalidToken = NewDefault(CodeInvalidToken) + ErrTokenExpired = NewDefault(CodeTokenExpired) + ErrInvalidParam = NewDefault(CodeInvalidParam) + ErrMissingParam = NewDefault(CodeMissingParam) + + // 验证码相关错误 + ErrVerifyCodeExpired = NewDefault(CodeVerifyCodeExpired) + ErrVerifyCodeInvalid = NewDefault(CodeVerifyCodeInvalid) + ErrVerifyCodeLocked = NewDefault(CodeVerifyCodeLocked) + ErrVerifyCodeTooFrequent = NewDefault(CodeVerifyCodeTooFrequent) + + ErrUnsupportedFormat = NewDefault(CodeUnsupportedFormat) + ErrFileTooLarge = NewDefault(CodeFileTooLarge) + ErrFileParseFailed = NewDefault(CodeFileParseFailed) + ErrWebScrapeFailed = NewDefault(CodeWebScrapeFailed) + ErrASTranscriptionFailed = NewDefault(CodeASTranscriptionFailed) + ErrSearchQuotaExhausted = NewDefault(CodeSearchQuotaExhausted) + ErrSearchProviderNotConfigured = NewDefault(CodeSearchProviderNotConfigured) + ErrSearchInvalidAPIKey = NewDefault(CodeSearchInvalidAPIKey) + ErrSearchInvalidResponse = NewDefault(CodeSearchInvalidResponse) + ErrInvalidYoudaoAPIKey = NewDefault(CodeInvalidYoudaoAPIKey) + ErrDuplicateImport = NewDefault(CodeDuplicateImport) + ErrPreviewExpired = NewDefault(CodePreviewExpired) + ErrSearchRequestTimeout = NewDefault(CodeSearchRequestTimeout) + ErrSearchProviderUnavailable = NewDefault(CodeSearchProviderUnavailable) + ErrSearchProviderEmptyResult = NewDefault(CodeSearchProviderEmptyResult) + ErrSearchNormalizedEmptyResult = NewDefault(CodeSearchNormalizedEmptyResult) + + // 搜索 Agent 相关错误 + ErrLLMNotConfigured = NewDefault(CodeLLMNotConfigured) + ErrLLMCallFailed = NewDefault(CodeLLMCallFailed) + ErrLLMResponseInvalid = NewDefault(CodeLLMResponseInvalid) + ErrSearchAgentTimeout = NewDefault(CodeSearchAgentTimeout) + + // 其他服务未配置 + ErrEmbeddingNotConfigured = NewDefault(CodeEmbeddingNotConfigured) + ErrASRNotConfigured = NewDefault(CodeASRNotConfigured) + + ErrInternalServiceError = NewDefault(CodeInternalServiceError) +) diff --git a/pkg/jwt/claims.go b/pkg/jwt/claims.go index be7e18e..6d38629 100644 --- a/pkg/jwt/claims.go +++ b/pkg/jwt/claims.go @@ -1,34 +1,34 @@ -package jwt - -import "github.com/golang-jwt/jwt/v5" - -// TokenType token 类型 -type TokenType string - -const ( - AccessToken TokenType = "access" - RefreshToken TokenType = "refresh" -) - -// CustomClaims 自定义 JWT Claims -type CustomClaims struct { - UserID uint `json:"user_id"` - Username string `json:"username"` - TokenType TokenType `json:"token_type"` - jwt.RegisteredClaims -} - -// GetUserID 获取用户 ID -func (c *CustomClaims) GetUserID() uint { - return c.UserID -} - -// GetUsername 获取用户名 -func (c *CustomClaims) GetUsername() string { - return c.Username -} - -// GetTokenType 获取 token 类型 -func (c *CustomClaims) GetTokenType() TokenType { - return c.TokenType -} +package jwt + +import "github.com/golang-jwt/jwt/v5" + +// TokenType token 类型 +type TokenType string + +const ( + AccessToken TokenType = "access" + RefreshToken TokenType = "refresh" +) + +// CustomClaims 自定义 JWT Claims +type CustomClaims struct { + UserID uint `json:"user_id"` + Username string `json:"username"` + TokenType TokenType `json:"token_type"` + jwt.RegisteredClaims +} + +// GetUserID 获取用户 ID +func (c *CustomClaims) GetUserID() uint { + return c.UserID +} + +// GetUsername 获取用户名 +func (c *CustomClaims) GetUsername() string { + return c.Username +} + +// GetTokenType 获取 token 类型 +func (c *CustomClaims) GetTokenType() TokenType { + return c.TokenType +} diff --git a/pkg/jwt/jwt.go b/pkg/jwt/jwt.go index e1a21fb..b97bccd 100644 --- a/pkg/jwt/jwt.go +++ b/pkg/jwt/jwt.go @@ -1,147 +1,157 @@ -package jwt - -import ( - "YoudaoNoteLm/pkg/config" - "crypto/rand" - "encoding/hex" - "errors" - "fmt" - "time" - - "github.com/golang-jwt/jwt/v5" -) - -// GetParser 获取 JWT Parser(用于解析 token 提取 claims,不做有效性校验) -func GetParser() *jwt.Parser { - return jwt.NewParser() -} - -// generateJTI 生成唯一的 Token ID -func generateJTI() (string, error) { - b := make([]byte, 16) - if _, err := rand.Read(b); err != nil { - return "", fmt.Errorf("生成 JTI 失败: %w", err) - } - return hex.EncodeToString(b), nil -} - -var ( - ErrTokenInvalid = errors.New("token 无效") - ErrTokenExpired = errors.New("token 已过期") - ErrTokenTypeInvalid = errors.New("token 类型错误") -) - -// TokenPair 双 token 结构 -type TokenPair struct { - AccessToken string `json:"access_token"` - RefreshToken string `json:"refresh_token"` -} - -// GenerateAccessToken 生成 Access Token(15 分钟) -func GenerateAccessToken(userID uint, username string) (string, error) { - cfg := config.Get().JWT - exp := cfg.GetAccessTokenExp() - - jti, err := generateJTI() - if err != nil { - return "", err - } - - claims := CustomClaims{ - UserID: userID, - Username: username, - TokenType: AccessToken, - RegisteredClaims: jwt.RegisteredClaims{ - ID: jti, - ExpiresAt: jwt.NewNumericDate(time.Now().Add(exp)), - IssuedAt: jwt.NewNumericDate(time.Now()), - NotBefore: jwt.NewNumericDate(time.Now()), - Issuer: cfg.GetIssuer(), - }, - } - - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - return token.SignedString([]byte(cfg.Secret)) -} - -// GenerateRefreshToken 生成 Refresh Token(7 天) -func GenerateRefreshToken(userID uint, username string) (string, error) { - cfg := config.Get().JWT - exp := cfg.GetRefreshTokenExp() - - jti, err := generateJTI() - if err != nil { - return "", err - } - - claims := CustomClaims{ - UserID: userID, - Username: username, - TokenType: RefreshToken, - RegisteredClaims: jwt.RegisteredClaims{ - ID: jti, - ExpiresAt: jwt.NewNumericDate(time.Now().Add(exp)), - IssuedAt: jwt.NewNumericDate(time.Now()), - NotBefore: jwt.NewNumericDate(time.Now()), - Issuer: cfg.GetIssuer(), - }, - } - - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - return token.SignedString([]byte(cfg.Secret)) -} - -// GenerateTokenPair 生成 Access + Refresh Token 对 -func GenerateTokenPair(userID uint, username string) (*TokenPair, error) { - accessToken, err := GenerateAccessToken(userID, username) - if err != nil { - return nil, err - } - - refreshToken, err := GenerateRefreshToken(userID, username) - if err != nil { - return nil, err - } - - return &TokenPair{ - AccessToken: accessToken, - RefreshToken: refreshToken, - }, nil -} - -// ParseToken 解析 JWT Token -func ParseToken(tokenString string) (*CustomClaims, error) { - cfg := config.Get().JWT - - token, err := jwt.ParseWithClaims(tokenString, &CustomClaims{}, func(token *jwt.Token) (interface{}, error) { - return []byte(cfg.Secret), nil - }) - - if err != nil { - if errors.Is(err, jwt.ErrTokenExpired) { - return nil, ErrTokenExpired - } - return nil, ErrTokenInvalid - } - - if claims, ok := token.Claims.(*CustomClaims); ok && token.Valid { - return claims, nil - } - - return nil, ErrTokenInvalid -} - -// RefreshAccessToken 用 Refresh Token 换取新的 Access Token -func RefreshAccessToken(refreshTokenString string) (*TokenPair, error) { - claims, err := ParseToken(refreshTokenString) - if err != nil { - return nil, err - } - - // 必须是 refresh token - if claims.TokenType != RefreshToken { - return nil, ErrTokenTypeInvalid - } - - return GenerateTokenPair(claims.UserID, claims.Username) -} +package jwt + +import ( + "YoudaoNoteLm/pkg/config" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +// GetParser 获取 JWT Parser(用于解析 token 提取 claims,不做有效性校验) +func GetParser() *jwt.Parser { + return jwt.NewParser() +} + +// generateJTI 生成唯一的 Token ID +func generateJTI() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("生成 JTI 失败: %w", err) + } + return hex.EncodeToString(b), nil +} + +var ( + ErrTokenInvalid = errors.New("token 无效") + ErrTokenExpired = errors.New("token 已过期") + ErrTokenTypeInvalid = errors.New("token 类型错误") +) + +// TokenPair 双 token 结构 +type TokenPair struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` +} + +// GenerateAccessToken 生成 Access Token(15 分钟) +func GenerateAccessToken(userID uint, username string) (string, error) { + cfg := config.Get().JWT + exp := cfg.GetAccessTokenExp() + + jti, err := generateJTI() + if err != nil { + return "", err + } + + claims := CustomClaims{ + UserID: userID, + Username: username, + TokenType: AccessToken, + RegisteredClaims: jwt.RegisteredClaims{ + ID: jti, + ExpiresAt: jwt.NewNumericDate(time.Now().Add(exp)), + IssuedAt: jwt.NewNumericDate(time.Now()), + NotBefore: jwt.NewNumericDate(time.Now()), + Issuer: cfg.GetIssuer(), + }, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + return token.SignedString([]byte(cfg.Secret)) +} + +// GenerateRefreshToken 生成 Refresh Token(7 天) +func GenerateRefreshToken(userID uint, username string) (string, error) { + cfg := config.Get().JWT + exp := cfg.GetRefreshTokenExp() + + jti, err := generateJTI() + if err != nil { + return "", err + } + + claims := CustomClaims{ + UserID: userID, + Username: username, + TokenType: RefreshToken, + RegisteredClaims: jwt.RegisteredClaims{ + ID: jti, + ExpiresAt: jwt.NewNumericDate(time.Now().Add(exp)), + IssuedAt: jwt.NewNumericDate(time.Now()), + NotBefore: jwt.NewNumericDate(time.Now()), + Issuer: cfg.GetIssuer(), + }, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + return token.SignedString([]byte(cfg.Secret)) +} + +// GenerateTokenPair 生成 Access + Refresh Token 对 +func GenerateTokenPair(userID uint, username string) (*TokenPair, error) { + accessToken, err := GenerateAccessToken(userID, username) + if err != nil { + return nil, err + } + + refreshToken, err := GenerateRefreshToken(userID, username) + if err != nil { + return nil, err + } + + return &TokenPair{ + AccessToken: accessToken, + RefreshToken: refreshToken, + }, nil +} + +// ParseToken 解析 JWT Token +func ParseToken(tokenString string) (*CustomClaims, error) { + cfg := config.Get().JWT + + token, err := jwt.ParseWithClaims(tokenString, &CustomClaims{}, func(token *jwt.Token) (interface{}, error) { + return []byte(cfg.Secret), nil + }) + + if err != nil { + if errors.Is(err, jwt.ErrTokenExpired) { + return nil, ErrTokenExpired + } + return nil, ErrTokenInvalid + } + + if claims, ok := token.Claims.(*CustomClaims); ok && token.Valid { + return claims, nil + } + + return nil, ErrTokenInvalid +} + +// ParseUnverified 解析 token 提取 claims,不做签名和有效期校验 +// 用于从 token 字符串提取 jti/user_id 等信息(如登录后登记到用户 token 集合、登出时从已过期 token 拿 uid) +func ParseUnverified(tokenString string) (*CustomClaims, error) { + claims := &CustomClaims{} + if _, _, err := jwt.NewParser().ParseUnverified(tokenString, claims); err != nil { + return nil, fmt.Errorf("解析 token 失败: %w", err) + } + return claims, nil +} + +// RefreshAccessToken 用 Refresh Token 换取新的 Access Token +func RefreshAccessToken(refreshTokenString string) (*TokenPair, error) { + claims, err := ParseToken(refreshTokenString) + if err != nil { + return nil, err + } + + // 必须是 refresh token + if claims.TokenType != RefreshToken { + return nil, ErrTokenTypeInvalid + } + + return GenerateTokenPair(claims.UserID, claims.Username) +} diff --git a/pkg/logger/context.go b/pkg/logger/context.go index 57dbb85..22e393f 100644 --- a/pkg/logger/context.go +++ b/pkg/logger/context.go @@ -1,24 +1,24 @@ -package logger - -import ( - "YoudaoNoteLm/pkg/config" -) - -// InitFromConfig 从全局配置初始化日志 -func InitFromConfig() error { - cfg := config.Get().Log - return Init(&cfg) -} - -// InitDefault 使用默认配置初始化日志 -func InitDefault() error { - cfg := &config.LogConfig{ - Level: "debug", - Filename: "logs/app.log", - MaxSize: 100, - MaxBackups: 3, - MaxAge: 28, - Compress: true, - } - return Init(cfg) -} +package logger + +import ( + "YoudaoNoteLm/pkg/config" +) + +// InitFromConfig 从全局配置初始化日志 +func InitFromConfig() error { + cfg := config.Get().Log + return Init(&cfg) +} + +// InitDefault 使用默认配置初始化日志 +func InitDefault() error { + cfg := &config.LogConfig{ + Level: "debug", + Filename: "logs/app.log", + MaxSize: 100, + MaxBackups: 3, + MaxAge: 28, + Compress: true, + } + return Init(cfg) +} diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 380a4db..fcae1dc 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -1,151 +1,151 @@ -package logger - -import ( - "YoudaoNoteLm/pkg/config" - "os" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "gopkg.in/natefinch/lumberjack.v2" -) - -var ( - log = zap.NewNop() - sugar = log.Sugar() -) - -func init() { - // 提供默认 logger,防止 init() 阶段(logger.Init 之前)调用日志函数导致 nil panic - // 后续 Init() 调用会覆盖为正式配置的 logger - log = zap.NewNop() - sugar = log.Sugar() -} - -// Init 初始化日志系统 -func Init(cfg *config.LogConfig) error { - // 日志编码器配置 - encoderConfig := zapcore.EncoderConfig{ - TimeKey: "time", - LevelKey: "level", - NameKey: "logger", - CallerKey: "caller", - FunctionKey: zapcore.OmitKey, - MessageKey: "msg", - StacktraceKey: "stacktrace", - LineEnding: zapcore.DefaultLineEnding, - EncodeLevel: zapcore.LowercaseLevelEncoder, - EncodeTime: zapcore.ISO8601TimeEncoder, - EncodeDuration: zapcore.SecondsDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - } - - // 日志级别 - level := zapcore.InfoLevel - switch cfg.Level { - case "debug": - level = zapcore.DebugLevel - case "info": - level = zapcore.InfoLevel - case "warn": - level = zapcore.WarnLevel - case "error": - level = zapcore.ErrorLevel - } - - // 文件输出 - fileWriter := &lumberjack.Logger{ - Filename: cfg.Filename, - MaxSize: cfg.MaxSize, - MaxBackups: cfg.MaxBackups, - MaxAge: cfg.MaxAge, - Compress: cfg.Compress, - } - - // 创建多个输出(文件 + 控制台) - var writers []zapcore.WriteSyncer - writers = append(writers, zapcore.AddSync(fileWriter)) - writers = append(writers, zapcore.AddSync(os.Stdout)) - - // 核心 - core := zapcore.NewCore( - zapcore.NewJSONEncoder(encoderConfig), - zapcore.NewMultiWriteSyncer(writers...), - level, - ) - - // 创建 logger - log = zap.New(core, zap.AddCaller(), zap.AddCallerSkip(1), zap.AddStacktrace(zapcore.ErrorLevel)) - sugar = log.Sugar() - - return nil -} - -// Debug 调试日志 -func Debug(msg string, fields ...zap.Field) { - log.Debug(msg, fields...) -} - -// Info 信息日志 -func Info(msg string, fields ...zap.Field) { - log.Info(msg, fields...) -} - -// Warn 警告日志 -func Warn(msg string, fields ...zap.Field) { - log.Warn(msg, fields...) -} - -// Error 错误日志 -func Error(msg string, fields ...zap.Field) { - log.Error(msg, fields...) -} - -// Fatal 致命错误日志 -func Fatal(msg string, fields ...zap.Field) { - log.Fatal(msg, fields...) -} - -// Debugf 格式化调试日志 -func Debugf(format string, args ...interface{}) { - sugar.Debugf(format, args...) -} - -// Infof 格式化信息日志 -func Infof(format string, args ...interface{}) { - sugar.Infof(format, args...) -} - -// Warnf 格式化警告日志 -func Warnf(format string, args ...interface{}) { - sugar.Warnf(format, args...) -} - -// Errorf 格式化错误日志 -func Errorf(format string, args ...interface{}) { - sugar.Errorf(format, args...) -} - -// Fatalf 格式化致命错误日志 -func Fatalf(format string, args ...interface{}) { - sugar.Fatalf(format, args...) -} - -// With 创建带字段的 logger -func With(fields ...zap.Field) *zap.Logger { - return log.With(fields...) -} - -// Sync 同步日志缓冲区 -func Sync() error { - return log.Sync() -} - -// GetLogger 获取原始 logger -func GetLogger() *zap.Logger { - return log -} - -// GetSugaredLogger 获取 sugared logger -func GetSugaredLogger() *zap.SugaredLogger { - return sugar -} +package logger + +import ( + "YoudaoNoteLm/pkg/config" + "os" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "gopkg.in/natefinch/lumberjack.v2" +) + +var ( + log = zap.NewNop() + sugar = log.Sugar() +) + +func init() { + // 提供默认 logger,防止 init() 阶段(logger.Init 之前)调用日志函数导致 nil panic + // 后续 Init() 调用会覆盖为正式配置的 logger + log = zap.NewNop() + sugar = log.Sugar() +} + +// Init 初始化日志系统 +func Init(cfg *config.LogConfig) error { + // 日志编码器配置 + encoderConfig := zapcore.EncoderConfig{ + TimeKey: "time", + LevelKey: "level", + NameKey: "logger", + CallerKey: "caller", + FunctionKey: zapcore.OmitKey, + MessageKey: "msg", + StacktraceKey: "stacktrace", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.LowercaseLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeDuration: zapcore.SecondsDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + } + + // 日志级别 + level := zapcore.InfoLevel + switch cfg.Level { + case "debug": + level = zapcore.DebugLevel + case "info": + level = zapcore.InfoLevel + case "warn": + level = zapcore.WarnLevel + case "error": + level = zapcore.ErrorLevel + } + + // 文件输出 + fileWriter := &lumberjack.Logger{ + Filename: cfg.Filename, + MaxSize: cfg.MaxSize, + MaxBackups: cfg.MaxBackups, + MaxAge: cfg.MaxAge, + Compress: cfg.Compress, + } + + // 创建多个输出(文件 + 控制台) + var writers []zapcore.WriteSyncer + writers = append(writers, zapcore.AddSync(fileWriter)) + writers = append(writers, zapcore.AddSync(os.Stdout)) + + // 核心 + core := zapcore.NewCore( + zapcore.NewJSONEncoder(encoderConfig), + zapcore.NewMultiWriteSyncer(writers...), + level, + ) + + // 创建 logger + log = zap.New(core, zap.AddCaller(), zap.AddCallerSkip(1), zap.AddStacktrace(zapcore.ErrorLevel)) + sugar = log.Sugar() + + return nil +} + +// Debug 调试日志 +func Debug(msg string, fields ...zap.Field) { + log.Debug(msg, fields...) +} + +// Info 信息日志 +func Info(msg string, fields ...zap.Field) { + log.Info(msg, fields...) +} + +// Warn 警告日志 +func Warn(msg string, fields ...zap.Field) { + log.Warn(msg, fields...) +} + +// Error 错误日志 +func Error(msg string, fields ...zap.Field) { + log.Error(msg, fields...) +} + +// Fatal 致命错误日志 +func Fatal(msg string, fields ...zap.Field) { + log.Fatal(msg, fields...) +} + +// Debugf 格式化调试日志 +func Debugf(format string, args ...interface{}) { + sugar.Debugf(format, args...) +} + +// Infof 格式化信息日志 +func Infof(format string, args ...interface{}) { + sugar.Infof(format, args...) +} + +// Warnf 格式化警告日志 +func Warnf(format string, args ...interface{}) { + sugar.Warnf(format, args...) +} + +// Errorf 格式化错误日志 +func Errorf(format string, args ...interface{}) { + sugar.Errorf(format, args...) +} + +// Fatalf 格式化致命错误日志 +func Fatalf(format string, args ...interface{}) { + sugar.Fatalf(format, args...) +} + +// With 创建带字段的 logger +func With(fields ...zap.Field) *zap.Logger { + return log.With(fields...) +} + +// Sync 同步日志缓冲区 +func Sync() error { + return log.Sync() +} + +// GetLogger 获取原始 logger +func GetLogger() *zap.Logger { + return log +} + +// GetSugaredLogger 获取 sugared logger +func GetSugaredLogger() *zap.SugaredLogger { + return sugar +} diff --git a/pkg/response/response.go b/pkg/response/response.go index 961392e..761c05d 100644 --- a/pkg/response/response.go +++ b/pkg/response/response.go @@ -1,115 +1,118 @@ -package response - -import ( - "YoudaoNoteLm/pkg/errors" - "math" - - "github.com/gin-gonic/gin" -) - -// Response 统一响应结构 -type Response struct { - Code int `json:"code"` - Message string `json:"message"` - Data interface{} `json:"data,omitempty"` -} - -// Success 成功响应 -func Success(c *gin.Context, data interface{}) { - c.JSON(200, Response{ - Code: errors.CodeSuccess, - Message: errors.GetMessage(errors.CodeSuccess), - Data: data, - }) -} - -// SuccessWithMessage 成功响应(自定义消息) -func SuccessWithMessage(c *gin.Context, message string, data interface{}) { - c.JSON(200, Response{ - Code: errors.CodeSuccess, - Message: message, - Data: data, - }) -} - -// Error 错误响应 -func Error(c *gin.Context, code int, message string) { - c.JSON(200, Response{ - Code: code, - Message: message, - }) -} - -// ErrorWithData 错误响应(带数据) -func ErrorWithData(c *gin.Context, code int, message string, data interface{}) { - c.JSON(200, Response{ - Code: code, - Message: message, - Data: data, - }) -} - -// BizError 业务错误响应 -func BizError(c *gin.Context, err error) { - if bizErr, ok := err.(*errors.BizError); ok { - c.JSON(200, Response{ - Code: bizErr.Code, - Message: bizErr.Message, - }) - return - } - // 其他错误类型,返回错误信息给前端 - Error(c, errors.CodeInternalError, err.Error()) -} - -// BadRequest 400 错误 -func BadRequest(c *gin.Context, message string) { - Error(c, errors.CodeBadRequest, message) -} - -// Unauthorized 401 错误 -func Unauthorized(c *gin.Context, message string) { - Error(c, errors.CodeUnauthorized, message) -} - -// Forbidden 403 错误 -func Forbidden(c *gin.Context, message string) { - Error(c, errors.CodeForbidden, message) -} - -// NotFound 404 错误 -func NotFound(c *gin.Context, message string) { - Error(c, errors.CodeNotFound, message) -} - -// InternalError 500 错误 -func InternalError(c *gin.Context, message string) { - Error(c, errors.CodeInternalError, message) -} - -// PageRequest 分页请求参数 -type PageRequest struct { - Page int `form:"page" binding:"required,min=1"` // 页码,从1开始 - Size int `form:"size" binding:"required,min=1,max=100"` // 每页大小,最大100 -} - -// PageResponse 分页响应结构 -type PageResponse struct { - List interface{} `json:"list"` // 数据列表 - Total int64 `json:"total"` // 总记录数 - Page int `json:"page"` // 当前页码 - Size int `json:"size"` // 每页大小 - TotalPage int `json:"total_page"` // 总页数 -} - -// NewPageResponse 创建分页响应 -func NewPageResponse(list interface{}, total int64, page, size int) *PageResponse { - totalPage := int(math.Ceil(float64(total) / float64(size))) - return &PageResponse{ - List: list, - Total: total, - Page: page, - Size: size, - TotalPage: totalPage, - } -} +package response + +import ( + stderrors "errors" + "math" + + "YoudaoNoteLm/pkg/errors" + + "github.com/gin-gonic/gin" +) + +// Response 统一响应结构 +type Response struct { + Code int `json:"code"` + Message string `json:"message"` + Data interface{} `json:"data,omitempty"` +} + +// Success 成功响应 +func Success(c *gin.Context, data interface{}) { + c.JSON(200, Response{ + Code: errors.CodeSuccess, + Message: errors.GetMessage(errors.CodeSuccess), + Data: data, + }) +} + +// SuccessWithMessage 成功响应(自定义消息) +func SuccessWithMessage(c *gin.Context, message string, data interface{}) { + c.JSON(200, Response{ + Code: errors.CodeSuccess, + Message: message, + Data: data, + }) +} + +// Error 错误响应 +func Error(c *gin.Context, code int, message string) { + c.JSON(200, Response{ + Code: code, + Message: message, + }) +} + +// ErrorWithData 错误响应(带数据) +func ErrorWithData(c *gin.Context, code int, message string, data interface{}) { + c.JSON(200, Response{ + Code: code, + Message: message, + Data: data, + }) +} + +// BizError 业务错误响应 +func BizError(c *gin.Context, err error) { + var bizErr *errors.BizError + if stderrors.As(err, &bizErr) { + c.JSON(200, Response{ + Code: bizErr.Code, + Message: bizErr.Message, + }) + return + } + // 其他错误类型,返回错误信息给前端 + Error(c, errors.CodeInternalError, err.Error()) +} + +// BadRequest 400 错误 +func BadRequest(c *gin.Context, message string) { + Error(c, errors.CodeBadRequest, message) +} + +// Unauthorized 401 错误 +func Unauthorized(c *gin.Context, message string) { + Error(c, errors.CodeUnauthorized, message) +} + +// Forbidden 403 错误 +func Forbidden(c *gin.Context, message string) { + Error(c, errors.CodeForbidden, message) +} + +// NotFound 404 错误 +func NotFound(c *gin.Context, message string) { + Error(c, errors.CodeNotFound, message) +} + +// InternalError 500 错误 +func InternalError(c *gin.Context, message string) { + Error(c, errors.CodeInternalError, message) +} + +// PageRequest 分页请求参数 +type PageRequest struct { + Page int `form:"page" binding:"required,min=1"` // 页码,从1开始 + Size int `form:"size" binding:"required,min=1,max=100"` // 每页大小,最大100 +} + +// PageResponse 分页响应结构 +type PageResponse struct { + List interface{} `json:"list"` // 数据列表 + Total int64 `json:"total"` // 总记录数 + Page int `json:"page"` // 当前页码 + Size int `json:"size"` // 每页大小 + TotalPage int `json:"total_page"` // 总页数 +} + +// NewPageResponse 创建分页响应 +func NewPageResponse(list interface{}, total int64, page, size int) *PageResponse { + totalPage := int(math.Ceil(float64(total) / float64(size))) + return &PageResponse{ + List: list, + Total: total, + Page: page, + Size: size, + TotalPage: totalPage, + } +} diff --git a/pkg/response/validation.go b/pkg/response/validation.go index 2587087..4704c64 100644 --- a/pkg/response/validation.go +++ b/pkg/response/validation.go @@ -1,111 +1,111 @@ -package response - -import ( - "errors" - "fmt" - "strings" - - "github.com/go-playground/validator/v10" -) - -// 字段名中文映射(key 全小写,匹配时忽略大小写) -var fieldLabelMap = map[string]string{ - "name": "配置名称", - "provider": "服务商", - "api_key": "API Key", - "apiurl": "API 地址", - "api_url": "API 地址", - "model": "模型名称", - "dimensions": "向量维度", - "daily_quota": "每日配额", - "extra_config": "扩展配置", - "enabled": "启用状态", - "email": "邮箱", - "password": "密码", - "nickname": "昵称", - "username": "用户名", - "code": "验证码", - "refresh_token": "刷新令牌", -} - -// 验证 tag 中文提示 -var tagMessageMap = map[string]string{ - "required": "不能为空", - "min": "长度不足", - "max": "长度超出限制", - "email": "格式不正确", - "oneof": "值不合法", - "len": "长度不正确", - "gt": "值太小", - "gte": "值太小", - "lt": "值太大", - "lte": "值太大", - "url": "格式不正确,请输入有效的 URL", - "uuid": "格式不正确", -} - -// ParseValidationErrors 将 validator.ValidationErrors 转为友好的中文错误信息 -// 返回格式如:"配置名称不能为空;API Key 不能为空" -func ParseValidationErrors(err error) string { - var validationErrors validator.ValidationErrors - if !errors.As(err, &validationErrors) { - // 非 validator 错误(如 JSON 解析错误),原样返回 - return err.Error() - } - - var msgs []string - for _, fieldErr := range validationErrors { - // fieldErr.Field() 可能返回 "UserConfigRequest.Name" 这种带命名空间的格式 - // fieldErr.StructField() 返回 "Name" - label := resolveFieldLabel(fieldErr) - msg := buildFieldMessage(label, fieldErr) - msgs = append(msgs, msg) - } - - return strings.Join(msgs, ";") -} - -// resolveFieldLabel 从 validator.FieldError 解析出中文字段标签 -func resolveFieldLabel(fieldErr validator.FieldError) string { - // 优先用 StructField()(不带命名空间),如 "Name" - structField := fieldErr.StructField() - lower := strings.ToLower(structField) - - if label, ok := fieldLabelMap[lower]; ok { - return label - } - return structField -} - -// buildFieldMessage 根据验证 tag 构建错误提示 -func buildFieldMessage(label string, fieldErr validator.FieldError) string { - tag := fieldErr.Tag() - - switch tag { - case "required": - return fmt.Sprintf("%s不能为空", label) - case "min": - return fmt.Sprintf("%s长度至少为 %s 个字符", label, fieldErr.Param()) - case "max": - return fmt.Sprintf("%s长度不能超过 %s 个字符", label, fieldErr.Param()) - case "email": - return fmt.Sprintf("%s格式不正确", label) - case "oneof": - return fmt.Sprintf("%s的值必须是 [%s] 之一", label, fieldErr.Param()) - case "url": - return fmt.Sprintf("%s请输入有效的 URL", label) - case "gt": - return fmt.Sprintf("%s必须大于 %s", label, fieldErr.Param()) - case "gte": - return fmt.Sprintf("%s必须大于等于 %s", label, fieldErr.Param()) - case "lt": - return fmt.Sprintf("%s必须小于 %s", label, fieldErr.Param()) - case "lte": - return fmt.Sprintf("%s必须小于等于 %s", label, fieldErr.Param()) - default: - if msg, ok := tagMessageMap[tag]; ok { - return fmt.Sprintf("%s%s", label, msg) - } - return fmt.Sprintf("%s验证失败(%s)", label, tag) - } -} +package response + +import ( + "errors" + "fmt" + "strings" + + "github.com/go-playground/validator/v10" +) + +// 字段名中文映射(key 全小写,匹配时忽略大小写) +var fieldLabelMap = map[string]string{ + "name": "配置名称", + "provider": "服务商", + "api_key": "API Key", + "apiurl": "API 地址", + "api_url": "API 地址", + "model": "模型名称", + "dimensions": "向量维度", + "daily_quota": "每日配额", + "extra_config": "扩展配置", + "enabled": "启用状态", + "email": "邮箱", + "password": "密码", + "nickname": "昵称", + "username": "用户名", + "code": "验证码", + "refresh_token": "刷新令牌", +} + +// 验证 tag 中文提示 +var tagMessageMap = map[string]string{ + "required": "不能为空", + "min": "长度不足", + "max": "长度超出限制", + "email": "格式不正确", + "oneof": "值不合法", + "len": "长度不正确", + "gt": "值太小", + "gte": "值太小", + "lt": "值太大", + "lte": "值太大", + "url": "格式不正确,请输入有效的 URL", + "uuid": "格式不正确", +} + +// ParseValidationErrors 将 validator.ValidationErrors 转为友好的中文错误信息 +// 返回格式如:"配置名称不能为空;API Key 不能为空" +func ParseValidationErrors(err error) string { + var validationErrors validator.ValidationErrors + if !errors.As(err, &validationErrors) { + // 非 validator 错误(如 JSON 解析错误),原样返回 + return err.Error() + } + + var msgs []string + for _, fieldErr := range validationErrors { + // fieldErr.Field() 可能返回 "UserConfigRequest.Name" 这种带命名空间的格式 + // fieldErr.StructField() 返回 "Name" + label := resolveFieldLabel(fieldErr) + msg := buildFieldMessage(label, fieldErr) + msgs = append(msgs, msg) + } + + return strings.Join(msgs, ";") +} + +// resolveFieldLabel 从 validator.FieldError 解析出中文字段标签 +func resolveFieldLabel(fieldErr validator.FieldError) string { + // 优先用 StructField()(不带命名空间),如 "Name" + structField := fieldErr.StructField() + lower := strings.ToLower(structField) + + if label, ok := fieldLabelMap[lower]; ok { + return label + } + return structField +} + +// buildFieldMessage 根据验证 tag 构建错误提示 +func buildFieldMessage(label string, fieldErr validator.FieldError) string { + tag := fieldErr.Tag() + + switch tag { + case "required": + return fmt.Sprintf("%s不能为空", label) + case "min": + return fmt.Sprintf("%s长度至少为 %s 个字符", label, fieldErr.Param()) + case "max": + return fmt.Sprintf("%s长度不能超过 %s 个字符", label, fieldErr.Param()) + case "email": + return fmt.Sprintf("%s格式不正确", label) + case "oneof": + return fmt.Sprintf("%s的值必须是 [%s] 之一", label, fieldErr.Param()) + case "url": + return fmt.Sprintf("%s请输入有效的 URL", label) + case "gt": + return fmt.Sprintf("%s必须大于 %s", label, fieldErr.Param()) + case "gte": + return fmt.Sprintf("%s必须大于等于 %s", label, fieldErr.Param()) + case "lt": + return fmt.Sprintf("%s必须小于 %s", label, fieldErr.Param()) + case "lte": + return fmt.Sprintf("%s必须小于等于 %s", label, fieldErr.Param()) + default: + if msg, ok := tagMessageMap[tag]; ok { + return fmt.Sprintf("%s%s", label, msg) + } + return fmt.Sprintf("%s验证失败(%s)", label, tag) + } +} diff --git a/pkg/utils/audio.go b/pkg/utils/audio.go index 5fd6c75..6545cdf 100644 --- a/pkg/utils/audio.go +++ b/pkg/utils/audio.go @@ -1,285 +1,285 @@ -package utils - -import ( - "bytes" - "fmt" - "io" - "math" - "os" - "path/filepath" - "strings" - - "github.com/go-audio/audio" - "github.com/go-audio/wav" - "github.com/hajimehoshi/go-mp3" -) - -const ( - // ASRTargetSampleRate 阿里云 ASR 要求的采样率 - ASRTargetSampleRate = 16000 - // ASRChannels 声道数(单声道) - ASRChannels = 1 - // ASRBitDepth 位深度 - ASRBitDepth = 16 -) - -// ConvertFileToASRFormat 将音频文件转换为阿里云 ASR 兼容格式(16kHz 单声道 WAV) -// 输入: 文件路径 -// 输出: 转换后的 WAV 文件路径 -func ConvertFileToASRFormat(inputPath string) (string, error) { - ext := strings.ToLower(filepath.Ext(inputPath)) - - // 读取并解码音频 - pcmData, sampleRate, channels, err := decodeAudioFile(inputPath, ext) - if err != nil { - return "", fmt.Errorf("解码音频失败: %w", err) - } - - // 如果已经是目标格式,直接写入 WAV - if sampleRate == ASRTargetSampleRate && channels == ASRChannels { - return writeWAV(inputPath, pcmData, sampleRate, channels) - } - - // 转换为单声道 - if channels > 1 { - pcmData = convertToMono(pcmData, channels) - } - - // 重采样到目标采样率 - if sampleRate != ASRTargetSampleRate { - pcmData = resample(pcmData, sampleRate, ASRTargetSampleRate) - } - - // 写入 WAV 文件 - return writeWAV(inputPath, pcmData, ASRTargetSampleRate, ASRChannels) -} - -// ConvertBytesToASRFormat 将音频字节数据转换为 ASR 兼容格式 -func ConvertBytesToASRFormat(data []byte, originalExt string) ([]byte, error) { - ext := strings.ToLower(originalExt) - - // 解码 - pcmData, sampleRate, channels, err := decodeAudioBytes(data, ext) - if err != nil { - return nil, fmt.Errorf("解码音频失败: %w", err) - } - - // 转换单声道 - if channels > 1 { - pcmData = convertToMono(pcmData, channels) - } - - // 重采样 - if sampleRate != ASRTargetSampleRate { - pcmData = resample(pcmData, sampleRate, ASRTargetSampleRate) - } - - // 编码为 WAV 字节 - return encodeToWAVBytes(pcmData, ASRTargetSampleRate, ASRChannels) -} - -// decodeAudioFile 根据扩展名解码音频文件 -func decodeAudioFile(path, ext string) ([]int, int, int, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, 0, 0, err - } - return decodeAudioBytes(data, ext) -} - -// decodeAudioBytes 根据格式解码音频字节 -func decodeAudioBytes(data []byte, ext string) ([]int, int, int, error) { - switch ext { - case ".mp3": - return decodeMP3(data) - case ".wav": - return decodeWAV(data) - default: - return nil, 0, 0, fmt.Errorf("不支持的音频格式: %s", ext) - } -} - -// decodeMP3 解码 MP3 数据 -func decodeMP3(data []byte) ([]int, int, int, error) { - decoder, err := mp3.NewDecoder(bytes.NewReader(data)) - if err != nil { - return nil, 0, 0, fmt.Errorf("MP3 解码器创建失败: %w", err) - } - - sampleRate := decoder.SampleRate() - - // 读取所有 PCM 数据(16bit signed little-endian) - var pcmBytes []byte - buf := make([]byte, 4096) - for { - n, err := decoder.Read(buf) - if n > 0 { - pcmBytes = append(pcmBytes, buf[:n]...) - } - if err == io.EOF { - break - } - if err != nil { - return nil, 0, 0, fmt.Errorf("MP3 读取失败: %w", err) - } - } - - // MP3 解码输出是 16bit stereo (2 channels) - channels := 2 - // 转换为 int 切片 - pcmData := make([]int, len(pcmBytes)/2) - for i := 0; i < len(pcmData); i++ { - // little-endian 16bit signed - sample := int16(pcmBytes[i*2]) | int16(pcmBytes[i*2+1])<<8 - pcmData[i] = int(sample) - } - - return pcmData, sampleRate, channels, nil -} - -// decodeWAV 解码 WAV 数据 -func decodeWAV(data []byte) ([]int, int, int, error) { - decoder := wav.NewDecoder(bytes.NewReader(data)) - if !decoder.IsValidFile() { - return nil, 0, 0, fmt.Errorf("无效的 WAV 文件") - } - - buf := &audio.IntBuffer{ - Format: &audio.Format{ - NumChannels: int(decoder.NumChans), - SampleRate: int(decoder.SampleRate), - }, - } - // 读取所有采样 - for { - chunk, err := decoder.FullPCMBuffer() - if err == io.EOF { - break - } - if err != nil { - return nil, 0, 0, fmt.Errorf("WAV 读取失败: %w", err) - } - buf.Data = append(buf.Data, chunk.Data...) - break // FullPCMBuffer 一次读完 - } - - if len(buf.Data) == 0 { - return nil, 0, 0, fmt.Errorf("WAV 文件无音频数据") - } - - return buf.Data, int(decoder.SampleRate), int(decoder.NumChans), nil -} - -// convertToMono 多声道转单声道(取平均值) -func convertToMono(data []int, channels int) []int { - if channels == 1 { - return data - } - - monoLen := len(data) / channels - mono := make([]int, monoLen) - for i := 0; i < monoLen; i++ { - sum := 0 - for ch := 0; ch < channels; ch++ { - sum += data[i*channels+ch] - } - mono[i] = sum / channels - } - return mono -} - -// resample 线性插值重采样 -func resample(data []int, fromRate, toRate int) []int { - if fromRate == toRate { - return data - } - - ratio := float64(fromRate) / float64(toRate) - newLen := int(float64(len(data)) / ratio) - result := make([]int, newLen) - - for i := 0; i < newLen; i++ { - srcPos := float64(i) * ratio - srcIdx := int(srcPos) - frac := srcPos - float64(srcIdx) - - if srcIdx+1 < len(data) { - // 线性插值 - result[i] = int(math.Round(float64(data[srcIdx])*(1-frac) + float64(data[srcIdx+1])*frac)) - } else if srcIdx < len(data) { - result[i] = data[srcIdx] - } - } - - return result -} - -// writeWAV 将 PCM 数据写入 WAV 文件 -func writeWAV(originalPath string, data []int, sampleRate, channels int) (string, error) { - ext := filepath.Ext(originalPath) - outputPath := originalPath[:len(originalPath)-len(ext)] + "_16k.wav" - - f, err := os.Create(outputPath) - if err != nil { - return "", fmt.Errorf("创建输出文件失败: %w", err) - } - defer f.Close() - - encoder := wav.NewEncoder(f, sampleRate, ASRBitDepth, channels, 1) - - // 转换为 audio.IntBuffer - buf := &audio.IntBuffer{ - Format: &audio.Format{ - NumChannels: channels, - SampleRate: sampleRate, - }, - Data: data, - } - - if err := encoder.Write(buf); err != nil { - return "", fmt.Errorf("写入 WAV 失败: %w", err) - } - - if err := encoder.Close(); err != nil { - return "", fmt.Errorf("关闭 WAV 编码器失败: %w", err) - } - - return outputPath, nil -} - -// encodeToWAVBytes 将 PCM 数据编码为 WAV 字节 -func encodeToWAVBytes(data []int, sampleRate, channels int) ([]byte, error) { - // WAV 编码器需要 Seek 支持,使用临时文件 - tmpFile, err := os.CreateTemp("", "asr-wav-*.wav") - if err != nil { - return nil, fmt.Errorf("创建临时文件失败: %w", err) - } - defer os.Remove(tmpFile.Name()) - defer tmpFile.Close() - - encoder := wav.NewEncoder(tmpFile, sampleRate, ASRBitDepth, channels, 1) - - intBuf := &audio.IntBuffer{ - Format: &audio.Format{ - NumChannels: channels, - SampleRate: sampleRate, - }, - Data: data, - } - - if err := encoder.Write(intBuf); err != nil { - return nil, fmt.Errorf("编码 WAV 失败: %w", err) - } - - if err := encoder.Close(); err != nil { - return nil, fmt.Errorf("关闭 WAV 编码器失败: %w", err) - } - - // 读取文件内容 - result, err := os.ReadFile(tmpFile.Name()) - if err != nil { - return nil, fmt.Errorf("读取 WAV 文件失败: %w", err) - } - - return result, nil -} +package utils + +import ( + "bytes" + "fmt" + "io" + "math" + "os" + "path/filepath" + "strings" + + "github.com/go-audio/audio" + "github.com/go-audio/wav" + "github.com/hajimehoshi/go-mp3" +) + +const ( + // ASRTargetSampleRate 阿里云 ASR 要求的采样率 + ASRTargetSampleRate = 16000 + // ASRChannels 声道数(单声道) + ASRChannels = 1 + // ASRBitDepth 位深度 + ASRBitDepth = 16 +) + +// ConvertFileToASRFormat 将音频文件转换为阿里云 ASR 兼容格式(16kHz 单声道 WAV) +// 输入: 文件路径 +// 输出: 转换后的 WAV 文件路径 +func ConvertFileToASRFormat(inputPath string) (string, error) { + ext := strings.ToLower(filepath.Ext(inputPath)) + + // 读取并解码音频 + pcmData, sampleRate, channels, err := decodeAudioFile(inputPath, ext) + if err != nil { + return "", fmt.Errorf("解码音频失败: %w", err) + } + + // 如果已经是目标格式,直接写入 WAV + if sampleRate == ASRTargetSampleRate && channels == ASRChannels { + return writeWAV(inputPath, pcmData, sampleRate, channels) + } + + // 转换为单声道 + if channels > 1 { + pcmData = convertToMono(pcmData, channels) + } + + // 重采样到目标采样率 + if sampleRate != ASRTargetSampleRate { + pcmData = resample(pcmData, sampleRate, ASRTargetSampleRate) + } + + // 写入 WAV 文件 + return writeWAV(inputPath, pcmData, ASRTargetSampleRate, ASRChannels) +} + +// ConvertBytesToASRFormat 将音频字节数据转换为 ASR 兼容格式 +func ConvertBytesToASRFormat(data []byte, originalExt string) ([]byte, error) { + ext := strings.ToLower(originalExt) + + // 解码 + pcmData, sampleRate, channels, err := decodeAudioBytes(data, ext) + if err != nil { + return nil, fmt.Errorf("解码音频失败: %w", err) + } + + // 转换单声道 + if channels > 1 { + pcmData = convertToMono(pcmData, channels) + } + + // 重采样 + if sampleRate != ASRTargetSampleRate { + pcmData = resample(pcmData, sampleRate, ASRTargetSampleRate) + } + + // 编码为 WAV 字节 + return encodeToWAVBytes(pcmData, ASRTargetSampleRate, ASRChannels) +} + +// decodeAudioFile 根据扩展名解码音频文件 +func decodeAudioFile(path, ext string) ([]int, int, int, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, 0, 0, err + } + return decodeAudioBytes(data, ext) +} + +// decodeAudioBytes 根据格式解码音频字节 +func decodeAudioBytes(data []byte, ext string) ([]int, int, int, error) { + switch ext { + case ".mp3": + return decodeMP3(data) + case ".wav": + return decodeWAV(data) + default: + return nil, 0, 0, fmt.Errorf("不支持的音频格式: %s", ext) + } +} + +// decodeMP3 解码 MP3 数据 +func decodeMP3(data []byte) ([]int, int, int, error) { + decoder, err := mp3.NewDecoder(bytes.NewReader(data)) + if err != nil { + return nil, 0, 0, fmt.Errorf("MP3 解码器创建失败: %w", err) + } + + sampleRate := decoder.SampleRate() + + // 读取所有 PCM 数据(16bit signed little-endian) + var pcmBytes []byte + buf := make([]byte, 4096) + for { + n, err := decoder.Read(buf) + if n > 0 { + pcmBytes = append(pcmBytes, buf[:n]...) + } + if err == io.EOF { + break + } + if err != nil { + return nil, 0, 0, fmt.Errorf("MP3 读取失败: %w", err) + } + } + + // MP3 解码输出是 16bit stereo (2 channels) + channels := 2 + // 转换为 int 切片 + pcmData := make([]int, len(pcmBytes)/2) + for i := 0; i < len(pcmData); i++ { + // little-endian 16bit signed + sample := int16(pcmBytes[i*2]) | int16(pcmBytes[i*2+1])<<8 + pcmData[i] = int(sample) + } + + return pcmData, sampleRate, channels, nil +} + +// decodeWAV 解码 WAV 数据 +func decodeWAV(data []byte) ([]int, int, int, error) { + decoder := wav.NewDecoder(bytes.NewReader(data)) + if !decoder.IsValidFile() { + return nil, 0, 0, fmt.Errorf("无效的 WAV 文件") + } + + buf := &audio.IntBuffer{ + Format: &audio.Format{ + NumChannels: int(decoder.NumChans), + SampleRate: int(decoder.SampleRate), + }, + } + // 读取所有采样 + for { + chunk, err := decoder.FullPCMBuffer() + if err == io.EOF { + break + } + if err != nil { + return nil, 0, 0, fmt.Errorf("WAV 读取失败: %w", err) + } + buf.Data = append(buf.Data, chunk.Data...) + break // FullPCMBuffer 一次读完 + } + + if len(buf.Data) == 0 { + return nil, 0, 0, fmt.Errorf("WAV 文件无音频数据") + } + + return buf.Data, int(decoder.SampleRate), int(decoder.NumChans), nil +} + +// convertToMono 多声道转单声道(取平均值) +func convertToMono(data []int, channels int) []int { + if channels == 1 { + return data + } + + monoLen := len(data) / channels + mono := make([]int, monoLen) + for i := 0; i < monoLen; i++ { + sum := 0 + for ch := 0; ch < channels; ch++ { + sum += data[i*channels+ch] + } + mono[i] = sum / channels + } + return mono +} + +// resample 线性插值重采样 +func resample(data []int, fromRate, toRate int) []int { + if fromRate == toRate { + return data + } + + ratio := float64(fromRate) / float64(toRate) + newLen := int(float64(len(data)) / ratio) + result := make([]int, newLen) + + for i := 0; i < newLen; i++ { + srcPos := float64(i) * ratio + srcIdx := int(srcPos) + frac := srcPos - float64(srcIdx) + + if srcIdx+1 < len(data) { + // 线性插值 + result[i] = int(math.Round(float64(data[srcIdx])*(1-frac) + float64(data[srcIdx+1])*frac)) + } else if srcIdx < len(data) { + result[i] = data[srcIdx] + } + } + + return result +} + +// writeWAV 将 PCM 数据写入 WAV 文件 +func writeWAV(originalPath string, data []int, sampleRate, channels int) (string, error) { + ext := filepath.Ext(originalPath) + outputPath := originalPath[:len(originalPath)-len(ext)] + "_16k.wav" + + f, err := os.Create(outputPath) + if err != nil { + return "", fmt.Errorf("创建输出文件失败: %w", err) + } + defer f.Close() + + encoder := wav.NewEncoder(f, sampleRate, ASRBitDepth, channels, 1) + + // 转换为 audio.IntBuffer + buf := &audio.IntBuffer{ + Format: &audio.Format{ + NumChannels: channels, + SampleRate: sampleRate, + }, + Data: data, + } + + if err := encoder.Write(buf); err != nil { + return "", fmt.Errorf("写入 WAV 失败: %w", err) + } + + if err := encoder.Close(); err != nil { + return "", fmt.Errorf("关闭 WAV 编码器失败: %w", err) + } + + return outputPath, nil +} + +// encodeToWAVBytes 将 PCM 数据编码为 WAV 字节 +func encodeToWAVBytes(data []int, sampleRate, channels int) ([]byte, error) { + // WAV 编码器需要 Seek 支持,使用临时文件 + tmpFile, err := os.CreateTemp("", "asr-wav-*.wav") + if err != nil { + return nil, fmt.Errorf("创建临时文件失败: %w", err) + } + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + + encoder := wav.NewEncoder(tmpFile, sampleRate, ASRBitDepth, channels, 1) + + intBuf := &audio.IntBuffer{ + Format: &audio.Format{ + NumChannels: channels, + SampleRate: sampleRate, + }, + Data: data, + } + + if err := encoder.Write(intBuf); err != nil { + return nil, fmt.Errorf("编码 WAV 失败: %w", err) + } + + if err := encoder.Close(); err != nil { + return nil, fmt.Errorf("关闭 WAV 编码器失败: %w", err) + } + + // 读取文件内容 + result, err := os.ReadFile(tmpFile.Name()) + if err != nil { + return nil, fmt.Errorf("读取 WAV 文件失败: %w", err) + } + + return result, nil +} diff --git a/pkg/utils/crypto.go b/pkg/utils/crypto.go index 0e69ab5..05c284b 100644 --- a/pkg/utils/crypto.go +++ b/pkg/utils/crypto.go @@ -1,82 +1,82 @@ -package utils - -import ( - "crypto/aes" - "crypto/cipher" - "crypto/rand" - "encoding/base64" - "errors" - "io" -) - -// Encrypt 使用 AES-GCM 加密数据 -func Encrypt(plaintext string, key []byte) (string, error) { - if plaintext == "" { - return "", nil - } - - block, err := aes.NewCipher(key) - if err != nil { - return "", err - } - - aesGCM, err := cipher.NewGCM(block) - if err != nil { - return "", err - } - - nonce := make([]byte, aesGCM.NonceSize()) - if _, err := io.ReadFull(rand.Reader, nonce); err != nil { - return "", err - } - - ciphertext := aesGCM.Seal(nonce, nonce, []byte(plaintext), nil) - return base64.StdEncoding.EncodeToString(ciphertext), nil -} - -// DecryptAPIKey 解密 API Key,失败时返回原值 -func DecryptAPIKey(apiKey string, encryptionKey []byte) string { - if apiKey == "" || len(encryptionKey) == 0 { - return apiKey - } - decrypted, err := Decrypt(apiKey, encryptionKey) - if err != nil { - return apiKey // 解密失败返回原值 - } - return decrypted -} - -// Decrypt 使用 AES-GCM 解密数据 -func Decrypt(ciphertext string, key []byte) (string, error) { - if ciphertext == "" { - return "", nil - } - - data, err := base64.StdEncoding.DecodeString(ciphertext) - if err != nil { - return "", err - } - - block, err := aes.NewCipher(key) - if err != nil { - return "", err - } - - aesGCM, err := cipher.NewGCM(block) - if err != nil { - return "", err - } - - nonceSize := aesGCM.NonceSize() - if len(data) < nonceSize { - return "", errors.New("ciphertext too short") - } - - nonce, ciphertextBytes := data[:nonceSize], data[nonceSize:] - plaintext, err := aesGCM.Open(nil, nonce, ciphertextBytes, nil) - if err != nil { - return "", err - } - - return string(plaintext), nil -} +package utils + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "errors" + "io" +) + +// Encrypt 使用 AES-GCM 加密数据 +func Encrypt(plaintext string, key []byte) (string, error) { + if plaintext == "" { + return "", nil + } + + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + + aesGCM, err := cipher.NewGCM(block) + if err != nil { + return "", err + } + + nonce := make([]byte, aesGCM.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return "", err + } + + ciphertext := aesGCM.Seal(nonce, nonce, []byte(plaintext), nil) + return base64.StdEncoding.EncodeToString(ciphertext), nil +} + +// DecryptAPIKey 解密 API Key,失败时返回原值 +func DecryptAPIKey(apiKey string, encryptionKey []byte) string { + if apiKey == "" || len(encryptionKey) == 0 { + return apiKey + } + decrypted, err := Decrypt(apiKey, encryptionKey) + if err != nil { + return apiKey // 解密失败返回原值 + } + return decrypted +} + +// Decrypt 使用 AES-GCM 解密数据 +func Decrypt(ciphertext string, key []byte) (string, error) { + if ciphertext == "" { + return "", nil + } + + data, err := base64.StdEncoding.DecodeString(ciphertext) + if err != nil { + return "", err + } + + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + + aesGCM, err := cipher.NewGCM(block) + if err != nil { + return "", err + } + + nonceSize := aesGCM.NonceSize() + if len(data) < nonceSize { + return "", errors.New("ciphertext too short") + } + + nonce, ciphertextBytes := data[:nonceSize], data[nonceSize:] + plaintext, err := aesGCM.Open(nil, nonce, ciphertextBytes, nil) + if err != nil { + return "", err + } + + return string(plaintext), nil +} diff --git a/pkg/utils/string.go b/pkg/utils/string.go index 6b594ad..f7665e5 100644 --- a/pkg/utils/string.go +++ b/pkg/utils/string.go @@ -1,72 +1,72 @@ -package utils - -import ( - "crypto/rand" - "encoding/base64" - "math/big" - "strings" -) - -// Charset 字符集类型 -type Charset string - -const ( - // Numeric 纯数字字符集 - Numeric Charset = "numeric" - // AlphaNumeric 字母数字字符集 - AlphaNumeric Charset = "alphanumeric" -) - -var charsetMap = map[Charset]string{ - Numeric: "0123456789", - AlphaNumeric: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", -} - -// GenerateRandomString 生成随机字符串 -func GenerateRandomString(length int, charset ...Charset) (string, error) { - if len(charset) == 0 { - // 默认使用 base64 方式 - bytes := make([]byte, length) - if _, err := rand.Read(bytes); err != nil { - return "", err - } - return base64.URLEncoding.EncodeToString(bytes)[:length], nil - } - - // 使用指定字符集 - chars := charsetMap[charset[0]] - if chars == "" { - chars = charsetMap[Numeric] - } - - result := make([]byte, length) - max := big.NewInt(int64(len(chars))) - for i := 0; i < length; i++ { - n, err := rand.Int(rand.Reader, max) - if err != nil { - return "", err - } - result[i] = chars[n.Int64()] - } - return string(result), nil -} - -// Contains 检查字符串是否在切片中 -func Contains(slice []string, item string) bool { - for _, s := range slice { - if s == item { - return true - } - } - return false -} - -// TrimSpace 去除首尾空格 -func TrimSpace(s string) string { - return strings.TrimSpace(s) -} - -// IsEmpty 检查字符串是否为空 -func IsEmpty(s string) bool { - return TrimSpace(s) == "" -} +package utils + +import ( + "crypto/rand" + "encoding/base64" + "math/big" + "strings" +) + +// Charset 字符集类型 +type Charset string + +const ( + // Numeric 纯数字字符集 + Numeric Charset = "numeric" + // AlphaNumeric 字母数字字符集 + AlphaNumeric Charset = "alphanumeric" +) + +var charsetMap = map[Charset]string{ + Numeric: "0123456789", + AlphaNumeric: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", +} + +// GenerateRandomString 生成随机字符串 +func GenerateRandomString(length int, charset ...Charset) (string, error) { + if len(charset) == 0 { + // 默认使用 base64 方式 + bytes := make([]byte, length) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return base64.URLEncoding.EncodeToString(bytes)[:length], nil + } + + // 使用指定字符集 + chars := charsetMap[charset[0]] + if chars == "" { + chars = charsetMap[Numeric] + } + + result := make([]byte, length) + max := big.NewInt(int64(len(chars))) + for i := 0; i < length; i++ { + n, err := rand.Int(rand.Reader, max) + if err != nil { + return "", err + } + result[i] = chars[n.Int64()] + } + return string(result), nil +} + +// Contains 检查字符串是否在切片中 +func Contains(slice []string, item string) bool { + for _, s := range slice { + if s == item { + return true + } + } + return false +} + +// TrimSpace 去除首尾空格 +func TrimSpace(s string) string { + return strings.TrimSpace(s) +} + +// IsEmpty 检查字符串是否为空 +func IsEmpty(s string) bool { + return TrimSpace(s) == "" +} diff --git a/pkg/utils/time.go b/pkg/utils/time.go index d52ce35..ce87227 100644 --- a/pkg/utils/time.go +++ b/pkg/utils/time.go @@ -1,23 +1,23 @@ -package utils - -import "time" - -// GetCurrentTimestamp 获取当前时间戳(秒) -func GetCurrentTimestamp() int64 { - return time.Now().Unix() -} - -// GetCurrentMilliTimestamp 获取当前时间戳(毫秒) -func GetCurrentMilliTimestamp() int64 { - return time.Now().UnixMilli() -} - -// FormatTime 格式化时间 -func FormatTime(t time.Time) string { - return t.Format("2006-01-02 15:04:05") -} - -// ParseTime 解析时间字符串 -func ParseTime(s string) (time.Time, error) { - return time.Parse("2006-01-02 15:04:05", s) -} +package utils + +import "time" + +// GetCurrentTimestamp 获取当前时间戳(秒) +func GetCurrentTimestamp() int64 { + return time.Now().Unix() +} + +// GetCurrentMilliTimestamp 获取当前时间戳(毫秒) +func GetCurrentMilliTimestamp() int64 { + return time.Now().UnixMilli() +} + +// FormatTime 格式化时间 +func FormatTime(t time.Time) string { + return t.Format("2006-01-02 15:04:05") +} + +// ParseTime 解析时间字符串 +func ParseTime(s string) (time.Time, error) { + return time.Parse("2006-01-02 15:04:05", s) +}