From b15ad482c69376dd8f3cdede5aa8403db26a63d2 Mon Sep 17 00:00:00 2001 From: Maurice Date: Mon, 23 Feb 2026 21:48:09 +0100 Subject: [PATCH 1/4] add base64 image capabilities local llm --- raven/ai/agents_integration.py | 12 +++-- raven/ai/ai.py | 84 ++++++++++++++++++++++++++++++---- 2 files changed, 84 insertions(+), 12 deletions(-) diff --git a/raven/ai/agents_integration.py b/raven/ai/agents_integration.py index 5611c0eb3..ecd896e9c 100644 --- a/raven/ai/agents_integration.py +++ b/raven/ai/agents_integration.py @@ -570,8 +570,11 @@ async def handle_ai_request_async( {"role": msg["role"], "content": [{"type": "text", "text": msg["content"]}]} ) - # Add current user message - messages.append({"role": "user", "content": [{"type": "text", "text": message}]}) + # Add current user message (may be a list for vision content with images) + if isinstance(message, list): + messages.append({"role": "user", "content": message}) + else: + messages.append({"role": "user", "content": [{"type": "text", "text": message}]}) # Create the API call with or without tools api_params = { @@ -620,7 +623,10 @@ async def handle_ai_request_async( messages = [ {"role": "system", "content": [{"type": "text", "text": agent.instructions}]}, - {"role": "user", "content": [{"type": "text", "text": str(full_input)}]}, + { + "role": "user", + "content": message if isinstance(message, list) else [{"type": "text", "text": str(full_input)}], + }, assistant_message, ] diff --git a/raven/ai/ai.py b/raven/ai/ai.py index b7926edf0..398f1fe73 100644 --- a/raven/ai/ai.py +++ b/raven/ai/ai.py @@ -353,6 +353,55 @@ def extract_file_content_for_agent(file_url: str, file_extension: str, bot, file return extracted_content +def _build_vision_content(file_url: str, user_text: str = "") -> list: + """ + Build an OpenAI-compatible vision content list with a base64-encoded image. + Used for local LLMs with vision capability (Ollama, LM Studio, etc.) that accept + the OpenAI chat completions format but cannot fetch server-side file paths. + """ + import base64 + import mimetypes + + content = [] + + if user_text: + content.append({"type": "text", "text": user_text}) + + try: + file_doc = frappe.get_doc("File", {"file_url": file_url}) + file_path = file_doc.get_full_path() + + with open(file_path, "rb") as f: + image_data = base64.b64encode(f.read()).decode("utf-8") + + mime_type, _ = mimetypes.guess_type(file_path) + if not mime_type: + ext = file_path.rsplit(".", 1)[-1].lower() if "." in file_path else "jpeg" + mime_map = { + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "png": "image/png", + "gif": "image/gif", + "webp": "image/webp", + } + mime_type = mime_map.get(ext, "image/jpeg") + + content.append( + { + "type": "image_url", + "image_url": {"url": f"data:{mime_type};base64,{image_data}"}, + } + ) + except Exception as e: + frappe.log_error( + f"Error encoding image for Local LLM vision: {str(e)}\nFile: {file_url}", + "Image Vision Encoding Error", + ) + content.append({"type": "text", "text": f"[Image: {file_url}]"}) + + return content + + def process_message_with_agent( message, bot, channel_id: str, is_new_conversation: bool, channel=None ): @@ -428,11 +477,21 @@ def process_message_with_agent( if message.text or message.content: content += f"\n\nUser's question: {message.text or message.content}" else: - content = f"[User uploaded an image: {file_url}]" + user_text = "" if extracted_content: - content += f"\n\nExtracted content from the image:\n{extracted_content}" + user_text = f"Extracted content from the image:\n{extracted_content}\n\n" if message.text or message.content: - content += f"\n\nUser's question: {message.text or message.content}" + user_text += message.text or message.content + + if bot.model_provider == "Local LLM": + # Encode image as base64 so local vision models (Ollama etc.) can see it + content = _build_vision_content(file_url, user_text or None) + else: + content = f"[User uploaded an image: {file_url}]" + if extracted_content: + content += f"\n\nExtracted content from the image:\n{extracted_content}" + if message.text or message.content: + content += f"\n\nUser's question: {message.text or message.content}" else: content = message.text or message.content or "" @@ -446,13 +505,20 @@ def process_message_with_agent( file_extension = file_url.split(".")[-1].lower() if "." in file_url else "" extracted_content = extract_file_content_for_agent(file_url, file_extension, bot, file_handler) - file_prefix = f"[User uploaded a {'file' if recent_file_message.message_type == 'File' else 'image'}: {file_url}]" - if extracted_content: - file_prefix += f"\n\nExtracted content from the file:\n{extracted_content}\n\n" + if recent_file_message.message_type == "Image" and bot.model_provider == "Local LLM": + # Encode image as base64 for local vision models, combining with the user's text + vision_text = "" + if extracted_content: + vision_text = f"Extracted content from the image:\n{extracted_content}\n\n" + vision_text += content + content = _build_vision_content(file_url, vision_text) else: - file_prefix += "\n" - - content = file_prefix + content + file_prefix = f"[User uploaded a {'file' if recent_file_message.message_type == 'File' else 'image'}: {file_url}]" + if extracted_content: + file_prefix += f"\n\nExtracted content from the file:\n{extracted_content}\n\n" + else: + file_prefix += "\n" + content = file_prefix + content # Get conversation history if this is an existing thread conversation_history = [] From 53fdd3e420d0668acd52d666dcfd38801e608d6c Mon Sep 17 00:00:00 2001 From: Maurice Date: Mon, 23 Feb 2026 22:05:43 +0100 Subject: [PATCH 2/4] add extra button frontend ai agent settings for base64 encoding --- .../ai/bots/BotDocumentProcessorsForm.tsx | 44 ++++++++++++++----- frontend/src/types/RavenBot/RavenBot.ts | 2 + packages/types/RavenBot/RavenBot.ts | 2 + raven/ai/ai.py | 4 +- .../doctype/raven_bot/raven_bot.json | 7 +++ 5 files changed, 46 insertions(+), 13 deletions(-) diff --git a/frontend/src/components/feature/settings/ai/bots/BotDocumentProcessorsForm.tsx b/frontend/src/components/feature/settings/ai/bots/BotDocumentProcessorsForm.tsx index 4888dc907..74138bcea 100644 --- a/frontend/src/components/feature/settings/ai/bots/BotDocumentProcessorsForm.tsx +++ b/frontend/src/components/feature/settings/ai/bots/BotDocumentProcessorsForm.tsx @@ -68,9 +68,37 @@ export const BotDocumentProcessorsForm = () => { } ) - if (!isGoogleApisEnabled) { - return ( - + const hasExistingProcessors = existingProcessors?.message && existingProcessors.message.length > 0 + + return ( + + {/* Local LLM vision toggle — always visible, no Google APIs required */} + + + + ( + field.onChange(v ? 1 : 0)} + /> + )} + /> + Enable base64 image encoding for Local LLM vision + + + + When images are sent to a Local LLM agent, encode them as base64 and include them directly + in the request. Enable this for vision-capable local models (Ollama, LM Studio, etc.). + + + + + + {/* Google Document AI section */} + {!isGoogleApisEnabled ? ( @@ -79,14 +107,7 @@ export const BotDocumentProcessorsForm = () => { Document Processors require Google Cloud APIs to be enabled in your Raven settings. - - ) - } - - const hasExistingProcessors = existingProcessors?.message && existingProcessors.message.length > 0 - - return ( - + ) : ( @@ -108,6 +129,7 @@ export const BotDocumentProcessorsForm = () => { to process the document and send its results to the agent for better context. + )} {useDocumentParser ? ( <> diff --git a/frontend/src/types/RavenBot/RavenBot.ts b/frontend/src/types/RavenBot/RavenBot.ts index 9e551ca07..b3b0fd97f 100644 --- a/frontend/src/types/RavenBot/RavenBot.ts +++ b/frontend/src/types/RavenBot/RavenBot.ts @@ -66,4 +66,6 @@ File search enables the assistant with knowledge from files that you upload. Onc use_google_document_parser?: 0 | 1 /** Google Document Processor ID : Data */ google_document_processor_id?: string + /** Enable base64 image encoding for Local LLM vision : Check - When images are sent to a Local LLM agent, encode them as base64 and include them directly in the request. */ + enable_vision_base64?: 0 | 1 } \ No newline at end of file diff --git a/packages/types/RavenBot/RavenBot.ts b/packages/types/RavenBot/RavenBot.ts index afca165f5..9c3535567 100644 --- a/packages/types/RavenBot/RavenBot.ts +++ b/packages/types/RavenBot/RavenBot.ts @@ -45,4 +45,6 @@ File search enables the assistant with knowledge from files that you upload. Onc dynamic_instructions?: 0 | 1 /** Bot Functions : Table - Raven Bot Functions */ bot_functions?: RavenBotFunctions[] + /** Enable base64 image encoding for Local LLM vision : Check - When images are sent to a Local LLM agent, encode them as base64 and include them directly in the request. */ + enable_vision_base64?: 0 | 1 } \ No newline at end of file diff --git a/raven/ai/ai.py b/raven/ai/ai.py index 398f1fe73..598e16c27 100644 --- a/raven/ai/ai.py +++ b/raven/ai/ai.py @@ -483,7 +483,7 @@ def process_message_with_agent( if message.text or message.content: user_text += message.text or message.content - if bot.model_provider == "Local LLM": + if bot.model_provider == "Local LLM" and getattr(bot, "enable_vision_base64", 1): # Encode image as base64 so local vision models (Ollama etc.) can see it content = _build_vision_content(file_url, user_text or None) else: @@ -505,7 +505,7 @@ def process_message_with_agent( file_extension = file_url.split(".")[-1].lower() if "." in file_url else "" extracted_content = extract_file_content_for_agent(file_url, file_extension, bot, file_handler) - if recent_file_message.message_type == "Image" and bot.model_provider == "Local LLM": + if recent_file_message.message_type == "Image" and bot.model_provider == "Local LLM" and getattr(bot, "enable_vision_base64", 1): # Encode image as base64 for local vision models, combining with the user's text vision_text = "" if extracted_content: diff --git a/raven/raven_bot/doctype/raven_bot/raven_bot.json b/raven/raven_bot/doctype/raven_bot/raven_bot.json index 878b03751..2e06063d6 100644 --- a/raven/raven_bot/doctype/raven_bot/raven_bot.json +++ b/raven/raven_bot/doctype/raven_bot/raven_bot.json @@ -244,6 +244,13 @@ "fieldtype": "Data", "label": "Google Document Processor ID", "length": 400 + }, + { + "default": "1", + "description": "When images are sent to a Local LLM agent, encode them as base64 and include them directly in the request. Enable this for vision-capable local models (Ollama, LM Studio, etc.).", + "fieldname": "enable_vision_base64", + "fieldtype": "Check", + "label": "Enable base64 image encoding for Local LLM vision" } ], "grid_page_length": 50, From 9c90d45fcc8d357549f76944b709c32a61639600 Mon Sep 17 00:00:00 2001 From: Maurice Date: Mon, 23 Feb 2026 22:17:06 +0100 Subject: [PATCH 3/4] moved checkbox to AI section --- .../settings/ai/bots/AIFeaturesBotForm.tsx | 41 +++++++++++++---- .../ai/bots/BotDocumentProcessorsForm.tsx | 44 +++++-------------- 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/frontend/src/components/feature/settings/ai/bots/AIFeaturesBotForm.tsx b/frontend/src/components/feature/settings/ai/bots/AIFeaturesBotForm.tsx index 18a2064ca..ab4274514 100644 --- a/frontend/src/components/feature/settings/ai/bots/AIFeaturesBotForm.tsx +++ b/frontend/src/components/feature/settings/ai/bots/AIFeaturesBotForm.tsx @@ -139,15 +139,38 @@ const AIFeaturesBotForm = (props: Props) => { )} {isLocalLLM && ( - - - - - - Currently, code interpreter features are not available for Local LLM providers. - These features require OpenAI's infrastructure. - - + <> + + + + + + Currently, code interpreter features are not available for Local LLM providers. + These features require OpenAI's infrastructure. + + + + + + ( + field.onChange(v ? 1 : 0)} + /> + )} + /> + Enable vision (base64 image encoding) + + + + When images are sent to this agent, encode them as base64 and include them directly + in the request. Enable this for vision-capable models (Ollama, LM Studio, etc.). + + + )} Advanced diff --git a/frontend/src/components/feature/settings/ai/bots/BotDocumentProcessorsForm.tsx b/frontend/src/components/feature/settings/ai/bots/BotDocumentProcessorsForm.tsx index 74138bcea..4888dc907 100644 --- a/frontend/src/components/feature/settings/ai/bots/BotDocumentProcessorsForm.tsx +++ b/frontend/src/components/feature/settings/ai/bots/BotDocumentProcessorsForm.tsx @@ -68,37 +68,9 @@ export const BotDocumentProcessorsForm = () => { } ) - const hasExistingProcessors = existingProcessors?.message && existingProcessors.message.length > 0 - - return ( - - {/* Local LLM vision toggle — always visible, no Google APIs required */} - - - - ( - field.onChange(v ? 1 : 0)} - /> - )} - /> - Enable base64 image encoding for Local LLM vision - - - - When images are sent to a Local LLM agent, encode them as base64 and include them directly - in the request. Enable this for vision-capable local models (Ollama, LM Studio, etc.). - - - - - - {/* Google Document AI section */} - {!isGoogleApisEnabled ? ( + if (!isGoogleApisEnabled) { + return ( + @@ -107,7 +79,14 @@ export const BotDocumentProcessorsForm = () => { Document Processors require Google Cloud APIs to be enabled in your Raven settings. - ) : ( + + ) + } + + const hasExistingProcessors = existingProcessors?.message && existingProcessors.message.length > 0 + + return ( + @@ -129,7 +108,6 @@ export const BotDocumentProcessorsForm = () => { to process the document and send its results to the agent for better context. - )} {useDocumentParser ? ( <> From 6e39b366c7bdcf6cafddc1081269d77b022900a3 Mon Sep 17 00:00:00 2001 From: Maurice Date: Mon, 23 Feb 2026 22:37:29 +0100 Subject: [PATCH 4/4] change default to 0 (off) --- raven/raven_bot/doctype/raven_bot/raven_bot.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raven/raven_bot/doctype/raven_bot/raven_bot.json b/raven/raven_bot/doctype/raven_bot/raven_bot.json index 2e06063d6..59970e9e0 100644 --- a/raven/raven_bot/doctype/raven_bot/raven_bot.json +++ b/raven/raven_bot/doctype/raven_bot/raven_bot.json @@ -246,7 +246,7 @@ "length": 400 }, { - "default": "1", + "default": "0", "description": "When images are sent to a Local LLM agent, encode them as base64 and include them directly in the request. Enable this for vision-capable local models (Ollama, LM Studio, etc.).", "fieldname": "enable_vision_base64", "fieldtype": "Check",