feat: add base64 vision encoding for Local LLM agents

frontend/src/components/feature/settings/ai/bots/AIFeaturesBotForm.tsx

@@ -139,15 +139,38 @@ const AIFeaturesBotForm = (props: Props) => {
             )}
 
             {isLocalLLM && (
-                <Callout.Root size="1">
-                    <Callout.Icon>
-                        <BiInfoCircle />
-                    </Callout.Icon>
-                    <Callout.Text>
-                        Currently, code interpreter features are not available for Local LLM providers.
-                        These features require OpenAI's infrastructure.
-                    </Callout.Text>
-                </Callout.Root>
+                <>
+                    <Callout.Root size="1">
+                        <Callout.Icon>
+                            <BiInfoCircle />
+                        </Callout.Icon>
+                        <Callout.Text>
+                            Currently, code interpreter features are not available for Local LLM providers.
+                            These features require OpenAI's infrastructure.
+                        </Callout.Text>
+                    </Callout.Root>
+                    <Stack maxWidth={'480px'}>
+                        <Text as="label" size="2">
+                            <HStack align='center'>
+                                <Controller
+                                    control={control}
+                                    name='enable_vision_base64'
+                                    render={({ field }) => (
+                                        <Checkbox
+                                            checked={field.value ? true : false}
+                                            onCheckedChange={(v) => field.onChange(v ? 1 : 0)}
+                                        />
+                                    )}
+                                />
+                                <span>Enable vision (base64 image encoding)</span>
+                            </HStack>
+                        </Text>
+                        <HelperText>
+                            When images are sent to this agent, encode them as base64 and include them directly
+                            in the request. Enable this for vision-capable models (Ollama, LM Studio, etc.).
+                        </HelperText>
+                    </Stack>
+                </>
             )}
 
             <Heading as='h5' size='3' className='not-cal' weight='bold'>Advanced</Heading>

frontend/src/types/RavenBot/RavenBot.ts

@@ -66,4 +66,6 @@ File search enables the assistant with knowledge from files that you upload. Onc
 	use_google_document_parser?: 0 | 1
 	/**	Google Document Processor ID : Data	*/
 	google_document_processor_id?: string
+	/**	Enable base64 image encoding for Local LLM vision : Check - When images are sent to a Local LLM agent, encode them as base64 and include them directly in the request.	*/
+	enable_vision_base64?: 0 | 1
 }

packages/types/RavenBot/RavenBot.ts

@@ -45,4 +45,6 @@ File search enables the assistant with knowledge from files that you upload. Onc
 	dynamic_instructions?: 0 | 1
 	/**	Bot Functions : Table - Raven Bot Functions	*/
 	bot_functions?: RavenBotFunctions[]
+	/**	Enable base64 image encoding for Local LLM vision : Check - When images are sent to a Local LLM agent, encode them as base64 and include them directly in the request.	*/
+	enable_vision_base64?: 0 | 1
 }

raven/ai/agents_integration.py

@@ -570,8 +570,11 @@ async def handle_ai_request_async(
 								{"role": msg["role"], "content": [{"type": "text", "text": msg["content"]}]}
 							)
 
-					# Add current user message
-					messages.append({"role": "user", "content": [{"type": "text", "text": message}]})
+					# Add current user message (may be a list for vision content with images)
+					if isinstance(message, list):
+						messages.append({"role": "user", "content": message})
+					else:
+						messages.append({"role": "user", "content": [{"type": "text", "text": message}]})
 
 					# Create the API call with or without tools
 					api_params = {
@@ -620,7 +623,10 @@ async def handle_ai_request_async(
 
 								messages = [
 									{"role": "system", "content": [{"type": "text", "text": agent.instructions}]},
-									{"role": "user", "content": [{"type": "text", "text": str(full_input)}]},
+									{
+										"role": "user",
+										"content": message if isinstance(message, list) else [{"type": "text", "text": str(full_input)}],
+									},
 									assistant_message,
 								]
 

raven/ai/ai.py

@@ -353,6 +353,55 @@ def extract_file_content_for_agent(file_url: str, file_extension: str, bot, file
 	return extracted_content
 
 
+def _build_vision_content(file_url: str, user_text: str = "") -> list:
+	"""
+	Build an OpenAI-compatible vision content list with a base64-encoded image.
+	Used for local LLMs with vision capability (Ollama, LM Studio, etc.) that accept
+	the OpenAI chat completions format but cannot fetch server-side file paths.
+	"""
+	import base64
+	import mimetypes
+
+	content = []
+
+	if user_text:
+		content.append({"type": "text", "text": user_text})
+
+	try:
+		file_doc = frappe.get_doc("File", {"file_url": file_url})
+		file_path = file_doc.get_full_path()
+
+		with open(file_path, "rb") as f:
+			image_data = base64.b64encode(f.read()).decode("utf-8")
+
+		mime_type, _ = mimetypes.guess_type(file_path)
+		if not mime_type:
+			ext = file_path.rsplit(".", 1)[-1].lower() if "." in file_path else "jpeg"
+			mime_map = {
+				"jpg": "image/jpeg",
+				"jpeg": "image/jpeg",
+				"png": "image/png",
+				"gif": "image/gif",
+				"webp": "image/webp",
+			}
+			mime_type = mime_map.get(ext, "image/jpeg")
+
+		content.append(
+			{
+				"type": "image_url",
+				"image_url": {"url": f"data:{mime_type};base64,{image_data}"},
+			}
+		)
+	except Exception as e:
+		frappe.log_error(
+			f"Error encoding image for Local LLM vision: {str(e)}\nFile: {file_url}",
+			"Image Vision Encoding Error",
+		)
+		content.append({"type": "text", "text": f"[Image: {file_url}]"})
+
+	return content
+
+
 def process_message_with_agent(
 	message, bot, channel_id: str, is_new_conversation: bool, channel=None
 ):
@@ -428,11 +477,21 @@ def process_message_with_agent(
 			if message.text or message.content:
 				content += f"\n\nUser's question: {message.text or message.content}"
 		else:
-			content = f"[User uploaded an image: {file_url}]"
+			user_text = ""
 			if extracted_content:
-				content += f"\n\nExtracted content from the image:\n{extracted_content}"
+				user_text = f"Extracted content from the image:\n{extracted_content}\n\n"
 			if message.text or message.content:
-				content += f"\n\nUser's question: {message.text or message.content}"
+				user_text += message.text or message.content
+
+			if bot.model_provider == "Local LLM" and getattr(bot, "enable_vision_base64", 1):
+				# Encode image as base64 so local vision models (Ollama etc.) can see it
+				content = _build_vision_content(file_url, user_text or None)
+			else:
+				content = f"[User uploaded an image: {file_url}]"
+				if extracted_content:
+					content += f"\n\nExtracted content from the image:\n{extracted_content}"
+				if message.text or message.content:
+					content += f"\n\nUser's question: {message.text or message.content}"
 	else:
 		content = message.text or message.content or ""
 
@@ -446,13 +505,20 @@ def process_message_with_agent(
 			file_extension = file_url.split(".")[-1].lower() if "." in file_url else ""
 			extracted_content = extract_file_content_for_agent(file_url, file_extension, bot, file_handler)
 
-			file_prefix = f"[User uploaded a {'file' if recent_file_message.message_type == 'File' else 'image'}: {file_url}]"
-			if extracted_content:
-				file_prefix += f"\n\nExtracted content from the file:\n{extracted_content}\n\n"
+			if recent_file_message.message_type == "Image" and bot.model_provider == "Local LLM" and getattr(bot, "enable_vision_base64", 1):
+				# Encode image as base64 for local vision models, combining with the user's text
+				vision_text = ""
+				if extracted_content:
+					vision_text = f"Extracted content from the image:\n{extracted_content}\n\n"
+				vision_text += content
+				content = _build_vision_content(file_url, vision_text)
 			else:
-				file_prefix += "\n"
-
-			content = file_prefix + content
+				file_prefix = f"[User uploaded a {'file' if recent_file_message.message_type == 'File' else 'image'}: {file_url}]"
+				if extracted_content:
+					file_prefix += f"\n\nExtracted content from the file:\n{extracted_content}\n\n"
+				else:
+					file_prefix += "\n"
+				content = file_prefix + content
 
 	# Get conversation history if this is an existing thread
 	conversation_history = []

raven/raven_bot/doctype/raven_bot/raven_bot.json

@@ -244,6 +244,13 @@
    "fieldtype": "Data",
    "label": "Google Document Processor ID",
    "length": 400
+  },
+  {
+   "default": "0",
+   "description": "When images are sent to a Local LLM agent, encode them as base64 and include them directly in the request. Enable this for vision-capable local models (Ollama, LM Studio, etc.).",
+   "fieldname": "enable_vision_base64",
+   "fieldtype": "Check",
+   "label": "Enable base64 image encoding for Local LLM vision"
   }
  ],
  "grid_page_length": 50,