From afd31c7548fde0b5c8fc36faf92cb211685a0213 Mon Sep 17 00:00:00 2001 From: Marc Doerflinger Date: Mon, 6 Jul 2026 20:01:40 +0000 Subject: [PATCH 1/2] feat(ocr): implement multi-image OCR processing with per-image opt-out flags --- .claude/settings.json | 7 + app/controllers/ocr_controller.rb | 132 +++++++++++------- .../controllers/dropzone_controller.js | 40 +++++- app/services/openai_service.rb | 107 ++++++++------ app/views/shared/_dropzone.html.erb | 7 +- config/locales/de.yml | 2 + config/locales/en.yml | 2 + config/prompts/mistral_markdown.txt | 1 + config/prompts/openai_markdown.txt | 1 + spec/controllers/ocr_controller_spec.rb | 130 +++++++++++++++-- 10 files changed, 323 insertions(+), 106 deletions(-) create mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..ebeb23ff --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": [ + "Bash(node --input-type=module --check)" + ] + } +} diff --git a/app/controllers/ocr_controller.rb b/app/controllers/ocr_controller.rb index a69255ba..fcf11409 100644 --- a/app/controllers/ocr_controller.rb +++ b/app/controllers/ocr_controller.rb @@ -1,35 +1,26 @@ class OcrController < ApplicationController include Secured + PAGE_BREAK_MARKER = "\n\n---\n**[NEW SCANNED PAGE]**\n---\n\n" + def scan - # Process only the first uploaded file - file = params[:files].first + files = Array(params[:files]) ai_method = params[:ai_method] || 'mistral_only' + ocr_flags = build_ocr_flags(files) - begin - # Process based on selected AI method - magic_data_json = if ai_method == 'mistral_only' - # Two-phase: Mistral OCR -> Mistral parsing - markdown = mistral_service.ocr_to_markdown(file.tempfile, file.content_type) - - if markdown.blank? - raise "Mistral OCR returned empty markdown" - end - - mistral_service.parse_markdown_to_recipes(markdown) - elsif ai_method == 'mistral_openai' - # Two-phase: Mistral OCR -> OpenAI parsing - markdown = mistral_service.ocr_to_markdown(file.tempfile, file.content_type) + ocr_files = [] + files.each_with_index do |file, index| + ocr_files << file if ocr_flags[index] + end - if markdown.blank? - raise "Mistral OCR returned empty markdown" - end + if ocr_files.empty? + render json: { success: false, error: I18n.t('ocr.errors.no_images_selected') } + return + end - openai_service.parse_markdown_to_recipes(markdown) - else - # Direct OpenAI OCR - openai_service.ocr(file.tempfile, file.content_type) - end + begin + files_with_types = ocr_files.map { |f| [f.tempfile, f.content_type] } + magic_data_json = run_ocr(files_with_types, ai_method) if magic_data_json.empty? raise "No recipes extracted from image" @@ -39,11 +30,11 @@ def scan # Save full OCR result array to database and store id in flash to avoid flash size limits ocrresult = OcrResult.create(result: magic_data_json.to_json, ai_method: ai_method) - ocrresult.image.attach(file) + ocrresult.image.attach(files.first) ocrresult.save - if params[:files].length > 1 - params[:files][1..].each { |f| ocrresult.extra_images.attach(f) } + if files.length > 1 + files[1..].each { |f| ocrresult.extra_images.attach(f) } end logger.debug "OCR data id stored in flash: #{ocrresult.id}" @@ -149,29 +140,7 @@ def reparse_image temp_file.write(image_file) temp_file.rewind - # Process based on selected AI method - magic_data_json = if ai_method == 'mistral_only' - # Two-phase: Mistral OCR -> Mistral parsing - markdown = mistral_service.ocr_to_markdown(temp_file, content_type) - - if markdown.blank? - raise "Mistral OCR returned empty markdown" - end - - mistral_service.parse_markdown_to_recipes(markdown) - elsif ai_method == 'mistral_openai' - # Two-phase: Mistral OCR -> OpenAI parsing - markdown = mistral_service.ocr_to_markdown(temp_file, content_type) - - if markdown.blank? - raise "Mistral OCR returned empty markdown" - end - - openai_service.parse_markdown_to_recipes(markdown) - else - # Direct OpenAI OCR - openai_service.ocr(temp_file, content_type) - end + magic_data_json = run_ocr([[temp_file, content_type]], ai_method) if magic_data_json.empty? raise "No recipes extracted from image" @@ -210,6 +179,69 @@ def reparse_image private + # Runs the selected AI pipeline across one or more images and returns the extracted recipes array. + # files_with_types is an array of [tempfile, content_type] pairs. + def run_ocr(files_with_types, ai_method) + case ai_method + when 'mistral_only' + run_mistral_only(files_with_types) + when 'mistral_openai' + run_mistral_then_openai(files_with_types) + else + run_openai_direct(files_with_types) + end + end + + # Two-phase: Mistral OCR on every image -> combined markdown parsed by Mistral. + def run_mistral_only(files_with_types) + markdown = combined_ocr_markdown(files_with_types) + mistral_service.parse_markdown_to_recipes(markdown) + end + + # Two-phase: Mistral OCR on every image -> combined markdown parsed by OpenAI. + def run_mistral_then_openai(files_with_types) + markdown = combined_ocr_markdown(files_with_types) + openai_service.parse_markdown_to_recipes(markdown) + end + + # Direct OpenAI OCR: all images are sent in a single vision request. + def run_openai_direct(files_with_types) + openai_service.ocr_multi(files_with_types) + end + + # Runs Mistral OCR on each image in turn, then joins the resulting markdown + # pages into one document (separated by PAGE_BREAK_MARKER) for a single parse step. + def combined_ocr_markdown(files_with_types) + markdowns = [] + + files_with_types.each_with_index do |(tempfile, content_type), index| + markdown = mistral_service.ocr_to_markdown(tempfile, content_type) + + if markdown.blank? + raise "Mistral OCR returned empty markdown for image #{index + 1}" + end + + markdowns << markdown + end + + markdowns.join(PAGE_BREAK_MARKER) + end + + # Builds a boolean array (same length/order as files) indicating which files should be sent for OCR. + # Missing or absent ocr_flags default to true (include all), preserving behavior for callers that + # don't send the flag (e.g. the single-image reparse flow, or older clients). + def build_ocr_flags(files) + raw_flags = Array(params[:ocr_flags]) + return Array.new(files.length, true) if raw_flags.empty? + + flags = [] + files.each_index do |index| + flag = raw_flags[index] + flags << (flag.nil? || ActiveModel::Type::Boolean.new.cast(flag)) + end + flags + end + def openai_service @openai_service ||= OpenaiService.new end diff --git a/app/javascript/controllers/dropzone_controller.js b/app/javascript/controllers/dropzone_controller.js index 0e552029..f5e4f419 100644 --- a/app/javascript/controllers/dropzone_controller.js +++ b/app/javascript/controllers/dropzone_controller.js @@ -3,7 +3,7 @@ import { Utils } from "../src/utils" export default class extends Controller { static targets = ["dropzone", "fileInput", "spinner", "successMessage", "errorMessage", "resetButton", "previewContainer", "uploadButton"] - static values = { url: String, previewMode: Boolean } + static values = { url: String, previewMode: Boolean, ocrBadgeTitle: String, noImagesSelectedError: String } connect() { this.maxFiles = 10 @@ -11,6 +11,7 @@ export default class extends Controller { this.acceptedExtensions = ['.jpg', '.jpeg', '.png', '.webp', '.heic', '.heif'] this.pendingFiles = [] this.pendingObjectUrls = [] + this.pendingOcrFlags = [] // Initialize button state in preview mode if (this.previewModeValue && this.hasUploadButtonTarget) { @@ -92,6 +93,7 @@ export default class extends Controller { if (this.previewModeValue) { for (let i = 0; i < files.length; i++) { this.pendingFiles.push(files[i]) + this.pendingOcrFlags.push(true) } this.renderPreviews() this.uploadButtonTarget.disabled = false @@ -121,6 +123,11 @@ export default class extends Controller { const objectUrl = URL.createObjectURL(file) this.pendingObjectUrls.push(objectUrl) + const included = this.pendingOcrFlags[index] + const badgeClass = included + ? 'bg-blue-500 text-white' + : 'bg-gray-300 text-gray-500 opacity-60' + const card = document.createElement('div') card.className = 'relative rounded-lg overflow-hidden border border-gray-200 bg-gray-50' card.innerHTML = ` @@ -133,7 +140,13 @@ export default class extends Controller { aria-label="Remove"> - ${index === 0 ? '
AI
' : ''} + ` grid.appendChild(card) }) @@ -145,6 +158,7 @@ export default class extends Controller { removeFile(event) { const index = parseInt(event.currentTarget.dataset.index, 10) this.pendingFiles.splice(index, 1) + this.pendingOcrFlags.splice(index, 1) if (this.pendingFiles.length === 0) { this.uploadButtonTarget.disabled = true @@ -157,19 +171,33 @@ export default class extends Controller { } } + toggleOcr(event) { + event.stopPropagation() + const index = parseInt(event.currentTarget.dataset.index, 10) + this.pendingOcrFlags[index] = !this.pendingOcrFlags[index] + this.renderPreviews() + } + submit() { + if (this.pendingOcrFlags.every(flag => !flag)) { + this.showError(this.noImagesSelectedErrorValue) + return + } + const filesToUpload = this.pendingFiles.slice() + const ocrFlagsToUpload = this.pendingOcrFlags.slice() this.pendingFiles = [] + this.pendingOcrFlags = [] this.uploadButtonTarget.disabled = true this.previewContainerTarget.classList.add('hidden') - this.uploadFiles(filesToUpload) + this.uploadFiles(filesToUpload, ocrFlagsToUpload) } escapeHtml(str) { return str.replace(/[&<>"']/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c])) } - uploadFiles(files) { + uploadFiles(files, ocrFlags = null) { // Hide dropzone this.dropzoneTarget.classList.add('hidden') @@ -180,6 +208,9 @@ export default class extends Controller { const formData = new FormData() for (let i = 0; i < files.length; i++) { formData.append('files[]', files[i]) + if (ocrFlags) { + formData.append('ocr_flags[]', ocrFlags[i] ? 'true' : 'false') + } } // Add AI method selection if present @@ -240,6 +271,7 @@ export default class extends Controller { // Clear any pending preview state this.pendingFiles = [] + this.pendingOcrFlags = [] this.pendingObjectUrls.forEach(url => URL.revokeObjectURL(url)) this.pendingObjectUrls = [] if (this.hasPreviewContainerTarget) { diff --git a/app/services/openai_service.rb b/app/services/openai_service.rb index 339edffc..326f865c 100644 --- a/app/services/openai_service.rb +++ b/app/services/openai_service.rb @@ -27,31 +27,20 @@ def cleanup(text, prompt) end def ocr(image_file, content_type) - # Read and encode the image file as base64 - image_data = if image_file.respond_to?(:read) - # Handle uploaded file (Tempfile) - Base64.strict_encode64(image_file.read) - elsif image_file.is_a?(String) - # Handle file path - File.open(image_file, 'rb') { |f| Base64.strict_encode64(f.read) } - else - raise ArgumentError, "Invalid image_file type" - end + ocr_multi([[image_file, content_type]]) + end - # Determine the image format from content_type or file extension - image_format = case content_type - when /jpeg|jpg/ then 'jpeg' - when /png/ then 'png' - when /webp/ then 'webp' - when /heic|heif/ then 'heic' - else 'jpeg' # default - end + def ocr_multi(image_files_with_types) + image_blocks = image_files_with_types.map { |image_file, content_type| build_image_content_block(image_file, content_type) } - # puts "Image format detected: #{image_format} for content type: #{content_type} content #{image_data[0..30]}..." - filedata = "data:image/#{image_format};base64,#{image_data}" model = Rails.configuration.openai.recipe_ocr_model system_prompt = File.read(Rails.root.join("config", "prompts", Rails.configuration.openai.recipe_ocr_prompt_file)) - Rails.logger.debug "Sending data to OpenAI API (model #{model}) -> #{filedata[0..100]}..." + prompt_text = if image_blocks.length > 1 + "Extract all data from these images. They are sequential pages of the same scan; a single recipe may span multiple images, or each image may contain one or more distinct recipes." + else + "Extract all data from this image" + end + Rails.logger.debug "Sending #{image_blocks.length} image(s) to OpenAI API (model #{model})" response = @client.responses.create( parameters: { @@ -66,12 +55,9 @@ def ocr(image_file, content_type) content: [ { type: "input_text", - text: "Extract all data from this image" + text: prompt_text }, - { - type: "input_image", - image_url: filedata - } + *image_blocks ] } ], @@ -88,23 +74,7 @@ def ocr(image_file, content_type) Rails.logger.debug "OpenAI OCR response: #{response}" - output = response.dig("output") || [] - message = output.find { |item| item["type"] == "message" } - llm_response_text = message&.dig("content", 0, "text") - - begin - parsed = JSON.parse(llm_response_text) - recipes = parsed['recipes'] || [] - Rails.logger.warn "No recipes found in OpenAI response" if recipes.empty? - Rails.logger.info "OpenAI OCR recipes: #{recipes}" - recipes - rescue JSON::ParserError => e - Rails.logger.error "Failed to parse OpenAI response: #{e.message}" - [] - rescue => e - Rails.logger.error "Unexpected error parsing recipes: #{e.message}" - [] - end + extract_recipes_from_response(response, "OpenAI OCR") end def parse_markdown_to_recipes(markdown_text) @@ -206,4 +176,55 @@ def parse_url_to_recipes(markdown_text) raise end end + + private + + def build_image_content_block(image_file, content_type) + # Read and encode the image file as base64 + image_data = if image_file.respond_to?(:read) + # Handle uploaded file (Tempfile) + Base64.strict_encode64(image_file.read) + elsif image_file.is_a?(String) + # Handle file path + File.open(image_file, 'rb') { |f| Base64.strict_encode64(f.read) } + else + raise ArgumentError, "Invalid image_file type" + end + + # Determine the image format from content_type or file extension + image_format = case content_type + when /jpeg|jpg/ then 'jpeg' + when /png/ then 'png' + when /webp/ then 'webp' + when /heic|heif/ then 'heic' + else 'jpeg' # default + end + + filedata = "data:image/#{image_format};base64,#{image_data}" + + { + type: "input_image", + image_url: filedata + } + end + + def extract_recipes_from_response(response, log_label) + output = response.dig("output") || [] + message = output.find { |item| item["type"] == "message" } + llm_response_text = message&.dig("content", 0, "text") + + begin + parsed = JSON.parse(llm_response_text) + recipes = parsed['recipes'] || [] + Rails.logger.warn "No recipes found in #{log_label} response" if recipes.empty? + Rails.logger.info "#{log_label} recipes: #{recipes}" + recipes + rescue JSON::ParserError => e + Rails.logger.error "Failed to parse #{log_label} response: #{e.message}" + [] + rescue => e + Rails.logger.error "Unexpected error parsing recipes: #{e.message}" + [] + end + end end diff --git a/app/views/shared/_dropzone.html.erb b/app/views/shared/_dropzone.html.erb index 9a59b7d6..0bb3c8a1 100644 --- a/app/views/shared/_dropzone.html.erb +++ b/app/views/shared/_dropzone.html.erb @@ -14,7 +14,12 @@ preview_mode ||= false %> -
+
'Multi-page Recipe' }] } + let(:captured_args) { [] } + + before do + allow(openai_service).to receive(:ocr_multi) { |args| captured_args.replace(args); single_recipe } + end + + it 'sends all files in a single ocr_multi call' do + post :scan, params: { files: [file, file2], ai_method: 'openai_direct' } + + expect(openai_service).to have_received(:ocr_multi).once + expect(captured_args.length).to eq(2) + end + + it 'attaches the first file as image and the rest as extra_images' do + post :scan, params: { files: [file, file2], ai_method: 'openai_direct' } + + ocrresult = OcrResult.last + expect(ocrresult.image).to be_attached + expect(ocrresult.extra_images.count).to eq(1) + end + end + + context 'with multiple uploaded images (mistral_only)' do + let(:single_recipe) { [{ 'title' => 'Multi-page Recipe' }] } + let(:captured_markdown) { [] } + + before do + allow(mistral_service).to receive(:ocr_to_markdown).and_return('page one markdown', 'page two markdown') + allow(mistral_service).to receive(:parse_markdown_to_recipes) { |markdown| captured_markdown << markdown; single_recipe } + end + + it 'OCRs every image and parses the combined, ordered markdown once' do + post :scan, params: { files: [file, file2], ai_method: 'mistral_only' } + + expect(mistral_service).to have_received(:ocr_to_markdown).twice + expect(mistral_service).to have_received(:parse_markdown_to_recipes).once + combined = captured_markdown.first + expect(combined).to include('page one markdown') + expect(combined).to include('page two markdown') + expect(combined.index('page one markdown')).to be < combined.index('page two markdown') + end + end + + context 'when one of several images fails OCR (mistral_only)' do + before do + call_count = 0 + allow(mistral_service).to receive(:ocr_to_markdown) do + call_count += 1 + call_count == 1 ? 'page one markdown' : raise(StandardError, 'Mistral API Error') + end + end + + it 'fails the whole request and does not create an OcrResult' do + expect { + post :scan, params: { files: [file, file2], ai_method: 'mistral_only' } + }.not_to change(OcrResult, :count) + + json_response = JSON.parse(response.body) + expect(json_response['success']).to be false + end + end + + context 'with per-image ocr_flags opt-out' do + let(:single_recipe) { [{ 'title' => 'Test Recipe' }] } + let(:captured_args) { [] } + + before do + allow(openai_service).to receive(:ocr_multi) { |args| captured_args.replace(args); single_recipe } + end + + it 'only sends flagged-in files for OCR but attaches all files' do + post :scan, params: { + files: [file, file2, file3], + ai_method: 'openai_direct', + ocr_flags: ['true', 'false', 'true'] + } + + expect(captured_args.length).to eq(2) + + ocrresult = OcrResult.last + expect(ocrresult.extra_images.count).to eq(2) + end + + it 'defaults to including all files when ocr_flags is omitted' do + post :scan, params: { files: [file, file2], ai_method: 'openai_direct' } + + expect(captured_args.length).to eq(2) + end + + it 'rejects the request when every image is excluded' do + expect { + post :scan, params: { + files: [file, file2], + ai_method: 'openai_direct', + ocr_flags: ['false', 'false'] + } + }.not_to change(OcrResult, :count) + + expect(openai_service).not_to have_received(:ocr_multi) + json_response = JSON.parse(response.body) + expect(json_response['success']).to be false + expect(json_response['error']).to be_present + end + end end describe 'POST #select_recipe' do From 4c70d06992d2233dae104a2c3b165877dd1d8288 Mon Sep 17 00:00:00 2001 From: Marc Doerflinger Date: Mon, 6 Jul 2026 20:31:20 +0000 Subject: [PATCH 2/2] feat(ocr): enhance OCR processing with shared pipeline --- .claude/settings.json | 5 +- app/controllers/ocr_controller.rb | 78 ++------------- app/controllers/recipes_controller.rb | 45 --------- .../controllers/dropzone_controller.js | 2 +- app/services/image_data_uri.rb | 24 +++++ app/services/mistralai_service.rb | 59 ++---------- app/services/ocr_pipeline.rb | 38 ++++++++ app/services/openai_service.rb | 96 +++---------------- app/services/recipe_json_parser.rb | 17 ++++ config/prompts/openai_ocr.txt | 14 +-- spec/controllers/ocr_controller_spec.rb | 13 +++ 11 files changed, 133 insertions(+), 258 deletions(-) create mode 100644 app/services/image_data_uri.rb create mode 100644 app/services/ocr_pipeline.rb create mode 100644 app/services/recipe_json_parser.rb diff --git a/.claude/settings.json b/.claude/settings.json index ebeb23ff..957ebebe 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,7 +1,10 @@ { "permissions": { "allow": [ - "Bash(node --input-type=module --check)" + "Bash(node --input-type=module --check)", + "Bash(bundle exec *)", + "Bash(git stash *)", + "Bash(bundle exec rails runner ' *)" ] } } diff --git a/app/controllers/ocr_controller.rb b/app/controllers/ocr_controller.rb index fcf11409..df7fd663 100644 --- a/app/controllers/ocr_controller.rb +++ b/app/controllers/ocr_controller.rb @@ -1,17 +1,11 @@ class OcrController < ApplicationController include Secured - PAGE_BREAK_MARKER = "\n\n---\n**[NEW SCANNED PAGE]**\n---\n\n" - def scan files = Array(params[:files]) ai_method = params[:ai_method] || 'mistral_only' ocr_flags = build_ocr_flags(files) - - ocr_files = [] - files.each_with_index do |file, index| - ocr_files << file if ocr_flags[index] - end + ocr_files = files.select.with_index { |_, index| ocr_flags[index] } if ocr_files.empty? render json: { success: false, error: I18n.t('ocr.errors.no_images_selected') } @@ -20,7 +14,7 @@ def scan begin files_with_types = ocr_files.map { |f| [f.tempfile, f.content_type] } - magic_data_json = run_ocr(files_with_types, ai_method) + magic_data_json = ocr_pipeline.extract_recipes(files_with_types, ai_method) if magic_data_json.empty? raise "No recipes extracted from image" @@ -140,7 +134,7 @@ def reparse_image temp_file.write(image_file) temp_file.rewind - magic_data_json = run_ocr([[temp_file, content_type]], ai_method) + magic_data_json = ocr_pipeline.extract_recipes([[temp_file, content_type]], ai_method) if magic_data_json.empty? raise "No recipes extracted from image" @@ -179,74 +173,22 @@ def reparse_image private - # Runs the selected AI pipeline across one or more images and returns the extracted recipes array. - # files_with_types is an array of [tempfile, content_type] pairs. - def run_ocr(files_with_types, ai_method) - case ai_method - when 'mistral_only' - run_mistral_only(files_with_types) - when 'mistral_openai' - run_mistral_then_openai(files_with_types) - else - run_openai_direct(files_with_types) - end - end - - # Two-phase: Mistral OCR on every image -> combined markdown parsed by Mistral. - def run_mistral_only(files_with_types) - markdown = combined_ocr_markdown(files_with_types) - mistral_service.parse_markdown_to_recipes(markdown) - end - - # Two-phase: Mistral OCR on every image -> combined markdown parsed by OpenAI. - def run_mistral_then_openai(files_with_types) - markdown = combined_ocr_markdown(files_with_types) - openai_service.parse_markdown_to_recipes(markdown) - end - - # Direct OpenAI OCR: all images are sent in a single vision request. - def run_openai_direct(files_with_types) - openai_service.ocr_multi(files_with_types) - end - - # Runs Mistral OCR on each image in turn, then joins the resulting markdown - # pages into one document (separated by PAGE_BREAK_MARKER) for a single parse step. - def combined_ocr_markdown(files_with_types) - markdowns = [] - - files_with_types.each_with_index do |(tempfile, content_type), index| - markdown = mistral_service.ocr_to_markdown(tempfile, content_type) - - if markdown.blank? - raise "Mistral OCR returned empty markdown for image #{index + 1}" - end - - markdowns << markdown - end - - markdowns.join(PAGE_BREAK_MARKER) - end + BOOLEAN_TYPE = ActiveModel::Type::Boolean.new # Builds a boolean array (same length/order as files) indicating which files should be sent for OCR. - # Missing or absent ocr_flags default to true (include all), preserving behavior for callers that - # don't send the flag (e.g. the single-image reparse flow, or older clients). + # Missing, blank, or unparseable ocr_flags default to true (include all); only an explicit false excludes. def build_ocr_flags(files) raw_flags = Array(params[:ocr_flags]) return Array.new(files.length, true) if raw_flags.empty? - flags = [] - files.each_index do |index| - flag = raw_flags[index] - flags << (flag.nil? || ActiveModel::Type::Boolean.new.cast(flag)) - end - flags + files.each_index.map { |index| BOOLEAN_TYPE.cast(raw_flags[index]) != false } end - def openai_service - @openai_service ||= OpenaiService.new + def ocr_pipeline + @ocr_pipeline ||= OcrPipeline.new end - def mistral_service - @mistral_service ||= MistralaiService.new + def openai_service + @openai_service ||= OpenaiService.new end end diff --git a/app/controllers/recipes_controller.rb b/app/controllers/recipes_controller.rb index 45f95e15..9f2539a6 100644 --- a/app/controllers/recipes_controller.rb +++ b/app/controllers/recipes_controller.rb @@ -121,51 +121,6 @@ def favorite render json: { redirect_url: recipe_path(recipe.id) } end - def reparse_image - @recipe = Recipe.find(params[:id]) - attachment_id = params[:attachment_id] - - begin - # Find the selected image attachment - attachment = @recipe.recipe_images.find(attachment_id) - - # Download the image blob and get content type - blob = attachment.blob - image_file = blob.download - content_type = blob.content_type - - # Create a temporary file for the OpenAI service - temp_file = Tempfile.new(['recipe_image', File.extname(blob.filename.to_s)]) - temp_file.binmode - temp_file.write(image_file) - temp_file.rewind - - # Call OpenAI service to parse the image - openai = OpenaiService.new - magic_data_json = openai.ocr(temp_file, content_type) - - # Save OCR result to database - ocrresult = OcrResult.create(result: magic_data_json.to_s) - ocrresult.image.attach(blob) - ocrresult.save - flash[:ocr_data] = ocrresult.id - - logger.debug "Reparse OCR data id stored in flash: #{flash[:ocr_data]}" - - redirect_to edit_recipe_path(@recipe) - rescue JSON::ParserError => e - logger.error "Reparse JSON parse error: #{e.message}" - flash[:error] = I18n.t('ocr.errors.parse_failed') - redirect_to recipe_path(@recipe) - rescue => e - logger.error "Reparse error: #{e.message}" - flash[:error] = I18n.t('ocr.errors.processing_failed') - redirect_to recipe_path(@recipe) - ensure - temp_file&.close - temp_file&.unlink - end - end def destroy @recipe = Recipe.find(params[:id]) diff --git a/app/javascript/controllers/dropzone_controller.js b/app/javascript/controllers/dropzone_controller.js index f5e4f419..736f6172 100644 --- a/app/javascript/controllers/dropzone_controller.js +++ b/app/javascript/controllers/dropzone_controller.js @@ -144,7 +144,7 @@ export default class extends Controller { class="absolute top-1 left-1 text-xs rounded px-1 leading-5 cursor-pointer ${badgeClass}" data-action="click->dropzone#toggleOcr" data-index="${index}" - title="${this.ocrBadgeTitleValue}"> + title="${this.escapeHtml(this.ocrBadgeTitleValue)}"> AI ` diff --git a/app/services/image_data_uri.rb b/app/services/image_data_uri.rb new file mode 100644 index 00000000..f424ca9d --- /dev/null +++ b/app/services/image_data_uri.rb @@ -0,0 +1,24 @@ +module ImageDataUri + def build_data_uri(image_file, content_type) + image_data = if image_file.respond_to?(:read) + image_file.rewind if image_file.respond_to?(:rewind) + data = Base64.strict_encode64(image_file.read) + image_file.rewind if image_file.respond_to?(:rewind) + data + elsif image_file.is_a?(String) + File.open(image_file, 'rb') { |f| Base64.strict_encode64(f.read) } + else + raise ArgumentError, "Invalid image_file type" + end + + image_format = case content_type + when /jpeg|jpg/ then 'jpeg' + when /png/ then 'png' + when /webp/ then 'webp' + when /heic|heif/ then 'heic' + else 'jpeg' # default + end + + "data:image/#{image_format};base64,#{image_data}" + end +end diff --git a/app/services/mistralai_service.rb b/app/services/mistralai_service.rb index 690cf723..c71272d3 100644 --- a/app/services/mistralai_service.rb +++ b/app/services/mistralai_service.rb @@ -1,4 +1,7 @@ class MistralaiService + include ImageDataUri + include RecipeJsonParser + def initialize # API key validation happens at runtime (not initialization) to allow # the service to be instantiated even when Mistral AI is not configured. @@ -10,31 +13,7 @@ def initialize end def ocr_to_markdown(image_file, content_type) - # Read and encode the image file as base64 - image_data = if image_file.respond_to?(:read) - # Handle uploaded file (Tempfile) - image_file.rewind - data = Base64.strict_encode64(image_file.read) - image_file.rewind # Reset for potential subsequent reads - data - elsif image_file.is_a?(String) - # Handle file path - File.open(image_file, 'rb') { |f| Base64.strict_encode64(f.read) } - else - raise ArgumentError, "Invalid image_file type" - end - - # Determine the image format from content_type or file extension - image_format = case content_type - when /jpeg|jpg/ then 'jpeg' - when /png/ then 'png' - when /webp/ then 'webp' - when /heic|heif/ then 'heic' - else 'jpeg' # default - end - - Rails.logger.debug "Image format detected: #{image_format} for content type: #{content_type}" - filedata = "data:image/#{image_format};base64,#{image_data}" + filedata = build_data_uri(image_file, content_type) response = @client.ocr(filedata, kind: :image) recognized_markdown = response.pages[0].markdown @@ -59,20 +38,7 @@ def parse_markdown_to_recipes(markdown_text) Rails.logger.debug "Mistral AI markdown parsing response: #{completion.text}" llm_response_text = completion.text.gsub(/```json/, '').gsub(/```/, '') - - begin - parsed = JSON.parse(llm_response_text) - recipes = parsed['recipes'] || [] - Rails.logger.warn "No recipes found in Mistral AI response" if recipes.empty? - Rails.logger.info "Mistral AI parsed #{recipes.length} recipes from markdown" - recipes - rescue JSON::ParserError => e - Rails.logger.error "Failed to parse Mistral AI markdown response: #{e.message}" - raise - rescue => e - Rails.logger.error "Unexpected error parsing recipes from markdown: #{e.message}" - raise - end + parse_recipes_json(llm_response_text, "Mistral AI markdown parsing", on_error: :raise) end def parse_url_to_recipes(markdown_text) @@ -89,19 +55,6 @@ def parse_url_to_recipes(markdown_text) Rails.logger.debug "Mistral AI URL parsing response: #{completion.text}" llm_response_text = completion.text.gsub(/```json/, '').gsub(/```/, '') - - begin - parsed = JSON.parse(llm_response_text) - recipes = parsed['recipes'] || [] - Rails.logger.warn "No recipes found in Mistral AI URL response" if recipes.empty? - Rails.logger.info "Mistral AI parsed #{recipes.length} recipes from URL markdown" - recipes - rescue JSON::ParserError => e - Rails.logger.error "Failed to parse Mistral AI URL response: #{e.message}" - raise - rescue => e - Rails.logger.error "Unexpected error parsing recipes from URL: #{e.message}" - raise - end + parse_recipes_json(llm_response_text, "Mistral AI URL parsing", on_error: :raise) end end diff --git a/app/services/ocr_pipeline.rb b/app/services/ocr_pipeline.rb new file mode 100644 index 00000000..7e24a729 --- /dev/null +++ b/app/services/ocr_pipeline.rb @@ -0,0 +1,38 @@ +class OcrPipeline + PAGE_BREAK_MARKER = "\n\n---\n**[NEW SCANNED PAGE]**\n---\n\n" + + # files_with_types is an array of [tempfile, content_type] pairs. + def extract_recipes(files_with_types, ai_method) + case ai_method + when 'mistral_only' + mistral_service.parse_markdown_to_recipes(combined_ocr_markdown(files_with_types)) + when 'mistral_openai' + openai_service.parse_markdown_to_recipes(combined_ocr_markdown(files_with_types)) + else + openai_service.ocr_multi(files_with_types) + end + end + + private + + # Runs Mistral OCR on each image in turn, then joins the resulting markdown + # pages into one document (separated by PAGE_BREAK_MARKER) for a single parse step. + def combined_ocr_markdown(files_with_types) + markdowns = files_with_types.each_with_index.map do |(tempfile, content_type), index| + markdown = mistral_service.ocr_to_markdown(tempfile, content_type) + raise "Mistral OCR returned empty markdown for image #{index + 1}" if markdown.blank? + + markdown + end + + markdowns.join(PAGE_BREAK_MARKER) + end + + def openai_service + @openai_service ||= OpenaiService.new + end + + def mistral_service + @mistral_service ||= MistralaiService.new + end +end diff --git a/app/services/openai_service.rb b/app/services/openai_service.rb index 326f865c..ecf1deca 100644 --- a/app/services/openai_service.rb +++ b/app/services/openai_service.rb @@ -1,4 +1,7 @@ class OpenaiService + include ImageDataUri + include RecipeJsonParser + def initialize # Try environment variable first, fallback to credentials for development api_key = ENV['OPENAI_API_KEY'] @@ -26,20 +29,11 @@ def cleanup(text, prompt) response.dig("choices", 0, "message", "content") end - def ocr(image_file, content_type) - ocr_multi([[image_file, content_type]]) - end - def ocr_multi(image_files_with_types) image_blocks = image_files_with_types.map { |image_file, content_type| build_image_content_block(image_file, content_type) } model = Rails.configuration.openai.recipe_ocr_model system_prompt = File.read(Rails.root.join("config", "prompts", Rails.configuration.openai.recipe_ocr_prompt_file)) - prompt_text = if image_blocks.length > 1 - "Extract all data from these images. They are sequential pages of the same scan; a single recipe may span multiple images, or each image may contain one or more distinct recipes." - else - "Extract all data from this image" - end Rails.logger.debug "Sending #{image_blocks.length} image(s) to OpenAI API (model #{model})" response = @client.responses.create( @@ -55,7 +49,7 @@ def ocr_multi(image_files_with_types) content: [ { type: "input_text", - text: prompt_text + text: "Extract all data from the following image(s)" }, *image_blocks ] @@ -108,23 +102,7 @@ def parse_markdown_to_recipes(markdown_text) Rails.logger.debug "OpenAI markdown parsing response: #{response}" - output = response.dig("output") || [] - message = output.find { |item| item["type"] == "message" } - llm_response_text = message&.dig("content", 0, "text") - - begin - parsed = JSON.parse(llm_response_text) - recipes = parsed['recipes'] || [] - Rails.logger.warn "No recipes found in OpenAI markdown parsing response" if recipes.empty? - Rails.logger.info "OpenAI parsed #{recipes.length} recipes from markdown" - recipes - rescue JSON::ParserError => e - Rails.logger.error "Failed to parse OpenAI markdown response: #{e.message}" - raise - rescue => e - Rails.logger.error "Unexpected error parsing recipes from markdown: #{e.message}" - raise - end + parse_recipes_json(extract_message_text(response), "OpenAI markdown parsing", on_error: :raise) end def parse_url_to_recipes(markdown_text) @@ -158,73 +136,25 @@ def parse_url_to_recipes(markdown_text) Rails.logger.debug "OpenAI URL parsing response: #{response}" - output = response.dig("output") || [] - message = output.find { |item| item["type"] == "message" } - llm_response_text = message&.dig("content", 0, "text") - - begin - parsed = JSON.parse(llm_response_text) - recipes = parsed['recipes'] || [] - Rails.logger.warn "No recipes found in OpenAI URL parsing response" if recipes.empty? - Rails.logger.info "OpenAI parsed #{recipes.length} recipes from URL markdown" - recipes - rescue JSON::ParserError => e - Rails.logger.error "Failed to parse OpenAI URL response: #{e.message}" - raise - rescue => e - Rails.logger.error "Unexpected error parsing recipes from URL: #{e.message}" - raise - end + parse_recipes_json(extract_message_text(response), "OpenAI URL parsing", on_error: :raise) end private def build_image_content_block(image_file, content_type) - # Read and encode the image file as base64 - image_data = if image_file.respond_to?(:read) - # Handle uploaded file (Tempfile) - Base64.strict_encode64(image_file.read) - elsif image_file.is_a?(String) - # Handle file path - File.open(image_file, 'rb') { |f| Base64.strict_encode64(f.read) } - else - raise ArgumentError, "Invalid image_file type" - end - - # Determine the image format from content_type or file extension - image_format = case content_type - when /jpeg|jpg/ then 'jpeg' - when /png/ then 'png' - when /webp/ then 'webp' - when /heic|heif/ then 'heic' - else 'jpeg' # default - end - - filedata = "data:image/#{image_format};base64,#{image_data}" - { type: "input_image", - image_url: filedata + image_url: build_data_uri(image_file, content_type) } end - def extract_recipes_from_response(response, log_label) + def extract_message_text(response) output = response.dig("output") || [] message = output.find { |item| item["type"] == "message" } - llm_response_text = message&.dig("content", 0, "text") - - begin - parsed = JSON.parse(llm_response_text) - recipes = parsed['recipes'] || [] - Rails.logger.warn "No recipes found in #{log_label} response" if recipes.empty? - Rails.logger.info "#{log_label} recipes: #{recipes}" - recipes - rescue JSON::ParserError => e - Rails.logger.error "Failed to parse #{log_label} response: #{e.message}" - [] - rescue => e - Rails.logger.error "Unexpected error parsing recipes: #{e.message}" - [] - end + message&.dig("content", 0, "text") + end + + def extract_recipes_from_response(response, log_label) + parse_recipes_json(extract_message_text(response), log_label) end end diff --git a/app/services/recipe_json_parser.rb b/app/services/recipe_json_parser.rb new file mode 100644 index 00000000..66338ee8 --- /dev/null +++ b/app/services/recipe_json_parser.rb @@ -0,0 +1,17 @@ +module RecipeJsonParser + def parse_recipes_json(text, log_label, on_error: :empty) + parsed = JSON.parse(text) + recipes = parsed['recipes'] || [] + Rails.logger.warn "No recipes found in #{log_label} response" if recipes.empty? + Rails.logger.info "#{log_label} recipes: #{recipes}" + recipes + rescue JSON::ParserError => e + Rails.logger.error "Failed to parse #{log_label} response: #{e.message}" + raise if on_error == :raise + [] + rescue => e + Rails.logger.error "Unexpected error parsing recipes from #{log_label}: #{e.message}" + raise if on_error == :raise + [] + end +end diff --git a/config/prompts/openai_ocr.txt b/config/prompts/openai_ocr.txt index 84c5b5f5..0fbd3dc2 100644 --- a/config/prompts/openai_ocr.txt +++ b/config/prompts/openai_ocr.txt @@ -1,7 +1,7 @@ -Extract all recipes from the input text or image (such as a scan from a cookbook or magazine), identifying and parsing each recipe individually according to the rules below, and return all results as a list of parsed recipes within a single JSON response. Always output a JSON object with a "recipes" key mapping to a list of extracted recipe objects—include all recipes present on the page. If only one recipe is found, output a list with just that one. Adhere to the following extraction and reasoning steps for each recipe: +Extract all recipes from the input text or image(s) (such as a scan from a cookbook or magazine), identifying and parsing each recipe individually according to the rules below, and return all results as a list of parsed recipes within a single JSON response. Always output a JSON object with a "recipes" key mapping to a list of extracted recipe objects—include all recipes present in the input. If only one recipe is found, output a list with just that one. Adhere to the following extraction and reasoning steps for each recipe: -- Accept user input as either typed/pasted text or an image of a recipe page. -- If input is an image, first run OCR to extract text. Then, identify boundaries between multiple recipes, and separate the content for each recipe. +- Accept user input as either typed/pasted text, or one or more images of recipe pages. +- If input is one or more images, first run OCR to extract text from each. The images are sequential pages of the same scan; a single recipe may span multiple images, or each image may contain one or more distinct recipes—use context and content continuity, not just image order, to decide recipe boundaries. Then, identify boundaries between multiple recipes, and separate the content for each recipe. - For each recipe identified, extract these fields: - Recipe title (if present) - Ingredients (each ingredient as a separate list item; do not modify or translate ingredients—retain exact original wording. If illegible, use "[unreadable]". If missing, use "[not found]".) @@ -19,8 +19,8 @@ Extract all recipes from the input text or image (such as a scan from a cookbook # Steps -1. (If input is an image) Run OCR to obtain text. -2. Identify where recipe boundaries occur, so you can separate multiple recipes on the page. +1. (If input is one or more images) Run OCR to obtain text from each. +2. Identify where recipe boundaries occur, so you can separate multiple recipes across the input. 3. For each recipe section, extract the fields listed above, using the specified error and inference markers as needed. 4. For ambiguous, missing, or illegible data, reason step-by-step about best possible field values. 5. Prepare a JSON response object with a "recipes" array, each element corresponding to a parsed recipe (in the order they appear). @@ -162,7 +162,7 @@ Steps: # Notes -- Recipes on a single page/input may be separated by titles, bold headings, or formatting; reason carefully about possible recipe boundaries. +- Recipes in the input may be separated by titles, bold headings, or formatting; reason carefully about possible recipe boundaries. - All fields must be present in each recipe object. Use "[not found]" or "[unreadable]" strictly according to rules. - Do not translate any text or tags; preserve original language. - The only output allowed is the JSON structure described above. @@ -170,4 +170,4 @@ Steps: # Task Reminder -Extract fields for all recipes on the input page—output as a list of recipes in a JSON object ("recipes": [ ... ]), applying step-by-step internal reasoning and marking all missing info per the specified conventions. If only one recipe is present, return a singleton list. Never output anything except the final JSON. \ No newline at end of file +Extract fields for all recipes in the input—output as a list of recipes in a JSON object ("recipes": [ ... ]), applying step-by-step internal reasoning and marking all missing info per the specified conventions. If only one recipe is present, return a singleton list. Never output anything except the final JSON. \ No newline at end of file diff --git a/spec/controllers/ocr_controller_spec.rb b/spec/controllers/ocr_controller_spec.rb index dbe85aab..23cba33f 100644 --- a/spec/controllers/ocr_controller_spec.rb +++ b/spec/controllers/ocr_controller_spec.rb @@ -209,6 +209,19 @@ expect(json_response['success']).to be false expect(json_response['error']).to be_present end + + it 'defaults a blank ocr_flags entry to include the image' do + post :scan, params: { + files: [file, file2], + ai_method: 'openai_direct', + ocr_flags: ['', 'false'] + } + + expect(captured_args.length).to eq(1) + + ocrresult = OcrResult.last + expect(ocrresult.extra_images.count).to eq(1) + end end end