[AI-3] Probe models for embedding support automatically

app/services/llm/probes/embeddings_probe.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# See COPYRIGHT and LICENSE files for more details.
+#++
+
+module Llm
+  module Probes
+    # Determines whether a model can produce embeddings, by asking it to.
+    #
+    # The 200 case is checked by shape rather than by status, because unknown
+    # parameters are silently dropped by vLLM, llama.cpp and Ollama alike: a 200
+    # on its own proves nothing.
+    class EmbeddingsProbe
+      PROBE_INPUT = "openproject"
+
+      # The server understood the request and refused it for this model. Anything
+      # else -- 5xx, throttling -- says something about the server, not the model.
+      REFUSED_STATUSES = [400].freeze
+
+      # Answers about the embeddings route rather than about a model, read as
+      # LlmServerValidator::MODELS_ENDPOINT_ABSENT reads them: a gateway may route
+      # chat completions and nothing else, and would refuse every model alike.
+      ENDPOINT_ABSENT_REASONS = [404, 405, 501].map { |status| "http_#{status}" }.freeze
+
+      Result = Data.define(:state, :detail)
+
+      def initialize(connection)
+        @connection = connection
+      end
+
+      def call(model_id)
+        classify(session.embed(PROBE_INPUT, model: model_id))
+      rescue Llm::Errors::ApiError => e
+        return unsupported(e.status) if e.status.in?(REFUSED_STATUSES)
+
+        unknown("http_#{e.status}")
+      rescue Llm::Errors::AuthenticationError
+        unknown("unauthorized")
+      rescue Llm::Errors::Error => e
+        unknown(e.class.name.demodulize.underscore)
+      end
+
+      private
+
+      attr_reader :connection
+
+      # Never retried: a refusal is the answer we are looking for, and repeating
+      # it would only slow the probe down.
+      def session
+        @session ||= Llm::Session.for(connection, timeout: Llm::Session::PROBE_TIMEOUT, max_retries: 0)
+      end
+
+      def classify(embedding)
+        vector = embedding.vectors
+        vector = vector.first if vector.is_a?(Array) && vector.first.is_a?(Array)
+
+        if vector.is_a?(Array) && vector.any? && vector.all?(Numeric)
+          Result.new(state: :supported, detail: { "dimensions" => vector.length })
+        else
+          # A 200 whose body is not an embedding response: the server accepted the
+          # request but answered with something else entirely.
+          unknown("unexpected_body")
+        end
+      end
+
+      def unsupported(status)
+        Result.new(state: :unsupported, detail: { "http_status" => status })
+      end
+
+      def unknown(reason)
+        Result.new(state: :unknown, detail: { "reason" => reason })
+      end
+    end
+  end
+end

app/services/llm_connections/detect_capabilities_service.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# See COPYRIGHT and LICENSE files for more details.
+#++
+
+module LlmConnections
+  # Records what a model on this server can do.
+  #
+  # Probing every listed model would be wrong: a gateway can list hundreds, each
+  # probe is a request, and some providers bill per request. So the models worth
+  # asking about are either the one an administrator is about to bind, or a small
+  # number whose names suggest they are embedding models.
+  class DetectCapabilitiesService
+    # Naming is a hint for which models are worth spending a probe on, never a
+    # verdict in itself.
+    EMBEDDING_NAME_HINT = %r{embed|bge|nomic|minilm|(^|[-_./])(e5|gte)}i
+    BACKGROUND_LIMIT = 10
+
+    def initialize(connection)
+      @connection = connection
+    end
+
+    # Probes a specific model, synchronously. Used when an administrator binds a
+    # model to a feature that requires embeddings -- the verdict that matters.
+    #
+    # @return [ServiceResult] carrying the verdict
+    def detect(model_id)
+      return ServiceResult.success(result: existing_admin_verdict(model_id)) if admin_asserted?(model_id)
+
+      result = probe.call(model_id)
+      ServiceResult.success(result: record(model_id, result))
+    end
+
+    # Pre-colours the model list after a connect, without spending a request per
+    # model. Everything not probed stays unknown, which never blocks.
+    #
+    # @return [ServiceResult] carrying the verdicts that were recorded
+    def detect_likely_embedding_models
+      recorded = []
+
+      candidates.each do |model_id|
+        result = probe.call(model_id)
+        recorded << record(model_id, result)
+        break if endpoint_absent?(result)
+      end
+
+      ServiceResult.success(result: recorded)
+    end
+
+    private
+
+    attr_reader :connection
+
+    def probe
+      @probe ||= Llm::Probes::EmbeddingsProbe.new(connection)
+    end
+
+    # A model an administrator switched off is not going to be bound to a
+    # feature, so a speculative probe against it is a request spent for nothing.
+    def candidates
+      connection.selectable_model_ids
+                .grep(EMBEDDING_NAME_HINT)
+                .reject { |model_id| admin_asserted?(model_id) }
+                .first(BACKGROUND_LIMIT)
+    end
+
+    # The server answered for the embeddings route rather than for the model, so
+    # the requests the rest of the batch would spend buy the same answer again.
+    # Read off the probe rather than the verdict, which may be an earlier and
+    # definite one that this inconclusive answer deliberately did not soften.
+    def endpoint_absent?(result)
+      result.state == :unknown && result.detail["reason"].in?(Llm::Probes::EmbeddingsProbe::ENDPOINT_ABSENT_REASONS)
+    end
+
+    # An administrator knows things about their deployment that a probe cannot
+    # determine, so their assertion is never overwritten by re-detection.
+    def admin_asserted?(model_id)
+      verdicts.for_model(model_id).for_capability(:embeddings).sticky.exists?
+    end
+
+    def existing_admin_verdict(model_id)
+      verdicts.for_model(model_id).for_capability(:embeddings).first
+    end
+
+    def record(model_id, result)
+      verdict = claim(model_id)
+
+      verdicts.transaction do
+        verdict.lock!
+        # Re-checked under the row lock: a probe runs for seconds, and an
+        # administrator may have asserted the capability in the meantime.
+        break verdict if verdict.source_admin?
+        # A probe that learned nothing must not soften a definite verdict:
+        # only :unsupported blocks, so downgrading it to :unknown on a transient
+        # failure would quietly make a rejected model usable again.
+        break verdict if result.state == :unknown && !verdict.unknown?
+
+        verdict.update!(state: result.state.to_s,
+                        source: "probe",
+                        detail: result.detail,
+                        checked_at: Time.current)
+        verdict
+      end
+    end
+
+    # FOR UPDATE has no row to lock before the first probe of a model, and a
+    # synchronous detection can run alongside the background pass, so the row is
+    # claimed through the unique index rather than built in memory.
+    def claim(model_id)
+      verdicts.insert_all([{ llm_connection_id: connection.id,
+                             model_id:,
+                             capability: "embeddings",
+                             state: "unknown",
+                             source: "probe",
+                             checked_at: Time.current }],
+                          unique_by: %i[llm_connection_id model_id capability])
+
+      verdicts.for_model(model_id).for_capability(:embeddings).first
+    end
+
+    def verdicts
+      connection.capability_verdicts
+    end
+  end
+end

app/services/llm_connections/enrich_capabilities_service.rb

@@ -66,13 +66,20 @@ def apply_metadata(llm_model, published)
       attributes = {}
       attributes[:display_name] = published[:display_name] if llm_model.display_name.blank?
 
-      if published[:context_window].present? && llm_model.raw_metadata["max_model_len"].blank?
+      if published[:context_window].present? && !server_sized?(llm_model)
         attributes[:raw_metadata] = llm_model.raw_metadata.merge("context_window" => published[:context_window])
       end
 
       llm_model.update!(attributes) if attributes.any?
     end
 
+    # Only what the server said counts here, not what the administrator
+    # overrode: clearing the override later has to reveal the published figure
+    # again rather than leave the window unknown.
+    def server_sized?(llm_model)
+      llm_model.raw_metadata.values_at("max_model_len", "context_window").any?(&:present?)
+    end
+
     def record(model_id, capability, state)
       verdict = connection.capability_verdicts.find_or_initialize_by(model_id:, capability: capability.to_s)
       # Anything an administrator or a probe established beats a published claim:

app/services/llm_connections/sync_models_service.rb

@@ -45,6 +45,7 @@ def call
       invalidate_a_different_deployment
 
       store(adapter.models)
+      Llm::DetectCapabilitiesJob.perform_later
 
       ServiceResult.success(result: connection)
     rescue Llm::Client::Error => e
@@ -91,7 +92,7 @@ def upsert(cards)
 
       cards.each do |card|
         model = connection.models.find_or_initialize_by(external_id: card.fetch(:id))
-        model.update!(display_name: card[:display_name].presence || model.display_name,
+        model.update!(display_name: display_name_for(model, card),
                       raw_metadata: merged_metadata(model, card),
                       last_seen_at: now,
                       active: true)
@@ -131,15 +132,31 @@ def forget_the_previous_deployment
     end
 
     # The server names the model, but only when it says so: the administrator's
-    # display name and context-window override are theirs, and a routine refresh
-    # must not silently discard them.
+    # display name is theirs, and a routine refresh must not silently discard it.
+    # A gateway listing in the OpenAI shape carries the name on the raw card,
+    # which the adapter has no vocabulary for.
+    def display_name_for(model, card)
+      card[:display_name].presence || card.dig(:raw, "name").presence || model.display_name
+    end
+
+    # The administrator's context-window override is theirs, and a routine
+    # refresh must not silently discard it.
     def merged_metadata(model, card)
-      raw = card.fetch(:raw, {})
+      raw = normalised_window(card.fetch(:raw, {}))
       admin_window = model.raw_metadata["admin_context_window"]
 
       admin_window ? raw.merge("admin_context_window" => admin_window) : raw
     end
 
+    # OpenRouter and gateways following it publish the window as
+    # +context_length+, where vLLM and SGLang report +max_model_len+, the
+    # operator's actual limit and therefore the better figure of the two.
+    def normalised_window(raw)
+      return raw if raw["context_length"].blank? || raw["max_model_len"].present?
+
+      raw.merge("context_window" => raw["context_length"])
+    end
+
     # Same deployment, but a model is gone. Its verdict is meaningless now,
     # except an administrator's assertion: an operator restarting a server must
     # not silently lose one.

app/workers/llm/detect_capabilities_job.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# See COPYRIGHT and LICENSE files for more details.
+#++
+
+module Llm
+  # Pre-colours the model list after a refresh, out of band so that fetching the
+  # catalogue does not wait on one request per candidate model.
+  class DetectCapabilitiesJob < ApplicationJob
+    def perform
+      LlmConnection.find_each do |connection|
+        LlmConnections::DetectCapabilitiesService.new(connection).detect_likely_embedding_models
+      end
+    end
+  end
+end