From 033a76b9042f643c1ae03df7181e39379dc55cd6 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Sat, 22 Aug 2026 18:12:58 +0200 Subject: [PATCH 1/8] [AI-3] Probe models for embedding support automatically Adds the one behavioural probe worth writing: a single embeddings request per candidate model, whose 200 response also carries the vector dimension count. Probing every listed model would be wrong, since a gateway can list hundreds and some providers bill per request, so the background pass after a credential change is capped at ten models whose names suggest they embed. The name is only a hint for where to spend a probe, never a verdict in itself. A definite refusal records unsupported, a vector records supported, and anything else records nothing at all, because servers silently drop unknown parameters and a 200 alone proves nothing. Administrator assertions are never overwritten. Part 8 of the AI-3 stack. https://community.openproject.org/work_packages/66020 --- app/services/llm/probes/embeddings_probe.rb | 97 +++++++++++++++ .../detect_capabilities_service.rb | 115 ++++++++++++++++++ .../llm_connections/update_service.rb | 1 + app/workers/llm/detect_capabilities_job.rb | 42 +++++++ .../llm/probes/embeddings_probe_spec.rb | 101 +++++++++++++++ .../detect_capabilities_service_spec.rb | 70 +++++++++++ 6 files changed, 426 insertions(+) create mode 100644 app/services/llm/probes/embeddings_probe.rb create mode 100644 app/services/llm_connections/detect_capabilities_service.rb create mode 100644 app/workers/llm/detect_capabilities_job.rb create mode 100644 spec/services/llm/probes/embeddings_probe_spec.rb create mode 100644 spec/services/llm_connections/detect_capabilities_service_spec.rb diff --git a/app/services/llm/probes/embeddings_probe.rb b/app/services/llm/probes/embeddings_probe.rb new file mode 100644 index 000000000000..fac80d9412b8 --- /dev/null +++ b/app/services/llm/probes/embeddings_probe.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Probes + # Determines whether a model can produce embeddings, by asking it to. + # + # The 200 case is checked by shape rather than by status, because unknown + # parameters are silently dropped by vLLM, llama.cpp and Ollama alike: a 200 + # on its own proves nothing. + class EmbeddingsProbe + PROBE_INPUT = "openproject" + + # The server understood the request and refused it for this model. Anything + # else -- 5xx, throttling -- says something about the server, not the model. + REFUSED_STATUSES = [400, 404, 405, 501].freeze + + Result = Data.define(:state, :detail) do + def supported? = state == :supported + end + + def initialize(connection) + @connection = connection + end + + def call(model_id) + classify(session.embed(PROBE_INPUT, model: model_id)) + rescue Llm::Errors::ApiError => e + return unsupported(e.status) if e.status.in?(REFUSED_STATUSES) + + unknown("http_#{e.status}") + rescue Llm::Errors::AuthenticationError + unknown("unauthorized") + rescue Llm::Errors::Error => e + unknown(e.class.name.demodulize.underscore) + end + + private + + attr_reader :connection + + # Never retried: a refusal is the answer we are looking for, and repeating + # it would only slow the probe down. + def session + @session ||= Llm::Session.for(connection, timeout: Llm::Session::PROBE_TIMEOUT, max_retries: 0) + end + + def classify(embedding) + vector = embedding.vectors + vector = vector.first if vector.is_a?(Array) && vector.first.is_a?(Array) + + if vector.is_a?(Array) && vector.any? && vector.all?(Numeric) + Result.new(state: :supported, detail: { "dimensions" => vector.length }) + else + # A 200 whose body is not an embedding response: the server accepted the + # request but answered with something else entirely. + unknown("unexpected_body") + end + end + + def unsupported(status) + Result.new(state: :unsupported, detail: { "http_status" => status }) + end + + def unknown(reason) + Result.new(state: :unknown, detail: { "reason" => reason }) + end + end + end +end diff --git a/app/services/llm_connections/detect_capabilities_service.rb b/app/services/llm_connections/detect_capabilities_service.rb new file mode 100644 index 000000000000..ec8d23a3bc7f --- /dev/null +++ b/app/services/llm_connections/detect_capabilities_service.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Records what a model on this server can do. + # + # Probing every listed model would be wrong: a gateway can list hundreds, each + # probe is a request, and some providers bill per request. So the models worth + # asking about are either the one an administrator is about to bind, or a small + # number whose names suggest they are embedding models. + class DetectCapabilitiesService + # Naming is a hint for which models are worth spending a probe on, never a + # verdict in itself. + EMBEDDING_NAME_HINT = /embed|bge|e5|gte|nomic|minilm/i + BACKGROUND_LIMIT = 10 + + def initialize(connection) + @connection = connection + end + + # Probes a specific model, synchronously. Used when an administrator binds a + # model to a feature that requires embeddings -- the verdict that matters. + # + # @return [ServiceResult] carrying the verdict + def detect(model_id) + return ServiceResult.success(result: existing_admin_verdict(model_id)) if admin_asserted?(model_id) + + result = probe.call(model_id) + ServiceResult.success(result: record(model_id, result)) + end + + # Pre-colours the model list after a connect, without spending a request per + # model. Everything not probed stays unknown, which never blocks. + # + # @return [ServiceResult] carrying the verdicts that were recorded + def detect_likely_embedding_models + ServiceResult.success(result: candidates.map { |model_id| detect(model_id).result }) + end + + private + + attr_reader :connection + + def probe + @probe ||= Llm::Probes::EmbeddingsProbe.new(connection) + end + + def candidates + connection.available_model_ids + .grep(EMBEDDING_NAME_HINT) + .reject { |model_id| admin_asserted?(model_id) } + .first(BACKGROUND_LIMIT) + end + + # An administrator knows things about their deployment that a probe cannot + # determine, so their assertion is never overwritten by re-detection. + def admin_asserted?(model_id) + verdicts.for_model(model_id).for_capability(:embeddings).sticky.exists? + end + + def existing_admin_verdict(model_id) + verdicts.for_model(model_id).for_capability(:embeddings).first + end + + def record(model_id, result) + verdicts.transaction do + verdict = verdicts.lock.find_or_initialize_by(model_id:, capability: "embeddings") + # Re-checked under the row lock: a probe runs for seconds, and an + # administrator may have asserted the capability in the meantime. + break verdict if verdict.persisted? && verdict.source_admin? + # A probe that learned nothing must not soften a definite verdict: + # only :unsupported blocks, so downgrading it to :unknown on a transient + # failure would quietly make a rejected model usable again. + break verdict if verdict.persisted? && result.state == :unknown + + verdict.update!(state: result.state.to_s, + source: "probe", + detail: result.detail, + checked_at: Time.current) + verdict + end + end + + def verdicts + connection.capability_verdicts + end + end +end diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb index 2b77bcb2f979..bdfe1f177bc9 100644 --- a/app/services/llm_connections/update_service.rb +++ b/app/services/llm_connections/update_service.rb @@ -43,6 +43,7 @@ def after_perform(service_call) next unless initial_fill?(service_call.result) SyncModelsService.new(service_call.result).call + Llm::DetectCapabilitiesJob.perform_later end end diff --git a/app/workers/llm/detect_capabilities_job.rb b/app/workers/llm/detect_capabilities_job.rb new file mode 100644 index 000000000000..5c37305b2ac7 --- /dev/null +++ b/app/workers/llm/detect_capabilities_job.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Pre-colours the model list after a connect, out of band so that saving the + # connection does not wait on one request per candidate model. + class DetectCapabilitiesJob < ApplicationJob + def perform + connection = LlmConnection.first + return if connection.nil? || !connection.configured? + + LlmConnections::DetectCapabilitiesService.new(connection).detect_likely_embedding_models + end + end +end diff --git a/spec/services/llm/probes/embeddings_probe_spec.rb b/spec/services/llm/probes/embeddings_probe_spec.rb new file mode 100644 index 000000000000..9fdfc0439b40 --- /dev/null +++ b/spec/services/llm/probes/embeddings_probe_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Probes::EmbeddingsProbe, :webmock do + subject(:result) { described_class.new(connection).call("some-model") } + + let(:connection) { build(:llm_connection, base_url: "https://example.com/v1") } + + def stub_embeddings(status:, body:) + stub_request(:post, "https://example.com/v1/embeddings") + .to_return(status:, headers: { "Content-Type" => "application/json" }, body: body.to_json) + end + + context "when the server returns a vector" do + before { stub_embeddings(status: 200, body: { data: [{ embedding: [0.1, 0.2, 0.3] }] }) } + + it "is supported and captures the dimension count" do + expect(result.state).to eq(:supported) + expect(result.detail["dimensions"]).to eq(3) + end + end + + # vLLM, llama.cpp and Ollama all silently drop parameters they do not + # understand, so a 200 on its own proves nothing about the model. + context "when the server returns 200 with something that is not an embedding" do + before { stub_embeddings(status: 200, body: { data: [{ message: "hello" }] }) } + + it "is unknown rather than supported" do + expect(result.state).to eq(:unknown) + expect(result.detail["reason"]).to eq("unexpected_body") + end + end + + context "when the server returns an empty data array" do + before { stub_embeddings(status: 200, body: { data: [] }) } + + it { expect(result.state).to eq(:unknown) } + end + + [400, 404, 501].each do |status| + context "when the server refuses the request with #{status}" do + before { stub_embeddings(status:, body: { error: "nope" }) } + + it "is unsupported" do + expect(result.state).to eq(:unsupported) + expect(result.detail["http_status"]).to eq(status) + end + end + end + + # A 5xx says something about the server, not about the model. + context "when the server errors" do + before { stub_embeddings(status: 500, body: { error: "boom" }) } + + it { expect(result.state).to eq(:unknown) } + end + + context "when the credentials are rejected" do + before { stub_embeddings(status: 401, body: { error: "no" }) } + + it "is unknown, since this says nothing about the model" do + expect(result.state).to eq(:unknown) + expect(result.detail["reason"]).to eq("unauthorized") + end + end + + context "when the server cannot be reached" do + before { stub_request(:post, "https://example.com/v1/embeddings").to_timeout } + + it { expect(result.state).to eq(:unknown) } + end +end diff --git a/spec/services/llm_connections/detect_capabilities_service_spec.rb b/spec/services/llm_connections/detect_capabilities_service_spec.rb new file mode 100644 index 000000000000..67eedbdb7624 --- /dev/null +++ b/spec/services/llm_connections/detect_capabilities_service_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe LlmConnections::DetectCapabilitiesService, :llm_server_helpers, :webmock do + subject(:service) { described_class.new(connection) } + + let(:base_url) { "https://example.com/v1" } + let(:connection) { create(:llm_connection, :with_models, :enabled, base_url:, api_key: "sk-test") } + + it "wraps the verdict in a ServiceResult" do + mock_llm_embeddings_response(base_url) + + result = service.detect("bge-m3") + + expect(result).to be_success + expect(result.result).to be_supported + end + + # Only :unsupported blocks, so downgrading a definite verdict to :unknown on + # a transient failure would quietly make a rejected model usable again. + it "does not overwrite a definite verdict with an inconclusive probe" do + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: 1.day.ago) + mock_llm_embeddings_response(base_url, response_code: 500) + + service.detect("bge-m3") + + expect(connection.capability_verdicts.find_by(model_id: "bge-m3")).to be_unsupported + end + + it "never overwrites an administrator's assertion" do + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "supported", source: "admin", checked_at: 1.day.ago) + mock_llm_embeddings_response(base_url, response_code: 404) + + verdict = service.detect("bge-m3").result + + expect(verdict).to be_source_admin + expect(verdict).to be_supported + end +end From e368d208a223c09aa69d3e87bc1e8042b7ed36a7 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 7 Sep 2026 09:30:00 +0200 Subject: [PATCH 2/8] [AI-3] Take names and windows from the live card or the registry Live discovery hands us the server's own card, which for OpenRouter and the gateways following it carries the model name and a context_length rather than the max_model_len that vLLM reports. Both were dropped on the floor, so an OpenRouter connection listed bare ids and an empty context window even though the server had just told us better. The card now names the model when the adapter cannot, its context_length fills the window unless the operator's own limit is reported, and the registry only fills in what neither the card nor the stored metadata already has, so a refresh no longer overwrites a figure that came from the deployment itself. --- .../enrich_capabilities_service.rb | 2 +- .../llm_connections/sync_models_service.rb | 24 ++++++++++--- .../sync_models_service_spec.rb | 35 +++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/app/services/llm_connections/enrich_capabilities_service.rb b/app/services/llm_connections/enrich_capabilities_service.rb index e5d53976d4b0..c8d8e20b878b 100644 --- a/app/services/llm_connections/enrich_capabilities_service.rb +++ b/app/services/llm_connections/enrich_capabilities_service.rb @@ -66,7 +66,7 @@ def apply_metadata(llm_model, published) attributes = {} attributes[:display_name] = published[:display_name] if llm_model.display_name.blank? - if published[:context_window].present? && llm_model.raw_metadata["max_model_len"].blank? + if published[:context_window].present? && llm_model.context_window.blank? attributes[:raw_metadata] = llm_model.raw_metadata.merge("context_window" => published[:context_window]) end diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb index 9301d7cb4cfe..34605b5a3271 100644 --- a/app/services/llm_connections/sync_models_service.rb +++ b/app/services/llm_connections/sync_models_service.rb @@ -91,7 +91,7 @@ def upsert(cards) cards.each do |card| model = connection.models.find_or_initialize_by(external_id: card.fetch(:id)) - model.update!(display_name: card[:display_name].presence || model.display_name, + model.update!(display_name: display_name_for(model, card), raw_metadata: merged_metadata(model, card), last_seen_at: now, active: true) @@ -131,15 +131,31 @@ def forget_the_previous_deployment end # The server names the model, but only when it says so: the administrator's - # display name and context-window override are theirs, and a routine refresh - # must not silently discard them. + # display name is theirs, and a routine refresh must not silently discard it. + # A gateway listing in the OpenAI shape carries the name on the raw card, + # which the adapter has no vocabulary for. + def display_name_for(model, card) + card[:display_name].presence || card.dig(:raw, "name").presence || model.display_name + end + + # The administrator's context-window override is theirs, and a routine + # refresh must not silently discard it. def merged_metadata(model, card) - raw = card.fetch(:raw, {}) + raw = normalised_window(card.fetch(:raw, {})) admin_window = model.raw_metadata["admin_context_window"] admin_window ? raw.merge("admin_context_window" => admin_window) : raw end + # OpenRouter and gateways following it publish the window as + # +context_length+, where vLLM and SGLang report +max_model_len+, the + # operator's actual limit and therefore the better figure of the two. + def normalised_window(raw) + return raw if raw["context_length"].blank? || raw["max_model_len"].present? + + raw.merge("context_window" => raw["context_length"]) + end + # Same deployment, but a model is gone. Its verdict is meaningless now, # except an administrator's assertion: an operator restarting a server must # not silently lose one. diff --git a/spec/services/llm_connections/sync_models_service_spec.rb b/spec/services/llm_connections/sync_models_service_spec.rb index f39d3ca53116..2f92f6b4fa25 100644 --- a/spec/services/llm_connections/sync_models_service_spec.rb +++ b/spec/services/llm_connections/sync_models_service_spec.rb @@ -126,4 +126,39 @@ expect(connection.capability_verdicts.pluck(:source)).to eq(["admin"]) end end + + describe "naming a model and sizing its context window" do + let(:connection) { create(:llm_connection, base_url:, api_key: "sk-test") } + + it "reads both off a card that names them in the gateway's own vocabulary" do + mock_llm_models_response(base_url, + models: [{ id: "openai/gpt-4o", name: "OpenAI: GPT-4o", context_length: 128_000 }]) + + service.call + + llm_model = connection.models.find_by(external_id: "openai/gpt-4o") + expect(llm_model.name).to eq("OpenAI: GPT-4o") + expect(llm_model.context_window).to eq(128_000) + end + + it "falls back to the registry for a server that lists bare ids" do + mock_llm_models_response(base_url, models: [{ id: "gpt-4o", object: "model" }]) + + service.call + + llm_model = connection.models.find_by(external_id: "gpt-4o") + expect(llm_model.name).to eq("GPT-4o") + expect(llm_model.context_window).to eq(128_000) + end + + it "keeps an administrator's display name over the one the registry publishes" do + mock_llm_models_response(base_url, models: [{ id: "gpt-4o", object: "model" }]) + service.call + connection.models.find_by(external_id: "gpt-4o").update!(display_name: "The house model") + + described_class.new(connection).call + + expect(connection.models.find_by(external_id: "gpt-4o").display_name).to eq("The house model") + end + end end From 1a53ccff08d779110cd2cc312ae119dbba79b1c8 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 7 Sep 2026 10:48:34 +0200 Subject: [PATCH 3/8] [AI-3] Detect capabilities after every successful model sync The UX review with Tom made the model refresh manual, so hanging capability detection off a settings change no longer matches when the list actually changes. Detection now follows the sync itself, whichever caller asked for it: the initial fill, the Refresh button on the AI models page and the environment seeder's background sync all end in a probe pass. The pass itself spends fewer requests. It reads the models an administrator still offers rather than every active one, so a model switched off is not probed, and the e5 and gte name hints only count on a separator, which keeps build hashes out of the candidate set. The job iterates the stored connections instead of taking the first one behind a guard that cannot fire. Before the first probe of a model there is no row for FOR UPDATE to lock, so two probes running at once both inserted one. The row is now claimed through the unique index first and the verdict written under the lock as before. --- .../detect_capabilities_service.rb | 29 +++++++++-- .../llm_connections/sync_models_service.rb | 1 + .../llm_connections/update_service.rb | 1 - app/workers/llm/detect_capabilities_job.rb | 11 ++--- .../detect_capabilities_service_spec.rb | 46 ++++++++++++++++++ .../sync_models_service_spec.rb | 12 +++++ .../llm/detect_capabilities_job_spec.rb | 48 +++++++++++++++++++ 7 files changed, 136 insertions(+), 12 deletions(-) create mode 100644 spec/workers/llm/detect_capabilities_job_spec.rb diff --git a/app/services/llm_connections/detect_capabilities_service.rb b/app/services/llm_connections/detect_capabilities_service.rb index ec8d23a3bc7f..9741b03e0643 100644 --- a/app/services/llm_connections/detect_capabilities_service.rb +++ b/app/services/llm_connections/detect_capabilities_service.rb @@ -38,7 +38,7 @@ module LlmConnections class DetectCapabilitiesService # Naming is a hint for which models are worth spending a probe on, never a # verdict in itself. - EMBEDDING_NAME_HINT = /embed|bge|e5|gte|nomic|minilm/i + EMBEDDING_NAME_HINT = %r{embed|bge|nomic|minilm|(^|[-_./])(e5|gte)}i BACKGROUND_LIMIT = 10 def initialize(connection) @@ -72,8 +72,10 @@ def probe @probe ||= Llm::Probes::EmbeddingsProbe.new(connection) end + # A model an administrator switched off is not going to be bound to a + # feature, so a speculative probe against it is a request spent for nothing. def candidates - connection.available_model_ids + connection.selectable_model_ids .grep(EMBEDDING_NAME_HINT) .reject { |model_id| admin_asserted?(model_id) } .first(BACKGROUND_LIMIT) @@ -90,15 +92,17 @@ def existing_admin_verdict(model_id) end def record(model_id, result) + verdict = claim(model_id) + verdicts.transaction do - verdict = verdicts.lock.find_or_initialize_by(model_id:, capability: "embeddings") + verdict.lock! # Re-checked under the row lock: a probe runs for seconds, and an # administrator may have asserted the capability in the meantime. - break verdict if verdict.persisted? && verdict.source_admin? + break verdict if verdict.source_admin? # A probe that learned nothing must not soften a definite verdict: # only :unsupported blocks, so downgrading it to :unknown on a transient # failure would quietly make a rejected model usable again. - break verdict if verdict.persisted? && result.state == :unknown + break verdict if result.state == :unknown && !verdict.unknown? verdict.update!(state: result.state.to_s, source: "probe", @@ -108,6 +112,21 @@ def record(model_id, result) end end + # FOR UPDATE has no row to lock before the first probe of a model, and a + # synchronous detection can run alongside the background pass, so the row is + # claimed through the unique index rather than built in memory. + def claim(model_id) + verdicts.insert_all([{ llm_connection_id: connection.id, + model_id:, + capability: "embeddings", + state: "unknown", + source: "probe", + checked_at: Time.current }], + unique_by: %i[llm_connection_id model_id capability]) + + verdicts.for_model(model_id).for_capability(:embeddings).first + end + def verdicts connection.capability_verdicts end diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb index 34605b5a3271..63edef6da799 100644 --- a/app/services/llm_connections/sync_models_service.rb +++ b/app/services/llm_connections/sync_models_service.rb @@ -45,6 +45,7 @@ def call invalidate_a_different_deployment store(adapter.models) + Llm::DetectCapabilitiesJob.perform_later ServiceResult.success(result: connection) rescue Llm::Client::Error => e diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb index bdfe1f177bc9..2b77bcb2f979 100644 --- a/app/services/llm_connections/update_service.rb +++ b/app/services/llm_connections/update_service.rb @@ -43,7 +43,6 @@ def after_perform(service_call) next unless initial_fill?(service_call.result) SyncModelsService.new(service_call.result).call - Llm::DetectCapabilitiesJob.perform_later end end diff --git a/app/workers/llm/detect_capabilities_job.rb b/app/workers/llm/detect_capabilities_job.rb index 5c37305b2ac7..ee31862a7385 100644 --- a/app/workers/llm/detect_capabilities_job.rb +++ b/app/workers/llm/detect_capabilities_job.rb @@ -29,14 +29,13 @@ #++ module Llm - # Pre-colours the model list after a connect, out of band so that saving the - # connection does not wait on one request per candidate model. + # Pre-colours the model list after a refresh, out of band so that fetching the + # catalogue does not wait on one request per candidate model. class DetectCapabilitiesJob < ApplicationJob def perform - connection = LlmConnection.first - return if connection.nil? || !connection.configured? - - LlmConnections::DetectCapabilitiesService.new(connection).detect_likely_embedding_models + LlmConnection.find_each do |connection| + LlmConnections::DetectCapabilitiesService.new(connection).detect_likely_embedding_models + end end end end diff --git a/spec/services/llm_connections/detect_capabilities_service_spec.rb b/spec/services/llm_connections/detect_capabilities_service_spec.rb index 67eedbdb7624..a8655e0fc3df 100644 --- a/spec/services/llm_connections/detect_capabilities_service_spec.rb +++ b/spec/services/llm_connections/detect_capabilities_service_spec.rb @@ -67,4 +67,50 @@ expect(verdict).to be_source_admin expect(verdict).to be_supported end + + it "adopts a verdict that appeared while its own probe was in flight" do + probe = instance_double(Llm::Probes::EmbeddingsProbe) + allow(Llm::Probes::EmbeddingsProbe).to receive(:new).and_return(probe) + allow(probe).to receive(:call) do + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "unknown", source: "probe", checked_at: Time.current) + Llm::Probes::EmbeddingsProbe::Result.new(state: :supported, detail: { "dimensions" => 4 }) + end + + service.detect("bge-m3") + + expect(connection.capability_verdicts.for_model("bge-m3").count).to eq(1) + expect(connection.capability_verdicts.find_by(model_id: "bge-m3")).to be_supported + end + + describe "#detect_likely_embedding_models" do + before { mock_llm_embeddings_response(base_url) } + + it "probes only the models whose names suggest they embed" do + create(:llm_model, llm_connection: connection, external_id: "nomic-embed-text") + create(:llm_model, llm_connection: connection, external_id: "solar-10.7b-v1.0-4e5f9c") + + service.detect_likely_embedding_models + + expect(connection.capability_verdicts.pluck(:model_id)).to contain_exactly("bge-m3", "nomic-embed-text") + end + + it "leaves out a model an administrator switched off" do + create(:llm_model, :deactivated, llm_connection: connection, external_id: "nomic-embed-text") + + service.detect_likely_embedding_models + + expect(connection.capability_verdicts.pluck(:model_id)).to eq(["bge-m3"]) + end + + it "spends no more than BACKGROUND_LIMIT requests" do + (described_class::BACKGROUND_LIMIT + 5).times do |index| + create(:llm_model, llm_connection: connection, external_id: "embed-#{index}") + end + + service.detect_likely_embedding_models + + expect(connection.capability_verdicts.count).to eq(described_class::BACKGROUND_LIMIT) + end + end end diff --git a/spec/services/llm_connections/sync_models_service_spec.rb b/spec/services/llm_connections/sync_models_service_spec.rb index 2f92f6b4fa25..12fe845f7f1a 100644 --- a/spec/services/llm_connections/sync_models_service_spec.rb +++ b/spec/services/llm_connections/sync_models_service_spec.rb @@ -127,6 +127,18 @@ end end + describe "the capability detection that follows" do + it "asks for it after a successful sync, whichever caller asked for the list" do + expect { service.call }.to have_enqueued_job(Llm::DetectCapabilitiesJob) + end + + it "asks for nothing when the server does not answer with a list" do + mock_llm_models_response(base_url, response_code: 404) + + expect { service.call }.not_to have_enqueued_job(Llm::DetectCapabilitiesJob) + end + end + describe "naming a model and sizing its context window" do let(:connection) { create(:llm_connection, base_url:, api_key: "sk-test") } diff --git a/spec/workers/llm/detect_capabilities_job_spec.rb b/spec/workers/llm/detect_capabilities_job_spec.rb new file mode 100644 index 000000000000..8bf38fbd4893 --- /dev/null +++ b/spec/workers/llm/detect_capabilities_job_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::DetectCapabilitiesJob, :llm_server_helpers, :webmock do + let(:base_url) { "https://example.com/v1" } + + it "probes the likely embedding models of every stored connection" do + connection = create(:llm_connection, :with_models, base_url:) + mock_llm_embeddings_response(base_url) + + described_class.perform_now + + expect(connection.capability_verdicts.pluck(:model_id, :source)).to eq([["bge-m3", "probe"]]) + end + + it "does nothing while no connection is stored" do + expect { described_class.perform_now }.not_to raise_error + end +end From 467bd70820af690a0834fa26886625805403b238 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 7 Sep 2026 11:33:19 +0200 Subject: [PATCH 4/8] [AI-3] Read only a 400 as a model refusing to embed Tom pointed out in the review that three of the four statuses the probe read as a refusal are statements about the server. A 404, 405 or 501 says the embeddings route is not there, which a gateway routing chat completions and nothing else answers for every model alike. Since only an unsupported verdict blocks, and a definite verdict is never softened again, that marked every candidate permanently incapable and left semantic search with nothing to bind to. Those three are now inconclusive, in the same reading LlmServerValidator already applies to the model list, and a batch that meets one stops: the remaining probes are billed for the same answer. A 400 stays the one per-model refusal. Result#supported? goes with it; nothing ever called it. --- app/services/llm/probes/embeddings_probe.rb | 11 ++++++---- .../detect_capabilities_service.rb | 15 ++++++++++++- .../llm/probes/embeddings_probe_spec.rb | 21 ++++++++++++++----- .../detect_capabilities_service_spec.rb | 9 ++++++++ 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/app/services/llm/probes/embeddings_probe.rb b/app/services/llm/probes/embeddings_probe.rb index fac80d9412b8..ce7f4b08719d 100644 --- a/app/services/llm/probes/embeddings_probe.rb +++ b/app/services/llm/probes/embeddings_probe.rb @@ -40,11 +40,14 @@ class EmbeddingsProbe # The server understood the request and refused it for this model. Anything # else -- 5xx, throttling -- says something about the server, not the model. - REFUSED_STATUSES = [400, 404, 405, 501].freeze + REFUSED_STATUSES = [400].freeze - Result = Data.define(:state, :detail) do - def supported? = state == :supported - end + # Answers about the embeddings route rather than about a model, read as + # LlmServerValidator::MODELS_ENDPOINT_ABSENT reads them: a gateway may route + # chat completions and nothing else, and would refuse every model alike. + ENDPOINT_ABSENT_REASONS = [404, 405, 501].map { |status| "http_#{status}" }.freeze + + Result = Data.define(:state, :detail) def initialize(connection) @connection = connection diff --git a/app/services/llm_connections/detect_capabilities_service.rb b/app/services/llm_connections/detect_capabilities_service.rb index 9741b03e0643..cb4acf38ad9f 100644 --- a/app/services/llm_connections/detect_capabilities_service.rb +++ b/app/services/llm_connections/detect_capabilities_service.rb @@ -61,7 +61,14 @@ def detect(model_id) # # @return [ServiceResult] carrying the verdicts that were recorded def detect_likely_embedding_models - ServiceResult.success(result: candidates.map { |model_id| detect(model_id).result }) + recorded = [] + + candidates.each do |model_id| + recorded << detect(model_id).result + break if endpoint_absent?(recorded.last) + end + + ServiceResult.success(result: recorded) end private @@ -81,6 +88,12 @@ def candidates .first(BACKGROUND_LIMIT) end + # The server answered for the embeddings route rather than for the model, so + # the requests the rest of the batch would spend buy the same answer again. + def endpoint_absent?(verdict) + verdict.unknown? && verdict.detail["reason"].in?(Llm::Probes::EmbeddingsProbe::ENDPOINT_ABSENT_REASONS) + end + # An administrator knows things about their deployment that a probe cannot # determine, so their assertion is never overwritten by re-detection. def admin_asserted?(model_id) diff --git a/spec/services/llm/probes/embeddings_probe_spec.rb b/spec/services/llm/probes/embeddings_probe_spec.rb index 9fdfc0439b40..7cb328ddf029 100644 --- a/spec/services/llm/probes/embeddings_probe_spec.rb +++ b/spec/services/llm/probes/embeddings_probe_spec.rb @@ -66,13 +66,24 @@ def stub_embeddings(status:, body:) it { expect(result.state).to eq(:unknown) } end - [400, 404, 501].each do |status| - context "when the server refuses the request with #{status}" do + context "when the server refuses the request with 400" do + before { stub_embeddings(status: 400, body: { error: "nope" }) } + + it "is unsupported" do + expect(result.state).to eq(:unsupported) + expect(result.detail["http_status"]).to eq(400) + end + end + + # A gateway routing chat completions and nothing else answers these for every + # model, so reading them as a refusal would mark them all incapable for good. + [404, 405, 501].each do |status| + context "when the server has no embeddings route and says so with #{status}" do before { stub_embeddings(status:, body: { error: "nope" }) } - it "is unsupported" do - expect(result.state).to eq(:unsupported) - expect(result.detail["http_status"]).to eq(status) + it "is unknown rather than a refusal by the model" do + expect(result.state).to eq(:unknown) + expect(result.detail["reason"]).to eq("http_#{status}") end end end diff --git a/spec/services/llm_connections/detect_capabilities_service_spec.rb b/spec/services/llm_connections/detect_capabilities_service_spec.rb index a8655e0fc3df..06c6b6b0e521 100644 --- a/spec/services/llm_connections/detect_capabilities_service_spec.rb +++ b/spec/services/llm_connections/detect_capabilities_service_spec.rb @@ -103,6 +103,15 @@ expect(connection.capability_verdicts.pluck(:model_id)).to eq(["bge-m3"]) end + it "stops the batch when the server has no embeddings route at all" do + create(:llm_model, llm_connection: connection, external_id: "nomic-embed-text") + request = mock_llm_embeddings_response(base_url, response_code: 404) + + service.detect_likely_embedding_models + + expect(request).to have_been_made.once + end + it "spends no more than BACKGROUND_LIMIT requests" do (described_class::BACKGROUND_LIMIT + 5).times do |index| create(:llm_model, llm_connection: connection, external_id: "embed-#{index}") From 180d5bda03dddb2a4a84cef55ae280143357a5b5 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 7 Sep 2026 12:49:35 +0200 Subject: [PATCH 5/8] [AI-3] Keep the registry context window under an override Enrichment skipped the published window whenever the model already had one, and an administrator's override counts as one. A refresh against a server that lists bare model ids drops the registry figure from the metadata, so clearing the override afterwards left the window unknown until the next refresh. The guard now looks at what the server reported, not at the effective window. Reported in the review of the follow-up commits (finding 4). --- .../enrich_capabilities_service.rb | 9 ++++++++- .../llm_connections/sync_models_service_spec.rb | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/app/services/llm_connections/enrich_capabilities_service.rb b/app/services/llm_connections/enrich_capabilities_service.rb index c8d8e20b878b..d6d4e76764d1 100644 --- a/app/services/llm_connections/enrich_capabilities_service.rb +++ b/app/services/llm_connections/enrich_capabilities_service.rb @@ -66,13 +66,20 @@ def apply_metadata(llm_model, published) attributes = {} attributes[:display_name] = published[:display_name] if llm_model.display_name.blank? - if published[:context_window].present? && llm_model.context_window.blank? + if published[:context_window].present? && !server_sized?(llm_model) attributes[:raw_metadata] = llm_model.raw_metadata.merge("context_window" => published[:context_window]) end llm_model.update!(attributes) if attributes.any? end + # Only what the server said counts here, not what the administrator + # overrode: clearing the override later has to reveal the published figure + # again rather than leave the window unknown. + def server_sized?(llm_model) + llm_model.raw_metadata.values_at("max_model_len", "context_window").any?(&:present?) + end + def record(model_id, capability, state) verdict = connection.capability_verdicts.find_or_initialize_by(model_id:, capability: capability.to_s) # Anything an administrator or a probe established beats a published claim: diff --git a/spec/services/llm_connections/sync_models_service_spec.rb b/spec/services/llm_connections/sync_models_service_spec.rb index 12fe845f7f1a..3fc11e5c7939 100644 --- a/spec/services/llm_connections/sync_models_service_spec.rb +++ b/spec/services/llm_connections/sync_models_service_spec.rb @@ -163,6 +163,21 @@ expect(llm_model.context_window).to eq(128_000) end + it "keeps the published window under an administrator's override" do + mock_llm_models_response(base_url, models: [{ id: "gpt-4o", object: "model" }]) + service.call + llm_model = connection.models.find_by(external_id: "gpt-4o") + llm_model.update!(admin_context_window: 8_000) + + described_class.new(connection).call + + expect(llm_model.reload.context_window).to eq(8_000) + + llm_model.update!(admin_context_window: nil) + + expect(llm_model.reload.context_window).to eq(128_000) + end + it "keeps an administrator's display name over the one the registry publishes" do mock_llm_models_response(base_url, models: [{ id: "gpt-4o", object: "model" }]) service.call From d31cea8dce69cc9b46fa4329c7bff15e537a256e Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 7 Sep 2026 14:07:18 +0200 Subject: [PATCH 6/8] [AI-3] Stop probing on a missing endpoint even with cached verdicts The batch decided whether to stop from the verdict it had just recorded, but an inconclusive probe never softens a definite verdict, so a model with a cached supported or unsupported answer hid the 404 and the batch kept spending requests up to its limit. The decision now reads the probe result itself. Reported in the review of the follow-up commits (finding 6). --- .../llm_connections/detect_capabilities_service.rb | 11 +++++++---- .../detect_capabilities_service_spec.rb | 12 ++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/app/services/llm_connections/detect_capabilities_service.rb b/app/services/llm_connections/detect_capabilities_service.rb index cb4acf38ad9f..e63633dadeb4 100644 --- a/app/services/llm_connections/detect_capabilities_service.rb +++ b/app/services/llm_connections/detect_capabilities_service.rb @@ -64,8 +64,9 @@ def detect_likely_embedding_models recorded = [] candidates.each do |model_id| - recorded << detect(model_id).result - break if endpoint_absent?(recorded.last) + result = probe.call(model_id) + recorded << record(model_id, result) + break if endpoint_absent?(result) end ServiceResult.success(result: recorded) @@ -90,8 +91,10 @@ def candidates # The server answered for the embeddings route rather than for the model, so # the requests the rest of the batch would spend buy the same answer again. - def endpoint_absent?(verdict) - verdict.unknown? && verdict.detail["reason"].in?(Llm::Probes::EmbeddingsProbe::ENDPOINT_ABSENT_REASONS) + # Read off the probe rather than the verdict, which may be an earlier and + # definite one that this inconclusive answer deliberately did not soften. + def endpoint_absent?(result) + result.state == :unknown && result.detail["reason"].in?(Llm::Probes::EmbeddingsProbe::ENDPOINT_ABSENT_REASONS) end # An administrator knows things about their deployment that a probe cannot diff --git a/spec/services/llm_connections/detect_capabilities_service_spec.rb b/spec/services/llm_connections/detect_capabilities_service_spec.rb index 06c6b6b0e521..6d8e77ff8b3d 100644 --- a/spec/services/llm_connections/detect_capabilities_service_spec.rb +++ b/spec/services/llm_connections/detect_capabilities_service_spec.rb @@ -112,6 +112,18 @@ expect(request).to have_been_made.once end + it "stops the batch even where an earlier verdict survives the 404" do + create(:llm_model, llm_connection: connection, external_id: "nomic-embed-text") + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "supported", source: "probe", checked_at: 1.day.ago) + request = mock_llm_embeddings_response(base_url, response_code: 404) + + service.detect_likely_embedding_models + + expect(request).to have_been_made.once + expect(connection.capability_verdicts.find_by(model_id: "bge-m3")).to be_supported + end + it "spends no more than BACKGROUND_LIMIT requests" do (described_class::BACKGROUND_LIMIT + 5).times do |index| create(:llm_model, llm_connection: connection, external_id: "embed-#{index}") From 0ebfcf75b1ec538de34cd418b91fb3832f902a83 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 7 Sep 2026 15:10:23 +0200 Subject: [PATCH 7/8] [AI-3] Drop a comment that live discovery made false The example was annotated with the claim that only registry-backed adapters report a name. Since this part reads the name out of the live card as well, a server speaking the OpenAI API does supply one whenever it lists more than ids. --- spec/services/llm_connections/sync_models_service_spec.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/spec/services/llm_connections/sync_models_service_spec.rb b/spec/services/llm_connections/sync_models_service_spec.rb index 3fc11e5c7939..31fbc654ff87 100644 --- a/spec/services/llm_connections/sync_models_service_spec.rb +++ b/spec/services/llm_connections/sync_models_service_spec.rb @@ -96,8 +96,6 @@ expect(llm_model.reload.display_name).to eq("The house model") end - # Only the registry-backed adapters report a name; a server speaking the - # OpenAI API lists ids and nothing else. it "adopts the display name the adapter reports" do llm_model = connection.models.find_by(external_id: "qwen3.6-27b") llm_model.update!(display_name: "The house model") From 861d35f2dd98c2e8224163bca162700a6ab03a14 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 7 Sep 2026 15:20:00 +0200 Subject: [PATCH 8/8] [AI-3] Build the probe's connection without the removed trait The capability detection spec still asked the factory for a connection with LLMs switched on, which was a column on the row before that switch became a setting. The probe does not read it either way. --- .../llm_connections/detect_capabilities_service_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/services/llm_connections/detect_capabilities_service_spec.rb b/spec/services/llm_connections/detect_capabilities_service_spec.rb index 6d8e77ff8b3d..29a2f77e81a0 100644 --- a/spec/services/llm_connections/detect_capabilities_service_spec.rb +++ b/spec/services/llm_connections/detect_capabilities_service_spec.rb @@ -34,7 +34,7 @@ subject(:service) { described_class.new(connection) } let(:base_url) { "https://example.com/v1" } - let(:connection) { create(:llm_connection, :with_models, :enabled, base_url:, api_key: "sk-test") } + let(:connection) { create(:llm_connection, :with_models, base_url:, api_key: "sk-test") } it "wraps the verdict in a ServiceResult" do mock_llm_embeddings_response(base_url)