diff --git a/app/services/llm/probes/embeddings_probe.rb b/app/services/llm/probes/embeddings_probe.rb new file mode 100644 index 000000000000..ce7f4b08719d --- /dev/null +++ b/app/services/llm/probes/embeddings_probe.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Probes + # Determines whether a model can produce embeddings, by asking it to. + # + # The 200 case is checked by shape rather than by status, because unknown + # parameters are silently dropped by vLLM, llama.cpp and Ollama alike: a 200 + # on its own proves nothing. + class EmbeddingsProbe + PROBE_INPUT = "openproject" + + # The server understood the request and refused it for this model. Anything + # else -- 5xx, throttling -- says something about the server, not the model. + REFUSED_STATUSES = [400].freeze + + # Answers about the embeddings route rather than about a model, read as + # LlmServerValidator::MODELS_ENDPOINT_ABSENT reads them: a gateway may route + # chat completions and nothing else, and would refuse every model alike. + ENDPOINT_ABSENT_REASONS = [404, 405, 501].map { |status| "http_#{status}" }.freeze + + Result = Data.define(:state, :detail) + + def initialize(connection) + @connection = connection + end + + def call(model_id) + classify(session.embed(PROBE_INPUT, model: model_id)) + rescue Llm::Errors::ApiError => e + return unsupported(e.status) if e.status.in?(REFUSED_STATUSES) + + unknown("http_#{e.status}") + rescue Llm::Errors::AuthenticationError + unknown("unauthorized") + rescue Llm::Errors::Error => e + unknown(e.class.name.demodulize.underscore) + end + + private + + attr_reader :connection + + # Never retried: a refusal is the answer we are looking for, and repeating + # it would only slow the probe down. + def session + @session ||= Llm::Session.for(connection, timeout: Llm::Session::PROBE_TIMEOUT, max_retries: 0) + end + + def classify(embedding) + vector = embedding.vectors + vector = vector.first if vector.is_a?(Array) && vector.first.is_a?(Array) + + if vector.is_a?(Array) && vector.any? && vector.all?(Numeric) + Result.new(state: :supported, detail: { "dimensions" => vector.length }) + else + # A 200 whose body is not an embedding response: the server accepted the + # request but answered with something else entirely. + unknown("unexpected_body") + end + end + + def unsupported(status) + Result.new(state: :unsupported, detail: { "http_status" => status }) + end + + def unknown(reason) + Result.new(state: :unknown, detail: { "reason" => reason }) + end + end + end +end diff --git a/app/services/llm_connections/detect_capabilities_service.rb b/app/services/llm_connections/detect_capabilities_service.rb new file mode 100644 index 000000000000..e63633dadeb4 --- /dev/null +++ b/app/services/llm_connections/detect_capabilities_service.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Records what a model on this server can do. + # + # Probing every listed model would be wrong: a gateway can list hundreds, each + # probe is a request, and some providers bill per request. So the models worth + # asking about are either the one an administrator is about to bind, or a small + # number whose names suggest they are embedding models. + class DetectCapabilitiesService + # Naming is a hint for which models are worth spending a probe on, never a + # verdict in itself. + EMBEDDING_NAME_HINT = %r{embed|bge|nomic|minilm|(^|[-_./])(e5|gte)}i + BACKGROUND_LIMIT = 10 + + def initialize(connection) + @connection = connection + end + + # Probes a specific model, synchronously. Used when an administrator binds a + # model to a feature that requires embeddings -- the verdict that matters. + # + # @return [ServiceResult] carrying the verdict + def detect(model_id) + return ServiceResult.success(result: existing_admin_verdict(model_id)) if admin_asserted?(model_id) + + result = probe.call(model_id) + ServiceResult.success(result: record(model_id, result)) + end + + # Pre-colours the model list after a connect, without spending a request per + # model. Everything not probed stays unknown, which never blocks. + # + # @return [ServiceResult] carrying the verdicts that were recorded + def detect_likely_embedding_models + recorded = [] + + candidates.each do |model_id| + result = probe.call(model_id) + recorded << record(model_id, result) + break if endpoint_absent?(result) + end + + ServiceResult.success(result: recorded) + end + + private + + attr_reader :connection + + def probe + @probe ||= Llm::Probes::EmbeddingsProbe.new(connection) + end + + # A model an administrator switched off is not going to be bound to a + # feature, so a speculative probe against it is a request spent for nothing. + def candidates + connection.selectable_model_ids + .grep(EMBEDDING_NAME_HINT) + .reject { |model_id| admin_asserted?(model_id) } + .first(BACKGROUND_LIMIT) + end + + # The server answered for the embeddings route rather than for the model, so + # the requests the rest of the batch would spend buy the same answer again. + # Read off the probe rather than the verdict, which may be an earlier and + # definite one that this inconclusive answer deliberately did not soften. + def endpoint_absent?(result) + result.state == :unknown && result.detail["reason"].in?(Llm::Probes::EmbeddingsProbe::ENDPOINT_ABSENT_REASONS) + end + + # An administrator knows things about their deployment that a probe cannot + # determine, so their assertion is never overwritten by re-detection. + def admin_asserted?(model_id) + verdicts.for_model(model_id).for_capability(:embeddings).sticky.exists? + end + + def existing_admin_verdict(model_id) + verdicts.for_model(model_id).for_capability(:embeddings).first + end + + def record(model_id, result) + verdict = claim(model_id) + + verdicts.transaction do + verdict.lock! + # Re-checked under the row lock: a probe runs for seconds, and an + # administrator may have asserted the capability in the meantime. + break verdict if verdict.source_admin? + # A probe that learned nothing must not soften a definite verdict: + # only :unsupported blocks, so downgrading it to :unknown on a transient + # failure would quietly make a rejected model usable again. + break verdict if result.state == :unknown && !verdict.unknown? + + verdict.update!(state: result.state.to_s, + source: "probe", + detail: result.detail, + checked_at: Time.current) + verdict + end + end + + # FOR UPDATE has no row to lock before the first probe of a model, and a + # synchronous detection can run alongside the background pass, so the row is + # claimed through the unique index rather than built in memory. + def claim(model_id) + verdicts.insert_all([{ llm_connection_id: connection.id, + model_id:, + capability: "embeddings", + state: "unknown", + source: "probe", + checked_at: Time.current }], + unique_by: %i[llm_connection_id model_id capability]) + + verdicts.for_model(model_id).for_capability(:embeddings).first + end + + def verdicts + connection.capability_verdicts + end + end +end diff --git a/app/services/llm_connections/enrich_capabilities_service.rb b/app/services/llm_connections/enrich_capabilities_service.rb index e5d53976d4b0..d6d4e76764d1 100644 --- a/app/services/llm_connections/enrich_capabilities_service.rb +++ b/app/services/llm_connections/enrich_capabilities_service.rb @@ -66,13 +66,20 @@ def apply_metadata(llm_model, published) attributes = {} attributes[:display_name] = published[:display_name] if llm_model.display_name.blank? - if published[:context_window].present? && llm_model.raw_metadata["max_model_len"].blank? + if published[:context_window].present? && !server_sized?(llm_model) attributes[:raw_metadata] = llm_model.raw_metadata.merge("context_window" => published[:context_window]) end llm_model.update!(attributes) if attributes.any? end + # Only what the server said counts here, not what the administrator + # overrode: clearing the override later has to reveal the published figure + # again rather than leave the window unknown. + def server_sized?(llm_model) + llm_model.raw_metadata.values_at("max_model_len", "context_window").any?(&:present?) + end + def record(model_id, capability, state) verdict = connection.capability_verdicts.find_or_initialize_by(model_id:, capability: capability.to_s) # Anything an administrator or a probe established beats a published claim: diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb index 9301d7cb4cfe..63edef6da799 100644 --- a/app/services/llm_connections/sync_models_service.rb +++ b/app/services/llm_connections/sync_models_service.rb @@ -45,6 +45,7 @@ def call invalidate_a_different_deployment store(adapter.models) + Llm::DetectCapabilitiesJob.perform_later ServiceResult.success(result: connection) rescue Llm::Client::Error => e @@ -91,7 +92,7 @@ def upsert(cards) cards.each do |card| model = connection.models.find_or_initialize_by(external_id: card.fetch(:id)) - model.update!(display_name: card[:display_name].presence || model.display_name, + model.update!(display_name: display_name_for(model, card), raw_metadata: merged_metadata(model, card), last_seen_at: now, active: true) @@ -131,15 +132,31 @@ def forget_the_previous_deployment end # The server names the model, but only when it says so: the administrator's - # display name and context-window override are theirs, and a routine refresh - # must not silently discard them. + # display name is theirs, and a routine refresh must not silently discard it. + # A gateway listing in the OpenAI shape carries the name on the raw card, + # which the adapter has no vocabulary for. + def display_name_for(model, card) + card[:display_name].presence || card.dig(:raw, "name").presence || model.display_name + end + + # The administrator's context-window override is theirs, and a routine + # refresh must not silently discard it. def merged_metadata(model, card) - raw = card.fetch(:raw, {}) + raw = normalised_window(card.fetch(:raw, {})) admin_window = model.raw_metadata["admin_context_window"] admin_window ? raw.merge("admin_context_window" => admin_window) : raw end + # OpenRouter and gateways following it publish the window as + # +context_length+, where vLLM and SGLang report +max_model_len+, the + # operator's actual limit and therefore the better figure of the two. + def normalised_window(raw) + return raw if raw["context_length"].blank? || raw["max_model_len"].present? + + raw.merge("context_window" => raw["context_length"]) + end + # Same deployment, but a model is gone. Its verdict is meaningless now, # except an administrator's assertion: an operator restarting a server must # not silently lose one. diff --git a/app/workers/llm/detect_capabilities_job.rb b/app/workers/llm/detect_capabilities_job.rb new file mode 100644 index 000000000000..ee31862a7385 --- /dev/null +++ b/app/workers/llm/detect_capabilities_job.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Pre-colours the model list after a refresh, out of band so that fetching the + # catalogue does not wait on one request per candidate model. + class DetectCapabilitiesJob < ApplicationJob + def perform + LlmConnection.find_each do |connection| + LlmConnections::DetectCapabilitiesService.new(connection).detect_likely_embedding_models + end + end + end +end diff --git a/spec/services/llm/probes/embeddings_probe_spec.rb b/spec/services/llm/probes/embeddings_probe_spec.rb new file mode 100644 index 000000000000..7cb328ddf029 --- /dev/null +++ b/spec/services/llm/probes/embeddings_probe_spec.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Probes::EmbeddingsProbe, :webmock do + subject(:result) { described_class.new(connection).call("some-model") } + + let(:connection) { build(:llm_connection, base_url: "https://example.com/v1") } + + def stub_embeddings(status:, body:) + stub_request(:post, "https://example.com/v1/embeddings") + .to_return(status:, headers: { "Content-Type" => "application/json" }, body: body.to_json) + end + + context "when the server returns a vector" do + before { stub_embeddings(status: 200, body: { data: [{ embedding: [0.1, 0.2, 0.3] }] }) } + + it "is supported and captures the dimension count" do + expect(result.state).to eq(:supported) + expect(result.detail["dimensions"]).to eq(3) + end + end + + # vLLM, llama.cpp and Ollama all silently drop parameters they do not + # understand, so a 200 on its own proves nothing about the model. + context "when the server returns 200 with something that is not an embedding" do + before { stub_embeddings(status: 200, body: { data: [{ message: "hello" }] }) } + + it "is unknown rather than supported" do + expect(result.state).to eq(:unknown) + expect(result.detail["reason"]).to eq("unexpected_body") + end + end + + context "when the server returns an empty data array" do + before { stub_embeddings(status: 200, body: { data: [] }) } + + it { expect(result.state).to eq(:unknown) } + end + + context "when the server refuses the request with 400" do + before { stub_embeddings(status: 400, body: { error: "nope" }) } + + it "is unsupported" do + expect(result.state).to eq(:unsupported) + expect(result.detail["http_status"]).to eq(400) + end + end + + # A gateway routing chat completions and nothing else answers these for every + # model, so reading them as a refusal would mark them all incapable for good. + [404, 405, 501].each do |status| + context "when the server has no embeddings route and says so with #{status}" do + before { stub_embeddings(status:, body: { error: "nope" }) } + + it "is unknown rather than a refusal by the model" do + expect(result.state).to eq(:unknown) + expect(result.detail["reason"]).to eq("http_#{status}") + end + end + end + + # A 5xx says something about the server, not about the model. + context "when the server errors" do + before { stub_embeddings(status: 500, body: { error: "boom" }) } + + it { expect(result.state).to eq(:unknown) } + end + + context "when the credentials are rejected" do + before { stub_embeddings(status: 401, body: { error: "no" }) } + + it "is unknown, since this says nothing about the model" do + expect(result.state).to eq(:unknown) + expect(result.detail["reason"]).to eq("unauthorized") + end + end + + context "when the server cannot be reached" do + before { stub_request(:post, "https://example.com/v1/embeddings").to_timeout } + + it { expect(result.state).to eq(:unknown) } + end +end diff --git a/spec/services/llm_connections/detect_capabilities_service_spec.rb b/spec/services/llm_connections/detect_capabilities_service_spec.rb new file mode 100644 index 000000000000..29a2f77e81a0 --- /dev/null +++ b/spec/services/llm_connections/detect_capabilities_service_spec.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe LlmConnections::DetectCapabilitiesService, :llm_server_helpers, :webmock do + subject(:service) { described_class.new(connection) } + + let(:base_url) { "https://example.com/v1" } + let(:connection) { create(:llm_connection, :with_models, base_url:, api_key: "sk-test") } + + it "wraps the verdict in a ServiceResult" do + mock_llm_embeddings_response(base_url) + + result = service.detect("bge-m3") + + expect(result).to be_success + expect(result.result).to be_supported + end + + # Only :unsupported blocks, so downgrading a definite verdict to :unknown on + # a transient failure would quietly make a rejected model usable again. + it "does not overwrite a definite verdict with an inconclusive probe" do + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: 1.day.ago) + mock_llm_embeddings_response(base_url, response_code: 500) + + service.detect("bge-m3") + + expect(connection.capability_verdicts.find_by(model_id: "bge-m3")).to be_unsupported + end + + it "never overwrites an administrator's assertion" do + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "supported", source: "admin", checked_at: 1.day.ago) + mock_llm_embeddings_response(base_url, response_code: 404) + + verdict = service.detect("bge-m3").result + + expect(verdict).to be_source_admin + expect(verdict).to be_supported + end + + it "adopts a verdict that appeared while its own probe was in flight" do + probe = instance_double(Llm::Probes::EmbeddingsProbe) + allow(Llm::Probes::EmbeddingsProbe).to receive(:new).and_return(probe) + allow(probe).to receive(:call) do + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "unknown", source: "probe", checked_at: Time.current) + Llm::Probes::EmbeddingsProbe::Result.new(state: :supported, detail: { "dimensions" => 4 }) + end + + service.detect("bge-m3") + + expect(connection.capability_verdicts.for_model("bge-m3").count).to eq(1) + expect(connection.capability_verdicts.find_by(model_id: "bge-m3")).to be_supported + end + + describe "#detect_likely_embedding_models" do + before { mock_llm_embeddings_response(base_url) } + + it "probes only the models whose names suggest they embed" do + create(:llm_model, llm_connection: connection, external_id: "nomic-embed-text") + create(:llm_model, llm_connection: connection, external_id: "solar-10.7b-v1.0-4e5f9c") + + service.detect_likely_embedding_models + + expect(connection.capability_verdicts.pluck(:model_id)).to contain_exactly("bge-m3", "nomic-embed-text") + end + + it "leaves out a model an administrator switched off" do + create(:llm_model, :deactivated, llm_connection: connection, external_id: "nomic-embed-text") + + service.detect_likely_embedding_models + + expect(connection.capability_verdicts.pluck(:model_id)).to eq(["bge-m3"]) + end + + it "stops the batch when the server has no embeddings route at all" do + create(:llm_model, llm_connection: connection, external_id: "nomic-embed-text") + request = mock_llm_embeddings_response(base_url, response_code: 404) + + service.detect_likely_embedding_models + + expect(request).to have_been_made.once + end + + it "stops the batch even where an earlier verdict survives the 404" do + create(:llm_model, llm_connection: connection, external_id: "nomic-embed-text") + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "supported", source: "probe", checked_at: 1.day.ago) + request = mock_llm_embeddings_response(base_url, response_code: 404) + + service.detect_likely_embedding_models + + expect(request).to have_been_made.once + expect(connection.capability_verdicts.find_by(model_id: "bge-m3")).to be_supported + end + + it "spends no more than BACKGROUND_LIMIT requests" do + (described_class::BACKGROUND_LIMIT + 5).times do |index| + create(:llm_model, llm_connection: connection, external_id: "embed-#{index}") + end + + service.detect_likely_embedding_models + + expect(connection.capability_verdicts.count).to eq(described_class::BACKGROUND_LIMIT) + end + end +end diff --git a/spec/services/llm_connections/sync_models_service_spec.rb b/spec/services/llm_connections/sync_models_service_spec.rb index f39d3ca53116..31fbc654ff87 100644 --- a/spec/services/llm_connections/sync_models_service_spec.rb +++ b/spec/services/llm_connections/sync_models_service_spec.rb @@ -96,8 +96,6 @@ expect(llm_model.reload.display_name).to eq("The house model") end - # Only the registry-backed adapters report a name; a server speaking the - # OpenAI API lists ids and nothing else. it "adopts the display name the adapter reports" do llm_model = connection.models.find_by(external_id: "qwen3.6-27b") llm_model.update!(display_name: "The house model") @@ -126,4 +124,66 @@ expect(connection.capability_verdicts.pluck(:source)).to eq(["admin"]) end end + + describe "the capability detection that follows" do + it "asks for it after a successful sync, whichever caller asked for the list" do + expect { service.call }.to have_enqueued_job(Llm::DetectCapabilitiesJob) + end + + it "asks for nothing when the server does not answer with a list" do + mock_llm_models_response(base_url, response_code: 404) + + expect { service.call }.not_to have_enqueued_job(Llm::DetectCapabilitiesJob) + end + end + + describe "naming a model and sizing its context window" do + let(:connection) { create(:llm_connection, base_url:, api_key: "sk-test") } + + it "reads both off a card that names them in the gateway's own vocabulary" do + mock_llm_models_response(base_url, + models: [{ id: "openai/gpt-4o", name: "OpenAI: GPT-4o", context_length: 128_000 }]) + + service.call + + llm_model = connection.models.find_by(external_id: "openai/gpt-4o") + expect(llm_model.name).to eq("OpenAI: GPT-4o") + expect(llm_model.context_window).to eq(128_000) + end + + it "falls back to the registry for a server that lists bare ids" do + mock_llm_models_response(base_url, models: [{ id: "gpt-4o", object: "model" }]) + + service.call + + llm_model = connection.models.find_by(external_id: "gpt-4o") + expect(llm_model.name).to eq("GPT-4o") + expect(llm_model.context_window).to eq(128_000) + end + + it "keeps the published window under an administrator's override" do + mock_llm_models_response(base_url, models: [{ id: "gpt-4o", object: "model" }]) + service.call + llm_model = connection.models.find_by(external_id: "gpt-4o") + llm_model.update!(admin_context_window: 8_000) + + described_class.new(connection).call + + expect(llm_model.reload.context_window).to eq(8_000) + + llm_model.update!(admin_context_window: nil) + + expect(llm_model.reload.context_window).to eq(128_000) + end + + it "keeps an administrator's display name over the one the registry publishes" do + mock_llm_models_response(base_url, models: [{ id: "gpt-4o", object: "model" }]) + service.call + connection.models.find_by(external_id: "gpt-4o").update!(display_name: "The house model") + + described_class.new(connection).call + + expect(connection.models.find_by(external_id: "gpt-4o").display_name).to eq("The house model") + end + end end diff --git a/spec/workers/llm/detect_capabilities_job_spec.rb b/spec/workers/llm/detect_capabilities_job_spec.rb new file mode 100644 index 000000000000..8bf38fbd4893 --- /dev/null +++ b/spec/workers/llm/detect_capabilities_job_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::DetectCapabilitiesJob, :llm_server_helpers, :webmock do + let(:base_url) { "https://example.com/v1" } + + it "probes the likely embedding models of every stored connection" do + connection = create(:llm_connection, :with_models, base_url:) + mock_llm_embeddings_response(base_url) + + described_class.perform_now + + expect(connection.capability_verdicts.pluck(:model_id, :source)).to eq([["bge-m3", "probe"]]) + end + + it "does nothing while no connection is stored" do + expect { described_class.perform_now }.not_to raise_error + end +end