diff --git a/app/components/llm_connections/delete_api_key_dialog_component.rb b/app/components/llm_connections/delete_api_key_dialog_component.rb index aaf46b1574ae..4e0f49fc9805 100644 --- a/app/components/llm_connections/delete_api_key_dialog_component.rb +++ b/app/components/llm_connections/delete_api_key_dialog_component.rb @@ -31,8 +31,7 @@ module LlmConnections # Confirms removing the stored API key. # - # No confirmation checkbox: the key itself can simply be pasted again. The - # dialog exists for what is *not* recoverable. + # No confirmation checkbox: the key itself can simply be pasted again. class DeleteApiKeyDialogComponent < ApplicationComponent include OpTurbo::Streamable include OpPrimer::ComponentHelpers diff --git a/app/components/llm_connections/disconnect_dialog_component.html.erb b/app/components/llm_connections/disconnect_dialog_component.html.erb index b7aa5a26855b..bbcda28dc1fa 100644 --- a/app/components/llm_connections/disconnect_dialog_component.html.erb +++ b/app/components/llm_connections/disconnect_dialog_component.html.erb @@ -20,7 +20,8 @@ content_tag(:ul) do safe_join( [ - content_tag(:li, t("admin.llm_connections.disconnect.keeps_settings")) + content_tag(:li, t("admin.llm_connections.disconnect.keeps_settings")), + content_tag(:li, t("admin.llm_connections.disconnect.keeps_models")) ].compact ) end diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb index 01f8a2b09fed..f98f48338ff1 100644 --- a/app/controllers/admin/llm_connections_controller.rb +++ b/app/controllers/admin/llm_connections_controller.rb @@ -87,8 +87,19 @@ def require_feature render_404 unless OpenProject::FeatureDecisions.llm_connection_active? end + # A connection can be perfectly usable without offering a model list, so the + # save succeeds either way; the administrator is told what to do next rather + # than being left with an empty table and no explanation. def redirect_after_save - redirect_with_notice(@connection.llm_features_enabled ? t(".success") : t(".disabled")) + return redirect_with_notice(t(".disabled")) unless @connection.llm_features_enabled + + if @connection.reload.models.none? + flash[:warning] = t(".no_models") + else + flash[:notice] = t(".success") + end + + redirect_to llm_connection_path, status: :see_other end def render_form_with_errors diff --git a/app/models/llm_capability_verdict.rb b/app/models/llm_capability_verdict.rb new file mode 100644 index 000000000000..b6fe7d1e73df --- /dev/null +++ b/app/models/llm_capability_verdict.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# What we know about one capability of one model on one server. +# +# Three states rather than a boolean, because the honest answer is very often +# "we cannot tell": the OpenAI model list carries no capability information at +# all, and only some servers offer a non-standard endpoint that does. +# +# The rule the rest of the system depends on: only :unsupported blocks. An +# :unknown verdict warns and lets the administrator proceed, because refusing +# on "we could not tell" would make most self-hosted servers unusable. +class LlmCapabilityVerdict < ApplicationRecord + belongs_to :llm_connection + + enum :state, { supported: "supported", unsupported: "unsupported", unknown: "unknown" }, validate: true + + # Where the verdict came from. Orthogonal to the state, so that an + # administrator's assertion can be shown as such without adding a fourth state + # that every caller would have to handle. + enum :source, + { metadata: "metadata", probe: "probe", admin: "admin", observed: "observed" }, + prefix: true, + validate: true + + validates :model_id, presence: true + validates :capability, presence: true, uniqueness: { scope: %i[llm_connection_id model_id] } + + scope :for_model, ->(model_id) { where(model_id:) } + scope :for_capability, ->(capability) { where(capability: capability.to_s) } + + # An administrator's assertion survives re-detection: they know something about + # their deployment that we could not determine. + scope :sticky, -> { where(source: "admin") } + + def blocking? + unsupported? + end + + def dimensions + detail["dimensions"] + end +end diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb index 8080d7beaa3b..9912b52d0de0 100644 --- a/app/models/llm_connection.rb +++ b/app/models/llm_connection.rb @@ -47,6 +47,11 @@ class LlmConnection < ApplicationRecord scope :active, -> { where(active: true) } + has_many :models, class_name: "LlmModel", dependent: :delete_all + + belongs_to :default_chat_model, class_name: "LlmModel", optional: true + belongs_to :default_embedding_model, class_name: "LlmModel", optional: true + has_many :capability_verdicts, class_name: "LlmCapabilityVerdict", dependent: :delete_all validates :base_url, presence: true validate :single_active_connection, if: :active? @@ -76,6 +81,28 @@ def configured? base_url.present? end + # Every model that can be addressed today: discovered and still offered, plus + # anything an administrator entered by hand. + # + # Deliberately includes models an administrator has deactivated. This is what + # Llm::Runtime resolves against, and hiding a model from the pickers must not + # break a feature that is already bound to it. + def available_model_ids + models.active.by_identifier.pluck(:external_id) + end + + # Identifies the deployment the models were fetched from. Recorded by + # LlmConnections::SyncModelsService as +connection_fingerprint+. + def settings_fingerprint + Digest::SHA256.hexdigest("#{api_format}\0#{base_url}\0#{api_key}") + end + + # The stored models were fetched from another deployment than the one + # configured now, so the list may no longer describe what the server offers. + def models_stale? + connection_fingerprint.present? && connection_fingerprint != settings_fingerprint + end + def server_flavour options["server_flavour"].presence&.to_sym end diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb new file mode 100644 index 000000000000..dbec6362c8bb --- /dev/null +++ b/app/models/llm_model.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# A model this connection can address. +# +# Rows come from two places: discovered from the server's model list, or entered +# by an administrator. Both are addressed by +external_id+, which is whatever the +# deployment calls the model -- provider-specific and not comparable across +# vendors, which is why it is never used as a lookup key into a public catalogue. +class LlmModel < ApplicationRecord + belongs_to :llm_connection + + validates :external_id, presence: true, uniqueness: { scope: :llm_connection_id } + + scope :active, -> { where(active: true) } + scope :discovered, -> { where(manual: false) } + scope :manual, -> { where(manual: true) } + scope :by_identifier, -> { order(:external_id) } + + def name = display_name.presence || external_id + + # Precedence: what an administrator set, then what the server reported (vLLM + # and SGLang publish the operator's actual --max-model-len), then what a + # registry believes about the model in general. + def context_window + raw_metadata["admin_context_window"] || + raw_metadata["max_model_len"] || + raw_metadata["context_window"] + end + + def context_window_source + return :admin if raw_metadata["admin_context_window"].present? + return :server if raw_metadata["max_model_len"].present? + return :registry if raw_metadata["context_window"].present? + + nil + end + + # A model is an embedding model when its embeddings verdict says so, and a + # chat model otherwise. There is no third kind, and no model is both. + def embedding? = verdict_for(:embeddings)&.state == "supported" + + def model_type = embedding? ? :embedding : :chat + + def verdict_for(capability) + llm_connection.capability_verdicts + .for_model(external_id) + .for_capability(capability) + .first + end + + # Discovered models that the server stopped offering are deactivated rather + # than deleted, so a binding or verdict pointing at one still has something to + # name. Manual entries are never deactivated by a refresh: nothing confirms + # them, so nothing can un-confirm them either. + def withdrawn? = !active? && !manual? +end diff --git a/app/services/llm/capabilities.rb b/app/services/llm/capabilities.rb new file mode 100644 index 000000000000..edd4a40727c8 --- /dev/null +++ b/app/services/llm/capabilities.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # The capabilities a model may have, and how to read the published ones. + module Capabilities + ALL = %i[embeddings function_calling structured_output vision reasoning].freeze + + # Capabilities a chat model can be asked about. +embeddings+ is deliberately + # absent: it is a different kind of model, not a feature of a chat one. + CHAT = (ALL - %i[embeddings]).freeze + EMBEDDING = %i[embeddings].freeze + + module_function + + # What a public registry says about this model id. + # + # Advisory only. The registry describes a model as some vendor deploys it, + # which is not the same as this deployment: the same weights are catalogued + # with contradictory capability flags and context windows an order of + # magnitude apart across providers. Verdicts derived from it are therefore + # recorded with source "metadata", and an administrator can overrule them. + # + # @return [Hash{Symbol => Symbol}, nil] capability => :supported / :unsupported, + # or nil when the registry does not know the model -- the normal case for a + # self-hosted server. + def published_for(model_id) + info = RubyLLM.models.find(model_id) + + { states: states_from(info), context_window: info.context_window, display_name: info.name } + rescue RubyLLM::ModelNotFoundError + nil + rescue StandardError => e + # Registry lookup is an enrichment; it must never break a model sync. + Rails.logger.info { "LLM capability lookup for #{model_id} failed: #{e.class} #{e.message}" } + nil + end + + def states_from(info) + embedding = info.type.to_s == "embedding" + published = Array(info.capabilities).map(&:to_sym) + relevant = embedding ? EMBEDDING : CHAT + # An entry that lists no capability at all says nothing about the model, + # which is not the same as saying it can do nothing. + unpublished = published.empty? ? :unknown : :unsupported + + states = relevant.index_with { |capability| published.include?(capability) ? :supported : unpublished } + states.merge(embeddings: embedding ? :supported : :unsupported) + end + + def label(capability) + I18n.t("llm.capabilities.#{capability}.label") + end + end +end diff --git a/app/services/llm_connections/enrich_capabilities_service.rb b/app/services/llm_connections/enrich_capabilities_service.rb new file mode 100644 index 000000000000..e5d53976d4b0 --- /dev/null +++ b/app/services/llm_connections/enrich_capabilities_service.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Fills in what a public registry publishes about the models a connection offers. + # + # This is the "the provider tells us" path. A hosted provider's model ids are + # catalogued, so capabilities arrive without asking the server anything. A + # self-hosted deployment naming its model "default" or "my-finetune-v3" is not + # catalogued, nothing is filled in, and the administrator enters capabilities + # by hand instead. + # + # An administrator's assertion is never overwritten: they know things about + # their deployment that no registry can. + class EnrichCapabilitiesService + def initialize(connection) + @connection = connection + end + + def call + connection.models.active.find_each { |llm_model| enrich(llm_model) } + + ServiceResult.success(result: connection) + end + + private + + attr_reader :connection + + def enrich(llm_model) + published = Llm::Capabilities.published_for(llm_model.external_id) + return if published.nil? + + apply_metadata(llm_model, published) + published[:states].each { |capability, state| record(llm_model.external_id, capability, state) } + end + + def apply_metadata(llm_model, published) + attributes = {} + attributes[:display_name] = published[:display_name] if llm_model.display_name.blank? + + if published[:context_window].present? && llm_model.raw_metadata["max_model_len"].blank? + attributes[:raw_metadata] = llm_model.raw_metadata.merge("context_window" => published[:context_window]) + end + + llm_model.update!(attributes) if attributes.any? + end + + def record(model_id, capability, state) + verdict = connection.capability_verdicts.find_or_initialize_by(model_id:, capability: capability.to_s) + # Anything an administrator or a probe established beats a published claim: + # both looked at this deployment, the registry did not. + return if verdict.persisted? && verdict.source.in?(%w[admin probe]) + + verdict.update!(state: state.to_s, source: "metadata", checked_at: Time.current) + end + end +end diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb new file mode 100644 index 000000000000..9301d7cb4cfe --- /dev/null +++ b/app/services/llm_connections/sync_models_service.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Refreshes the model list from the remote server. + # + # Kept separate from the contract probe so the same path serves the "Refresh + # models" button, the update service and the environment seeder. + class SyncModelsService + def initialize(connection) + @connection = connection + end + + def call + # Before the fetch, so that a deployment change invalidates the old state + # even when the new server refuses the model list: keeping the previous + # deployment's models and verdicts under new credentials would be wrong. + invalidate_a_different_deployment + + store(adapter.models) + + ServiceResult.success(result: connection) + rescue Llm::Client::Error => e + Rails.logger.info { "LLM model sync for #{connection.base_url} failed: #{e.class} #{e.message}" } + ServiceResult.failure(errors: e.message) + end + + private + + attr_reader :connection + + def adapter + @adapter ||= Llm::Adapters.for(connection) + end + + def store(cards) + ActiveRecord::Base.transaction do + connection.update!(connection_attributes) + upsert(cards) + withdraw_models_absent_from(cards) + discard_verdicts_for_vanished_models + end + + EnrichCapabilitiesService.new(connection).call + end + + def connection_attributes + now = Time.current + + { + catalogue_fetched_at: now, + last_connected_at: now, + connection_fingerprint: fingerprint, + options: connection.options.merge("server_flavour" => adapter.server_flavour) + } + end + + def fingerprint + @fingerprint ||= connection.settings_fingerprint + end + + def upsert(cards) + now = Time.current + + cards.each do |card| + model = connection.models.find_or_initialize_by(external_id: card.fetch(:id)) + model.update!(display_name: card[:display_name].presence || model.display_name, + raw_metadata: merged_metadata(model, card), + last_seen_at: now, + active: true) + end + end + + # Deactivated rather than deleted, so a binding or verdict pointing at one + # still has something to name. Manual entries are left alone: the server was + # never the thing that confirmed them. + # where.not against an empty id list matches nothing, so an empty catalogue + # needs its own branch to withdraw everything discovered. + def withdraw_models_absent_from(cards) + ids = cards.map { |card| card.fetch(:id) } + scope = connection.models.discovered + scope = scope.where.not(external_id: ids) if ids.any? + + scope.update_all(active: false) + end + + # A changed base URL or key means a different deployment, so what the + # registry and the probes established about the old one is void. What an + # administrator asserted is theirs and survives, as it does on a refresh. + def invalidate_a_different_deployment + return if connection.connection_fingerprint.blank? + return if connection.connection_fingerprint == fingerprint + + forget_the_previous_deployment + end + + # Only a successful fetch records the fingerprint (see + # +connection_attributes+), so a failed refresh leaves the list stale. + def forget_the_previous_deployment + ActiveRecord::Base.transaction do + connection.capability_verdicts.where.not(source: "admin").delete_all + connection.models.discovered.update_all(active: false) + end + end + + # The server names the model, but only when it says so: the administrator's + # display name and context-window override are theirs, and a routine refresh + # must not silently discard them. + def merged_metadata(model, card) + raw = card.fetch(:raw, {}) + admin_window = model.raw_metadata["admin_context_window"] + + admin_window ? raw.merge("admin_context_window" => admin_window) : raw + end + + # Same deployment, but a model is gone. Its verdict is meaningless now, + # except an administrator's assertion: an operator restarting a server must + # not silently lose one. + def discard_verdicts_for_vanished_models + known = connection.available_model_ids + scope = connection.capability_verdicts.where.not(source: "admin") + scope = scope.where.not(model_id: known) if known.any? + + scope.delete_all + end + end +end diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb index a8e8e1982474..2b77bcb2f979 100644 --- a/app/services/llm_connections/update_service.rb +++ b/app/services/llm_connections/update_service.rb @@ -32,10 +32,25 @@ module LlmConnections class UpdateService < BaseServices::Update private - def after_perform(call) - Setting.llm_features_enabled = model.llm_features_enabled if call.success? + # The contract has already proven the server reachable when the credentials + # changed, so refreshing the catalogue here cannot be the thing that fails + # the save. A sync failure is therefore logged, not surfaced. + def after_perform(service_call) + super.tap do + next unless service_call.success? - call + Setting.llm_features_enabled = model.llm_features_enabled + next unless initial_fill?(service_call.result) + + SyncModelsService.new(service_call.result).call + end + end + + # The only automatic refresh: nothing is stored yet, so nothing an + # administrator curated can be lost. Every later refresh is asked for. + def initial_fill?(connection) + connection.saved_changes.keys.intersect?(LlmServerValidator::CONNECTION_ATTRIBUTES) && + connection.models.none? end end end diff --git a/app/workers/llm/sync_models_job.rb b/app/workers/llm/sync_models_job.rb new file mode 100644 index 000000000000..9cb300e24624 --- /dev/null +++ b/app/workers/llm/sync_models_job.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Refreshes the cached model catalogue out of band. + # + # Used by the environment seeder, which must not block on -- or fail because of + # -- an LLM server that has not finished starting. + class SyncModelsJob < ApplicationJob + def perform + LlmConnection.find_each do |connection| + LlmConnections::SyncModelsService.new(connection).call + end + end + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 0d09a862ed07..34d0c0c8bf8b 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -201,6 +201,10 @@ en: onthefly: "Automatic user creation" port: "Port" tls_certificate_string: "LDAP server SSL certificate" + llm_capability_verdict: + capability: "Capability" + model_id: "Model" + state: "State" llm_connection: api_format: "API format" api_key: "API key" @@ -208,6 +212,10 @@ en: # ActiveRecord::Base.human_attribute_name strips the _id suffix, so these # keys deliberately do not carry it (see lib/open_project/patches/active_record_i18n.rb). llm_features_enabled: "Enable LLMs for this instance" + llm_model: + display_name: "Display name" + # human_attribute_name strips the _id suffix, so the key omits it. + external: "Model name" mcp_configuration: description: Description enabled: Enabled @@ -1708,6 +1716,7 @@ en: disconnect: description: "OpenProject will stop using the LLM server. AI features will be unavailable until you connect again." heading: "Disconnect from the LLM server?" + keeps_models: "The model list, including any models you added manually, is kept." keeps_settings: "The endpoint and API format are kept. Only the stored API key is removed." menu_label: "Disconnect" success: "OpenProject has disconnected from the LLM server." @@ -1728,6 +1737,7 @@ en: description: "Connect OpenProject to an LLM server so that AI features can use it." update: disabled: "Saved. LLM features are switched off for this instance." + no_models: "Saved, but no models are stored yet. Either the server offers no model list, or the endpoint is missing its API version segment (for example /v1)." success: "Successfully connected to the LLM server." mcp_configurations: index: diff --git a/db/migrate/20260811140000_create_llm_capability_verdicts.rb b/db/migrate/20260811140000_create_llm_capability_verdicts.rb new file mode 100644 index 000000000000..1e94a71fa5e0 --- /dev/null +++ b/db/migrate/20260811140000_create_llm_capability_verdicts.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmCapabilityVerdicts < ActiveRecord::Migration[8.1] + def change + create_table :llm_capability_verdicts do |t| + t.references :llm_connection, null: false, foreign_key: true + # A plain string, not a foreign key: the catalogue is a cache of a remote + # list, and a model may vanish from it without invalidating what we learned. + t.string :model_id, null: false + t.string :capability, null: false + t.string :state, null: false + t.string :source, null: false + t.jsonb :detail, null: false, default: {} + t.datetime :checked_at, null: false + + t.timestamps null: false + end + + add_index :llm_capability_verdicts, + %i[llm_connection_id model_id capability], + unique: true, + name: "index_llm_capability_verdicts_on_connection_model_capability" + end +end diff --git a/db/migrate/20260812090000_create_llm_models.rb b/db/migrate/20260812090000_create_llm_models.rb new file mode 100644 index 000000000000..2552d109b7f7 --- /dev/null +++ b/db/migrate/20260812090000_create_llm_models.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmModels < ActiveRecord::Migration[8.1] + def change + create_table :llm_models do |t| + t.references :llm_connection, null: false, foreign_key: true + # Whatever this deployment calls the model. Provider-specific: Scaleway + # serves "qwen3.6-35b-a3b" for weights another catalogue lists as + # "Qwen/Qwen3.6-35B-A3B". + t.string :external_id, null: false + t.string :display_name + t.boolean :active, null: false, default: true + # Entered by an administrator rather than discovered. Survives a refresh + # that cannot see it, which is what makes a server offering + # /v1/chat/completions but no /v1/models usable. + t.boolean :manual, null: false, default: false + t.datetime :last_seen_at + t.jsonb :raw_metadata, null: false, default: {} + + t.timestamps null: false + end + + add_index :llm_models, %i[llm_connection_id external_id], unique: true + + # The catalogue is superseded by the table above, and a model the server + # stops reporting keeps its row here, so a default can be a reference rather + # than an identifier that nothing resolves. Clearing it on delete is what the + # administrator is warned about before removing a model. + change_table :llm_connections, bulk: true do |t| + t.remove :catalogue, type: :jsonb, null: false, default: {} + t.remove :default_chat_model_id, type: :string + t.remove :default_embedding_model_id, type: :string + t.references :default_chat_model, null: true, + foreign_key: { to_table: :llm_models, on_delete: :nullify } + t.references :default_embedding_model, null: true, + foreign_key: { to_table: :llm_models, on_delete: :nullify } + end + end +end diff --git a/spec/factories/llm_connection_factory.rb b/spec/factories/llm_connection_factory.rb index 4dd95263d50d..c18f1b837c1e 100644 --- a/spec/factories/llm_connection_factory.rb +++ b/spec/factories/llm_connection_factory.rb @@ -33,5 +33,16 @@ sequence(:identifier) { |n| n == 1 ? LlmConnection::DEFAULT_IDENTIFIER : "connection-#{n}" } base_url { "https://example.com/v1" } api_key { "sk-test-key" } + trait :with_models do + catalogue_fetched_at { Time.current } + last_connected_at { Time.current } + + after(:create) do |connection| + create(:llm_model, llm_connection: connection, external_id: "qwen3.6-27b", + raw_metadata: { "owned_by" => "vllm", "max_model_len" => 262_144 }) + create(:llm_model, llm_connection: connection, external_id: "bge-m3", + raw_metadata: { "owned_by" => "vllm", "max_model_len" => 8_192 }) + end + end end end diff --git a/spec/factories/llm_model_factory.rb b/spec/factories/llm_model_factory.rb new file mode 100644 index 000000000000..59527ef65278 --- /dev/null +++ b/spec/factories/llm_model_factory.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +FactoryBot.define do + factory :llm_model do + llm_connection + sequence(:external_id) { |n| "model-#{n}" } + active { true } + manual { false } + last_seen_at { Time.current } + + trait :manual do + manual { true } + last_seen_at { nil } + end + + trait :withdrawn do + active { false } + end + + # Still offered by the server; hidden by an administrator. + trait :deactivated do + deactivated_at { Time.current } + end + end +end diff --git a/spec/models/llm_connection_spec.rb b/spec/models/llm_connection_spec.rb index eec473a46473..26a4b4251922 100644 --- a/spec/models/llm_connection_spec.rb +++ b/spec/models/llm_connection_spec.rb @@ -93,4 +93,41 @@ expect(described_class.active_connection).to eq(connection) end end + + describe "#settings_fingerprint" do + subject(:connection) { build(:llm_connection, base_url: "https://example.com/v1", api_key: "sk-test") } + + it "changes with the API format" do + expect { connection.api_format = "anthropic" }.to change(connection, :settings_fingerprint) + end + + it "changes with the host URL" do + expect { connection.base_url = "https://elsewhere.example/v1" }.to change(connection, :settings_fingerprint) + end + + it "changes with the API key" do + expect { connection.api_key = "sk-rotated" }.to change(connection, :settings_fingerprint) + end + end + + describe "#models_stale?" do + subject(:connection) { create(:llm_connection, base_url: "https://example.com/v1", api_key: "sk-test") } + + it "is false while no model list has been fetched" do + expect(connection).not_to be_models_stale + end + + it "is false while the settings still match the fetched list" do + connection.update!(connection_fingerprint: connection.settings_fingerprint) + + expect(connection).not_to be_models_stale + end + + it "is true once a connection setting changed" do + connection.update!(connection_fingerprint: connection.settings_fingerprint) + connection.update!(api_key: "sk-rotated") + + expect(connection).to be_models_stale + end + end end diff --git a/spec/models/llm_model_spec.rb b/spec/models/llm_model_spec.rb new file mode 100644 index 000000000000..b9143bef6ace --- /dev/null +++ b/spec/models/llm_model_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe LlmModel do + let(:connection) { create(:llm_connection) } + let(:llm_model) { create(:llm_model, llm_connection: connection, external_id: "bge-m3") } + + describe "#model_type" do + it "is chat while nothing says the model embeds" do + expect(llm_model).not_to be_embedding + expect(llm_model.model_type).to eq(:chat) + end + + it "is embedding once the embeddings verdict says so" do + connection.capability_verdicts.create!(model_id: llm_model.external_id, capability: "embeddings", + state: "supported", source: "admin", checked_at: Time.current) + + expect(llm_model).to be_embedding + expect(llm_model.model_type).to eq(:embedding) + end + + it "is chat when the embeddings verdict is unknown" do + connection.capability_verdicts.create!(model_id: llm_model.external_id, capability: "embeddings", + state: "unknown", source: "metadata", checked_at: Time.current) + + expect(llm_model.model_type).to eq(:chat) + end + end +end diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb index 2263f4ffa131..0a02eec4d73a 100644 --- a/spec/requests/admin/llm_connections_spec.rb +++ b/spec/requests/admin/llm_connections_spec.rb @@ -117,12 +117,55 @@ expect(connection.api_key).to eq("sk-test") end + it "fills the catalogue once when none is stored" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-test" } } + + expect(LlmConnection.first.available_model_ids).to contain_exactly("qwen3.6-27b", "bge-m3") + end + it "confirms the connection once LLMs are switched on" do patch llm_connection_path, params: { llm_connection: { llm_features_enabled: "1", base_url:, api_key: "sk-test" } } expect(flash[:notice]).to eq(I18n.t("admin.llm_connections.update.success")) end + + context "when models are already stored" do + let!(:connection) { create(:llm_connection, :with_models, base_url:, api_key: "sk-test") } + + before do + connection.update!(connection_fingerprint: connection.settings_fingerprint) + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "supported", source: "admin", checked_at: Time.current) + end + + it "leaves the stored catalogue alone when the host URL changes" do + elsewhere = "https://elsewhere.example/v1" + mock_llm_models_response(elsewhere, models: [{ id: "llama4-8b", object: "model", owned_by: "vllm" }]) + + patch llm_connection_path, params: { llm_connection: { base_url: elsewhere } } + + connection.reload + expect(connection.available_model_ids).to contain_exactly("qwen3.6-27b", "bge-m3") + expect(connection.capability_verdicts.pluck(:source)).to eq(["admin"]) + expect(connection).to be_models_stale + end + end + end + + # The case that matters for OpenProject's own gateway: chat completions are + # routed, the model list is not. + context "with a server that exposes no model list" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: 404) } + + it "still saves the connection and says models must be added by hand" do + patch llm_connection_path, + params: { llm_connection: { llm_features_enabled: "1", base_url:, api_key: "sk-test" } } + + expect(response).to have_http_status(:see_other) + expect(LlmConnection.first.base_url).to eq(base_url) + expect(flash[:warning]).to be_present + end end context "when the administrator switches LLMs off" do @@ -226,9 +269,25 @@ end end + describe "GET /admin/llm_connection/delete_api_key_dialog" do + let!(:connection) { create(:llm_connection, base_url: "https://example.com/v1", api_key: "sk-test") } + + before { login_as admin } + + it "offers the confirmation" do + # Requested by the async-dialog Stimulus controller, which asks for a + # turbo stream rather than HTML. + get delete_api_key_dialog_llm_connection_path, + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Remove the stored API key?") + end + end + describe "disconnecting" do let!(:connection) do - create(:llm_connection, + create(:llm_connection, :with_models, base_url: "https://example.com/v1", api_key: "sk-test") end @@ -253,6 +312,7 @@ expect(connection.api_key).to be_blank expect(Setting.llm_features_enabled?).to be(false) expect(connection.base_url).to eq("https://example.com/v1") + expect(connection.models.count).to eq(2) end it "is refused to a non-admin" do diff --git a/spec/services/llm/capabilities_spec.rb b/spec/services/llm/capabilities_spec.rb new file mode 100644 index 000000000000..73c3b075655a --- /dev/null +++ b/spec/services/llm/capabilities_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Capabilities do + describe ".states_from" do + subject(:states) { described_class.states_from(info) } + + context "with an embedding entry" do + let(:info) { instance_double(RubyLLM::Model::Info, type: "embedding", capabilities: []) } + + it "reports embeddings only" do + expect(states).to eq(embeddings: :supported) + end + end + + context "with a chat entry listing its capabilities" do + let(:info) { instance_double(RubyLLM::Model::Info, type: "chat", capabilities: %w[function_calling vision]) } + + it "reports the listed ones as supported and the rest as unsupported" do + expect(states).to eq(embeddings: :unsupported, + function_calling: :supported, + structured_output: :unsupported, + vision: :supported, + reasoning: :unsupported) + end + end + + context "with a chat entry listing no capability at all" do + let(:info) { instance_double(RubyLLM::Model::Info, type: "chat", capabilities: []) } + + it "leaves the chat capabilities unknown rather than denying them" do + expect(states).to eq(embeddings: :unsupported, + function_calling: :unknown, + structured_output: :unknown, + vision: :unknown, + reasoning: :unknown) + end + end + end +end diff --git a/spec/services/llm_connections/sync_models_service_spec.rb b/spec/services/llm_connections/sync_models_service_spec.rb new file mode 100644 index 000000000000..f39d3ca53116 --- /dev/null +++ b/spec/services/llm_connections/sync_models_service_spec.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe LlmConnections::SyncModelsService, :llm_server_helpers, :webmock do + subject(:service) { described_class.new(connection) } + + let(:base_url) { "https://example.com/v1" } + let(:connection) { create(:llm_connection, :with_models, base_url:, api_key: "sk-test") } + + before { mock_llm_models_response(base_url) } + + describe "switching to a different deployment" do + before do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "supported", source: "admin", checked_at: Time.current) + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "vision", + state: "unsupported", source: "metadata", checked_at: Time.current) + connection.update_columns(base_url: "https://elsewhere.example/v1", + connection_fingerprint: "the-previous-deployment") + end + + it "invalidates the old models and verdicts even when the new server offers no list" do + mock_llm_models_response("https://elsewhere.example/v1", response_code: 404) + + result = service.call + + expect(result).to be_failure + expect(connection.capability_verdicts.pluck(:capability, :source)).to eq([%w[embeddings admin]]) + expect(connection.models.active).to be_empty + end + + it "keeps warning about a stale list when the new server refuses it" do + mock_llm_models_response("https://elsewhere.example/v1", response_code: 404) + + expect(service.call).to be_failure + expect(connection.reload).to be_models_stale + end + + it "keeps administrator assertions and re-activates what the new server reports" do + mock_llm_models_response("https://elsewhere.example/v1") + + described_class.new(connection).call + + expect(connection.capability_verdicts.where(source: "admin").pluck(:capability)).to eq(["embeddings"]) + expect(connection.models.active.pluck(:external_id)).to contain_exactly("qwen3.6-27b", "bge-m3") + end + end + + describe "refreshing the same deployment" do + before { service.call } + + it "keeps an administrator's context window override across refreshes" do + llm_model = connection.models.find_by(external_id: "qwen3.6-27b") + llm_model.update!(raw_metadata: llm_model.raw_metadata.merge("admin_context_window" => 4096)) + + described_class.new(connection).call + + expect(llm_model.reload.context_window).to eq(4096) + end + + it "keeps an administrator's display name when the server reports none" do + llm_model = connection.models.find_by(external_id: "qwen3.6-27b") + llm_model.update!(display_name: "The house model") + + described_class.new(connection).call + + expect(llm_model.reload.display_name).to eq("The house model") + end + + # Only the registry-backed adapters report a name; a server speaking the + # OpenAI API lists ids and nothing else. + it "adopts the display name the adapter reports" do + llm_model = connection.models.find_by(external_id: "qwen3.6-27b") + llm_model.update!(display_name: "The house model") + allow(Llm::Adapters).to receive(:for).and_return( + instance_double(Llm::Adapters::RegistryBacked, + models: [{ id: "qwen3.6-27b", display_name: "Qwen 3.6 27B", raw: {} }], + server_flavour: "anthropic") + ) + + described_class.new(connection).call + + expect(llm_model.reload.display_name).to eq("Qwen 3.6 27B") + end + + it "drops every non-admin verdict when the catalogue comes back empty" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "supported", source: "probe", checked_at: Time.current) + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "supported", source: "admin", checked_at: Time.current) + mock_llm_models_response(base_url, body: { object: "list", data: [] }.to_json) + + # A fresh instance, as every caller builds one: the adapter memoises the + # fetched list within a run. + described_class.new(connection).call + + expect(connection.capability_verdicts.pluck(:source)).to eq(["admin"]) + end + end +end diff --git a/spec/workers/llm/sync_models_job_spec.rb b/spec/workers/llm/sync_models_job_spec.rb new file mode 100644 index 000000000000..90ed7c2ab257 --- /dev/null +++ b/spec/workers/llm/sync_models_job_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::SyncModelsJob, :llm_server_helpers, :webmock do + let(:base_url) { "https://example.com/v1" } + + it "refreshes the model list of every stored connection" do + connection = create(:llm_connection, base_url:) + request = mock_llm_models_response(base_url) + + described_class.perform_now + + expect(request).to have_been_made.once + expect(connection.reload.available_model_ids).to contain_exactly("qwen3.6-27b", "bge-m3") + end + + it "does nothing while no connection is stored" do + expect { described_class.perform_now }.not_to raise_error + end +end