From 281818a923ff599462011f5a4dd11cc58b837946 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Sat, 22 Aug 2026 18:12:19 +0200 Subject: [PATCH] [AI-3] Store the model catalogue and what is known about each model's capabilities Persists the models a connection offers as LlmModel records, refreshed by LlmConnections::SyncModelsService after every credential change and on demand through the new Refresh models action. Models the server stops offering are deactivated rather than deleted, so anything pointing at one still has something to name. What is known about each model is stored as LlmCapabilityVerdict rows in three states: supported, unsupported and unknown, because the OpenAI model list carries no capability information and the honest answer is very often that we cannot tell. Verdicts are seeded from the ruby_llm registry where the model id is catalogued, survive re-detection when an administrator asserted them, and are discarded wholesale when the credentials point at a different deployment. No UI beyond the refresh action yet; the model list and manual entries follow in later parts. Part 4 of the AI-3 stack. https://community.openproject.org/work_packages/66020 --- .../delete_api_key_dialog_component.html.erb | 8 +- .../delete_api_key_dialog_component.rb | 11 +- .../disconnect_dialog_component.html.erb | 3 +- .../admin/llm_connections_controller.rb | 21 ++- app/models/llm_capability_verdict.rb | 70 ++++++++ app/models/llm_connection.rb | 14 +- app/models/llm_model.rb | 78 +++++++++ app/services/llm/capabilities.rb | 79 +++++++++ .../enrich_capabilities_service.rb | 85 ++++++++++ .../llm_connections/sync_models_service.rb | 152 ++++++++++++++++++ .../llm_connections/update_service.rb | 17 ++ app/workers/llm/sync_models_job.rb | 44 +++++ config/locales/en.yml | 14 ++ config/routes.rb | 1 + ...11140000_create_llm_capability_verdicts.rb | 52 ++++++ .../20260812090000_create_llm_models.rb | 56 +++++++ spec/factories/llm_connection_factory.rb | 12 ++ spec/factories/llm_model_factory.rb | 53 ++++++ spec/requests/admin/llm_connections_spec.rb | 59 ++++++- .../sync_models_service_spec.rb | 86 ++++++++++ 20 files changed, 908 insertions(+), 7 deletions(-) create mode 100644 app/models/llm_capability_verdict.rb create mode 100644 app/models/llm_model.rb create mode 100644 app/services/llm/capabilities.rb create mode 100644 app/services/llm_connections/enrich_capabilities_service.rb create mode 100644 app/services/llm_connections/sync_models_service.rb create mode 100644 app/workers/llm/sync_models_job.rb create mode 100644 db/migrate/20260811140000_create_llm_capability_verdicts.rb create mode 100644 db/migrate/20260812090000_create_llm_models.rb create mode 100644 spec/factories/llm_model_factory.rb create mode 100644 spec/services/llm_connections/sync_models_service_spec.rb diff --git a/app/components/llm_connections/delete_api_key_dialog_component.html.erb b/app/components/llm_connections/delete_api_key_dialog_component.html.erb index 73d7bc0542b1..0c317a9534ec 100644 --- a/app/components/llm_connections/delete_api_key_dialog_component.html.erb +++ b/app/components/llm_connections/delete_api_key_dialog_component.html.erb @@ -10,7 +10,13 @@ ) do |dialog| dialog.with_confirmation_message do |message| message.with_heading(tag: :h2) { t("admin.llm_connections.delete_api_key.heading") } - message.with_description_content(t("admin.llm_connections.delete_api_key.description")) + message.with_description_content( + if loses_admin_verdicts? + t("admin.llm_connections.delete_api_key.description_verdicts") + else + t("admin.llm_connections.delete_api_key.description") + end + ) end end %> diff --git a/app/components/llm_connections/delete_api_key_dialog_component.rb b/app/components/llm_connections/delete_api_key_dialog_component.rb index aaf46b1574ae..6c0aa7280fee 100644 --- a/app/components/llm_connections/delete_api_key_dialog_component.rb +++ b/app/components/llm_connections/delete_api_key_dialog_component.rb @@ -32,7 +32,7 @@ module LlmConnections # Confirms removing the stored API key. # # No confirmation checkbox: the key itself can simply be pasted again. The - # dialog exists for what is *not* recoverable. + # dialog exists for what is *not* recoverable -- see #loses_admin_verdicts?. class DeleteApiKeyDialogComponent < ApplicationComponent include OpTurbo::Streamable include OpPrimer::ComponentHelpers @@ -44,5 +44,14 @@ class DeleteApiKeyDialogComponent < ApplicationComponent def form_arguments { action: url_helpers.api_key_llm_connection_path, method: :delete } end + + # The catalogue sync fingerprints base_url and api_key together, so the next + # refresh after the key changes treats the endpoint as a different deployment + # and discards every capability verdict -- including the ones an + # administrator asserted by hand, which nothing else in the system throws + # away. Worth saying out loud before the key goes. + def loses_admin_verdicts? + connection.capability_verdicts.exists?(source: "admin") + end end end diff --git a/app/components/llm_connections/disconnect_dialog_component.html.erb b/app/components/llm_connections/disconnect_dialog_component.html.erb index 12d4533ee064..20ed7bbbddbe 100644 --- a/app/components/llm_connections/disconnect_dialog_component.html.erb +++ b/app/components/llm_connections/disconnect_dialog_component.html.erb @@ -19,7 +19,8 @@ content_tag(:ul) do safe_join( [ - content_tag(:li, t("admin.llm_connections.disconnect.keeps_settings")) + content_tag(:li, t("admin.llm_connections.disconnect.keeps_settings")), + content_tag(:li, t("admin.llm_connections.disconnect.keeps_models")) ].compact ) end diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb index 1ddda37a4cc3..9a311665748a 100644 --- a/app/controllers/admin/llm_connections_controller.rb +++ b/app/controllers/admin/llm_connections_controller.rb @@ -51,6 +51,16 @@ def update result.on_failure { render_form_with_errors } end + def refresh_models + result = ::LlmConnections::SyncModelsService.new(@connection).call + + if result.success? + redirect_with_notice(t(".success")) + else + redirect_with_error(t(".failure")) + end + end + def disconnect_dialog respond_with_dialog LlmConnections::DisconnectDialogComponent.new(@connection) end @@ -85,8 +95,17 @@ def require_feature render_404 unless OpenProject::FeatureDecisions.llm_connection_active? end + # A connection can be perfectly usable without offering a model list, so the + # save succeeds either way; the administrator is told what to do next rather + # than being left with an empty table and no explanation. def redirect_after_save - redirect_with_notice(t(".success")) + if @connection.reload.models.none? + flash[:warning] = t(".no_models") + else + flash[:notice] = t(".success") + end + + redirect_to llm_connection_path, status: :see_other end def render_form_with_errors diff --git a/app/models/llm_capability_verdict.rb b/app/models/llm_capability_verdict.rb new file mode 100644 index 000000000000..b6fe7d1e73df --- /dev/null +++ b/app/models/llm_capability_verdict.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# What we know about one capability of one model on one server. +# +# Three states rather than a boolean, because the honest answer is very often +# "we cannot tell": the OpenAI model list carries no capability information at +# all, and only some servers offer a non-standard endpoint that does. +# +# The rule the rest of the system depends on: only :unsupported blocks. An +# :unknown verdict warns and lets the administrator proceed, because refusing +# on "we could not tell" would make most self-hosted servers unusable. +class LlmCapabilityVerdict < ApplicationRecord + belongs_to :llm_connection + + enum :state, { supported: "supported", unsupported: "unsupported", unknown: "unknown" }, validate: true + + # Where the verdict came from. Orthogonal to the state, so that an + # administrator's assertion can be shown as such without adding a fourth state + # that every caller would have to handle. + enum :source, + { metadata: "metadata", probe: "probe", admin: "admin", observed: "observed" }, + prefix: true, + validate: true + + validates :model_id, presence: true + validates :capability, presence: true, uniqueness: { scope: %i[llm_connection_id model_id] } + + scope :for_model, ->(model_id) { where(model_id:) } + scope :for_capability, ->(capability) { where(capability: capability.to_s) } + + # An administrator's assertion survives re-detection: they know something about + # their deployment that we could not determine. + scope :sticky, -> { where(source: "admin") } + + def blocking? + unsupported? + end + + def dimensions + detail["dimensions"] + end +end diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb index f3252ecca1f9..92b9a2c088cf 100644 --- a/app/models/llm_connection.rb +++ b/app/models/llm_connection.rb @@ -38,9 +38,9 @@ class LlmConnection < ApplicationRecord include Redmine::Ciphering SINGLETON_NAME = "default" - has_many :health_reports, as: :subject, dependent: :delete_all - + has_many :models, class_name: "LlmModel", dependent: :delete_all + has_many :capability_verdicts, class_name: "LlmCapabilityVerdict", dependent: :delete_all validates :base_url, presence: true validate :only_one_connection, on: :create @@ -82,6 +82,16 @@ def configured? base_url.present? end + # Every model that can be addressed today: discovered and still offered, plus + # anything an administrator entered by hand. + # + # Deliberately includes models an administrator has deactivated. This is what + # Llm::Runtime resolves against, and hiding a model from the pickers must not + # break a feature that is already bound to it. + def available_model_ids + models.active.by_identifier.pluck(:external_id) + end + def server_flavour options["server_flavour"].presence&.to_sym end diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb new file mode 100644 index 000000000000..eff2b53c2ae9 --- /dev/null +++ b/app/models/llm_model.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# A model this connection can address. +# +# Rows come from two places: discovered from the server's model list, or entered +# by an administrator. Both are addressed by +external_id+, which is whatever the +# deployment calls the model -- provider-specific and not comparable across +# vendors, which is why it is never used as a lookup key into a public catalogue. +class LlmModel < ApplicationRecord + belongs_to :llm_connection + + validates :external_id, presence: true, uniqueness: { scope: :llm_connection_id } + + scope :active, -> { where(active: true) } + scope :discovered, -> { where(manual: false) } + scope :manual, -> { where(manual: true) } + scope :by_identifier, -> { order(:external_id) } + + def name = display_name.presence || external_id + + # Precedence: what an administrator set, then what the server reported (vLLM + # and SGLang publish the operator's actual --max-model-len), then what a + # registry believes about the model in general. + def context_window + raw_metadata["admin_context_window"] || + raw_metadata["max_model_len"] || + raw_metadata["context_window"] + end + + def context_window_source + return :admin if raw_metadata["admin_context_window"].present? + return :server if raw_metadata["max_model_len"].present? + return :registry if raw_metadata["context_window"].present? + + nil + end + + def verdict_for(capability) + llm_connection.capability_verdicts + .for_model(external_id) + .for_capability(capability) + .first + end + + # Discovered models that the server stopped offering are deactivated rather + # than deleted, so a binding or verdict pointing at one still has something to + # name. Manual entries are never deactivated by a refresh: nothing confirms + # them, so nothing can un-confirm them either. + def withdrawn? = !active? && !manual? +end diff --git a/app/services/llm/capabilities.rb b/app/services/llm/capabilities.rb new file mode 100644 index 000000000000..cf610a3067c3 --- /dev/null +++ b/app/services/llm/capabilities.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # The capabilities a model may have, and how to read the published ones. + module Capabilities + ALL = %i[embeddings function_calling structured_output vision reasoning].freeze + + # Capabilities a chat model can be asked about. +embeddings+ is deliberately + # absent: it is a different kind of model, not a feature of a chat one. + CHAT = (ALL - %i[embeddings]).freeze + EMBEDDING = %i[embeddings].freeze + + module_function + + # What a public registry says about this model id. + # + # Advisory only. The registry describes a model as some vendor deploys it, + # which is not the same as this deployment: the same weights are catalogued + # with contradictory capability flags and context windows an order of + # magnitude apart across providers. Verdicts derived from it are therefore + # recorded with source "metadata", and an administrator can overrule them. + # + # @return [Hash{Symbol => Symbol}, nil] capability => :supported / :unsupported, + # or nil when the registry does not know the model -- the normal case for a + # self-hosted server. + def published_for(model_id) + info = RubyLLM.models.find(model_id) + + { states: states_from(info), context_window: info.context_window, display_name: info.name } + rescue RubyLLM::ModelNotFoundError + nil + rescue StandardError => e + # Registry lookup is an enrichment; it must never break a model sync. + Rails.logger.info { "LLM capability lookup for #{model_id} failed: #{e.class} #{e.message}" } + nil + end + + def states_from(info) + embedding = info.type.to_s == "embedding" + published = Array(info.capabilities).map(&:to_sym) + relevant = embedding ? EMBEDDING : CHAT + + states = relevant.index_with { |capability| published.include?(capability) ? :supported : :unsupported } + states.merge(embeddings: embedding ? :supported : :unsupported) + end + + def label(capability) + I18n.t("llm.capabilities.#{capability}.label") + end + end +end diff --git a/app/services/llm_connections/enrich_capabilities_service.rb b/app/services/llm_connections/enrich_capabilities_service.rb new file mode 100644 index 000000000000..e5d53976d4b0 --- /dev/null +++ b/app/services/llm_connections/enrich_capabilities_service.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Fills in what a public registry publishes about the models a connection offers. + # + # This is the "the provider tells us" path. A hosted provider's model ids are + # catalogued, so capabilities arrive without asking the server anything. A + # self-hosted deployment naming its model "default" or "my-finetune-v3" is not + # catalogued, nothing is filled in, and the administrator enters capabilities + # by hand instead. + # + # An administrator's assertion is never overwritten: they know things about + # their deployment that no registry can. + class EnrichCapabilitiesService + def initialize(connection) + @connection = connection + end + + def call + connection.models.active.find_each { |llm_model| enrich(llm_model) } + + ServiceResult.success(result: connection) + end + + private + + attr_reader :connection + + def enrich(llm_model) + published = Llm::Capabilities.published_for(llm_model.external_id) + return if published.nil? + + apply_metadata(llm_model, published) + published[:states].each { |capability, state| record(llm_model.external_id, capability, state) } + end + + def apply_metadata(llm_model, published) + attributes = {} + attributes[:display_name] = published[:display_name] if llm_model.display_name.blank? + + if published[:context_window].present? && llm_model.raw_metadata["max_model_len"].blank? + attributes[:raw_metadata] = llm_model.raw_metadata.merge("context_window" => published[:context_window]) + end + + llm_model.update!(attributes) if attributes.any? + end + + def record(model_id, capability, state) + verdict = connection.capability_verdicts.find_or_initialize_by(model_id:, capability: capability.to_s) + # Anything an administrator or a probe established beats a published claim: + # both looked at this deployment, the registry did not. + return if verdict.persisted? && verdict.source.in?(%w[admin probe]) + + verdict.update!(state: state.to_s, source: "metadata", checked_at: Time.current) + end + end +end diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb new file mode 100644 index 000000000000..95c6256c8bac --- /dev/null +++ b/app/services/llm_connections/sync_models_service.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Refreshes the model list from the remote server. + # + # Kept separate from the contract probe so the same path serves the "Refresh + # models" button, the update service and the environment seeder. + class SyncModelsService + def initialize(connection) + @connection = connection + end + + def call + # Before the fetch, so that a deployment change invalidates the old state + # even when the new server refuses the model list: keeping the previous + # deployment's models and verdicts under new credentials would be wrong. + invalidate_a_different_deployment + + store(adapter.models) + + ServiceResult.success(result: connection) + rescue Llm::Client::Error => e + Rails.logger.info { "LLM model sync for #{connection.base_url} failed: #{e.class} #{e.message}" } + ServiceResult.failure(errors: e.message) + end + + private + + attr_reader :connection + + def adapter + @adapter ||= Llm::Adapters.for(connection) + end + + def store(cards) + ActiveRecord::Base.transaction do + connection.update!(connection_attributes) + upsert(cards) + withdraw_models_absent_from(cards) + discard_verdicts_for_vanished_models + end + + EnrichCapabilitiesService.new(connection).call + end + + def connection_attributes + now = Time.current + + { + catalogue_fetched_at: now, + last_connected_at: now, + connection_fingerprint: fingerprint, + options: connection.options.merge("server_flavour" => adapter.server_flavour) + } + end + + def fingerprint + @fingerprint ||= Digest::SHA256.hexdigest("#{connection.api_format}\0#{connection.base_url}\0#{connection.api_key}") + end + + def upsert(cards) + now = Time.current + + cards.each do |card| + model = connection.models.find_or_initialize_by(external_id: card.fetch(:id)) + model.update!(display_name: card[:display_name], + raw_metadata: merged_metadata(model, card), + last_seen_at: now, + active: true) + end + end + + # Deactivated rather than deleted, so a binding or verdict pointing at one + # still has something to name. Manual entries are left alone: the server was + # never the thing that confirmed them. + # where.not against an empty id list matches nothing, so an empty catalogue + # needs its own branch to withdraw everything discovered. + def withdraw_models_absent_from(cards) + ids = cards.map { |card| card.fetch(:id) } + scope = connection.models.discovered + scope = scope.where.not(external_id: ids) if ids.any? + + scope.update_all(active: false) + end + + # A changed base URL or key means a different deployment, so everything we + # learned about the old one is void -- including administrator assertions, + # which were about that deployment. + def invalidate_a_different_deployment + return if connection.connection_fingerprint.blank? + return if connection.connection_fingerprint == fingerprint + + forget_the_previous_deployment + end + + def forget_the_previous_deployment + ActiveRecord::Base.transaction do + connection.capability_verdicts.delete_all + connection.models.discovered.update_all(active: false) + connection.update!(connection_fingerprint: fingerprint) + end + end + + # The server names the model; the administrator's context-window override is + # theirs, and a routine refresh must not silently discard it. + def merged_metadata(model, card) + raw = card.fetch(:raw, {}) + admin_window = model.raw_metadata["admin_context_window"] + + admin_window ? raw.merge("admin_context_window" => admin_window) : raw + end + + # Same deployment, but a model is gone. Its verdict is meaningless now, + # except an administrator's assertion: an operator restarting a server must + # not silently lose one. + def discard_verdicts_for_vanished_models + known = connection.available_model_ids + scope = connection.capability_verdicts.where.not(source: "admin") + scope = scope.where.not(model_id: known) if known.any? + + scope.delete_all + end + end +end diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb index d82f98369938..b21271d13637 100644 --- a/app/services/llm_connections/update_service.rb +++ b/app/services/llm_connections/update_service.rb @@ -30,5 +30,22 @@ module LlmConnections class UpdateService < BaseServices::Update + private + + # The contract has already proven the server reachable when the credentials + # changed, so refreshing the catalogue here cannot be the thing that fails + # the save. A sync failure is therefore logged, not surfaced. + def after_perform(service_call) + super.tap do + next unless service_call.success? + next unless connection_changed?(service_call.result) + + SyncModelsService.new(service_call.result).call + end + end + + def connection_changed?(connection) + connection.saved_changes.keys.intersect?(LlmServerValidator::CONNECTION_ATTRIBUTES) + end end end diff --git a/app/workers/llm/sync_models_job.rb b/app/workers/llm/sync_models_job.rb new file mode 100644 index 000000000000..eb72d9a6075b --- /dev/null +++ b/app/workers/llm/sync_models_job.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Refreshes the cached model catalogue out of band. + # + # Used by the environment seeder, which must not block on -- or fail because of + # -- an LLM server that has not finished starting. + class SyncModelsJob < ApplicationJob + def perform + connection = LlmConnection.first + return if connection.nil? || !connection.configured? + + LlmConnections::SyncModelsService.new(connection).call + end + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 19472ddbcad1..296dc1d69414 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -197,6 +197,10 @@ en: onthefly: "Automatic user creation" port: "Port" tls_certificate_string: "LDAP server SSL certificate" + llm_capability_verdict: + capability: "Capability" + model_id: "Model" + state: "State" llm_connection: api_format: "API format" api_key: "API key" @@ -205,6 +209,10 @@ en: # keys deliberately do not carry it (see lib/open_project/patches/active_record_i18n.rb). enabled: "Enable LLMs for this instance" last_connected_at: "Last connected" + llm_model: + display_name: "Display name" + # human_attribute_name strips the _id suffix, so the key omits it. + external: "Model name" mcp_configuration: description: Description enabled: Enabled @@ -1694,6 +1702,7 @@ en: delete_api_key: configured_from_env: "This connection is configured through the environment and cannot be changed here." description: "OpenProject will stop sending an API key with its requests. Features will keep working only if the server requires no authentication. You can enter a new key at any time." + description_verdicts: "OpenProject will stop sending an API key with its requests, and the next model refresh will discard what is known about each model's capabilities, including the assertions you made yourself. You can enter a new key at any time." heading: "Remove the stored API key?" menu_label: "Remove API key" success: "The API key has been removed." @@ -1702,6 +1711,7 @@ en: configured_from_env: "This connection is configured through the environment and cannot be changed here." description: "OpenProject will stop using the LLM server. AI features will be unavailable until you connect again." heading: "Disconnect from the LLM server?" + keeps_models: "The model list, including any models you added manually, is kept." keeps_settings: "The endpoint and API format are kept. Only the stored API key is removed." menu_label: "Disconnect" success: "OpenProject has disconnected from the LLM server." @@ -1715,12 +1725,16 @@ en: button_connect: "Connect" enabled_caption: "When turned off, all LLM-backed AI features stop working. The connection settings are kept." label_connecting: "Contacting the LLM server…" + refresh_models: + failure: "The model list could not be refreshed. Please check that the LLM server is still reachable." + success: "The model list has been refreshed." show: add_model_description: "If the server does not expose a model list, enter the model name exactly as the server expects it. Manually added models are kept when the list is refreshed." add_model_submit: "Add model" description: "Connect OpenProject to a server that speaks the OpenAI API, so AI features can use it." refresh_models: "Refresh models" update: + no_models: "Saved, but the server did not return a model list. Either it does not offer one, or the endpoint is wrong — a URL missing its API version segment (for example /v1) looks exactly the same from here. Add the models you want to use below, then run the health checks to confirm the server answers." success: "Successfully connected to the LLM server." mcp_configurations: index: diff --git a/config/routes.rb b/config/routes.rb index 29f1932ea11f..8b2b6f0eb139 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -743,6 +743,7 @@ end resource :llm_connection, only: %i[show update], controller: "admin/llm_connections" do + post :refresh_models delete :api_key, action: :delete_api_key get :delete_api_key_dialog get :disconnect_dialog diff --git a/db/migrate/20260811140000_create_llm_capability_verdicts.rb b/db/migrate/20260811140000_create_llm_capability_verdicts.rb new file mode 100644 index 000000000000..1e94a71fa5e0 --- /dev/null +++ b/db/migrate/20260811140000_create_llm_capability_verdicts.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmCapabilityVerdicts < ActiveRecord::Migration[8.1] + def change + create_table :llm_capability_verdicts do |t| + t.references :llm_connection, null: false, foreign_key: true + # A plain string, not a foreign key: the catalogue is a cache of a remote + # list, and a model may vanish from it without invalidating what we learned. + t.string :model_id, null: false + t.string :capability, null: false + t.string :state, null: false + t.string :source, null: false + t.jsonb :detail, null: false, default: {} + t.datetime :checked_at, null: false + + t.timestamps null: false + end + + add_index :llm_capability_verdicts, + %i[llm_connection_id model_id capability], + unique: true, + name: "index_llm_capability_verdicts_on_connection_model_capability" + end +end diff --git a/db/migrate/20260812090000_create_llm_models.rb b/db/migrate/20260812090000_create_llm_models.rb new file mode 100644 index 000000000000..9e8d7298c944 --- /dev/null +++ b/db/migrate/20260812090000_create_llm_models.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmModels < ActiveRecord::Migration[8.1] + def change + create_table :llm_models do |t| + t.references :llm_connection, null: false, foreign_key: true + # Whatever this deployment calls the model. Provider-specific: Scaleway + # serves "qwen3.6-35b-a3b" for weights another catalogue lists as + # "Qwen/Qwen3.6-35B-A3B". + t.string :external_id, null: false + t.string :display_name + t.boolean :active, null: false, default: true + # Entered by an administrator rather than discovered. Survives a refresh + # that cannot see it, which is what makes a server offering + # /v1/chat/completions but no /v1/models usable. + t.boolean :manual, null: false, default: false + t.datetime :last_seen_at + t.jsonb :raw_metadata, null: false, default: {} + + t.timestamps null: false + end + + add_index :llm_models, %i[llm_connection_id external_id], unique: true + + # Superseded by the table above. + remove_column :llm_connections, :catalogue, :jsonb, null: false, default: {} + end +end diff --git a/spec/factories/llm_connection_factory.rb b/spec/factories/llm_connection_factory.rb index 74a3f2179440..7fd84f91a744 100644 --- a/spec/factories/llm_connection_factory.rb +++ b/spec/factories/llm_connection_factory.rb @@ -39,5 +39,17 @@ trait :enabled do enabled { true } end + + trait :with_models do + catalogue_fetched_at { Time.current } + last_connected_at { Time.current } + + after(:create) do |connection| + create(:llm_model, llm_connection: connection, external_id: "qwen3.6-27b", + raw_metadata: { "owned_by" => "vllm", "max_model_len" => 262_144 }) + create(:llm_model, llm_connection: connection, external_id: "bge-m3", + raw_metadata: { "owned_by" => "vllm", "max_model_len" => 8_192 }) + end + end end end diff --git a/spec/factories/llm_model_factory.rb b/spec/factories/llm_model_factory.rb new file mode 100644 index 000000000000..59527ef65278 --- /dev/null +++ b/spec/factories/llm_model_factory.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +FactoryBot.define do + factory :llm_model do + llm_connection + sequence(:external_id) { |n| "model-#{n}" } + active { true } + manual { false } + last_seen_at { Time.current } + + trait :manual do + manual { true } + last_seen_at { nil } + end + + trait :withdrawn do + active { false } + end + + # Still offered by the server; hidden by an administrator. + trait :deactivated do + deactivated_at { Time.current } + end + end +end diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb index 092e34823e7d..d09528b88fd5 100644 --- a/spec/requests/admin/llm_connections_spec.rb +++ b/spec/requests/admin/llm_connections_spec.rb @@ -81,11 +81,23 @@ connection = LlmConnection.first expect(connection.base_url).to eq(base_url) expect(connection.api_key).to eq("sk-test") + expect(connection.available_model_ids).to contain_exactly("qwen3.6-27b", "bge-m3") end end # The case that matters for OpenProject's own gateway: chat completions are # routed, the model list is not. + context "with a server that exposes no model list" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: 404) } + + it "still saves the connection and says models must be added by hand" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-test" } } + + expect(response).to have_http_status(:see_other) + expect(LlmConnection.first.base_url).to eq(base_url) + expect(flash[:warning]).to be_present + end + end context "with an unreachable server" do let!(:models_request) { mock_llm_models_response(base_url, timeout: true) } @@ -141,9 +153,53 @@ end end + describe "POST /admin/llm_connection/refresh_models" do + before { login_as admin } + + it "refetches the catalogue" do + create(:llm_connection, base_url:) + request = mock_llm_models_response(base_url) + + post refresh_models_llm_connection_path + + expect(request).to have_been_made.once + expect(LlmConnection.first.available_model_ids).to include("bge-m3") + end + end + + describe "GET /admin/llm_connection/delete_api_key_dialog" do + let!(:connection) { create(:llm_connection, base_url: "https://example.com/v1", api_key: "sk-test") } + + before { login_as admin } + + it "offers the confirmation" do + # Requested by the async-dialog Stimulus controller, which asks for a + # turbo stream rather than HTML. + get delete_api_key_dialog_llm_connection_path, + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Remove the stored API key?") + end + + # The catalogue sync fingerprints base_url and api_key together, so changing + # the key discards every verdict -- including hand-made ones, which nothing + # else throws away. + it "warns when hand-made capability assertions would be lost" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "supported", source: "admin", + checked_at: Time.current) + + get delete_api_key_dialog_llm_connection_path, + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response.body).to include("assertions you made yourself") + end + end + describe "disconnecting" do let!(:connection) do - create(:llm_connection, :enabled, + create(:llm_connection, :with_models, :enabled, base_url: "https://example.com/v1", api_key: "sk-test") end @@ -165,6 +221,7 @@ expect(connection.api_key).to be_blank expect(connection).not_to be_enabled expect(connection.base_url).to eq("https://example.com/v1") + expect(connection.models.count).to eq(2) end it "is refused to a non-admin" do diff --git a/spec/services/llm_connections/sync_models_service_spec.rb b/spec/services/llm_connections/sync_models_service_spec.rb new file mode 100644 index 000000000000..63eaf0313fa6 --- /dev/null +++ b/spec/services/llm_connections/sync_models_service_spec.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe LlmConnections::SyncModelsService, :llm_server_helpers, :webmock do + subject(:service) { described_class.new(connection) } + + let(:base_url) { "https://example.com/v1" } + let(:connection) { create(:llm_connection, :with_models, base_url:, api_key: "sk-test") } + + before { mock_llm_models_response(base_url) } + + describe "switching to a different deployment" do + before do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "supported", source: "admin", checked_at: Time.current) + connection.update_columns(base_url: "https://elsewhere.example/v1", + connection_fingerprint: "the-previous-deployment") + end + + it "invalidates the old models and verdicts even when the new server offers no list" do + mock_llm_models_response("https://elsewhere.example/v1", response_code: 404) + + result = service.call + + expect(result).to be_failure + expect(connection.capability_verdicts).to be_empty + expect(connection.models.active).to be_empty + end + end + + describe "refreshing the same deployment" do + before { service.call } + + it "keeps an administrator's context window override across refreshes" do + llm_model = connection.models.find_by(external_id: "qwen3.6-27b") + llm_model.update!(raw_metadata: llm_model.raw_metadata.merge("admin_context_window" => 4096)) + + described_class.new(connection).call + + expect(llm_model.reload.context_window).to eq(4096) + end + + it "drops every non-admin verdict when the catalogue comes back empty" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "supported", source: "probe", checked_at: Time.current) + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "supported", source: "admin", checked_at: Time.current) + mock_llm_models_response(base_url, body: { object: "list", data: [] }.to_json) + + # A fresh instance, as every caller builds one: the adapter memoises the + # fetched list within a run. + described_class.new(connection).call + + expect(connection.capability_verdicts.pluck(:source)).to eq(["admin"]) + end + end +end