From 6d1c73df8c2286396d3aaf6b4f932ef07123ebb2 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 6 Aug 2026 09:14:22 +0100 Subject: [PATCH 01/44] [#66020] Add LlmConnection model and llm_connection feature flag Introduces the persistence layer for the LLM server connection: base URL, API key, enabled flag, the designated default chat/embedding models and a verbatim copy of the remote model catalogue. The API key is ciphered through Redmine::Ciphering, matching LdapAuthSource. Note this is a no-op unless database_cipher_key is configured, which no packaged install does today. Only one connection is supported for now, enforced by a validation rather than by the schema so that lifting the restriction later needs no migration. Adds the llm_connection setting (writable: false, format: :hash) as the ENV carrier for headless provisioning, and the llm_connection feature flag. https://community.openproject.org/work_packages/66020 --- app/models/llm_connection.rb | 101 ++++++++++++++++++ config/constants/settings/definition.rb | 7 ++ config/initializers/feature_decisions.rb | 4 + config/locales/en.yml | 16 +++ .../20260811090000_create_llm_connections.rb | 55 ++++++++++ 5 files changed, 183 insertions(+) create mode 100644 app/models/llm_connection.rb create mode 100644 db/migrate/20260811090000_create_llm_connections.rb diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb new file mode 100644 index 000000000000..8f6ce8d7247c --- /dev/null +++ b/app/models/llm_connection.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# The connection to an OpenAI-API-compatible LLM server. +# +# Only a single connection is supported today. That is enforced by a validation +# rather than by the schema, so lifting the restriction later is a one-line change: +# every association is already scoped by +llm_connection_id+ and the STI +type+ +# column is in place. +class LlmConnection < ApplicationRecord + include Redmine::Ciphering + + SINGLETON_NAME = "default" + + has_many :health_reports, as: :subject, dependent: :delete_all + + validates :base_url, presence: true + validate :only_one_connection, on: :create + + class << self + # The connection record, whether or not it has been persisted yet. + def instance + first || new(name: SINGLETON_NAME, type: name) + end + + # Cheap enough to call from a menu visibility lambda. + def enabled? + exists?(enabled: true) + end + + # Whether LLM-backed features may run right now. This is the predicate + # sibling features gate on; see #77783. + def available? + OpenProject::FeatureDecisions.llm_connection_active? && + enabled? && + instance.configured? + end + end + + def api_key + read_ciphered_attribute(:api_key) + end + + def api_key=(value) + write_ciphered_attribute(:api_key, value) + end + + # Deliberately does not consider +last_connected_at+: a connection provisioned + # from the environment is never probed, and must still count as configured. + def configured? + base_url.present? + end + + def configured_from_env? + Setting.llm_connection.present? + end + + # The model ids the remote server last reported, in the order it reported them. + def catalogue_model_ids + Array(catalogue["data"]).filter_map { |model| model["id"] } + end + + def server_flavour + options["server_flavour"].presence&.to_sym + end + + private + + def only_one_connection + return unless self.class.where.not(id:).exists? + + errors.add(:base, :singleton) + end +end diff --git a/config/constants/settings/definition.rb b/config/constants/settings/definition.rb index 7d7b29b6e896..b804f1a31428 100644 --- a/config/constants/settings/definition.rb +++ b/config/constants/settings/definition.rb @@ -722,6 +722,13 @@ class Definition format: :boolean, default: false }, + llm_connection: { + description: "Configure the connection to an OpenAI-API-compatible LLM server through environment variables", + writable: false, + default: {}, + format: :hash, + string_values: true + }, log_level: { description: "Set the OpenProject logger level", default: Rails.env.development? ? "debug" : "info", diff --git a/config/initializers/feature_decisions.rb b/config/initializers/feature_decisions.rb index 729eed49f7c9..6cc5e6c10313 100644 --- a/config/initializers/feature_decisions.rb +++ b/config/initializers/feature_decisions.rb @@ -56,6 +56,10 @@ OpenProject::FeatureDecisions.add :subtypes, description: "Enables work package subtypes." +OpenProject::FeatureDecisions.add :llm_connection, + description: "Enables the administration page connecting OpenProject to an " \ + "OpenAI-API-compatible LLM server, and the AI features built on it." + OpenProject::FeatureDecisions.add :work_package_multiple_versions, description: "Enables assigning multiple (target) versions to a work package. " \ "Experimental; the user-facing setting and admin switch follow later." diff --git a/config/locales/en.yml b/config/locales/en.yml index bf1f2749fc08..39adf01aad93 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -655,6 +655,22 @@ en: tls_certificate_string: format: "%{message}" invalid_certificate: "The provided SSL certificate is invalid: %{additional_message}" + llm_connection: + attributes: + api_key: + invalid_api_key: "was rejected by the LLM server." + unknown_error: "could not be validated with the LLM server. Please verify that the connection is functioning properly." + base: + singleton: "Only one LLM connection can be configured." + base_url: + cannot_be_connected_to: "could not be reached. Please ensure the LLM server is running and reachable from OpenProject." + not_openai_compatible: "did not return a valid model list. Please ensure the URL points at an OpenAI-API-compatible endpoint, including the API version segment (for example https://example.com/v1)." + request_timed_out: "did not respond in time. Please ensure the LLM server is reachable and not overloaded." + ssrf_filtered: "resolves to a blocked address. If the LLM server runs on an internal network, allow its IP via the %{env_name} environment variable." + default_chat_model_id: + not_available: "is not offered by the configured LLM server." + enabled: + requires_connection: "cannot be turned on before a connection has been configured." meeting: error_conflict: "Unable to save because the meeting was updated by someone else in the meantime. Please reload the page." member: diff --git a/db/migrate/20260811090000_create_llm_connections.rb b/db/migrate/20260811090000_create_llm_connections.rb new file mode 100644 index 000000000000..4028aef13393 --- /dev/null +++ b/db/migrate/20260811090000_create_llm_connections.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmConnections < ActiveRecord::Migration[8.1] + def change + create_table :llm_connections do |t| + t.string :name, null: false, index: { unique: true } + t.string :type, null: false, index: true + t.boolean :enabled, null: false, default: false + t.string :base_url, null: false + # Ciphered through Redmine::Ciphering when database_cipher_key is configured. + # Nullable: an unauthenticated self-hosted server needs no key. + t.string :api_key + t.jsonb :options, null: false, default: {} + # Raw /v1/models payload plus any server-specific metadata, stored verbatim. + t.jsonb :catalogue, null: false, default: {} + t.datetime :catalogue_fetched_at + t.string :connection_fingerprint + # Model references are strings, never foreign keys: a selection must survive + # the model disappearing from the remote catalogue. + t.string :default_chat_model_id + t.string :default_embedding_model_id + t.datetime :last_connected_at + + t.timestamps null: false + end + end +end From cd3a9108bd68f4866f4c47765857fa67fdacda14 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 6 Aug 2026 10:47:05 +0100 Subject: [PATCH 02/44] [#66020] Add Llm::Client for OpenAI-compatible servers A thin client exposing #models against a base URL that already carries the API version segment, as every provider documents and every OpenAI client library expects. The catalogue is returned verbatim rather than normalised: vLLM adds max_model_len and root to each model card, and that is the only trustworthy source for a deployment's real context window. Two details worth knowing: * transport failures are HTTPX::ErrorResponse instances, while a response carrying a 4xx also populates #error (it delegates to #raise_for_status). The response class, not #error, distinguishes the two -- checking #error first swallows every status and makes the error taxonomy unreachable. * the body is parsed directly rather than through HTTPX's #json, which insists on a JSON content type that self-hosted servers behind a proxy do not reliably set. Timeouts are overridden explicitly; the global httpx defaults (connect 3s, read 3s, request 10s) are tuned for storage calls and cannot serve inference. https://community.openproject.org/work_packages/66020 --- app/services/llm/client.rb | 161 +++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 app/services/llm/client.rb diff --git a/app/services/llm/client.rb b/app/services/llm/client.rb new file mode 100644 index 000000000000..275e13a74db6 --- /dev/null +++ b/app/services/llm/client.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # A thin client for an OpenAI-API-compatible server. + # + # The configured base URL is expected to already contain the API version segment + # (for example +https://example.com/v1+), matching what every provider documents + # and what every OpenAI client library expects. This client only appends the + # endpoint path. + # + # Errors are raised as a small taxonomy so that callers can map them onto + # per-attribute contract errors rather than leaking transport detail into the UI. + # Response bodies are never included in error messages: an OpenAI-compatible + # gateway routinely echoes the submitted Authorization header, upstream provider + # URLs and internal hostnames in its error payloads. + class Client + class Error < StandardError; end + + # The server could not be reached at all. + class ConnectionError < Error; end + # The host resolved to an address blocked by the SSRF policy. + class SsrfError < ConnectionError; end + # The server took too long to answer. + class TimeoutError < ConnectionError; end + # The server answered, but rejected our credentials. + class AuthenticationError < Error; end + + # The server answered with an unexpected status. + class ApiError < Error + attr_reader :status + + def initialize(message, status: nil) + super(message) + @status = status + end + end + + # The server answered successfully with something that is not an OpenAI model list. + class ParseError < Error; end + + # The global httpx defaults (connect 3s / read 3s / request 10s, all + # writable: false) are tuned for storage and webhook calls and are far too + # tight for an inference endpoint. Every call site must override them. + PROBE_TIMEOUT = { + timeout: { connect_timeout: 5, read_timeout: 15, request_timeout: 20 } + }.freeze + + INFERENCE_TIMEOUT = { + timeout: { connect_timeout: 5, read_timeout: 120, request_timeout: 180 } + }.freeze + + def initialize(base_url:, api_key: nil, timeout: PROBE_TIMEOUT) + @base_url = base_url.to_s.chomp("/") + @api_key = api_key + @timeout = timeout + end + + # The model catalogue as the server reports it, verbatim. + # + # Kept verbatim on purpose: vLLM adds +max_model_len+ and +root+ to each card, + # which is the only trustworthy source for a deployment's real context window. + # + # @return [Hash] the parsed +GET /models+ body + def models + body = get("/models") + + raise ParseError, "Response does not contain a model list" unless body.is_a?(Hash) && body["data"].is_a?(Array) + + body + end + + private + + attr_reader :base_url, :api_key, :timeout + + def get(path) + response = session.get(uri_for(path)) + # A connection-level failure yields an HTTPX::ErrorResponse. A real response + # carrying a 4xx/5xx is an ordinary HTTPX::Response — note that its #error + # is also populated (it delegates to #raise_for_status), so the response + # class, not #error, is what distinguishes the two. + handle_transport_error(response) if response.is_a?(HTTPX::ErrorResponse) + handle_status(response) + parse(response) + rescue OpenProject::HttpxSsrfFilter::ServerSideRequestForgeryError + # Raised from HttpxSsrfFilter#addresses=; the throw/catch path surfaces as + # response.error instead and is handled in #handle_transport_error. + raise SsrfError, "Host resolves to a blocked address" + end + + def session + request = OpenProject.httpx.with(timeout) + api_key.present? ? request.plugin(:auth).bearer_auth(api_key) : request + end + + def uri_for(path) + URI.parse("#{base_url}#{path}") + rescue URI::InvalidURIError + raise ConnectionError, "Invalid URL" + end + + def handle_transport_error(response) + error = response.error + + case error + when OpenProject::HttpxSsrfFilter::ServerSideRequestForgeryError + raise SsrfError, "Host resolves to a blocked address" + when HTTPX::TimeoutError + raise TimeoutError, "Request timed out" + else + raise ConnectionError, error.class.name + end + end + + def handle_status(response) + status = response.status + return if status.in?(200..299) + + raise AuthenticationError, "Server rejected the API key (#{status})" if status.in?([401, 403]) + + raise ApiError.new("Server responded with #{status}", status:) + end + + # Parses the body directly rather than through HTTPX's +#json+, which insists on a + # JSON content type. Self-hosted servers behind a proxy do not reliably set one, + # and a content-type mismatch is not a reason to call a working server broken. + def parse(response) + JSON.parse(response.body.to_s) + rescue JSON::ParserError + raise ParseError, "Response is not valid JSON" + end + end +end From e27705b5919c5cf17b43b9f95c56089b6b80b8e5 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 6 Aug 2026 11:32:48 +0100 Subject: [PATCH 03/44] [#66020] Verify the LLM server inside the contract Saving the connection now proves the server is reachable and the credentials are accepted before anything is written, so a failed connect persists nothing. This follows the storages precedent, where the Nextcloud credentials validator adds a contract error on 401 and stops the write. Two constraints shape where the probe lives: * It is declared on UpdateContract, never on BaseContract. Provisioning from the environment reuses the base contract, and a `validates` line cannot be un-declared by a subclass -- an outbound request during seeding would fail the boot of a container whose LLM server has not started yet. EnvironmentUpdateContract therefore inherits from BaseContract. * The probe is guarded on the credential attributes. Without that guard every unrelated save, and every form render that builds a model through SetAttributesService, would fire an outbound request. The base URL is normalised only by stripping whitespace and a trailing slash. The /v1 segment is deliberately neither added nor removed: silently rewriting an administrator's URL makes the eventual failure harder to diagnose. https://community.openproject.org/work_packages/66020 --- .../llm_connections/base_contract.rb | 80 +++++++++++++ .../environment_update_contract.rb | 41 +++++++ .../llm_connections/update_contract.rb | 45 ++++++++ app/models/llm_connection.rb | 6 +- .../llm_connections/set_attributes_service.rb | 60 ++++++++++ .../llm_connections/sync_models_service.rb | 88 +++++++++++++++ .../llm_connections/update_service.rb | 50 +++++++++ app/validators/llm_server_validator.rb | 105 ++++++++++++++++++ config/locales/en.yml | 11 ++ 9 files changed, 485 insertions(+), 1 deletion(-) create mode 100644 app/contracts/llm_connections/base_contract.rb create mode 100644 app/contracts/llm_connections/environment_update_contract.rb create mode 100644 app/contracts/llm_connections/update_contract.rb create mode 100644 app/services/llm_connections/set_attributes_service.rb create mode 100644 app/services/llm_connections/sync_models_service.rb create mode 100644 app/services/llm_connections/update_service.rb create mode 100644 app/validators/llm_server_validator.rb diff --git a/app/contracts/llm_connections/base_contract.rb b/app/contracts/llm_connections/base_contract.rb new file mode 100644 index 000000000000..648aae65d695 --- /dev/null +++ b/app/contracts/llm_connections/base_contract.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Validations that hold for every write, including provisioning from the + # environment. Deliberately makes no network request -- see UpdateContract. + class BaseContract < ModelContract + attribute :enabled + attribute :base_url + attribute :api_key + attribute :default_chat_model_id + attribute :default_embedding_model_id + + validates :base_url, presence: true + # http is deliberately allowed: an on-premise LLM server on an internal + # network commonly terminates TLS elsewhere or not at all. + validates :base_url, url: { allowed_protocols: %w[http https] }, unless: -> { base_url.blank? } + + validate :enabled_requires_connection + validate :default_models_offered_by_server + validate :not_configured_from_env + + def not_configured_from_env + return unless model.configured_from_env? + + errors.add :base, :configured_via_env + end + + private + + def enabled_requires_connection + return unless model.enabled? + return if model.base_url.present? + + errors.add :enabled, :requires_connection + end + + # A designated default must be a model the server actually reported. Validated + # only when it changes, so a catalogue that shrinks underneath a stored + # selection does not block every unrelated save; the dangling state is + # surfaced in the UI instead. + def default_models_offered_by_server + %i[default_chat_model_id default_embedding_model_id].each do |attribute| + value = model.public_send(attribute) + next if value.blank? + next unless model.changed_attributes.include?(attribute.to_s) + next if model.catalogue_model_ids.include?(value) + + errors.add attribute, :not_available + end + end + end +end diff --git a/app/contracts/llm_connections/environment_update_contract.rb b/app/contracts/llm_connections/environment_update_contract.rb new file mode 100644 index 000000000000..a393c655c5b8 --- /dev/null +++ b/app/contracts/llm_connections/environment_update_contract.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Used when the connection is provisioned from the environment. + # + # Inherits from BaseContract, not UpdateContract: seeding must never reach out + # to the LLM server, because the container it runs in may well start before the + # server does. It also lifts the "configured from environment is read-only" + # guard, since this is the code path that legitimately writes those values. + class EnvironmentUpdateContract < BaseContract + def not_configured_from_env = nil + end +end diff --git a/app/contracts/llm_connections/update_contract.rb b/app/contracts/llm_connections/update_contract.rb new file mode 100644 index 000000000000..9c4ec676e034 --- /dev/null +++ b/app/contracts/llm_connections/update_contract.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # The contract used when an administrator saves the form. + # + # This is the only contract that talks to the remote server, so that a failed + # probe persists nothing. It must NOT move to BaseContract: provisioning from + # the environment reuses that base, and a validates line cannot be un-declared + # by a subclass -- an outbound request during seeding would fail the boot of a + # container whose LLM server has not started yet. + # + # The probe itself is guarded on changed attributes, so toggling +enabled+ or + # picking a default model contacts nothing. + class UpdateContract < BaseContract + validates :base_url, llm_server: true, unless: -> { errors.include?(:base_url) } + end +end diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb index 8f6ce8d7247c..879f23fc109d 100644 --- a/app/models/llm_connection.rb +++ b/app/models/llm_connection.rb @@ -46,8 +46,12 @@ class LlmConnection < ApplicationRecord class << self # The connection record, whether or not it has been persisted yet. + # + # Identifying attributes are left unset here and filled in by + # LlmConnections::SetAttributesService as system changes, so that they do not + # register as user-made changes to non-writable attributes. def instance - first || new(name: SINGLETON_NAME, type: name) + first || new end # Cheap enough to call from a menu visibility lambda. diff --git a/app/services/llm_connections/set_attributes_service.rb b/app/services/llm_connections/set_attributes_service.rb new file mode 100644 index 000000000000..4a24ffe2bb37 --- /dev/null +++ b/app/services/llm_connections/set_attributes_service.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class SetAttributesService < BaseServices::SetAttributes + private + + def set_attributes(params) + super + + model.base_url = normalized_base_url if model.base_url.present? + set_singleton_defaults + end + + # +name+ and +type+ identify the record but are never user-editable, so they + # are set as system changes: the contract's readonly check only looks at + # attributes the user changed. + def set_singleton_defaults + model.change_by_system do + model.name ||= LlmConnection::SINGLETON_NAME + model.type ||= LlmConnection.name + end + end + + # Only trailing whitespace and slashes are removed. The /v1 segment is + # deliberately not added or stripped: silently rewriting an administrator's + # URL makes the eventual failure harder to diagnose, so a wrong shape is + # reported by the probe instead. + def normalized_base_url + model.base_url.strip.chomp("/") + end + end +end diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb new file mode 100644 index 000000000000..d9770a273399 --- /dev/null +++ b/app/services/llm_connections/sync_models_service.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Refreshes the cached model catalogue from the remote server. + # + # Kept separate from the contract probe so that the same code path serves the + # "Refresh models" button, the update service and the environment seeder. + class SyncModelsService + def initialize(connection) + @connection = connection + end + + def call + connection.update!(catalogue_attributes(client.models)) + + ServiceResult.success(result: connection) + rescue Llm::Client::Error => e + Rails.logger.info { "LLM model sync for #{connection.base_url} failed: #{e.class} #{e.message}" } + ServiceResult.failure(errors: e.message) + end + + private + + attr_reader :connection + + def catalogue_attributes(catalogue) + now = Time.current + + { + catalogue:, + catalogue_fetched_at: now, + last_connected_at: now, + connection_fingerprint: fingerprint, + options: connection.options.merge("server_flavour" => detect_server_flavour(catalogue)) + } + end + + def client + Llm::Client.new(base_url: connection.base_url, api_key: connection.api_key) + end + + def fingerprint + Digest::SHA256.hexdigest("#{connection.base_url}\0#{connection.api_key}") + end + + # Which server we are talking to decides which non-standard metadata endpoint + # is worth asking later. +owned_by+ is the documented hint; the structural + # fallback catches servers whose operator overrode it. + def detect_server_flavour(catalogue) + cards = Array(catalogue["data"]) + owner = cards.first&.dig("owned_by").to_s.downcase + + case owner + when "vllm", "sglang", "llamacpp", "openai" then owner + else + cards.any? { |card| card.key?("max_model_len") || card.key?("root") } ? "vllm" : "unknown" + end + end + end +end diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb new file mode 100644 index 000000000000..582e1c3fa7a5 --- /dev/null +++ b/app/services/llm_connections/update_service.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class UpdateService < BaseServices::Update + private + + # The contract has already proven the server reachable when the credentials + # changed, so refreshing the catalogue here cannot be the thing that fails + # the save. A sync failure is therefore logged, not surfaced. + def after_perform(service_call) + super.tap do + next unless service_call.success? + + SyncModelsService.new(service_call.result).call if credentials_changed?(service_call.result) + end + end + + def credentials_changed?(connection) + connection.saved_changes.keys.intersect?(LlmServerValidator::CREDENTIAL_ATTRIBUTES) + end + end +end diff --git a/app/validators/llm_server_validator.rb b/app/validators/llm_server_validator.rb new file mode 100644 index 000000000000..b827ea38466d --- /dev/null +++ b/app/validators/llm_server_validator.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Verifies that the configured URL really is an OpenAI-API-compatible server that +# accepts the configured credentials, by fetching its model list. +# +# Runs inside the contract so that a server which cannot be reached never gets +# persisted. It is guarded on the credential attributes: without that guard every +# unrelated save -- and every form render that builds a model through +# SetAttributesService -- would fire an outbound HTTP request. +class LlmServerValidator < ActiveModel::EachValidator + CREDENTIAL_ATTRIBUTES = %w[base_url api_key].freeze + + def validate_each(contract, attribute, value) + return if value.blank? + return unless credentials_changed?(contract) + return unless host_allowed?(contract, attribute, value) + + probe(contract, attribute, value) + end + + private + + def credentials_changed?(contract) + contract.model.changed_attributes.keys.intersect?(CREDENTIAL_ATTRIBUTES) + end + + # A pre-flight check purely so the administrator gets an actionable message + # naming the environment variable, rather than a bare connection failure from + # the transport-level SSRF filter. + def host_allowed?(contract, attribute, value) + host = URI.parse(value).host + return false if host.blank? + return true if OpenProject::SsrfProtection.safe_ip?(host) + + contract.errors.add(attribute, :ssrf_filtered, env_name: ssrf_allowlist_env_name) + false + rescue URI::InvalidURIError + false + end + + def ssrf_allowlist_env_name + Settings::Definition[:ssrf_protection_ip_allowlist].env_name + end + + def probe(contract, attribute, value) + client(contract, value).models + rescue Llm::Client::Error => e + log(value, e) + add_error(contract, attribute, e) + end + + # Order matters: SsrfError and TimeoutError are both ConnectionError subclasses. + def add_error(contract, attribute, error) + case error + when Llm::Client::SsrfError + contract.errors.add(attribute, :ssrf_filtered, env_name: ssrf_allowlist_env_name) + when Llm::Client::AuthenticationError + contract.errors.add(:api_key, :invalid_api_key) + when Llm::Client::TimeoutError + contract.errors.add(attribute, :request_timed_out) + when Llm::Client::ConnectionError + contract.errors.add(attribute, :cannot_be_connected_to) + else + contract.errors.add(attribute, :not_openai_compatible) + end + end + + def client(contract, base_url) + Llm::Client.new(base_url:, api_key: contract.model.api_key) + end + + # The upstream message is logged but never surfaced: an OpenAI-compatible + # gateway routinely echoes credentials and internal hostnames in error bodies. + def log(base_url, error) + Rails.logger.info { "LLM connection probe to #{base_url} failed: #{error.class} #{error.message}" } + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 39adf01aad93..7ca9e2d1f028 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -200,6 +200,15 @@ en: onthefly: "Automatic user creation" port: "Port" tls_certificate_string: "LDAP server SSL certificate" + llm_connection: + api_key: "API key" + base_url: "Host URL" + # ActiveRecord::Base.human_attribute_name strips the _id suffix, so these + # keys deliberately do not carry it (see lib/open_project/patches/active_record_i18n.rb). + default_chat_model: "Default chat model" + default_embedding_model: "Default embedding model" + enabled: "Enable LLMs for this instance" + last_connected_at: "Last connected" mcp_configuration: description: Description enabled: Enabled @@ -669,6 +678,8 @@ en: ssrf_filtered: "resolves to a blocked address. If the LLM server runs on an internal network, allow its IP via the %{env_name} environment variable." default_chat_model_id: not_available: "is not offered by the configured LLM server." + default_embedding_model_id: + not_available: "is not offered by the configured LLM server." enabled: requires_connection: "cannot be turned on before a connection has been configured." meeting: From 6b06bee7b7f7423ac5505f213f1fc9f0ee230197 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 6 Aug 2026 14:05:11 +0100 Subject: [PATCH 04/44] [#66020] Provision the LLM connection from the environment On-premise and containerised deployments configure the connection through OPENPROJECT_LLM__CONNECTION_* variables, so an instance comes up connected without anyone opening the administration UI. Seeding never contacts the LLM server. The catalogue refresh is enqueued as Llm::SyncModelsJob instead, which is what lets a container whose LLM sidecar has not started yet finish seeding: with a dead server the seed completes in about a tenth of a second rather than blocking on a probe. Unknown keys raise the same actionable error the LDAP seeder gives, including the single-vs-double underscore explanation -- writing BASE_URL instead of BASE__URL parses as a nested base.url hash and would otherwise be silently ignored. Once provisioned this way the connection is read-only in the UI, guarded server-side by the existing configured_via_env contract error. https://community.openproject.org/work_packages/66020 --- app/seeders/env_data/llm_connection_seeder.rb | 87 +++++++++++++++++++ app/seeders/env_data_seeder.rb | 1 + .../llm_connections/env_sync_service.rb | 65 ++++++++++++++ .../llm_connections/update_service.rb | 11 ++- app/workers/llm/sync_models_job.rb | 44 ++++++++++ 5 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 app/seeders/env_data/llm_connection_seeder.rb create mode 100644 app/services/llm_connections/env_sync_service.rb create mode 100644 app/workers/llm/sync_models_job.rb diff --git a/app/seeders/env_data/llm_connection_seeder.rb b/app/seeders/env_data/llm_connection_seeder.rb new file mode 100644 index 000000000000..c62948cf72b7 --- /dev/null +++ b/app/seeders/env_data/llm_connection_seeder.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module EnvData + # Provisions the LLM connection from OPENPROJECT_LLM__CONNECTION_* variables so + # a container comes up connected without anyone opening the administration UI. + # + # Never contacts the LLM server: the catalogue refresh is enqueued, so seeding + # succeeds even when the server starts after OpenProject does. + class LlmConnectionSeeder < Seeder + KNOWN_KEYS = %w[base_url api_key default_chat_model default_embedding_model enabled].freeze + + def seed_data! + print_status " ↳ Creating LLM connection" do + validate_options!(config) + + result = LlmConnections::EnvSyncService.new(config).call + raise result.errors.full_messages.join(", ") if result.failure? + + Llm::SyncModelsJob.perform_later + end + end + + def applicable? + config.present? + end + + def not_applicable_message + "No LLM connection configured through environment variables." + end + + private + + def config + Setting.llm_connection + end + + def validate_options!(options) + check_unknown_keys!(options, KNOWN_KEYS) + return if options["base_url"].present? + + raise "LLM connection: #{env_form('base_url')} is required." + end + + def check_unknown_keys!(options, known_keys) + unknown = options.keys - known_keys + return if unknown.empty? + + raise <<~MSG.strip + LLM connection: unknown configuration key(s): #{unknown.map { |k| env_form(k) }.join(', ')}. + Accepted keys: #{known_keys.map { |k| env_form(k) }.join(', ')}. + Note: in environment variable names, single underscores split path segments and double underscores encode a literal underscore (e.g. BASE__URL, not BASE_URL). + MSG + end + + def env_form(key) + key.gsub("_", "__").upcase + end + end +end diff --git a/app/seeders/env_data_seeder.rb b/app/seeders/env_data_seeder.rb index 48a5959cced8..91300692b014 100644 --- a/app/seeders/env_data_seeder.rb +++ b/app/seeders/env_data_seeder.rb @@ -31,6 +31,7 @@ def data_seeder_classes [ EnvData::CustomDesignSeeder, EnvData::LdapSeeder, + EnvData::LlmConnectionSeeder, EnvData::ScimClientSeeder, EnvData::TokenSeeder ] diff --git a/app/services/llm_connections/env_sync_service.rb b/app/services/llm_connections/env_sync_service.rb new file mode 100644 index 000000000000..f86ace88ea0f --- /dev/null +++ b/app/services/llm_connections/env_sync_service.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Applies an environment-provided configuration to the connection record. + # + # Uses EnvironmentUpdateContract, which lifts the "configured from environment + # is read-only" guard and, crucially, does not probe the LLM server: the + # container running the seed may well start before the server does. + class EnvSyncService + def initialize(env_config) + @config = env_config.deep_symbolize_keys + end + + def call + UpdateService + .new(user: User.system, + model: LlmConnection.instance, + contract_class: EnvironmentUpdateContract, + sync_models: false) + .call(**attributes) + end + + private + + attr_reader :config + + def attributes + { + base_url: config.fetch(:base_url), + api_key: config[:api_key], + default_chat_model_id: config[:default_chat_model], + default_embedding_model_id: config[:default_embedding_model], + enabled: ActiveRecord::Type::Boolean.new.deserialize(config.fetch(:enabled, true)) + }.compact + end + end +end diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb index 582e1c3fa7a5..d6b3111dc6d2 100644 --- a/app/services/llm_connections/update_service.rb +++ b/app/services/llm_connections/update_service.rb @@ -30,6 +30,15 @@ module LlmConnections class UpdateService < BaseServices::Update + # @param sync_models [Boolean] whether to refresh the model catalogue inline + # after a successful save. Provisioning from the environment passes false: + # seeding must not block on an LLM server that has not started yet, and + # enqueues Llm::SyncModelsJob instead. + def initialize(*, sync_models: true, **) + super(*, **) + @sync_models = sync_models + end + private # The contract has already proven the server reachable when the credentials @@ -37,7 +46,7 @@ class UpdateService < BaseServices::Update # the save. A sync failure is therefore logged, not surfaced. def after_perform(service_call) super.tap do - next unless service_call.success? + next unless @sync_models && service_call.success? SyncModelsService.new(service_call.result).call if credentials_changed?(service_call.result) end diff --git a/app/workers/llm/sync_models_job.rb b/app/workers/llm/sync_models_job.rb new file mode 100644 index 000000000000..eb72d9a6075b --- /dev/null +++ b/app/workers/llm/sync_models_job.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Refreshes the cached model catalogue out of band. + # + # Used by the environment seeder, which must not block on -- or fail because of + # -- an LLM server that has not finished starting. + class SyncModelsJob < ApplicationJob + def perform + connection = LlmConnection.first + return if connection.nil? || !connection.configured? + + LlmConnections::SyncModelsService.new(connection).call + end + end +end From d78bcfba16b05968599193a497b249cb87c81a3f Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 6 Aug 2026 16:41:37 +0100 Subject: [PATCH 05/44] [#66020] Add the LLM settings administration page Adds Administration -> AI -> LLM settings, a sibling of the MCP page, behind the llm_connection feature flag. Saving is connecting: the contract proves the server reachable, so there is no separate "test" button that could report green without persisting anything. The form shows a spinner while that round trip happens. The API key is write-only. The stored value is never sent to the browser, a blank submission keeps the current key, and a separate action removes it -- without which a key could never be cleared once set. The Stimulus controller also wipes the field on turbo:before-cache, so the back button cannot restore a typed secret, and marks the submit button aria-disabled rather than disabled, which would move focus to and drop keyboard users to the top of the page. Departing from the mockup, the host and key fields are always visible instead of being revealed by the Enable checkbox. Hiding them behind the checkbox deadlocks: the fields only appear once enabled is saved, but enabling without a configured connection is rejected. Enable is a pure kill switch here. The model list renders from the cached catalogue, so opening the page never issues an HTTP request; refreshing is explicit. Context windows come from vLLM's max_model_len, which reflects the operator's actual --max-model-len. https://community.openproject.org/work_packages/66020 --- .../llm_connections/form_component.html.erb | 34 ++++++ .../llm_connections/form_component.rb | 60 +++++++++ .../llm_connections/models_row_component.rb | 51 ++++++++ .../llm_connections/models_table_component.rb | 63 ++++++++++ .../llm_connections/base_contract.rb | 5 +- .../admin/llm_connections_controller.rb | 99 +++++++++++++++ app/forms/llm_connections/connection_form.rb | 115 ++++++++++++++++++ app/views/admin/llm_connections/show.html.erb | 67 ++++++++++ config/initializers/menus.rb | 6 + config/locales/en.yml | 28 +++++ config/routes.rb | 5 + .../admin/llm-connection-form.controller.ts | 71 +++++++++++ 12 files changed, 602 insertions(+), 2 deletions(-) create mode 100644 app/components/llm_connections/form_component.html.erb create mode 100644 app/components/llm_connections/form_component.rb create mode 100644 app/components/llm_connections/models_row_component.rb create mode 100644 app/components/llm_connections/models_table_component.rb create mode 100644 app/controllers/admin/llm_connections_controller.rb create mode 100644 app/forms/llm_connections/connection_form.rb create mode 100644 app/views/admin/llm_connections/show.html.erb create mode 100644 frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts diff --git a/app/components/llm_connections/form_component.html.erb b/app/components/llm_connections/form_component.html.erb new file mode 100644 index 000000000000..401606ee473a --- /dev/null +++ b/app/components/llm_connections/form_component.html.erb @@ -0,0 +1,34 @@ +<%= + component_wrapper(tag: "turbo-frame", **wrapper_options) do + if connection.configured_from_env? + concat( + render(Primer::Alpha::Banner.new(mb: 3, icon: :info)) do + t("admin.banners.environment_configured_readonly") + end + ) + end + + concat( + settings_primer_form_with(**form_options) do |f| + render(LlmConnections::ConnectionForm.new(f)) + end + ) + + concat( + render( + Primer::Beta::Text.new( + tag: :div, + mt: 3, + hidden: true, + display: :flex, + align_items: :center, + color: :muted, + data: { "admin--llm-connection-form-target": "progressBanner" } + ) + ) do + concat(render(Primer::Beta::Spinner.new(size: :small, mr: 2))) + concat(t("admin.llm_connections.form.label_connecting")) + end + ) + end +%> diff --git a/app/components/llm_connections/form_component.rb b/app/components/llm_connections/form_component.rb new file mode 100644 index 000000000000..18313947a162 --- /dev/null +++ b/app/components/llm_connections/form_component.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class FormComponent < ApplicationComponent + include ApplicationHelper + include OpPrimer::ComponentHelpers + include OpTurbo::Streamable + + def self.wrapper_key = :llm_connection_form + + alias_method :connection, :model + + private + + def wrapper_options + { + data: { + controller: "admin--llm-connection-form", + test_selector: "llm-connection--form" + } + } + end + + def form_options + { + model: connection, + url: llm_connection_path, + method: :patch + } + end + end +end diff --git a/app/components/llm_connections/models_row_component.rb b/app/components/llm_connections/models_row_component.rb new file mode 100644 index 000000000000..2d98470c6e41 --- /dev/null +++ b/app/components/llm_connections/models_row_component.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # A row of the model catalogue. +model+ here is a raw model card as the server + # reported it, not an ActiveRecord object. + class ModelsRowComponent < OpPrimer::BorderBoxRowComponent + alias_method :card, :model + + def identifier + render(Primer::Beta::Text.new(font_weight: :bold)) { card["id"] } + end + + # vLLM and SGLang report the operator's real --max-model-len here, which is + # more trustworthy than any published figure for the model. Servers that do + # not report it simply show nothing rather than a guess. + def context_window + window = card["max_model_len"] + return render(Primer::Beta::Text.new(color: :muted)) { "—" } if window.blank? + + number_with_delimiter(window) + end + end +end diff --git a/app/components/llm_connections/models_table_component.rb b/app/components/llm_connections/models_table_component.rb new file mode 100644 index 000000000000..2d03302d01f6 --- /dev/null +++ b/app/components/llm_connections/models_table_component.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Lists the models the remote server reported, read from the cached catalogue. + # + # Rendering never issues an HTTP request: the catalogue is refreshed explicitly + # through the "Refresh models" action. + class ModelsTableComponent < OpPrimer::BorderBoxTableComponent + columns :identifier, :context_window + + mobile_columns :identifier + + def initial_sort = %i[identifier asc] + + def has_footer? = false + + def mobile_title = I18n.t("admin.llm_connections.show.models_heading") + + # The row class is otherwise derived by convention as LlmConnections::RowComponent. + def row_class = ModelsRowComponent + + def headers + [ + [:identifier, { caption: I18n.t("admin.llm_connections.models.identifier") }], + [:context_window, { caption: I18n.t("admin.llm_connections.models.context_window") }] + ] + end + + def blank_title = I18n.t("admin.llm_connections.models.blank_title") + + def blank_description = I18n.t("admin.llm_connections.models.blank_description") + + def blank_icon = :sparkle + end +end diff --git a/app/contracts/llm_connections/base_contract.rb b/app/contracts/llm_connections/base_contract.rb index 648aae65d695..8e99b7e25e75 100644 --- a/app/contracts/llm_connections/base_contract.rb +++ b/app/contracts/llm_connections/base_contract.rb @@ -39,9 +39,10 @@ class BaseContract < ModelContract attribute :default_embedding_model_id validates :base_url, presence: true + # Resolves to the validate_url gem, which defaults to http and https. Plain # http is deliberately allowed: an on-premise LLM server on an internal - # network commonly terminates TLS elsewhere or not at all. - validates :base_url, url: { allowed_protocols: %w[http https] }, unless: -> { base_url.blank? } + # network commonly terminates TLS elsewhere, or not at all. + validates :base_url, url: { message: :invalid_url }, unless: -> { base_url.blank? } validate :enabled_requires_connection validate :default_models_offered_by_server diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb new file mode 100644 index 000000000000..52cdaac7f13c --- /dev/null +++ b/app/controllers/admin/llm_connections_controller.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + class LlmConnectionsController < ApplicationController + include OpTurbo::ComponentStream + + layout "admin" + menu_item :llm_connection + + before_action :require_admin + before_action :set_connection + + def show; end + + def update + result = ::LlmConnections::UpdateService + .new(user: current_user, model: @connection) + .call(**llm_connection_params) + + result.on_success { redirect_with_notice(t(".success")) } + result.on_failure { render_form_with_errors } + end + + def refresh_models + result = ::LlmConnections::SyncModelsService.new(@connection).call + + if result.success? + redirect_with_notice(t(".success")) + else + redirect_with_error(t(".failure")) + end + end + + def delete_api_key + @connection.update!(api_key: nil) + + redirect_with_notice(t(".success")) + end + + private + + def set_connection + @connection = LlmConnection.instance + end + + def render_form_with_errors + update_via_turbo_stream(component: ::LlmConnections::FormComponent.new(@connection)) + respond_with_turbo_streams { |format| format.html { render :show } } + end + + def redirect_with_notice(message) + flash[:notice] = message + redirect_to llm_connection_path, status: :see_other + end + + def redirect_with_error(message) + flash[:error] = message + redirect_to llm_connection_path, status: :see_other + end + + # A blank API key means "keep the stored one": the form never renders the + # saved value, so submitting it unchanged posts an empty string. + def llm_connection_params + permitted = params.expect( + llm_connection: %i[enabled base_url api_key default_chat_model_id default_embedding_model_id] + ) + permitted.delete(:api_key) if permitted[:api_key].blank? + permitted.to_h.symbolize_keys + end + end +end diff --git a/app/forms/llm_connections/connection_form.rb b/app/forms/llm_connections/connection_form.rb new file mode 100644 index 000000000000..e3d3fb64fa1c --- /dev/null +++ b/app/forms/llm_connections/connection_form.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class ConnectionForm < ApplicationForm + form do |f| + f.check_box( + name: :enabled, + label: LlmConnection.human_attribute_name(:enabled), + caption: I18n.t("admin.llm_connections.form.enabled_caption"), + disabled: read_only? + ) + + f.text_field( + name: :base_url, + label: LlmConnection.human_attribute_name(:base_url), + caption: I18n.t("admin.llm_connections.form.base_url_caption"), + placeholder: "https://example.com/v1", + required: true, + type: :url, + input_width: :large, + disabled: read_only? + ) + + f.text_field( + name: :api_key, + label: LlmConnection.human_attribute_name(:api_key), + caption: api_key_caption, + # The stored key is never sent to the browser. A blank submission means + # "keep the current key", handled in the controller. + value: nil, + placeholder: api_key_placeholder, + type: :password, + autocomplete: "off", + input_width: :large, + disabled: read_only?, + data: { "admin--llm-connection-form-target": "secretInput" } + ) + + if models_available? + f.select_list( + name: :default_chat_model_id, + label: LlmConnection.human_attribute_name(:default_chat_model_id), + caption: I18n.t("admin.llm_connections.form.default_chat_model_caption"), + include_blank: true, + input_width: :large, + disabled: read_only? + ) do |select| + model.catalogue_model_ids.each do |model_id| + select.option(value: model_id, label: model_id) + end + end + end + + unless read_only? + f.submit( + name: :submit, + label: submit_label, + scheme: :primary, + data: { "admin--llm-connection-form-target": "submitButton" } + ) + end + end + + private + + def read_only? + model.configured_from_env? + end + + def models_available? + model.catalogue_model_ids.any? + end + + def submit_label + model.persisted? ? I18n.t(:button_save) : I18n.t("admin.llm_connections.form.button_connect") + end + + def api_key_caption + key = model.persisted? && model.api_key.present? + I18n.t("admin.llm_connections.form.api_key_caption#{'_stored' if key}") + end + + def api_key_placeholder + "••••••••••••••••" if model.api_key.present? + end + end +end diff --git a/app/views/admin/llm_connections/show.html.erb b/app/views/admin/llm_connections/show.html.erb new file mode 100644 index 000000000000..939efeb2d3fc --- /dev/null +++ b/app/views/admin/llm_connections/show.html.erb @@ -0,0 +1,67 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection") %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t("menus.admin.llm_connection") } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + t("menus.admin.llm_connection")] + ) + end +%> + +<%= render(LlmConnections::FormComponent.new(@connection)) %> + +<% if @connection.catalogue_model_ids.any? %> + <%= + render(Primer::Beta::Subhead.new(mt: 4)) do |component| + component.with_heading(tag: :h3) { t(".models_heading") } + component.with_description { t(".models_description", fetched_at: format_time(@connection.catalogue_fetched_at)) } + component.with_actions do + render( + Primer::Beta::Button.new( + tag: :a, + href: refresh_models_llm_connection_path, + data: { turbo_method: :post, controller: "disable-when-clicked" } + ) + ) do |button| + button.with_leading_visual_icon(icon: :sync) + t(".refresh_models") + end + end + end + %> + + <%= render(LlmConnections::ModelsTableComponent.new(rows: @connection.catalogue["data"])) %> +<% end %> diff --git a/config/initializers/menus.rb b/config/initializers/menus.rb index 0d2a3d6dd43c..b30df08098ca 100644 --- a/config/initializers/menus.rb +++ b/config/initializers/menus.rb @@ -498,6 +498,12 @@ caption: I18n.t("menus.admin.ai"), icon: :sparkle + menu.push :llm_connection, + { controller: "/admin/llm_connections", action: :show }, + if: ->(_) { User.current.admin? && OpenProject::FeatureDecisions.llm_connection_active? }, + caption: I18n.t("menus.admin.llm_connection"), + parent: :ai + menu.push :mcp_configurations, { controller: "/admin/mcp_configurations", action: :index }, if: ->(_) { User.current.admin? }, diff --git a/config/locales/en.yml b/config/locales/en.yml index 7ca9e2d1f028..f83735a0708d 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1407,6 +1407,33 @@ en: caption_with_maximum: > User actions on a work package (changing description, status, values, or writing comments) are grouped if performed within this period. It also controls notification and [webhook](webhook_link) delays. The maximum is %{max} minutes. + llm_connections: + form: + api_key_caption: "The key OpenProject authenticates with. Leave blank if the server requires no authentication." + api_key_caption_stored: "A key is stored. Leave blank to keep it, or enter a new one to replace it." + base_url_caption: > + The full base URL of the server, including the API version segment, exactly as your provider documents it (for example https://example.com/v1). OpenProject appends only the endpoint path. + button_connect: "Connect" + default_chat_model_caption: "Used by AI features that do not select a model themselves." + enabled_caption: "When turned off, all LLM-backed AI features stop working. The connection settings are kept." + label_connecting: "Contacting the LLM server…" + delete_api_key: + success: "The API key has been removed." + models: + blank_description: "Save the connection to retrieve the models the server offers." + blank_title: "No models retrieved yet" + context_window: "Context window" + identifier: "Model" + refresh_models: + failure: "The model list could not be refreshed. Please check that the LLM server is still reachable." + success: "The model list has been refreshed." + show: + description: "Connect OpenProject to a server that speaks the OpenAI API, so AI features can use it." + models_description: "Reported by the server on %{fetched_at}." + models_heading: "Available models" + refresh_models: "Refresh models" + update: + success: "Successfully connected to the LLM server." mcp_configurations: index: description: "The model context protocol allows AI agents to provide its users with tools and resources exposed by this OpenProject instance. This feature is still in beta." @@ -4106,6 +4133,7 @@ en: aggregation: "Aggregation" ai: "Artificial Intelligence (AI)" api_and_webhooks: "API and webhooks" + llm_connection: "LLM settings" mail_notification: "Email notifications" mails_and_notifications: "Emails and notifications" mcp_configurations: "Model Context Protocol (MCP)" diff --git a/config/routes.rb b/config/routes.rb index 39580a9af273..19dce902894e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -691,6 +691,11 @@ end end + resource :llm_connection, only: %i[show update], controller: "admin/llm_connections" do + post :refresh_models + delete :api_key, action: :delete_api_key + end + resources :mcp_configurations, only: %i[index update], controller: "admin/mcp_configurations" do collection do post :multi_update diff --git a/frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts b/frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts new file mode 100644 index 000000000000..a6fc4e9173d7 --- /dev/null +++ b/frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts @@ -0,0 +1,71 @@ +/* + * -- copyright + * OpenProject is an open source project management software. + * Copyright (C) the OpenProject GmbH + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version 3. + * + * OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: + * Copyright (C) 2006-2013 Jean-Philippe Lang + * Copyright (C) 2010-2013 the ChiliProject Team + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * See COPYRIGHT and LICENSE files for more details. + * ++ + */ + +import { Controller } from '@hotwired/stimulus'; + +// Saving the connection contacts the LLM server, which can take seconds. This +// gives the administrator feedback while that happens, and keeps the typed API +// key out of Turbo's page cache. +export default class extends Controller { + static targets = ['submitButton', 'progressBanner', 'secretInput']; + + declare readonly submitButtonTargets:HTMLButtonElement[]; + declare readonly progressBannerTarget:HTMLElement; + declare readonly secretInputTargets:HTMLInputElement[]; + + connect():void { + this.element.addEventListener('submit', this.showProgress); + document.addEventListener('turbo:before-cache', this.clearSecrets); + } + + disconnect():void { + this.element.removeEventListener('submit', this.showProgress); + document.removeEventListener('turbo:before-cache', this.clearSecrets); + } + + // Turbo caches the rendered page for the back button. Without this the typed + // API key would be restored into the DOM on navigating back. + private clearSecrets = ():void => { + this.secretInputTargets.forEach((input:HTMLInputElement) => { + input.value = ''; + }); + }; + + private showProgress = ():void => { + this.progressBannerTarget.hidden = false; + + // aria-disabled rather than disabled: disabling a button while it holds + // focus moves focus to , silently dropping keyboard users to the top + // of the page. It also keeps the button in the submitted form data. + this.submitButtonTargets.forEach((button:HTMLButtonElement) => { + button.setAttribute('aria-disabled', 'true'); + }); + }; +} From 0b35abed5aeb65b3da165b402a6637005ed9f783 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 6 Aug 2026 19:58:03 +0100 Subject: [PATCH 06/44] [#66020] Add specs for the LLM connection Contract and request specs covering the connect flow, the error taxonomy and the API key lifecycle, plus a shared LlmServerHelpers module for stubbing an OpenAI-compatible server. Three of these pin behaviour that is easy to regress silently: * the probe fires only when the credentials change, asserted by resetting WebMock's executed requests and re-validating. Without the guard every unrelated save would reach out to the LLM server. * a failed connect leaves the database untouched. * a blank API key submission keeps the stored key rather than clearing it. Contract specs are tagged :check_errors_i18n so a missing locale key for any new symbolic error code fails the build. https://community.openproject.org/work_packages/66020 --- .../llm_connections/update_contract_spec.rb | 154 ++++++++++++++++++ spec/factories/llm_connection_factory.rb | 57 +++++++ spec/requests/admin/llm_connections_spec.rb | 140 ++++++++++++++++ spec/support/llm_server_helpers.rb | 66 ++++++++ 4 files changed, 417 insertions(+) create mode 100644 spec/contracts/llm_connections/update_contract_spec.rb create mode 100644 spec/factories/llm_connection_factory.rb create mode 100644 spec/requests/admin/llm_connections_spec.rb create mode 100644 spec/support/llm_server_helpers.rb diff --git a/spec/contracts/llm_connections/update_contract_spec.rb b/spec/contracts/llm_connections/update_contract_spec.rb new file mode 100644 index 000000000000..dd38fa258fd4 --- /dev/null +++ b/spec/contracts/llm_connections/update_contract_spec.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" +require_relative "../shared/model_contract_shared_context" + +RSpec.describe LlmConnections::UpdateContract, :check_errors_i18n, :llm_server_helpers, :webmock do + include_context "ModelContract shared context" + + let(:current_user) { build_stubbed(:admin) } + let(:base_url) { "https://example.com/v1" } + # Persisted, so that only the attributes an example changes are dirty. A + # freshly built record has every factory attribute in changed_attributes, + # which the contract would rightly reject as writes to non-writable columns. + let(:connection) { create(:llm_connection, base_url: "https://previous.example/v1") } + let(:contract) { described_class.new(connection, current_user) } + + let!(:models_request) { mock_llm_models_response(base_url) } + + before { connection.base_url = base_url } + + context "when the server answers with a model list" do + include_examples "contract is valid" + + it "probes the server exactly once" do + contract.validate + + expect(models_request).to have_been_made.once + end + end + + context "when the server rejects the credentials" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: 401) } + + include_examples "contract is invalid", api_key: :invalid_api_key + end + + context "when the server cannot be reached" do + let!(:models_request) { mock_llm_models_response(base_url, timeout: true) } + + include_examples "contract is invalid", base_url: :request_timed_out + end + + context "when the endpoint is not OpenAI-compatible" do + let!(:models_request) { mock_llm_models_response(base_url, body: "login") } + + include_examples "contract is invalid", base_url: :not_openai_compatible + end + + context "when the host is blocked by the SSRF policy" do + before { allow_llm_host("something.else") } + + include_examples "contract is invalid", base_url: :ssrf_filtered + + it "does not contact the server" do + contract.validate + + expect(models_request).not_to have_been_made + end + end + + context "when the base URL is not a URL at all" do + let(:base_url) { "not a url" } + + # The validate_url gem always records :url in errors.details; the + # message: :invalid_url option controls the rendered text, not the symbol. + include_examples "contract is invalid", base_url: :url + + it "does not contact the server" do + contract.validate + + expect(models_request).not_to have_been_made + end + end + + # Without the changed-attributes guard every unrelated save -- and every form + # render that builds a model through SetAttributesService -- would fire an + # outbound request at the LLM server. + describe "the changed-attributes guard" do + it "probes only when the credentials changed" do + contract.validate + expect(models_request).to have_been_made.once + + WebMock.reset_executed_requests! + connection.save! + contract.validate + + expect(models_request).not_to have_been_made + end + + it "probes again when only the API key changed" do + connection.save! + WebMock.reset_executed_requests! + + connection.api_key = "sk-rotated" + contract.validate + + expect(models_request).to have_been_made.once + end + + it "does not probe when only the enabled flag changed" do + connection.save! + WebMock.reset_executed_requests! + + connection.enabled = true + + expect(contract.validate).to be(true) + expect(models_request).not_to have_been_made + end + end + + describe "default model selection" do + let(:connection) { create(:llm_connection, :with_models, base_url:) } + + context "with a model the server offers" do + before { connection.default_chat_model_id = "bge-m3" } + + include_examples "contract is valid" + end + + context "with a model the server does not offer" do + before { connection.default_chat_model_id = "not-there" } + + include_examples "contract is invalid", default_chat_model_id: :not_available + end + end +end diff --git a/spec/factories/llm_connection_factory.rb b/spec/factories/llm_connection_factory.rb new file mode 100644 index 000000000000..36b5d298f6a8 --- /dev/null +++ b/spec/factories/llm_connection_factory.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +FactoryBot.define do + factory :llm_connection do + name { LlmConnection::SINGLETON_NAME } + type { "LlmConnection" } + base_url { "https://example.com/v1" } + api_key { "sk-test-key" } + enabled { false } + + trait :enabled do + enabled { true } + end + + trait :with_models do + catalogue do + { + "object" => "list", + "data" => [ + { "id" => "qwen3.6-27b", "object" => "model", "owned_by" => "vllm", "max_model_len" => 262_144 }, + { "id" => "bge-m3", "object" => "model", "owned_by" => "vllm", "max_model_len" => 8_192 } + ] + } + end + catalogue_fetched_at { Time.current } + last_connected_at { Time.current } + end + end +end diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb new file mode 100644 index 000000000000..eb92ede7fb9f --- /dev/null +++ b/spec/requests/admin/llm_connections_spec.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe "Admin LLM connection", :llm_server_helpers, :skip_csrf, :webmock, type: :rails_request do + let(:admin) { create(:admin) } + let(:non_admin) { create(:user) } + let(:base_url) { "https://example.com/v1" } + + describe "GET /admin/llm_connection" do + it "is not reachable for non-admins" do + login_as non_admin + get llm_connection_path + + expect(response).not_to have_http_status(:ok) + end + + context "when logged in as admin" do + before { login_as admin } + + it "renders without a connection configured" do + get llm_connection_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Host URL") + end + + it "lists the cached models without contacting the server" do + create(:llm_connection, :with_models, base_url:) + + get llm_connection_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("qwen3.6-27b") + expect(a_request(:get, "#{base_url}/models")).not_to have_been_made + end + end + end + + describe "PATCH /admin/llm_connection" do + before { login_as admin } + + context "with a reachable server" do + let!(:models_request) { mock_llm_models_response(base_url) } + + it "stores the connection and caches the catalogue" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-test" } } + + expect(response).to have_http_status(:see_other) + connection = LlmConnection.first + expect(connection.base_url).to eq(base_url) + expect(connection.api_key).to eq("sk-test") + expect(connection.catalogue_model_ids).to contain_exactly("qwen3.6-27b", "bge-m3") + end + end + + context "with an unreachable server" do + let!(:models_request) { mock_llm_models_response(base_url, timeout: true) } + + it "persists nothing" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-test" } } + + expect(LlmConnection.count).to eq(0) + end + end + + context "when an API key is already stored" do + let!(:connection) { create(:llm_connection, base_url:, api_key: "sk-original") } + let!(:models_request) { mock_llm_models_response(base_url) } + + it "keeps the stored key when the field is submitted blank" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "" } } + + expect(connection.reload.api_key).to eq("sk-original") + end + + it "replaces the stored key when a new one is submitted" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-rotated" } } + + expect(connection.reload.api_key).to eq("sk-rotated") + end + end + end + + describe "DELETE /admin/llm_connection/api_key" do + before { login_as admin } + + it "removes the key but keeps the connection" do + connection = create(:llm_connection, base_url:, api_key: "sk-original") + + delete api_key_llm_connection_path + + expect(response).to have_http_status(:see_other) + expect(connection.reload.api_key).to be_nil + expect(connection.base_url).to eq(base_url) + end + end + + describe "POST /admin/llm_connection/refresh_models" do + before { login_as admin } + + it "refetches the catalogue" do + create(:llm_connection, base_url:) + request = mock_llm_models_response(base_url) + + post refresh_models_llm_connection_path + + expect(request).to have_been_made.once + expect(LlmConnection.first.catalogue_model_ids).to include("bge-m3") + end + end +end diff --git a/spec/support/llm_server_helpers.rb b/spec/support/llm_server_helpers.rb new file mode 100644 index 000000000000..b4562a7771cd --- /dev/null +++ b/spec/support/llm_server_helpers.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmServerHelpers + DEFAULT_MODELS = [ + { id: "qwen3.6-27b", object: "model", owned_by: "vllm", max_model_len: 262_144 }, + { id: "bge-m3", object: "model", owned_by: "vllm", max_model_len: 8_192 } + ].freeze + + # Stubs GET /models. Returns the stub so specs can assert on how + # often it was called -- which is how the changed-attributes guard is pinned. + def mock_llm_models_response(base_url, + models: DEFAULT_MODELS, + response_code: 200, + body: nil, + timeout: false) + stub = stub_request(:get, "#{base_url.chomp('/')}/models") + + return stub.to_timeout if timeout + + stub.to_return( + status: response_code, + headers: { "Content-Type" => "application/json" }, + body: body || { object: "list", data: models }.to_json + ) + end + + # example.com resolves publicly, but a spec that needs a literal or private + # host has to say so explicitly rather than opening the allowlist to 0.0.0.0/0. + def allow_llm_host(*hosts) + allow(OpenProject::SsrfProtection).to receive(:safe_ip?) do |host| + hosts.include?(host) ? IPAddr.new("93.184.216.34") : nil + end + end +end + +RSpec.configure do |config| + config.include LlmServerHelpers, :llm_server_helpers +end From 39b9fe74491de7139d1dca650ace1066df40d65c Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 6 Aug 2026 22:18:54 +0100 Subject: [PATCH 07/44] [#66020] Add a registry of LLM-consuming features Features declare themselves, the kind of model they need and the capabilities they require, so the administration UI can show which models are usable for which job instead of offering an undifferentiated list. The registry lives in lib_static because it is populated from initializers, which run before eager loading. Constants under app/ would be unloaded on a development reload and lose their registrations -- the same reason OpenProject::FeatureDecisions lives there. The capability vocabulary is deliberately two entries. Only :embeddings is a hard gate: it cannot be emulated, being a different endpoint answering with vectors. :structured_output is advisory, because responses have to be validated and repaired whatever the server claims, which makes constrained decoding an optimisation rather than a requirement. Tool calling is absent on purpose -- no vLLM endpoint reports whether its tool parser is configured, so a verdict there could only ever be a guess on our own primary stack. https://community.openproject.org/work_packages/66020 --- config/initializers/llm_features.rb | 52 ++++++++++ config/locales/en.yml | 16 +++ lib_static/open_project/llm/features.rb | 126 ++++++++++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 config/initializers/llm_features.rb create mode 100644 lib_static/open_project/llm/features.rb diff --git a/config/initializers/llm_features.rb b/config/initializers/llm_features.rb new file mode 100644 index 000000000000..568d977063fa --- /dev/null +++ b/config/initializers/llm_features.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require_relative "../../lib_static/open_project/llm/features" + +# Features that send requests to the configured LLM server. +# +# Add a feature here (or from a module engine initializer) so that +# administrators can assign it a model on the "AI models" page. + +# The description assistant rewrites work package text on explicit user action. +# Plain chat completions only: no tools, no JSON mode, no streaming. Individual +# actions may override the model, which is why it is overridable. +OpenProject::Llm::Features.register :description_assistant, + kind: :chat, + prefers: %i[structured_output], + overridable: true + +# Semantic search embeds work packages into a pgvector index. Pinned because the +# stored vectors are meaningless under a different model: changing it is a +# destructive re-index rather than a swap. +OpenProject::Llm::Features.register :semantic_search, + kind: :embedding, + requires: %i[embeddings], + pinned: true diff --git a/config/locales/en.yml b/config/locales/en.yml index f83735a0708d..d6d8fd0e003c 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -3907,6 +3907,22 @@ en: wiki_child_pages: errors: page_not_found: "Cannot find the wiki page '%{name}'." + llm: + capabilities: + embeddings: + label: "Embeddings" + unsupported: "This model returned an error for embedding requests." + unknown: "The server does not report whether this model can produce embeddings." + structured_output: + label: "Structured output" + unknown: "The server does not report whether this model supports structured output." + features: + description_assistant: + caption: "Rewrites and restructures work package text on request." + label: "Description assistant" + semantic_search: + caption: "Indexes work packages so they can be found by meaning rather than by keyword." + label: "Semantic search" mail: actions: "Actions" digests: diff --git a/lib_static/open_project/llm/features.rb b/lib_static/open_project/llm/features.rb new file mode 100644 index 000000000000..529248b8a1c5 --- /dev/null +++ b/lib_static/open_project/llm/features.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module OpenProject + module Llm + class UnknownFeature < StandardError; end + + # A feature that sends requests to the configured LLM server. + # + # Features declare the capabilities they need so the administration UI can + # tell an administrator which models are usable for which job, and so a + # feature never silently runs against a model that cannot serve it. + Feature = Data.define(:key, :kind, :requires, :prefers, :overridable, :pinned, :available, :i18n_scope) do + def available? = available.call + + def chat? = kind == :chat + + def embedding? = kind == :embedding + + def label = I18n.t("label", scope: i18n_scope) + + def caption = I18n.t("caption", scope: i18n_scope, default: nil) + end + + # The registry of LLM-consuming features. + # + # Lives in lib_static because it is populated from initializers, which run + # before eager loading; constants defined under app/ would be unloaded on a + # development reload and lose their registrations. This is the same reason + # OpenProject::FeatureDecisions lives here. + # + # Register from config/initializers/llm_features.rb for core features, or + # from a module's engine: + # + # initializer "openproject_foo.llm_features" do + # OpenProject::Llm::Features.register :foo, kind: :chat + # end + module Features + module_function + + KINDS = %i[chat embedding].freeze + + # Deliberately small. Every capability is a probe, an admin-facing + # explanation and a maintenance burden, so one is only added when a + # shipping feature cannot work without it. + # + # :embeddings - the only hard gate. Cannot be emulated: it is a + # different endpoint answering with vectors. + # :structured_output - advisory only. Responses are validated and + # repaired regardless, so constrained decoding is + # an optimisation rather than a requirement. + CAPABILITIES = { + chat: %i[structured_output].freeze, + embedding: %i[embeddings].freeze + }.freeze + + def register(key, + kind:, + requires: [], + prefers: [], + overridable: false, + pinned: false, + available: -> { true }, + i18n_scope: nil) + key = key.to_sym + validate!(key, kind, requires + prefers) + + all[key] = Feature.new(key:, kind:, requires: requires.map(&:to_sym).freeze, + prefers: prefers.map(&:to_sym).freeze, + overridable:, pinned:, available:, + i18n_scope: i18n_scope || "llm.features.#{key}") + end + + def all = @all ||= {} + + def [](key) + all.fetch(key.to_sym) { raise UnknownFeature, key.to_s } + end + + def registered?(key) = all.key?(key.to_sym) + + # Features whose own toggle is on. A feature that is switched off keeps its + # stored binding: flipping a flag must not lose an administrator's choice. + def available = all.values.select(&:available?) + + def for_kind(kind) = available.select { |feature| feature.kind == kind } + + def validate!(key, kind, capabilities) + raise ArgumentError, "unknown kind #{kind.inspect}" unless KINDS.include?(kind) + raise ArgumentError, "LLM feature #{key} is already registered" if all.key?(key) + + unknown = capabilities.map(&:to_sym) - CAPABILITIES.fetch(kind) + return if unknown.empty? + + raise ArgumentError, "#{unknown.join(', ')} not valid for a #{kind} feature" + end + end + end +end From 5e23b4f60a7bb6d4da5925f471a00c1f9fe7511d Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Fri, 7 Aug 2026 08:52:19 +0100 Subject: [PATCH 08/44] [#66020] Store capability verdicts and feature bindings Two tables, both keyed on the model id as a plain string rather than a foreign key. The catalogue is a cache of a remote list, so a model can disappear from it; a binding has to survive that and say so, which a foreign key would make impossible. Verdicts carry three states, not a boolean, because the honest answer is very often "we cannot tell": the OpenAI model list carries no capability data at all and only some servers offer a non-standard endpoint that does. Only :unsupported blocks. Refusing on :unknown would make most self-hosted servers unusable. The source (metadata, probe, admin, observed) is orthogonal, so an administrator's assertion can be displayed as such without adding a fourth state every caller would have to handle. Bindings are one row per registered feature and are never destroyed when a feature deregisters, so flipping a feature flag does not lose the choice. Whether a binding is dangling is derived rather than stored: a status column would be a cache with no invalidation trigger, stale exactly when it matters. https://community.openproject.org/work_packages/66020 --- app/models/llm_capability_verdict.rb | 70 +++++++++++++++ app/models/llm_connection.rb | 2 + app/models/llm_feature_binding.rb | 88 +++++++++++++++++++ config/locales/en.yml | 15 ++++ ...11140000_create_llm_capability_verdicts.rb | 52 +++++++++++ ...60811140100_create_llm_feature_bindings.rb | 52 +++++++++++ 6 files changed, 279 insertions(+) create mode 100644 app/models/llm_capability_verdict.rb create mode 100644 app/models/llm_feature_binding.rb create mode 100644 db/migrate/20260811140000_create_llm_capability_verdicts.rb create mode 100644 db/migrate/20260811140100_create_llm_feature_bindings.rb diff --git a/app/models/llm_capability_verdict.rb b/app/models/llm_capability_verdict.rb new file mode 100644 index 000000000000..b6fe7d1e73df --- /dev/null +++ b/app/models/llm_capability_verdict.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# What we know about one capability of one model on one server. +# +# Three states rather than a boolean, because the honest answer is very often +# "we cannot tell": the OpenAI model list carries no capability information at +# all, and only some servers offer a non-standard endpoint that does. +# +# The rule the rest of the system depends on: only :unsupported blocks. An +# :unknown verdict warns and lets the administrator proceed, because refusing +# on "we could not tell" would make most self-hosted servers unusable. +class LlmCapabilityVerdict < ApplicationRecord + belongs_to :llm_connection + + enum :state, { supported: "supported", unsupported: "unsupported", unknown: "unknown" }, validate: true + + # Where the verdict came from. Orthogonal to the state, so that an + # administrator's assertion can be shown as such without adding a fourth state + # that every caller would have to handle. + enum :source, + { metadata: "metadata", probe: "probe", admin: "admin", observed: "observed" }, + prefix: true, + validate: true + + validates :model_id, presence: true + validates :capability, presence: true, uniqueness: { scope: %i[llm_connection_id model_id] } + + scope :for_model, ->(model_id) { where(model_id:) } + scope :for_capability, ->(capability) { where(capability: capability.to_s) } + + # An administrator's assertion survives re-detection: they know something about + # their deployment that we could not determine. + scope :sticky, -> { where(source: "admin") } + + def blocking? + unsupported? + end + + def dimensions + detail["dimensions"] + end +end diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb index 879f23fc109d..20dc5dcd853b 100644 --- a/app/models/llm_connection.rb +++ b/app/models/llm_connection.rb @@ -40,6 +40,8 @@ class LlmConnection < ApplicationRecord SINGLETON_NAME = "default" has_many :health_reports, as: :subject, dependent: :delete_all + has_many :capability_verdicts, class_name: "LlmCapabilityVerdict", dependent: :delete_all + has_many :feature_bindings, class_name: "LlmFeatureBinding", dependent: :delete_all validates :base_url, presence: true validate :only_one_connection, on: :create diff --git a/app/models/llm_feature_binding.rb b/app/models/llm_feature_binding.rb new file mode 100644 index 000000000000..0c1601fe2b25 --- /dev/null +++ b/app/models/llm_feature_binding.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Which model a registered feature uses. +# +# One row per feature, not per binding: rows are reconciled against the registry +# and are never destroyed when a feature deregisters, so flipping a feature flag +# does not lose the administrator's choice. +class LlmFeatureBinding < ApplicationRecord + belongs_to :llm_connection + + validates :feature_key, presence: true, uniqueness: { scope: :llm_connection_id } + validate :feature_registered + validate :pinned_model_unchanged + + def feature + OpenProject::Llm::Features[feature_key] + rescue OpenProject::Llm::UnknownFeature + nil + end + + # NULL means "use the connection default for this kind of model". + def resolved_model_id + model_id.presence || default_model_id + end + + def inherits_default? = model_id.blank? + + # Derived, never stored. A status column would be a cache with no invalidation + # trigger, and would be stale exactly when it matters -- right after the remote + # catalogue changed. + def dangling? + resolved = resolved_model_id + resolved.present? && llm_connection.catalogue_model_ids.exclude?(resolved) + end + + def locked? = locked_at.present? + + private + + def default_model_id + return if feature.nil? + + feature.embedding? ? llm_connection.default_embedding_model_id : llm_connection.default_chat_model_id + end + + def feature_registered + return if feature.present? + + errors.add(:feature_key, :not_registered) + end + + # Vectors written under one embedding model are meaningless under another, and + # the dimension count is baked into the index, so a locked binding can only be + # changed by an explicit re-index. + def pinned_model_unchanged + return unless locked? && model_id_changed? + + errors.add(:model_id, :locked) + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index d6d8fd0e003c..06f783590dfb 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -200,6 +200,15 @@ en: onthefly: "Automatic user creation" port: "Port" tls_certificate_string: "LDAP server SSL certificate" + llm_capability_verdict: + capability: "Capability" + model_id: "Model" + state: "State" + llm_feature_binding: + dimensions: "Dimensions" + feature_key: "Feature" + # ActiveRecord::Base.human_attribute_name strips the _id suffix. + model: "Model" llm_connection: api_key: "API key" base_url: "Host URL" @@ -664,6 +673,12 @@ en: tls_certificate_string: format: "%{message}" invalid_certificate: "The provided SSL certificate is invalid: %{additional_message}" + llm_feature_binding: + attributes: + feature_key: + not_registered: "does not belong to a known AI feature." + model_id: + locked: "cannot be changed while data indexed with it still exists. Re-index to switch models." llm_connection: attributes: api_key: diff --git a/db/migrate/20260811140000_create_llm_capability_verdicts.rb b/db/migrate/20260811140000_create_llm_capability_verdicts.rb new file mode 100644 index 000000000000..1e94a71fa5e0 --- /dev/null +++ b/db/migrate/20260811140000_create_llm_capability_verdicts.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmCapabilityVerdicts < ActiveRecord::Migration[8.1] + def change + create_table :llm_capability_verdicts do |t| + t.references :llm_connection, null: false, foreign_key: true + # A plain string, not a foreign key: the catalogue is a cache of a remote + # list, and a model may vanish from it without invalidating what we learned. + t.string :model_id, null: false + t.string :capability, null: false + t.string :state, null: false + t.string :source, null: false + t.jsonb :detail, null: false, default: {} + t.datetime :checked_at, null: false + + t.timestamps null: false + end + + add_index :llm_capability_verdicts, + %i[llm_connection_id model_id capability], + unique: true, + name: "index_llm_capability_verdicts_on_connection_model_capability" + end +end diff --git a/db/migrate/20260811140100_create_llm_feature_bindings.rb b/db/migrate/20260811140100_create_llm_feature_bindings.rb new file mode 100644 index 000000000000..b9bbc61683e1 --- /dev/null +++ b/db/migrate/20260811140100_create_llm_feature_bindings.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmFeatureBindings < ActiveRecord::Migration[8.1] + def change + create_table :llm_feature_bindings do |t| + t.references :llm_connection, null: false, foreign_key: true + t.string :feature_key, null: false + # NULL means "use the connection default for this kind of model". + t.string :model_id + # Embedding features only. Frozen together with model_id once vectors exist. + t.integer :dimensions + t.string :input_prefix + t.string :query_prefix + # Set once the binding has data depending on it, after which the model + # cannot be swapped without a destructive re-index. + t.datetime :locked_at + t.datetime :last_seen_at + + t.timestamps null: false + end + + add_index :llm_feature_bindings, %i[llm_connection_id feature_key], unique: true + end +end From 1a000b10e84f663f8bf42ecdc2fa504999351c39 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Fri, 7 Aug 2026 10:20:43 +0100 Subject: [PATCH 09/44] [#66020] Detect which models can produce embeddings Embeddings is the one capability worth probing: it cannot be emulated, and a server either returns a vector or it does not. The verdict also yields the dimension count, which semantic search needs to size its index. The 200 case is judged by the shape of the body, not by the status. vLLM, llama.cpp and Ollama all silently drop parameters they do not understand, so a 200 on its own proves nothing -- a response without a numeric vector is recorded as unknown rather than supported. There is deliberately no tool-calling probe. Several current vLLM releases return 200 with the tool call as plain text on a fully configured server, and they do it deterministically, so retrying would launder a wrong answer into a confident one instead of correcting it. Probing is rationed. A gateway can list hundreds of models and some providers bill per request, so only the model an administrator is binding gets probed synchronously; after a connect, at most ten models whose names suggest they are embedders are probed in the background. Everything else stays unknown, which never blocks. Verdicts are discarded when the base URL or key changes, because that is a different deployment -- including administrator assertions, which were about the old one. When a model merely disappears from the same server, only probed verdicts are dropped: an operator restarting a server must not silently lose an assertion they made. https://community.openproject.org/work_packages/66020 --- app/services/llm/client.rb | 20 ++++ app/services/llm/probes/embeddings_probe.rb | 100 +++++++++++++++++ .../detect_capabilities_service.rb | 101 ++++++++++++++++++ .../llm_connections/sync_models_service.rb | 30 +++++- .../llm_connections/update_service.rb | 5 +- app/workers/llm/detect_capabilities_job.rb | 42 ++++++++ 6 files changed, 296 insertions(+), 2 deletions(-) create mode 100644 app/services/llm/probes/embeddings_probe.rb create mode 100644 app/services/llm_connections/detect_capabilities_service.rb create mode 100644 app/workers/llm/detect_capabilities_job.rb diff --git a/app/services/llm/client.rb b/app/services/llm/client.rb index 275e13a74db6..6872e5307348 100644 --- a/app/services/llm/client.rb +++ b/app/services/llm/client.rb @@ -97,10 +97,30 @@ def models body end + # Requests an embedding vector for a single short input. + # + # Used to determine whether a model can serve embeddings at all: the model + # list says nothing about it, and posting a chat completion to an embedding + # model (or the reverse) is the only reliable way to find out. + # + # @return [Hash] the parsed +POST /embeddings+ body + def embeddings(model:, input:) + post("/embeddings", { model:, input: }) + end + private attr_reader :base_url, :api_key, :timeout + def post(path, payload) + response = session.post(uri_for(path), json: payload) + handle_transport_error(response) if response.is_a?(HTTPX::ErrorResponse) + handle_status(response) + parse(response) + rescue OpenProject::HttpxSsrfFilter::ServerSideRequestForgeryError + raise SsrfError, "Host resolves to a blocked address" + end + def get(path) response = session.get(uri_for(path)) # A connection-level failure yields an HTTPX::ErrorResponse. A real response diff --git a/app/services/llm/probes/embeddings_probe.rb b/app/services/llm/probes/embeddings_probe.rb new file mode 100644 index 000000000000..f362798e3ff0 --- /dev/null +++ b/app/services/llm/probes/embeddings_probe.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Probes + # Determines whether a model can produce embeddings, by asking it to. + # + # This is the only behavioural probe in the system, and it qualifies because + # it cannot lie: a server either returns a vector or it does not. Probes for + # tool calling deliberately do not exist -- several current vLLM releases + # return HTTP 200 with the tool call as plain text on a fully capable server, + # deterministically, so retries would launder a wrong answer into a confident + # one rather than correcting it. + # + # The 200 case is checked by shape rather than by status, because unknown + # parameters are silently dropped by vLLM, llama.cpp and Ollama alike: a 200 + # on its own proves nothing. + class EmbeddingsProbe + PROBE_INPUT = "openproject" + + Result = Data.define(:state, :detail) do + def supported? = state == :supported + end + + def initialize(connection) + @connection = connection + end + + def call(model_id) + body = client.embeddings(model: model_id, input: PROBE_INPUT) + classify(body) + rescue Llm::Client::ApiError => e + # The server understood the request and refused it for this model. + return unsupported(e.status) if e.status.in?([400, 404, 405, 501]) + + # 5xx and anything else says something about the server, not the model. + unknown("http_#{e.status}") + rescue Llm::Client::AuthenticationError + unknown("unauthorized") + rescue Llm::Client::Error => e + unknown(e.class.name.demodulize.underscore) + end + + private + + attr_reader :connection + + def client + @client ||= Llm::Client.new(base_url: connection.base_url, api_key: connection.api_key) + end + + def classify(body) + vector = Array(body["data"]).first&.dig("embedding") + + if vector.is_a?(Array) && vector.any? && vector.all?(Numeric) + Result.new(state: :supported, detail: { "dimensions" => vector.length }) + else + # A 200 whose body is not an embedding response: the server accepted the + # request but answered with something else entirely. + unknown("unexpected_body") + end + end + + def unsupported(status) + Result.new(state: :unsupported, detail: { "http_status" => status }) + end + + def unknown(reason) + Result.new(state: :unknown, detail: { "reason" => reason }) + end + end + end +end diff --git a/app/services/llm_connections/detect_capabilities_service.rb b/app/services/llm_connections/detect_capabilities_service.rb new file mode 100644 index 000000000000..731eb390a8a9 --- /dev/null +++ b/app/services/llm_connections/detect_capabilities_service.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Records what a model on this server can do. + # + # Probing every listed model would be wrong: a gateway can list hundreds, each + # probe is a request, and some providers bill per request. So the models worth + # asking about are either the one an administrator is about to bind, or a small + # number whose names suggest they are embedding models. + class DetectCapabilitiesService + # Naming is a hint for which models are worth spending a probe on, never a + # verdict in itself. + EMBEDDING_NAME_HINT = /embed|bge|e5|gte|nomic|minilm/i + BACKGROUND_LIMIT = 10 + + def initialize(connection) + @connection = connection + end + + # Probes a specific model, synchronously. Used when an administrator binds a + # model to a feature that requires embeddings -- the verdict that matters. + def detect(model_id) + return existing_admin_verdict(model_id) if admin_asserted?(model_id) + + result = probe.call(model_id) + record(model_id, result) + end + + # Pre-colours the model list after a connect, without spending a request per + # model. Everything not probed stays unknown, which never blocks. + def detect_likely_embedding_models + candidates.each { |model_id| detect(model_id) } + end + + private + + attr_reader :connection + + def probe + @probe ||= Llm::Probes::EmbeddingsProbe.new(connection) + end + + def candidates + connection.catalogue_model_ids + .grep(EMBEDDING_NAME_HINT) + .reject { |model_id| admin_asserted?(model_id) } + .first(BACKGROUND_LIMIT) + end + + # An administrator knows things about their deployment that a probe cannot + # determine, so their assertion is never overwritten by re-detection. + def admin_asserted?(model_id) + verdicts.for_model(model_id).for_capability(:embeddings).sticky.exists? + end + + def existing_admin_verdict(model_id) + verdicts.for_model(model_id).for_capability(:embeddings).first + end + + def record(model_id, result) + verdict = verdicts.find_or_initialize_by(model_id:, capability: "embeddings") + verdict.update!(state: result.state.to_s, + source: "probe", + detail: result.detail, + checked_at: Time.current) + verdict + end + + def verdicts + connection.capability_verdicts + end + end +end diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb index d9770a273399..7524f00fec1e 100644 --- a/app/services/llm_connections/sync_models_service.rb +++ b/app/services/llm_connections/sync_models_service.rb @@ -39,7 +39,7 @@ def initialize(connection) end def call - connection.update!(catalogue_attributes(client.models)) + store(catalogue_attributes(client.models)) ServiceResult.success(result: connection) rescue Llm::Client::Error => e @@ -51,6 +51,34 @@ def call attr_reader :connection + def store(attributes) + ActiveRecord::Base.transaction do + discard_verdicts_for_a_different_deployment(attributes[:connection_fingerprint]) + connection.update!(attributes) + discard_verdicts_for_vanished_models + end + end + + # A changed base URL or key means we are talking to a different deployment, + # so everything we learned about the old one is void -- including + # administrator assertions, which were about that deployment, not this one. + def discard_verdicts_for_a_different_deployment(fingerprint) + return if connection.connection_fingerprint.blank? + return if connection.connection_fingerprint == fingerprint + + connection.capability_verdicts.delete_all + end + + # Same deployment, but a model is gone. Its verdict is meaningless now, except + # an administrator's assertion: an operator restarting a server must not + # silently lose one. + def discard_verdicts_for_vanished_models + known = connection.catalogue_model_ids + return if known.empty? + + connection.capability_verdicts.where.not(model_id: known).where.not(source: "admin").delete_all + end + def catalogue_attributes(catalogue) now = Time.current diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb index d6b3111dc6d2..a9b0e94a2226 100644 --- a/app/services/llm_connections/update_service.rb +++ b/app/services/llm_connections/update_service.rb @@ -48,7 +48,10 @@ def after_perform(service_call) super.tap do next unless @sync_models && service_call.success? - SyncModelsService.new(service_call.result).call if credentials_changed?(service_call.result) + next unless credentials_changed?(service_call.result) + + SyncModelsService.new(service_call.result).call + Llm::DetectCapabilitiesJob.perform_later end end diff --git a/app/workers/llm/detect_capabilities_job.rb b/app/workers/llm/detect_capabilities_job.rb new file mode 100644 index 000000000000..5c37305b2ac7 --- /dev/null +++ b/app/workers/llm/detect_capabilities_job.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Pre-colours the model list after a connect, out of band so that saving the + # connection does not wait on one request per candidate model. + class DetectCapabilitiesJob < ApplicationJob + def perform + connection = LlmConnection.first + return if connection.nil? || !connection.configured? + + LlmConnections::DetectCapabilitiesService.new(connection).detect_likely_embedding_models + end + end +end From 1e49f10162e5fc4478cf290e94281372821128e5 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Fri, 7 Aug 2026 13:44:02 +0100 Subject: [PATCH 10/44] [#66020] Add Llm::Runtime, the single model resolution point Every consuming feature asks the same question -- which model do I use, and can I run right now -- so it is answered in one place and every feature agrees on what an unset value means: per-item override -> feature binding -> connection default -> unbound A blank value at any level inherits from the level below. Resolution fails closed and never substitutes. When the chosen model is no longer in the server's catalogue the status is model_missing, not a quiet fallback to the default: a text transform run through a different model is a different feature, and silently swapping it produces exactly the "why did the output change" report that nobody can diagnose. Only a definite unsupported verdict blocks. Unknown -- the normal state for a server that reports nothing about its models -- is surfaced as a warning and still runs, because refusing on "we could not tell" would make most self-hosted servers unusable. https://community.openproject.org/work_packages/66020 --- app/services/llm/runtime.rb | 117 ++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 app/services/llm/runtime.rb diff --git a/app/services/llm/runtime.rb b/app/services/llm/runtime.rb new file mode 100644 index 000000000000..8bdb82279ecd --- /dev/null +++ b/app/services/llm/runtime.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Answers "which model should this feature use, and can it run right now?". + # + # The single place model resolution happens, so that every feature agrees on + # what an unset value means: + # + # per-item override -> feature binding -> connection default -> unbound + # + # A blank value at any level means "inherit from the level below". + class Runtime + # :ready - go ahead + # :feature_disabled - the feature's own toggle is off + # :no_connection - no LLM server configured, or AI switched off globally + # :unbound - nothing has chosen a model for this feature yet + # :model_missing - the chosen model is not in the server's catalogue + # :incapable - the chosen model is known not to support what is needed + Resolution = Data.define(:feature, :connection, :model_id, :status, :missing_capabilities) do + def ready? = status == :ready + end + + class << self + # @param feature_key [Symbol] a key registered with OpenProject::Llm::Features + # @param override [String, nil] a per-item model choice, e.g. one stored on + # a description assistant action. Blank means inherit. + def for(feature_key, override: nil) + new(OpenProject::Llm::Features[feature_key], override:).call + end + end + + def initialize(feature, override: nil) + @feature = feature + @override = override + end + + def call + return resolution(:feature_disabled) unless feature.available? + return resolution(:no_connection) unless LlmConnection.available? + + model_id = resolved_model_id + return resolution(:unbound) if model_id.blank? + return resolution(:model_missing, model_id:) unless connection.catalogue_model_ids.include?(model_id) + + missing = unsupported_capabilities(model_id) + return resolution(:incapable, model_id:, missing_capabilities: missing) if missing.any? + + resolution(:ready, model_id:) + end + + private + + attr_reader :feature, :override + + def connection + @connection ||= LlmConnection.instance + end + + def resolved_model_id + override.presence || binding_model_id || connection_default + end + + def binding_model_id + connection.feature_bindings.find_by(feature_key: feature.key.to_s)&.model_id.presence + end + + def connection_default + feature.embedding? ? connection.default_embedding_model_id : connection.default_chat_model_id + end + + # Only a definite :unsupported blocks. An :unknown verdict -- which is the + # normal state for a server that reports nothing about its models -- is + # surfaced in the UI as a warning but never prevents a call. + def unsupported_capabilities(model_id) + return [] if feature.requires.empty? + + blocking = connection.capability_verdicts + .for_model(model_id) + .where(capability: feature.requires.map(&:to_s), state: "unsupported") + + blocking.pluck(:capability).map(&:to_sym) + end + + def resolution(status, model_id: nil, missing_capabilities: []) + Resolution.new(feature:, connection: status == :feature_disabled ? nil : connection, + model_id:, status:, missing_capabilities:) + end + end +end From 9cc5b5ff632c8f4f3fc0f9fd129ae3c522cd8f47 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Fri, 7 Aug 2026 17:09:26 +0100 Subject: [PATCH 11/44] [#66020] Add the AI models assignment page One page where every registered feature picks its model, with the instance default as the inherited choice. Models are never hidden. Hiding one produces the single support question nobody can answer -- "why can I not pick the model I know works" -- and it is exactly wrong when most verdicts are unknown, which is the normal state for a server that reports nothing about its models. Instead each option says where it stands: plain when usable, "not verified" when a required capability is unknown, and disabled with the missing capability named when it is known absent. Choosing a model for a feature that requires one probes it immediately, because that is the verdict that actually matters. Features that require nothing never trigger a request. A binding whose model has vanished from the catalogue is flagged rather than quietly repointed, and a binding locked by indexed data cannot be changed here at all -- switching an embedding model is a re-index, not a swap. Two shape notes for anyone extending this: the block form of Rails' select writes to the template's output buffer rather than into the select element, so choices are built as pairs; and ApplicationComponent already owns the name `options`, which silently swallows a memoised method of the same name. https://community.openproject.org/work_packages/66020 --- .../feature_binding_component.html.erb | 34 +++++ .../feature_binding_component.rb | 108 +++++++++++++++ .../admin/llm_feature_bindings_controller.rb | 88 ++++++++++++ .../selectable_models_query.rb | 85 ++++++++++++ .../admin/llm_feature_bindings/index.html.erb | 63 +++++++++ config/initializers/menus.rb | 6 + config/locales/en.yml | 14 ++ config/routes.rb | 4 + .../admin/llm_feature_bindings_spec.rb | 128 ++++++++++++++++++ 9 files changed, 530 insertions(+) create mode 100644 app/components/llm_connections/feature_binding_component.html.erb create mode 100644 app/components/llm_connections/feature_binding_component.rb create mode 100644 app/controllers/admin/llm_feature_bindings_controller.rb create mode 100644 app/services/llm_connections/selectable_models_query.rb create mode 100644 app/views/admin/llm_feature_bindings/index.html.erb create mode 100644 spec/requests/admin/llm_feature_bindings_spec.rb diff --git a/app/components/llm_connections/feature_binding_component.html.erb b/app/components/llm_connections/feature_binding_component.html.erb new file mode 100644 index 000000000000..034926139ca7 --- /dev/null +++ b/app/components/llm_connections/feature_binding_component.html.erb @@ -0,0 +1,34 @@ +<%= render(Primer::Box.new(border: true, border_radius: 2, p: 3, mb: 3)) do %> + <%= render(Primer::Beta::Text.new(tag: :h3, font_size: 4, font_weight: :bold, mb: 1)) { feature.label } %> + + <% if feature.caption.present? %> + <%= render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 2)) { feature.caption } %> + <% end %> + + <% if dangling? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :alert)) do %> + <%= t("admin.llm_feature_bindings.dangling", model: binding.resolved_model_id) %> + <% end %> + <% end %> + + <% if locked? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :lock)) do %> + <%= t("admin.llm_feature_bindings.locked", model: binding.model_id) %> + <% end %> + <% end %> + + <%= form_with(url: form_url, method: :patch) do %> + <%= label_tag(select_id, LlmFeatureBinding.human_attribute_name(:model_id), class: "FormControl-label") %> + <%= select_tag( + "llm_feature_binding[model_id]", + options_for_select(select_choices, selected: selected_model_id.to_s, disabled: disabled_choices), + id: select_id, + disabled: locked?, + class: "FormControl-select" + ) %> + + <% unless locked? %> + <%= render(Primer::Beta::Button.new(type: :submit, scheme: :secondary, mt: 2)) { t(:button_save) } %> + <% end %> + <% end %> +<% end %> diff --git a/app/components/llm_connections/feature_binding_component.rb b/app/components/llm_connections/feature_binding_component.rb new file mode 100644 index 000000000000..f301177c7f1d --- /dev/null +++ b/app/components/llm_connections/feature_binding_component.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # One feature's row on the model assignment page. + class FeatureBindingComponent < ApplicationComponent + include ApplicationHelper + include OpPrimer::ComponentHelpers + + def initialize(feature:, connection:, binding: nil) + super(feature) + @feature = feature + @connection = connection + @binding = binding + end + + private + + attr_reader :feature, :connection, :binding + + # Not named +options+: ApplicationComponent already owns that name and + # initialises it to an empty hash, which silently swallowed the memoisation. + def model_options + @model_options ||= SelectableModelsQuery.new(connection, feature).call + end + + def selected_model_id = binding&.model_id + + def default_model_id + feature.embedding? ? connection.default_embedding_model_id : connection.default_chat_model_id + end + + def inherit_label + if default_model_id.present? + I18n.t("admin.llm_feature_bindings.inherit_with_default", model: default_model_id) + else + I18n.t("admin.llm_feature_bindings.inherit_without_default") + end + end + + def locked? = binding&.locked? + + def dangling? = binding&.dangling? + + # Built as label/value pairs rather than through a block, because the block + # form of Rails' select writes to the template's output buffer instead of + # into the select element. + def select_choices + [[inherit_label, ""]] + model_options.map { |option| [option_label(option), option.model_id] } + end + + # Listed but not choosable: a model whose required capability is known to be + # missing. It stays visible so the reason is visible with it. + def disabled_choices + model_options.reject(&:selectable?).map(&:model_id) + end + + def option_label(option) + case option.state + when :unsupported + I18n.t("admin.llm_feature_bindings.option_unsupported", + model: option.model_id, + capability: capability_labels(option.reasons)) + when :unknown + I18n.t("admin.llm_feature_bindings.option_unknown", model: option.model_id) + else + option.model_id + end + end + + def capability_labels(capabilities) + capabilities.map { |capability| I18n.t("llm.capabilities.#{capability}.label") }.join(", ") + end + + def form_url + url_helpers.llm_feature_binding_path(feature.key) + end + + def select_id = "llm_feature_binding_model_id_#{feature.key}" + end +end diff --git a/app/controllers/admin/llm_feature_bindings_controller.rb b/app/controllers/admin/llm_feature_bindings_controller.rb new file mode 100644 index 000000000000..5824c871405f --- /dev/null +++ b/app/controllers/admin/llm_feature_bindings_controller.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + # Assigns a model to each registered AI feature. + class LlmFeatureBindingsController < ApplicationController + layout "admin" + menu_item :llm_feature_bindings + + before_action :require_admin + before_action :set_connection + + def index + @features = OpenProject::Llm::Features.available + @bindings = bindings_by_feature_key + end + + def update + feature = OpenProject::Llm::Features[params[:id]] + assign(feature) + + redirect_to llm_feature_bindings_path, status: :see_other + rescue OpenProject::Llm::UnknownFeature + render_404 + end + + private + + def set_connection + @connection = LlmConnection.instance + end + + def bindings_by_feature_key + @connection.feature_bindings.index_by(&:feature_key) + end + + def binding_for(feature) + @connection.feature_bindings.find_or_initialize_by(feature_key: feature.key.to_s) + end + + def assign(feature) + binding = binding_for(feature) + binding.model_id = params.dig(:llm_feature_binding, :model_id).presence + + if binding.save + probe_capabilities(feature, binding) + flash[:notice] = t("admin.llm_feature_bindings.update.success", feature: feature.label) + else + flash[:error] = binding.errors.full_messages.join(", ") + end + end + + # The verdict that actually matters is the one for the model an administrator + # just chose, so it is fetched now rather than left unknown until first use. + def probe_capabilities(feature, binding) + return if feature.requires.empty? || binding.model_id.blank? + + LlmConnections::DetectCapabilitiesService.new(@connection).detect(binding.model_id) + end + end +end diff --git a/app/services/llm_connections/selectable_models_query.rb b/app/services/llm_connections/selectable_models_query.rb new file mode 100644 index 000000000000..dfb585db1102 --- /dev/null +++ b/app/services/llm_connections/selectable_models_query.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # The models offerable to a feature, each with why it is or is not usable. + # + # Models are never hidden. Hiding one produces the single support question + # nobody can answer -- "why can I not pick the model I know works" -- and it is + # exactly wrong when most verdicts are unknown. Instead each option carries a + # state the UI renders: selectable, selectable with a warning, or disabled with + # a reason. + class SelectableModelsQuery + Option = Data.define(:model_id, :state, :reasons) do + def selectable? = state != :unsupported + + def warning? = state == :unknown + end + + def initialize(connection, feature) + @connection = connection + @feature = feature + end + + def call + connection.catalogue_model_ids.map { |model_id| option_for(model_id) } + end + + private + + attr_reader :connection, :feature + + def option_for(model_id) + states = feature.requires.index_with { |capability| verdict_state(model_id, capability) } + + if states.value?(:unsupported) + Option.new(model_id:, state: :unsupported, + reasons: states.select { |_, s| s == :unsupported }.keys) + elsif states.value?(:unknown) + Option.new(model_id:, state: :unknown, + reasons: states.select { |_, s| s == :unknown }.keys) + else + Option.new(model_id:, state: :supported, reasons: []) + end + end + + # No verdict at all is the same as an inconclusive one: we do not know. + def verdict_state(model_id, capability) + verdicts.dig(model_id, capability.to_s)&.to_sym || :unknown + end + + def verdicts + @verdicts ||= connection.capability_verdicts + .pluck(:model_id, :capability, :state) + .group_by(&:first) + .transform_values { |rows| rows.to_h { |(_, capability, state)| [capability, state] } } + end + end +end diff --git a/app/views/admin/llm_feature_bindings/index.html.erb b/app/views/admin/llm_feature_bindings/index.html.erb new file mode 100644 index 000000000000..7c026aac983d --- /dev/null +++ b/app/views/admin/llm_feature_bindings/index.html.erb @@ -0,0 +1,63 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_feature_bindings") %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t("menus.admin.llm_feature_bindings") } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + t("menus.admin.llm_feature_bindings")] + ) + end +%> + +<% if !@connection.configured? %> + <%= + render(Primer::Beta::Blankslate.new(border: true)) do |component| + component.with_visual_icon(icon: :sparkle) + component.with_heading(tag: :h2) { t(".blank_title") } + component.with_description { t(".blank_description") } + component.with_primary_action(href: llm_connection_path) { t("menus.admin.llm_connection") } + end + %> +<% else %> + <% @features.each do |feature| %> + <%= render( + LlmConnections::FeatureBindingComponent.new( + feature:, + connection: @connection, + binding: @bindings[feature.key.to_s] + ) + ) %> + <% end %> +<% end %> diff --git a/config/initializers/menus.rb b/config/initializers/menus.rb index b30df08098ca..c5bfff10bfb4 100644 --- a/config/initializers/menus.rb +++ b/config/initializers/menus.rb @@ -504,6 +504,12 @@ caption: I18n.t("menus.admin.llm_connection"), parent: :ai + menu.push :llm_feature_bindings, + { controller: "/admin/llm_feature_bindings", action: :index }, + if: ->(_) { User.current.admin? && OpenProject::FeatureDecisions.llm_connection_active? }, + caption: I18n.t("menus.admin.llm_feature_bindings"), + parent: :ai + menu.push :mcp_configurations, { controller: "/admin/mcp_configurations", action: :index }, if: ->(_) { User.current.admin? }, diff --git a/config/locales/en.yml b/config/locales/en.yml index 06f783590dfb..16dd8c4232b9 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1422,6 +1422,19 @@ en: caption_with_maximum: > User actions on a work package (changing description, status, values, or writing comments) are grouped if performed within this period. It also controls notification and [webhook](webhook_link) delays. The maximum is %{max} minutes. + llm_feature_bindings: + dangling: "%{model} is no longer offered by the LLM server. This feature will not run until another model is selected." + index: + blank_description: "Connect OpenProject to an LLM server first. Models can be assigned once the server reports which ones it offers." + blank_title: "No LLM server configured" + description: "Choose which model each AI feature uses. Features without a choice use the instance default." + inherit_with_default: "Use the default (%{model})" + inherit_without_default: "Use the default (none set)" + locked: "%{model} is in use by indexed data and cannot be changed here. Re-index to switch models." + option_unknown: "%{model} — not verified" + option_unsupported: "%{model} — no %{capability} support" + update: + success: "The model for %{feature} has been saved." llm_connections: form: api_key_caption: "The key OpenProject authenticates with. Leave blank if the server requires no authentication." @@ -4165,6 +4178,7 @@ en: ai: "Artificial Intelligence (AI)" api_and_webhooks: "API and webhooks" llm_connection: "LLM settings" + llm_feature_bindings: "AI models" mail_notification: "Email notifications" mails_and_notifications: "Emails and notifications" mcp_configurations: "Model Context Protocol (MCP)" diff --git a/config/routes.rb b/config/routes.rb index 19dce902894e..da7dbcf7d6c9 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -696,6 +696,10 @@ delete :api_key, action: :delete_api_key end + # Keyed by feature key rather than by record id: the binding is an attribute + # of a registered feature, and a feature may not have a row yet. + resources :llm_feature_bindings, only: %i[index update], controller: "admin/llm_feature_bindings" + resources :mcp_configurations, only: %i[index update], controller: "admin/mcp_configurations" do collection do post :multi_update diff --git a/spec/requests/admin/llm_feature_bindings_spec.rb b/spec/requests/admin/llm_feature_bindings_spec.rb new file mode 100644 index 000000000000..bddf4cd72bf0 --- /dev/null +++ b/spec/requests/admin/llm_feature_bindings_spec.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe "Admin AI model assignment", :llm_server_helpers, :skip_csrf, :webmock, type: :rails_request do + let(:admin) { create(:admin) } + let(:base_url) { "https://example.com/v1" } + + describe "GET /admin/llm_feature_bindings" do + before { login_as admin } + + it "prompts to configure a connection when there is none" do + get llm_feature_bindings_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("No LLM server configured") + end + + context "with a configured connection" do + let!(:connection) { create(:llm_connection, :with_models, base_url:) } + + it "lists every registered feature" do + get llm_feature_bindings_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Description assistant") + expect(response.body).to include("Semantic search") + end + + # Hiding an unusable model is the one thing that produces an unanswerable + # support question, so it stays listed and says why it cannot be chosen. + it "offers a model with no verdict, marked as unverified" do + get llm_feature_bindings_path + + expect(response.body).to include("qwen3.6-27b — not verified") + end + + it "disables a model known not to support a required capability" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: Time.current) + + get llm_feature_bindings_path + + expect(response.body).to include("qwen3.6-27b — no Embeddings support") + end + end + end + + describe "PATCH /admin/llm_feature_bindings/:id" do + let!(:connection) { create(:llm_connection, :with_models, base_url:) } + + before { login_as admin } + + it "stores the chosen model" do + patch llm_feature_binding_path("description_assistant"), + params: { llm_feature_binding: { model_id: "qwen3.6-27b" } } + + expect(response).to have_http_status(:see_other) + expect(connection.feature_bindings.find_by(feature_key: "description_assistant").model_id) + .to eq("qwen3.6-27b") + end + + it "treats a blank choice as inheriting the default" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + patch llm_feature_binding_path("description_assistant"), params: { llm_feature_binding: { model_id: "" } } + + expect(connection.feature_bindings.find_by(feature_key: "description_assistant").model_id).to be_nil + end + + # The verdict that matters is the one for the model just chosen, so it is + # fetched now rather than left unknown until the feature first runs. + it "probes the model when the feature requires a capability" do + request = stub_request(:post, "#{base_url}/embeddings") + .to_return(status: 200, + headers: { "Content-Type" => "application/json" }, + body: { data: [{ embedding: [0.1, 0.2] }] }.to_json) + + patch llm_feature_binding_path("semantic_search"), params: { llm_feature_binding: { model_id: "bge-m3" } } + + expect(request).to have_been_made.once + verdict = connection.capability_verdicts.find_by(model_id: "bge-m3", capability: "embeddings") + expect(verdict.state).to eq("supported") + expect(verdict.dimensions).to eq(2) + end + + it "does not probe for a feature that requires nothing" do + patch llm_feature_binding_path("description_assistant"), + params: { llm_feature_binding: { model_id: "qwen3.6-27b" } } + + expect(a_request(:post, "#{base_url}/embeddings")).not_to have_been_made + end + + it "404s for a feature that is not registered" do + patch llm_feature_binding_path("no_such_feature"), params: { llm_feature_binding: { model_id: "x" } } + + expect(response).to have_http_status(:not_found) + end + end +end From 59fc259e853a2884b86c54073e1f3ce6d906fd67 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Fri, 7 Aug 2026 20:37:15 +0100 Subject: [PATCH 12/44] [#66020] Add specs for the resolver and the embeddings probe The probe spec pins the decision rule that matters most: a 200 whose body is not an embedding response is recorded as unknown, not supported. vLLM, llama.cpp and Ollama all silently drop parameters they do not understand, so judging by status alone would mark every model on such a server as capable. The resolver spec pins the two behaviours features depend on: a model that has disappeared from the catalogue fails closed rather than falling back to the default, and an unknown capability verdict warns without blocking -- refusing on "we could not tell" would make most self-hosted servers unusable. https://community.openproject.org/work_packages/66020 --- .../llm/probes/embeddings_probe_spec.rb | 101 +++++++++++++++ spec/services/llm/runtime_spec.rb | 122 ++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 spec/services/llm/probes/embeddings_probe_spec.rb create mode 100644 spec/services/llm/runtime_spec.rb diff --git a/spec/services/llm/probes/embeddings_probe_spec.rb b/spec/services/llm/probes/embeddings_probe_spec.rb new file mode 100644 index 000000000000..9fdfc0439b40 --- /dev/null +++ b/spec/services/llm/probes/embeddings_probe_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Probes::EmbeddingsProbe, :webmock do + subject(:result) { described_class.new(connection).call("some-model") } + + let(:connection) { build(:llm_connection, base_url: "https://example.com/v1") } + + def stub_embeddings(status:, body:) + stub_request(:post, "https://example.com/v1/embeddings") + .to_return(status:, headers: { "Content-Type" => "application/json" }, body: body.to_json) + end + + context "when the server returns a vector" do + before { stub_embeddings(status: 200, body: { data: [{ embedding: [0.1, 0.2, 0.3] }] }) } + + it "is supported and captures the dimension count" do + expect(result.state).to eq(:supported) + expect(result.detail["dimensions"]).to eq(3) + end + end + + # vLLM, llama.cpp and Ollama all silently drop parameters they do not + # understand, so a 200 on its own proves nothing about the model. + context "when the server returns 200 with something that is not an embedding" do + before { stub_embeddings(status: 200, body: { data: [{ message: "hello" }] }) } + + it "is unknown rather than supported" do + expect(result.state).to eq(:unknown) + expect(result.detail["reason"]).to eq("unexpected_body") + end + end + + context "when the server returns an empty data array" do + before { stub_embeddings(status: 200, body: { data: [] }) } + + it { expect(result.state).to eq(:unknown) } + end + + [400, 404, 501].each do |status| + context "when the server refuses the request with #{status}" do + before { stub_embeddings(status:, body: { error: "nope" }) } + + it "is unsupported" do + expect(result.state).to eq(:unsupported) + expect(result.detail["http_status"]).to eq(status) + end + end + end + + # A 5xx says something about the server, not about the model. + context "when the server errors" do + before { stub_embeddings(status: 500, body: { error: "boom" }) } + + it { expect(result.state).to eq(:unknown) } + end + + context "when the credentials are rejected" do + before { stub_embeddings(status: 401, body: { error: "no" }) } + + it "is unknown, since this says nothing about the model" do + expect(result.state).to eq(:unknown) + expect(result.detail["reason"]).to eq("unauthorized") + end + end + + context "when the server cannot be reached" do + before { stub_request(:post, "https://example.com/v1/embeddings").to_timeout } + + it { expect(result.state).to eq(:unknown) } + end +end diff --git a/spec/services/llm/runtime_spec.rb b/spec/services/llm/runtime_spec.rb new file mode 100644 index 000000000000..dca4948f5a87 --- /dev/null +++ b/spec/services/llm/runtime_spec.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Runtime, with_flag: { llm_connection: true } do + subject(:resolution) { described_class.for(feature_key, override:) } + + let(:feature_key) { :description_assistant } + let(:override) { nil } + + context "without a connection" do + it { expect(resolution.status).to eq(:no_connection) } + end + + context "with a connection that is not enabled" do + before { create(:llm_connection, :with_models, enabled: false) } + + it { expect(resolution.status).to eq(:no_connection) } + end + + context "with an enabled connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled) } + + it "is unbound until a model is chosen" do + expect(resolution.status).to eq(:unbound) + expect(resolution.model_id).to be_nil + end + + it "falls back to the connection default" do + connection.update!(default_chat_model_id: "qwen3.6-27b") + + expect(resolution).to be_ready + expect(resolution.model_id).to eq("qwen3.6-27b") + end + + it "prefers the feature binding over the connection default" do + connection.update!(default_chat_model_id: "qwen3.6-27b") + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "bge-m3") + + expect(resolution.model_id).to eq("bge-m3") + end + + context "with a per-item override" do + let(:override) { "qwen3.6-27b" } + + it "wins over the binding" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "bge-m3") + + expect(resolution.model_id).to eq("qwen3.6-27b") + end + end + + # Substituting the default here would silently change the output of a + # transform an administrator configured deliberately. + context "when the chosen model is gone from the catalogue" do + let(:override) { "vanished-model" } + + before { connection.update!(default_chat_model_id: "qwen3.6-27b") } + + it "fails closed rather than falling back" do + expect(resolution.status).to eq(:model_missing) + expect(resolution.model_id).to eq("vanished-model") + end + end + end + + describe "capability gating" do + let(:feature_key) { :semantic_search } + let!(:connection) { create(:llm_connection, :with_models, :enabled) } + + before { connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "qwen3.6-27b") } + + it "blocks on a definite unsupported verdict" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: Time.current) + + expect(resolution.status).to eq(:incapable) + expect(resolution.missing_capabilities).to eq([:embeddings]) + end + + # Refusing on "we could not tell" would make most self-hosted servers + # unusable, since the model list carries no capability information at all. + it "does not block when the verdict is unknown" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "unknown", source: "probe", checked_at: Time.current) + + expect(resolution).to be_ready + end + + it "does not block when there is no verdict at all" do + expect(resolution).to be_ready + end + end +end From 8e85c20fa6e30ef1a3fdfc50b8e229a829820bea Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Fri, 7 Aug 2026 23:31:58 +0100 Subject: [PATCH 13/44] [#66020] Distinguish a missing model list from an incompatible server A server can speak the OpenAI API for chat and still not expose a model list. OpenProject's own hosted stack does exactly that today: its gateway answers 401 for /v1/chat/completions, so the route is known and needs a key, while /v1/models answers 404 Route Not Found -- the endpoint from #77512, which has not shipped yet. Reporting that as "not an OpenAI-API-compatible endpoint" sends the administrator looking for the wrong problem. A 404 now says the server was reached but has no model list at the path we tried, and names that path, which covers both a wrong version segment and a server that genuinely lacks the endpoint. https://community.openproject.org/work_packages/66020 --- app/validators/llm_server_validator.rb | 13 +++++++++++++ config/locales/en.yml | 1 + .../llm_connections/update_contract_spec.rb | 8 ++++++++ 3 files changed, 22 insertions(+) diff --git a/app/validators/llm_server_validator.rb b/app/validators/llm_server_validator.rb index b827ea38466d..cace33450ac2 100644 --- a/app/validators/llm_server_validator.rb +++ b/app/validators/llm_server_validator.rb @@ -88,6 +88,19 @@ def add_error(contract, attribute, error) contract.errors.add(attribute, :request_timed_out) when Llm::Client::ConnectionError contract.errors.add(attribute, :cannot_be_connected_to) + when Llm::Client::ApiError + add_api_error(contract, attribute, error) + else + contract.errors.add(attribute, :not_openai_compatible) + end + end + + # A 404 is worth separating: the server answered, so it is reachable and the + # credentials were not the problem. Either the URL is missing or carries the + # wrong version segment, or the server genuinely does not expose a model list. + def add_api_error(contract, attribute, error) + if error.status == 404 + contract.errors.add(attribute, :models_endpoint_missing, path: "#{contract.model.base_url}/models") else contract.errors.add(attribute, :not_openai_compatible) end diff --git a/config/locales/en.yml b/config/locales/en.yml index 16dd8c4232b9..4d51ff519911 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -688,6 +688,7 @@ en: singleton: "Only one LLM connection can be configured." base_url: cannot_be_connected_to: "could not be reached. Please ensure the LLM server is running and reachable from OpenProject." + models_endpoint_missing: "was reached, but has no model list at %{path}. Check that the URL includes the right API version segment, and that the server exposes a models endpoint." not_openai_compatible: "did not return a valid model list. Please ensure the URL points at an OpenAI-API-compatible endpoint, including the API version segment (for example https://example.com/v1)." request_timed_out: "did not respond in time. Please ensure the LLM server is reachable and not overloaded." ssrf_filtered: "resolves to a blocked address. If the LLM server runs on an internal network, allow its IP via the %{env_name} environment variable." diff --git a/spec/contracts/llm_connections/update_contract_spec.rb b/spec/contracts/llm_connections/update_contract_spec.rb index dd38fa258fd4..80ee17a9d8e5 100644 --- a/spec/contracts/llm_connections/update_contract_spec.rb +++ b/spec/contracts/llm_connections/update_contract_spec.rb @@ -68,6 +68,14 @@ include_examples "contract is invalid", base_url: :request_timed_out end + # A server can speak the OpenAI API for chat and still not expose a model list: + # OpenProject's own hosted stack does exactly that while #77512 is unreleased. + context "when the server has no model list at that path" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: 404) } + + include_examples "contract is invalid", base_url: :models_endpoint_missing + end + context "when the endpoint is not OpenAI-compatible" do let!(:models_request) { mock_llm_models_response(base_url, body: "login") } From 8b5cfb251d8364ad43dff1ac00d8652ded6c9f58 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 10 Aug 2026 09:40:13 +0100 Subject: [PATCH 14/44] [#66020] Replace the jsonb catalogue with an llm_models table The model list was stored as a verbatim jsonb blob on the connection, wholly replaced on every refresh. That shape has no room for anything an administrator owns, which is a problem the moment models can be entered by hand: a manual entry would be erased by the next sync. Models are now rows keyed [llm_connection_id, external_id]. external_id is whatever this deployment calls the model -- Scaleway serves qwen3.6-35b-a3b for weights another catalogue lists as Qwen/Qwen3.6-35B-A3B -- so it stays an opaque string and is never used to look anything up in a public registry. A model the server stops offering is deactivated rather than deleted, so a binding or verdict pointing at it still has something to name. Also introduces an adapter seam. There is no universal model-discovery standard: OpenAI-compatible servers answer GET /models with data[].id, Gemini uses /v1beta/models, Bedrock needs AWS signing, and Azure indirects through deployment names. Only the OpenAI adapter exists; api_format selects it. https://community.openproject.org/work_packages/66020 --- .../llm_connections/models_row_component.rb | 31 +++++- .../llm_connections/models_table_component.rb | 5 +- .../llm_connections/base_contract.rb | 2 +- app/forms/llm_connections/connection_form.rb | 4 +- app/models/llm_connection.rb | 8 +- app/models/llm_model.rb | 60 +++++++++++ app/services/llm/adapters.rb | 55 ++++++++++ app/services/llm/adapters/openai.rb | 85 +++++++++++++++ app/services/llm/client.rb | 6 +- app/services/llm/runtime.rb | 2 +- .../detect_capabilities_service.rb | 2 +- .../selectable_models_query.rb | 2 +- .../llm_connections/sync_models_service.rb | 101 ++++++++++-------- app/views/admin/llm_connections/show.html.erb | 22 +++- .../20260812090000_create_llm_models.rb | 56 ++++++++++ spec/factories/llm_connection_factory.rb | 16 ++- spec/factories/llm_model_factory.rb | 48 +++++++++ spec/requests/admin/llm_connections_spec.rb | 4 +- 18 files changed, 432 insertions(+), 77 deletions(-) create mode 100644 app/models/llm_model.rb create mode 100644 app/services/llm/adapters.rb create mode 100644 app/services/llm/adapters/openai.rb create mode 100644 db/migrate/20260812090000_create_llm_models.rb create mode 100644 spec/factories/llm_model_factory.rb diff --git a/app/components/llm_connections/models_row_component.rb b/app/components/llm_connections/models_row_component.rb index 2d98470c6e41..6ee82abdff0f 100644 --- a/app/components/llm_connections/models_row_component.rb +++ b/app/components/llm_connections/models_row_component.rb @@ -29,23 +29,44 @@ #++ module LlmConnections - # A row of the model catalogue. +model+ here is a raw model card as the server - # reported it, not an ActiveRecord object. + # A row of the model list. Note +model+ is the row's record (an LlmModel), + # not the connection -- aliased to avoid confusion with either. class ModelsRowComponent < OpPrimer::BorderBoxRowComponent - alias_method :card, :model + alias_method :llm_model, :model def identifier - render(Primer::Beta::Text.new(font_weight: :bold)) { card["id"] } + render(Primer::Beta::Text.new(font_weight: :bold)) { llm_model.name } end # vLLM and SGLang report the operator's real --max-model-len here, which is # more trustworthy than any published figure for the model. Servers that do # not report it simply show nothing rather than a guess. def context_window - window = card["max_model_len"] + window = llm_model.context_window return render(Primer::Beta::Text.new(color: :muted)) { "—" } if window.blank? number_with_delimiter(window) end + + def source + if llm_model.manual? + render(Primer::Beta::Label.new(scheme: :accent)) { I18n.t("admin.llm_connections.models.source_manual") } + elsif llm_model.withdrawn? + render(Primer::Beta::Label.new(scheme: :attention)) { I18n.t("admin.llm_connections.models.source_withdrawn") } + else + render(Primer::Beta::Label.new(scheme: :secondary)) { I18n.t("admin.llm_connections.models.source_discovered") } + end + end + + def button_links + return [] unless llm_model.manual? + + [ + link_to(helpers.op_icon("icon-delete"), + url_helpers.llm_model_path(llm_model), + data: { turbo_method: :delete, turbo_confirm: I18n.t(:text_are_you_sure) }, + title: I18n.t(:button_delete)) + ] + end end end diff --git a/app/components/llm_connections/models_table_component.rb b/app/components/llm_connections/models_table_component.rb index 2d03302d01f6..9dba37bb7a3c 100644 --- a/app/components/llm_connections/models_table_component.rb +++ b/app/components/llm_connections/models_table_component.rb @@ -34,7 +34,7 @@ module LlmConnections # Rendering never issues an HTTP request: the catalogue is refreshed explicitly # through the "Refresh models" action. class ModelsTableComponent < OpPrimer::BorderBoxTableComponent - columns :identifier, :context_window + columns :identifier, :context_window, :source mobile_columns :identifier @@ -50,7 +50,8 @@ def row_class = ModelsRowComponent def headers [ [:identifier, { caption: I18n.t("admin.llm_connections.models.identifier") }], - [:context_window, { caption: I18n.t("admin.llm_connections.models.context_window") }] + [:context_window, { caption: I18n.t("admin.llm_connections.models.context_window") }], + [:source, { caption: I18n.t("admin.llm_connections.models.source") }] ] end diff --git a/app/contracts/llm_connections/base_contract.rb b/app/contracts/llm_connections/base_contract.rb index 8e99b7e25e75..e0ef64789970 100644 --- a/app/contracts/llm_connections/base_contract.rb +++ b/app/contracts/llm_connections/base_contract.rb @@ -72,7 +72,7 @@ def default_models_offered_by_server value = model.public_send(attribute) next if value.blank? next unless model.changed_attributes.include?(attribute.to_s) - next if model.catalogue_model_ids.include?(value) + next if model.available_model_ids.include?(value) errors.add attribute, :not_available end diff --git a/app/forms/llm_connections/connection_form.rb b/app/forms/llm_connections/connection_form.rb index e3d3fb64fa1c..30e061fd2581 100644 --- a/app/forms/llm_connections/connection_form.rb +++ b/app/forms/llm_connections/connection_form.rb @@ -73,7 +73,7 @@ class ConnectionForm < ApplicationForm input_width: :large, disabled: read_only? ) do |select| - model.catalogue_model_ids.each do |model_id| + model.available_model_ids.each do |model_id| select.option(value: model_id, label: model_id) end end @@ -96,7 +96,7 @@ def read_only? end def models_available? - model.catalogue_model_ids.any? + model.available_model_ids.any? end def submit_label diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb index 20dc5dcd853b..6e393c890664 100644 --- a/app/models/llm_connection.rb +++ b/app/models/llm_connection.rb @@ -40,6 +40,7 @@ class LlmConnection < ApplicationRecord SINGLETON_NAME = "default" has_many :health_reports, as: :subject, dependent: :delete_all + has_many :models, class_name: "LlmModel", dependent: :delete_all has_many :capability_verdicts, class_name: "LlmCapabilityVerdict", dependent: :delete_all has_many :feature_bindings, class_name: "LlmFeatureBinding", dependent: :delete_all @@ -88,9 +89,10 @@ def configured_from_env? Setting.llm_connection.present? end - # The model ids the remote server last reported, in the order it reported them. - def catalogue_model_ids - Array(catalogue["data"]).filter_map { |model| model["id"] } + # Every model that can be addressed today: discovered and still offered, plus + # anything an administrator entered by hand. + def available_model_ids + models.active.by_identifier.pluck(:external_id) end def server_flavour diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb new file mode 100644 index 000000000000..623dc1ae5b03 --- /dev/null +++ b/app/models/llm_model.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# A model this connection can address. +# +# Rows come from two places: discovered from the server's model list, or entered +# by an administrator. Both are addressed by +external_id+, which is whatever the +# deployment calls the model -- provider-specific and not comparable across +# vendors, which is why it is never used as a lookup key into a public catalogue. +class LlmModel < ApplicationRecord + belongs_to :llm_connection + + validates :external_id, presence: true, uniqueness: { scope: :llm_connection_id } + + scope :active, -> { where(active: true) } + scope :discovered, -> { where(manual: false) } + scope :manual, -> { where(manual: true) } + scope :by_identifier, -> { order(:external_id) } + + def name = display_name.presence || external_id + + # vLLM and SGLang report the operator's real --max-model-len here, which is + # more trustworthy for this deployment than any published figure. + def context_window + raw_metadata["max_model_len"] + end + + # Discovered models that the server stopped offering are deactivated rather + # than deleted, so a binding or verdict pointing at one still has something to + # name. Manual entries are never deactivated by a refresh: nothing confirms + # them, so nothing can un-confirm them either. + def withdrawn? = !active? && !manual? +end diff --git a/app/services/llm/adapters.rb b/app/services/llm/adapters.rb new file mode 100644 index 000000000000..15948302d9ce --- /dev/null +++ b/app/services/llm/adapters.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Resolves a connection's api_format to the adapter that speaks it. + # + # There is no universal model-discovery standard, so each dialect needs its own + # translation: OpenAI-compatible servers answer GET /models with data[].id, + # Gemini uses /v1beta/models with richer metadata, Bedrock needs AWS signing + # rather than a bearer token, and Azure indirects through deployment names. + # + # Only the OpenAI adapter is implemented. The seam exists so that adding one is + # a new class rather than a migration. + module Adapters + class UnsupportedFormat < StandardError; end + + FORMATS = { + "openai" => "Llm::Adapters::Openai" + }.freeze + + def self.for(connection) + class_name = FORMATS[connection.api_format.to_s] + raise UnsupportedFormat, connection.api_format.to_s if class_name.nil? + + class_name.constantize.new(connection) + end + end +end diff --git a/app/services/llm/adapters/openai.rb b/app/services/llm/adapters/openai.rb new file mode 100644 index 000000000000..68da5bf06e2f --- /dev/null +++ b/app/services/llm/adapters/openai.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Adapters + # Servers speaking the OpenAI API: OpenAI itself, and the great majority of + # gateways and self-hosted inference servers. + class Openai + def initialize(connection) + @connection = connection + end + + # Normalised model cards. + # + # The raw card is kept alongside, because the fields worth having are the + # non-standard ones: the OpenAI schema itself carries only id, object, + # created and owned_by, while vLLM adds max_model_len -- the operator's + # actual --max-model-len, and the only trustworthy context window for this + # deployment. + # + # @return [Array] cards with :id and :raw + def models + @models ||= Array(client.models["data"]).filter_map do |card| + id = card["id"] + next if id.blank? + + { id:, raw: card } + end + end + + def embeddings(model_id:, input:) + client.embeddings(model: model_id, input:) + end + + # Which server we are talking to decides which non-standard metadata is + # worth reading later. +owned_by+ is the documented hint; the structural + # fallback catches an operator who overrode it. + def server_flavour + cards = models + owner = cards.first&.dig(:raw, "owned_by").to_s.downcase + + return owner if %w[vllm sglang llamacpp openai].include?(owner) + + cards.any? { |card| card[:raw].key?("max_model_len") || card[:raw].key?("root") } ? "vllm" : "unknown" + end + + private + + attr_reader :connection + + def client + @client ||= Llm::Client.new(base_url: connection.base_url, + api_key: connection.api_key, + headers: connection.custom_headers) + end + end + end +end diff --git a/app/services/llm/client.rb b/app/services/llm/client.rb index 6872e5307348..d442f4bd44a4 100644 --- a/app/services/llm/client.rb +++ b/app/services/llm/client.rb @@ -77,10 +77,11 @@ class ParseError < Error; end timeout: { connect_timeout: 5, read_timeout: 120, request_timeout: 180 } }.freeze - def initialize(base_url:, api_key: nil, timeout: PROBE_TIMEOUT) + def initialize(base_url:, api_key: nil, timeout: PROBE_TIMEOUT, headers: {}) @base_url = base_url.to_s.chomp("/") @api_key = api_key @timeout = timeout + @headers = (headers || {}).compact_blank end # The model catalogue as the server reports it, verbatim. @@ -110,7 +111,7 @@ def embeddings(model:, input:) private - attr_reader :base_url, :api_key, :timeout + attr_reader :base_url, :api_key, :timeout, :headers def post(path, payload) response = session.post(uri_for(path), json: payload) @@ -138,6 +139,7 @@ def get(path) def session request = OpenProject.httpx.with(timeout) + request = request.with(headers:) if headers.any? api_key.present? ? request.plugin(:auth).bearer_auth(api_key) : request end diff --git a/app/services/llm/runtime.rb b/app/services/llm/runtime.rb index 8bdb82279ecd..11835b68cfd2 100644 --- a/app/services/llm/runtime.rb +++ b/app/services/llm/runtime.rb @@ -68,7 +68,7 @@ def call model_id = resolved_model_id return resolution(:unbound) if model_id.blank? - return resolution(:model_missing, model_id:) unless connection.catalogue_model_ids.include?(model_id) + return resolution(:model_missing, model_id:) unless connection.available_model_ids.include?(model_id) missing = unsupported_capabilities(model_id) return resolution(:incapable, model_id:, missing_capabilities: missing) if missing.any? diff --git a/app/services/llm_connections/detect_capabilities_service.rb b/app/services/llm_connections/detect_capabilities_service.rb index 731eb390a8a9..c9e9efe78d4f 100644 --- a/app/services/llm_connections/detect_capabilities_service.rb +++ b/app/services/llm_connections/detect_capabilities_service.rb @@ -69,7 +69,7 @@ def probe end def candidates - connection.catalogue_model_ids + connection.available_model_ids .grep(EMBEDDING_NAME_HINT) .reject { |model_id| admin_asserted?(model_id) } .first(BACKGROUND_LIMIT) diff --git a/app/services/llm_connections/selectable_models_query.rb b/app/services/llm_connections/selectable_models_query.rb index dfb585db1102..0855892ba13a 100644 --- a/app/services/llm_connections/selectable_models_query.rb +++ b/app/services/llm_connections/selectable_models_query.rb @@ -49,7 +49,7 @@ def initialize(connection, feature) end def call - connection.catalogue_model_ids.map { |model_id| option_for(model_id) } + connection.available_model_ids.map { |model_id| option_for(model_id) } end private diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb index 7524f00fec1e..1aa269ed94a3 100644 --- a/app/services/llm_connections/sync_models_service.rb +++ b/app/services/llm_connections/sync_models_service.rb @@ -29,17 +29,17 @@ #++ module LlmConnections - # Refreshes the cached model catalogue from the remote server. + # Refreshes the model list from the remote server. # - # Kept separate from the contract probe so that the same code path serves the - # "Refresh models" button, the update service and the environment seeder. + # Kept separate from the contract probe so the same path serves the "Refresh + # models" button, the update service and the environment seeder. class SyncModelsService def initialize(connection) @connection = connection end def call - store(catalogue_attributes(client.models)) + store(adapter.models) ServiceResult.success(result: connection) rescue Llm::Client::Error => e @@ -51,66 +51,75 @@ def call attr_reader :connection - def store(attributes) + def adapter + @adapter ||= Llm::Adapters.for(connection) + end + + def store(cards) ActiveRecord::Base.transaction do - discard_verdicts_for_a_different_deployment(attributes[:connection_fingerprint]) - connection.update!(attributes) + discard_verdicts_for_a_different_deployment(fingerprint) + connection.update!(connection_attributes) + upsert(cards) + withdraw_models_absent_from(cards) discard_verdicts_for_vanished_models end end - # A changed base URL or key means we are talking to a different deployment, - # so everything we learned about the old one is void -- including - # administrator assertions, which were about that deployment, not this one. - def discard_verdicts_for_a_different_deployment(fingerprint) - return if connection.connection_fingerprint.blank? - return if connection.connection_fingerprint == fingerprint - - connection.capability_verdicts.delete_all - end - - # Same deployment, but a model is gone. Its verdict is meaningless now, except - # an administrator's assertion: an operator restarting a server must not - # silently lose one. - def discard_verdicts_for_vanished_models - known = connection.catalogue_model_ids - return if known.empty? - - connection.capability_verdicts.where.not(model_id: known).where.not(source: "admin").delete_all - end - - def catalogue_attributes(catalogue) + def connection_attributes now = Time.current { - catalogue:, catalogue_fetched_at: now, last_connected_at: now, connection_fingerprint: fingerprint, - options: connection.options.merge("server_flavour" => detect_server_flavour(catalogue)) + options: connection.options.merge("server_flavour" => adapter.server_flavour) } end - def client - Llm::Client.new(base_url: connection.base_url, api_key: connection.api_key) - end - def fingerprint - Digest::SHA256.hexdigest("#{connection.base_url}\0#{connection.api_key}") + @fingerprint ||= Digest::SHA256.hexdigest("#{connection.base_url}\0#{connection.api_key}") end - # Which server we are talking to decides which non-standard metadata endpoint - # is worth asking later. +owned_by+ is the documented hint; the structural - # fallback catches servers whose operator overrode it. - def detect_server_flavour(catalogue) - cards = Array(catalogue["data"]) - owner = cards.first&.dig("owned_by").to_s.downcase - - case owner - when "vllm", "sglang", "llamacpp", "openai" then owner - else - cards.any? { |card| card.key?("max_model_len") || card.key?("root") } ? "vllm" : "unknown" + def upsert(cards) + now = Time.current + + cards.each do |card| + model = connection.models.find_or_initialize_by(external_id: card.fetch(:id)) + model.update!(display_name: card[:display_name], + raw_metadata: card.fetch(:raw, {}), + last_seen_at: now, + active: true) end end + + # Deactivated rather than deleted, so a binding or verdict pointing at one + # still has something to name. Manual entries are left alone: the server was + # never the thing that confirmed them. + def withdraw_models_absent_from(cards) + connection.models + .discovered + .where.not(external_id: cards.map { |card| card.fetch(:id) }) + .update_all(active: false) + end + + # A changed base URL or key means a different deployment, so everything we + # learned about the old one is void -- including administrator assertions, + # which were about that deployment. + def discard_verdicts_for_a_different_deployment(new_fingerprint) + return if connection.connection_fingerprint.blank? + return if connection.connection_fingerprint == new_fingerprint + + connection.capability_verdicts.delete_all + end + + # Same deployment, but a model is gone. Its verdict is meaningless now, + # except an administrator's assertion: an operator restarting a server must + # not silently lose one. + def discard_verdicts_for_vanished_models + known = connection.available_model_ids + return if known.empty? + + connection.capability_verdicts.where.not(model_id: known).where.not(source: "admin").delete_all + end end end diff --git a/app/views/admin/llm_connections/show.html.erb b/app/views/admin/llm_connections/show.html.erb index 939efeb2d3fc..16998ca5867a 100644 --- a/app/views/admin/llm_connections/show.html.erb +++ b/app/views/admin/llm_connections/show.html.erb @@ -43,7 +43,7 @@ See COPYRIGHT and LICENSE files for more details. <%= render(LlmConnections::FormComponent.new(@connection)) %> -<% if @connection.catalogue_model_ids.any? %> +<% if @connection.persisted? %> <%= render(Primer::Beta::Subhead.new(mt: 4)) do |component| component.with_heading(tag: :h3) { t(".models_heading") } @@ -63,5 +63,23 @@ See COPYRIGHT and LICENSE files for more details. end %> - <%= render(LlmConnections::ModelsTableComponent.new(rows: @connection.catalogue["data"])) %> + <%= render(LlmConnections::ModelsTableComponent.new(rows: @connection.models.by_identifier)) %> + + <%= render(Primer::Box.new(mt: 3, p: 3, border: true, border_radius: 2)) do %> + <%= render(Primer::Beta::Text.new(tag: :h4, font_weight: :bold, mb: 1)) { t(".add_model_heading") } %> + <%= render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 2)) { t(".add_model_description") } %> + + <%= form_with(url: llm_models_path, method: :post) do %> + <%= label_tag("llm_model_external_id", LlmModel.human_attribute_name(:external_id), class: "FormControl-label") %> + <%= text_field_tag( + "llm_model[external_id]", nil, + id: "llm_model_external_id", + required: true, + autocomplete: "off", + placeholder: "qwen3.6-35b-a3b", + class: "FormControl-input" + ) %> + <%= render(Primer::Beta::Button.new(type: :submit, scheme: :secondary, ml: 2)) { t(".add_model_submit") } %> + <% end %> + <% end %> <% end %> diff --git a/db/migrate/20260812090000_create_llm_models.rb b/db/migrate/20260812090000_create_llm_models.rb new file mode 100644 index 000000000000..9e8d7298c944 --- /dev/null +++ b/db/migrate/20260812090000_create_llm_models.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmModels < ActiveRecord::Migration[8.1] + def change + create_table :llm_models do |t| + t.references :llm_connection, null: false, foreign_key: true + # Whatever this deployment calls the model. Provider-specific: Scaleway + # serves "qwen3.6-35b-a3b" for weights another catalogue lists as + # "Qwen/Qwen3.6-35B-A3B". + t.string :external_id, null: false + t.string :display_name + t.boolean :active, null: false, default: true + # Entered by an administrator rather than discovered. Survives a refresh + # that cannot see it, which is what makes a server offering + # /v1/chat/completions but no /v1/models usable. + t.boolean :manual, null: false, default: false + t.datetime :last_seen_at + t.jsonb :raw_metadata, null: false, default: {} + + t.timestamps null: false + end + + add_index :llm_models, %i[llm_connection_id external_id], unique: true + + # Superseded by the table above. + remove_column :llm_connections, :catalogue, :jsonb, null: false, default: {} + end +end diff --git a/spec/factories/llm_connection_factory.rb b/spec/factories/llm_connection_factory.rb index 36b5d298f6a8..7fd84f91a744 100644 --- a/spec/factories/llm_connection_factory.rb +++ b/spec/factories/llm_connection_factory.rb @@ -41,17 +41,15 @@ end trait :with_models do - catalogue do - { - "object" => "list", - "data" => [ - { "id" => "qwen3.6-27b", "object" => "model", "owned_by" => "vllm", "max_model_len" => 262_144 }, - { "id" => "bge-m3", "object" => "model", "owned_by" => "vllm", "max_model_len" => 8_192 } - ] - } - end catalogue_fetched_at { Time.current } last_connected_at { Time.current } + + after(:create) do |connection| + create(:llm_model, llm_connection: connection, external_id: "qwen3.6-27b", + raw_metadata: { "owned_by" => "vllm", "max_model_len" => 262_144 }) + create(:llm_model, llm_connection: connection, external_id: "bge-m3", + raw_metadata: { "owned_by" => "vllm", "max_model_len" => 8_192 }) + end end end end diff --git a/spec/factories/llm_model_factory.rb b/spec/factories/llm_model_factory.rb new file mode 100644 index 000000000000..2082eb3c0bbb --- /dev/null +++ b/spec/factories/llm_model_factory.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +FactoryBot.define do + factory :llm_model do + llm_connection + sequence(:external_id) { |n| "model-#{n}" } + active { true } + manual { false } + last_seen_at { Time.current } + + trait :manual do + manual { true } + last_seen_at { nil } + end + + trait :withdrawn do + active { false } + end + end +end diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb index eb92ede7fb9f..760323015825 100644 --- a/spec/requests/admin/llm_connections_spec.rb +++ b/spec/requests/admin/llm_connections_spec.rb @@ -78,7 +78,7 @@ connection = LlmConnection.first expect(connection.base_url).to eq(base_url) expect(connection.api_key).to eq("sk-test") - expect(connection.catalogue_model_ids).to contain_exactly("qwen3.6-27b", "bge-m3") + expect(connection.available_model_ids).to contain_exactly("qwen3.6-27b", "bge-m3") end end @@ -134,7 +134,7 @@ post refresh_models_llm_connection_path expect(request).to have_been_made.once - expect(LlmConnection.first.catalogue_model_ids).to include("bge-m3") + expect(LlmConnection.first.available_model_ids).to include("bge-m3") end end end From 08a6fabe68cc04f4f95033bc47a127b099e2f9ed Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 10 Aug 2026 11:15:47 +0100 Subject: [PATCH 15/44] [#66020] Let an administrator add a model by hand Not every OpenAI-compatible server exposes a model list. OpenProject's own hosted gateway is the case in point: it routes /v1/chat/completions and answers 404 Route Not Found for /v1/models, so the connection can serve requests that OpenProject cannot discover. A manually added model is flagged and survives a refresh that cannot see it -- the server was never the thing that confirmed it, so a refresh cannot withdraw it either. Discovered models remain the server's to add and remove, and cannot be deleted from the UI. This unblocks using a working server whose model list is missing or not yet routed, without weakening the discovery path for servers that do expose one. https://community.openproject.org/work_packages/66020 --- .../admin/llm_models_controller.rb | 74 ++++++++++++ config/locales/en.yml | 20 +++- config/routes.rb | 3 + spec/requests/admin/llm_models_spec.rb | 105 ++++++++++++++++++ 4 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 app/controllers/admin/llm_models_controller.rb create mode 100644 spec/requests/admin/llm_models_spec.rb diff --git a/app/controllers/admin/llm_models_controller.rb b/app/controllers/admin/llm_models_controller.rb new file mode 100644 index 000000000000..41465a7d39fe --- /dev/null +++ b/app/controllers/admin/llm_models_controller.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + # Models an administrator enters by hand. + # + # Necessary because not every OpenAI-compatible server exposes a model list. + # A gateway may route /v1/chat/completions and nothing else, in which case the + # operator knows the model name and OpenProject cannot discover it. + class LlmModelsController < ApplicationController + layout "admin" + menu_item :llm_connection + + before_action :require_admin + before_action :set_connection + + def create + llm_model = @connection.models.new(model_params.merge(manual: true)) + + if llm_model.save + flash[:notice] = t(".success", model: llm_model.external_id) + else + flash[:error] = llm_model.errors.full_messages.join(", ") + end + + redirect_to llm_connection_path, status: :see_other + end + + def destroy + llm_model = @connection.models.manual.find(params.expect(:id)) + llm_model.destroy! + + flash[:notice] = t(".success", model: llm_model.external_id) + redirect_to llm_connection_path, status: :see_other + end + + private + + def set_connection + @connection = LlmConnection.instance + end + + def model_params + params.expect(llm_model: %i[external_id display_name]) + end + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 4d51ff519911..cfc460527dae 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -200,6 +200,10 @@ en: onthefly: "Automatic user creation" port: "Port" tls_certificate_string: "LDAP server SSL certificate" + llm_model: + display_name: "Display name" + # human_attribute_name strips the _id suffix, so the key omits it. + external: "Model name" llm_capability_verdict: capability: "Capability" model_id: "Model" @@ -1449,20 +1453,32 @@ en: delete_api_key: success: "The API key has been removed." models: - blank_description: "Save the connection to retrieve the models the server offers." - blank_title: "No models retrieved yet" + blank_description: "The server reported no models. If it does not offer a model list, add the model names you want to use below." + blank_title: "No models available" context_window: "Context window" identifier: "Model" + source: "Source" + source_discovered: "Reported by server" + source_manual: "Added manually" + source_withdrawn: "No longer reported" refresh_models: failure: "The model list could not be refreshed. Please check that the LLM server is still reachable." success: "The model list has been refreshed." show: + add_model_description: "If the server does not expose a model list, enter the model name exactly as the server expects it. Manually added models are kept when the list is refreshed." + add_model_heading: "Add a model manually" + add_model_submit: "Add model" description: "Connect OpenProject to a server that speaks the OpenAI API, so AI features can use it." models_description: "Reported by the server on %{fetched_at}." models_heading: "Available models" refresh_models: "Refresh models" update: success: "Successfully connected to the LLM server." + llm_models: + create: + success: "%{model} has been added." + destroy: + success: "%{model} has been removed." mcp_configurations: index: description: "The model context protocol allows AI agents to provide its users with tools and resources exposed by this OpenProject instance. This feature is still in beta." diff --git a/config/routes.rb b/config/routes.rb index da7dbcf7d6c9..7792bc2f7590 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -696,6 +696,9 @@ delete :api_key, action: :delete_api_key end + # Manual entries only; discovered models are managed by the sync. + resources :llm_models, only: %i[create destroy], controller: "admin/llm_models" + # Keyed by feature key rather than by record id: the binding is an attribute # of a registered feature, and a feature may not have a row yet. resources :llm_feature_bindings, only: %i[index update], controller: "admin/llm_feature_bindings" diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb new file mode 100644 index 000000000000..4a4336ff36d8 --- /dev/null +++ b/spec/requests/admin/llm_models_spec.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +# Manual entry exists for servers that route /v1/chat/completions but expose no +# model list -- OpenProject's own hosted gateway does exactly that today. +RSpec.describe "Admin manual LLM models", :llm_server_helpers, :skip_csrf, :webmock, type: :rails_request do + let(:admin) { create(:admin) } + let(:base_url) { "https://example.com/v1" } + let!(:connection) { create(:llm_connection, base_url:) } + + before { login_as admin } + + describe "POST /admin/llm_models" do + it "adds a model an administrator names" do + post llm_models_path, params: { llm_model: { external_id: "qwen3.6-35b-a3b" } } + + expect(response).to have_http_status(:see_other) + llm_model = connection.models.find_by(external_id: "qwen3.6-35b-a3b") + expect(llm_model).to be_manual + expect(connection.available_model_ids).to include("qwen3.6-35b-a3b") + end + + it "rejects a duplicate" do + create(:llm_model, llm_connection: connection, external_id: "already-there") + + post llm_models_path, params: { llm_model: { external_id: "already-there" } } + + expect(connection.models.where(external_id: "already-there").count).to eq(1) + end + + it "makes the model bindable straight away" do + post llm_models_path, params: { llm_model: { external_id: "qwen3.6-35b-a3b" } } + + patch llm_feature_binding_path("description_assistant"), + params: { llm_feature_binding: { model_id: "qwen3.6-35b-a3b" } } + + expect(connection.feature_bindings.find_by(feature_key: "description_assistant").model_id) + .to eq("qwen3.6-35b-a3b") + end + end + + describe "a refresh that cannot see the manual model" do + it "keeps it, and withdraws discovered models instead" do + create(:llm_model, llm_connection: connection, external_id: "was-discovered") + post llm_models_path, params: { llm_model: { external_id: "hand-typed" } } + mock_llm_models_response(base_url) + + post refresh_models_llm_connection_path + + expect(connection.models.find_by(external_id: "hand-typed")).to be_active + expect(connection.models.find_by(external_id: "was-discovered")).not_to be_active + expect(connection.available_model_ids).to include("hand-typed", "qwen3.6-27b") + end + end + + describe "DELETE /admin/llm_models/:id" do + it "removes a manual model" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") + + delete llm_model_path(llm_model) + + expect(response).to have_http_status(:see_other) + expect(LlmModel.where(id: llm_model.id)).to be_empty + end + + # Discovered models are the server's to add and remove, not the administrator's. + it "refuses to remove a discovered model" do + llm_model = create(:llm_model, llm_connection: connection, external_id: "from-server") + + delete llm_model_path(llm_model) + + expect(response).to have_http_status(:not_found) + expect(LlmModel.where(id: llm_model.id)).to exist + end + end +end From ce987eda49cd82c9d797a4bda16ade3935d9ce5b Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 10 Aug 2026 12:58:30 +0100 Subject: [PATCH 16/44] [#66020] Add api_format and custom_headers to the connection api_format records which dialect the server speaks. Only "openai" is implemented and it is the default; the column exists so that adding Azure -- the one non-OpenAI format with a real constituency under #62215's bring-your-own-infrastructure goal -- is a new adapter rather than a migration. custom_headers is sent with every request, which is what Azure's api-version and gateway-specific headers will need. Note the class is spelled AddAPIFormatToLlmConnections: inflections.rb registers "API" as an acronym, so Rails resolves this filename to that constant and silently skips a class named any other way. The migration reported itself as pending while db:migrate did nothing. https://community.openproject.org/work_packages/66020 --- app/models/llm_feature_binding.rb | 2 +- ...90100_add_api_format_to_llm_connections.rb | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20260812090100_add_api_format_to_llm_connections.rb diff --git a/app/models/llm_feature_binding.rb b/app/models/llm_feature_binding.rb index 0c1601fe2b25..5ff1214d200d 100644 --- a/app/models/llm_feature_binding.rb +++ b/app/models/llm_feature_binding.rb @@ -58,7 +58,7 @@ def inherits_default? = model_id.blank? # catalogue changed. def dangling? resolved = resolved_model_id - resolved.present? && llm_connection.catalogue_model_ids.exclude?(resolved) + resolved.present? && llm_connection.available_model_ids.exclude?(resolved) end def locked? = locked_at.present? diff --git a/db/migrate/20260812090100_add_api_format_to_llm_connections.rb b/db/migrate/20260812090100_add_api_format_to_llm_connections.rb new file mode 100644 index 000000000000..97aeedb6978b --- /dev/null +++ b/db/migrate/20260812090100_add_api_format_to_llm_connections.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Named with a capital API: config/initializers/inflections.rb registers "API" +# as an acronym, so Rails resolves this file to AddAPIFormatToLlmConnections and +# silently skips a class spelled any other way. +class AddAPIFormatToLlmConnections < ActiveRecord::Migration[8.1] + def change + # Which dialect the server speaks. Only "openai" is implemented; the column + # exists so that adding Azure -- the one non-OpenAI format with a real + # constituency under #62215's bring-your-own-infrastructure goal -- is a new + # adapter rather than a migration. + change_table :llm_connections, bulk: true do |t| + t.string :api_format, null: false, default: "openai" + # Provider-specific headers sent with every request, e.g. Azure's + # api-version or a gateway's own key header. + t.jsonb :custom_headers, null: false, default: {} + end + end +end From 68d3e56221d8a21c0d42d3537ba8ecb7cfcbc121 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 10 Aug 2026 15:22:09 +0100 Subject: [PATCH 17/44] [#66020] Use RubyLLM for provider coverage and published capabilities Adds ruby_llm and uses it for two things: model discovery on providers that do not speak the OpenAI model-list API, and published capability metadata. api_format now selects between two discovery strategies. OpenAI-compatible endpoints are queried live, so the list is what that endpoint actually serves. Anthropic, Gemini, Bedrock and the rest each list models differently -- and Bedrock needs request signing rather than a bearer token -- so their lists come from RubyLLM's registry. That is a published catalogue rather than a live query, which is worth knowing: it describes what the provider offers in general, not what these particular credentials can reach. Capabilities are filled in automatically where the registry knows the model: gpt-4o arrives with tool calling, structured output and vision set, and a context window. A self-hosted id it has never heard of yields nothing, which is the case an administrator resolves by hand. Registry-derived verdicts are recorded with source "metadata" and never overwrite one from a probe or an administrator. Both of those looked at this deployment; the registry did not, and it disagrees with itself across providers -- the same weights are catalogued with contradictory capability flags and context windows an order of magnitude apart. The capability vocabulary grows to what these sources actually report: tool calling, structured output, vision and reasoning, alongside embeddings. https://community.openproject.org/work_packages/66020 --- Gemfile | 6 ++ Gemfile.lock | 24 ++++++ app/models/llm_model.rb | 6 +- app/services/llm/adapters.rb | 34 ++++++-- app/services/llm/adapters/registry_backed.rb | 68 +++++++++++++++ app/services/llm/capabilities.rb | 79 +++++++++++++++++ .../enrich_capabilities_service.rb | 85 +++++++++++++++++++ .../llm_connections/sync_models_service.rb | 2 + lib_static/open_project/llm/features.rb | 14 +-- 9 files changed, 299 insertions(+), 19 deletions(-) create mode 100644 app/services/llm/adapters/registry_backed.rb create mode 100644 app/services/llm/capabilities.rb create mode 100644 app/services/llm_connections/enrich_capabilities_service.rb diff --git a/Gemfile b/Gemfile index 9721d5a5ba88..eea23bd5dcf8 100644 --- a/Gemfile +++ b/Gemfile @@ -256,6 +256,12 @@ gem "turbo-rails", "~> 2.0.20" gem "httpx", "~> 1.7.5" +# Provider adapters and a model metadata registry for the AI features. Used as +# transport and as a source of published model capabilities; what a given +# connection actually offers is tracked in llm_models / llm_capability_verdicts, +# never in RubyLLM's application-wide registry. +gem "ruby_llm", "~> 1.16" + # Brings actual deep-freezing to most ruby objects gem "ice_nine" diff --git a/Gemfile.lock b/Gemfile.lock index 9f29dd02ac9f..fc75bce21a9a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -565,6 +565,7 @@ GEM escape_utils (1.3.0) et-orbi (1.4.0) tzinfo + event_stream_parser (1.0.0) eventmachine (1.2.7) eventmachine_httpserver (0.2.1) excon (1.5.0) @@ -580,8 +581,12 @@ GEM logger faraday-follow_redirects (0.5.0) faraday (>= 1, < 3) + faraday-multipart (1.2.0) + multipart-post (~> 2.0) faraday-net_http (3.4.4) net-http (~> 0.5) + faraday-retry (2.4.0) + faraday (~> 2.0) ferrum (0.17.2) addressable (~> 2.5) base64 (~> 0.2) @@ -842,6 +847,7 @@ GEM prism (~> 1.5) msgpack (1.8.3) multi_json (1.21.1) + multipart-post (2.4.1) mustermann (4.0.0) net-http (0.9.1) uri (>= 0.11.1) @@ -1382,6 +1388,17 @@ GEM ruby-vips (2.3.0) ffi (~> 1.12) logger + ruby_llm (1.16.0) + base64 + event_stream_parser (~> 1) + faraday (>= 1.10.0) + faraday-multipart (>= 1) + faraday-net_http (>= 1) + faraday-retry (>= 1) + marcel (~> 1) + ruby_llm-schema (~> 0) + zeitwerk (~> 2) + ruby_llm-schema (0.4.0) rubytree (2.2.0) json (~> 2.0, > 2.9) rubyzip (2.4.1) @@ -1737,6 +1754,7 @@ DEPENDENCIES ruby-duration (~> 3.2.0) ruby-prof ruby-progressbar (~> 1.13.0) + ruby_llm (~> 1.16) rubytree (~> 2.2.0) sanitize (~> 7.0.0) scimitar (~> 2.13) @@ -1901,6 +1919,7 @@ CHECKSUMS erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9 escape_utils (1.3.0) sha256=dffb7010922880ace6ceed642156c64e2a64620f27e0849f43bc4f68fd3c2c09 et-orbi (1.4.0) sha256=6c7e3c90779821f9e3b324c5e96fda9767f72995d6ae435b96678a4f3e2de8bc + event_stream_parser (1.0.0) sha256=a2683bab70126286f8184dc88f7968ffc4028f813161fb073ec90d171f7de3c8 eventmachine (1.2.7) sha256=994016e42aa041477ba9cff45cbe50de2047f25dd418eba003e84f0d16560972 eventmachine_httpserver (0.2.1) sha256=5db5e8a23754204d43592e5fcc2160457c57c870babe6307c4e61fc95019b809 excon (1.5.0) sha256=c503ad1d0123bc8ab2a062ff3789dc891ec368cb9e13765ab88a9c58c8bb6d50 @@ -1908,7 +1927,9 @@ CHECKSUMS factory_bot_rails (6.5.1) sha256=d3cc4851eae4dea8a665ec4a4516895045e710554d2b5ac9e68b94d351bc6d68 faraday (2.14.3) sha256=1882247e6766615c8220b4392bf1d27f6ebb63d8e28267587cef1fb0bf37f278 faraday-follow_redirects (0.5.0) sha256=5cde93c894b30943a5d2b93c2fe9284216a6b756f7af406a1e55f211d97d10ad + faraday-multipart (1.2.0) sha256=7d89a949693714176f612323ca13746a2ded204031a6ba528adee788694ef757 faraday-net_http (3.4.4) sha256=0e78af151747ed1b00f33e25973b4bc220d7f16c00c39676817c8b12331eb588 + faraday-retry (2.4.0) sha256=7b79c48fb7e56526faf247b12d94a680071ff40c9fda7cf1ec1549439ad11ebe ferrum (0.17.2) sha256=2c2540a850b211a46f4d81de21bfd62048f507e4c327d1807225c3823c17e6ee ffi (1.17.4-aarch64-linux-gnu) sha256=b208f06f91ffd8f5e1193da3cae3d2ccfc27fc36fba577baf698d26d91c080df ffi (1.17.4-aarch64-linux-musl) sha256=9286b7a615f2676245283aef0a0a3b475ae3aae2bb5448baace630bb77b91f39 @@ -2008,6 +2029,7 @@ CHECKSUMS minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1 msgpack (1.8.3) sha256=8bda4a6428d3244e50d6bd55854d354edbada88a4e1f4f5731a39a0f86bee6a1 multi_json (1.21.1) sha256=e6126a31808e3b4d19f483c775ceac34df190dffa62adfb63a165ee14ba68080 + multipart-post (2.4.1) sha256=9872d03a8e552020ca096adadbf5e3cb1cd1cdd6acd3c161136b8a5737cdb4a8 mustermann (4.0.0) sha256=91f67411bb208d1d93c41e6128cb3b0f8ddd9ec7c45966f1007e1c43c08040d7 my_page (1.0.0) net-http (0.9.1) sha256=25ba0b67c63e89df626ed8fac771d0ad24ad151a858af2cc8e6a716ca4336996 @@ -2230,6 +2252,8 @@ CHECKSUMS ruby-rc4 (0.1.5) sha256=00cc40a39d20b53f5459e7ea006a92cf584e9bc275e2a6f7aa1515510e896c03 ruby-saml (1.18.1) sha256=1b0e7a44aef150b4197955f5e015d593672e242cfdc5d06aa7554ec2350b9107 ruby-vips (2.3.0) sha256=e685ec02c13969912debbd98019e50492e12989282da5f37d05f5471442f5374 + ruby_llm (1.16.0) sha256=26bd5310cf2ce55f74a60f8aae0b0d0327b586ff4532c84828103c3b2b905a18 + ruby_llm-schema (0.4.0) sha256=e930f5a5316f9301bff3fb7fe572e44727d05bb8e50621001bbb49a47d63b8da rubytree (2.2.0) sha256=e312dc1ed814153583b57d9662e6caac1fd60830886a571b48bb06daef906eb6 rubyzip (2.4.1) sha256=8577c88edc1fde8935eb91064c5cb1aef9ad5494b940cf19c775ee833e075615 safety_net_attestation (0.5.0) sha256=c8cd01dd550dbe8553862918af6355a04672db11d218ec96104ce3955293f2aa diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb index 623dc1ae5b03..c5cf6b34b2ad 100644 --- a/app/models/llm_model.rb +++ b/app/models/llm_model.rb @@ -46,10 +46,10 @@ class LlmModel < ApplicationRecord def name = display_name.presence || external_id - # vLLM and SGLang report the operator's real --max-model-len here, which is - # more trustworthy for this deployment than any published figure. + # The server's own figure wins: vLLM and SGLang report the operator's actual + # --max-model-len, where a registry only knows what some vendor publishes. def context_window - raw_metadata["max_model_len"] + raw_metadata["max_model_len"] || raw_metadata["context_window"] end # Discovered models that the server stopped offering are deactivated rather diff --git a/app/services/llm/adapters.rb b/app/services/llm/adapters.rb index 15948302d9ce..f319abcfc1c6 100644 --- a/app/services/llm/adapters.rb +++ b/app/services/llm/adapters.rb @@ -41,15 +41,37 @@ module Llm module Adapters class UnsupportedFormat < StandardError; end - FORMATS = { - "openai" => "Llm::Adapters::Openai" - }.freeze + # Formats an administrator can choose. "openai" covers OpenAI itself and the + # great majority of gateways and self-hosted inference servers; the rest are + # RubyLLM providers whose model lists come from its registry. + OPENAI_COMPATIBLE = "openai" + + FORMATS = %w[ + openai + anthropic + gemini + mistral + deepseek + openrouter + perplexity + xai + ollama + gpustack + azure + bedrock + vertexai + ].freeze def self.for(connection) - class_name = FORMATS[connection.api_format.to_s] - raise UnsupportedFormat, connection.api_format.to_s if class_name.nil? + format = connection.api_format.to_s + raise UnsupportedFormat, format unless FORMATS.include?(format) - class_name.constantize.new(connection) + if format == OPENAI_COMPATIBLE + # Queried live, so the list is what this endpoint actually serves. + Openai.new(connection) + else + RegistryBacked.new(connection) + end end end end diff --git a/app/services/llm/adapters/registry_backed.rb b/app/services/llm/adapters/registry_backed.rb new file mode 100644 index 000000000000..f551f878cef7 --- /dev/null +++ b/app/services/llm/adapters/registry_backed.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Adapters + # Discovery for providers that do not speak the OpenAI model-list API. + # + # Anthropic, Gemini, Bedrock and the rest each list models differently -- and + # some, like Bedrock, need request signing rather than a bearer token. Rather + # than implement each, the model list comes from RubyLLM's registry for that + # provider, which is a published catalogue rather than a live query. + # + # The consequence is worth stating: this list is what the provider offers in + # general, not what these particular credentials can reach. An administrator + # can add anything missing by hand. + class RegistryBacked + def initialize(connection) + @connection = connection + end + + def models + RubyLLM.models.by_provider(connection.api_format.to_sym).map do |info| + { + id: info.id, + display_name: info.name, + raw: { "context_window" => info.context_window, "owned_by" => connection.api_format } + } + end + rescue StandardError => e + raise Llm::Client::ApiError.new("Model registry lookup failed: #{e.class} #{e.message}", status: nil) + end + + # Nothing is queried, so there is no server to characterise. + def server_flavour = connection.api_format + + private + + attr_reader :connection + end + end +end diff --git a/app/services/llm/capabilities.rb b/app/services/llm/capabilities.rb new file mode 100644 index 000000000000..cf610a3067c3 --- /dev/null +++ b/app/services/llm/capabilities.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # The capabilities a model may have, and how to read the published ones. + module Capabilities + ALL = %i[embeddings function_calling structured_output vision reasoning].freeze + + # Capabilities a chat model can be asked about. +embeddings+ is deliberately + # absent: it is a different kind of model, not a feature of a chat one. + CHAT = (ALL - %i[embeddings]).freeze + EMBEDDING = %i[embeddings].freeze + + module_function + + # What a public registry says about this model id. + # + # Advisory only. The registry describes a model as some vendor deploys it, + # which is not the same as this deployment: the same weights are catalogued + # with contradictory capability flags and context windows an order of + # magnitude apart across providers. Verdicts derived from it are therefore + # recorded with source "metadata", and an administrator can overrule them. + # + # @return [Hash{Symbol => Symbol}, nil] capability => :supported / :unsupported, + # or nil when the registry does not know the model -- the normal case for a + # self-hosted server. + def published_for(model_id) + info = RubyLLM.models.find(model_id) + + { states: states_from(info), context_window: info.context_window, display_name: info.name } + rescue RubyLLM::ModelNotFoundError + nil + rescue StandardError => e + # Registry lookup is an enrichment; it must never break a model sync. + Rails.logger.info { "LLM capability lookup for #{model_id} failed: #{e.class} #{e.message}" } + nil + end + + def states_from(info) + embedding = info.type.to_s == "embedding" + published = Array(info.capabilities).map(&:to_sym) + relevant = embedding ? EMBEDDING : CHAT + + states = relevant.index_with { |capability| published.include?(capability) ? :supported : :unsupported } + states.merge(embeddings: embedding ? :supported : :unsupported) + end + + def label(capability) + I18n.t("llm.capabilities.#{capability}.label") + end + end +end diff --git a/app/services/llm_connections/enrich_capabilities_service.rb b/app/services/llm_connections/enrich_capabilities_service.rb new file mode 100644 index 000000000000..e5d53976d4b0 --- /dev/null +++ b/app/services/llm_connections/enrich_capabilities_service.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Fills in what a public registry publishes about the models a connection offers. + # + # This is the "the provider tells us" path. A hosted provider's model ids are + # catalogued, so capabilities arrive without asking the server anything. A + # self-hosted deployment naming its model "default" or "my-finetune-v3" is not + # catalogued, nothing is filled in, and the administrator enters capabilities + # by hand instead. + # + # An administrator's assertion is never overwritten: they know things about + # their deployment that no registry can. + class EnrichCapabilitiesService + def initialize(connection) + @connection = connection + end + + def call + connection.models.active.find_each { |llm_model| enrich(llm_model) } + + ServiceResult.success(result: connection) + end + + private + + attr_reader :connection + + def enrich(llm_model) + published = Llm::Capabilities.published_for(llm_model.external_id) + return if published.nil? + + apply_metadata(llm_model, published) + published[:states].each { |capability, state| record(llm_model.external_id, capability, state) } + end + + def apply_metadata(llm_model, published) + attributes = {} + attributes[:display_name] = published[:display_name] if llm_model.display_name.blank? + + if published[:context_window].present? && llm_model.raw_metadata["max_model_len"].blank? + attributes[:raw_metadata] = llm_model.raw_metadata.merge("context_window" => published[:context_window]) + end + + llm_model.update!(attributes) if attributes.any? + end + + def record(model_id, capability, state) + verdict = connection.capability_verdicts.find_or_initialize_by(model_id:, capability: capability.to_s) + # Anything an administrator or a probe established beats a published claim: + # both looked at this deployment, the registry did not. + return if verdict.persisted? && verdict.source.in?(%w[admin probe]) + + verdict.update!(state: state.to_s, source: "metadata", checked_at: Time.current) + end + end +end diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb index 1aa269ed94a3..8421757cde71 100644 --- a/app/services/llm_connections/sync_models_service.rb +++ b/app/services/llm_connections/sync_models_service.rb @@ -63,6 +63,8 @@ def store(cards) withdraw_models_absent_from(cards) discard_verdicts_for_vanished_models end + + EnrichCapabilitiesService.new(connection).call end def connection_attributes diff --git a/lib_static/open_project/llm/features.rb b/lib_static/open_project/llm/features.rb index 529248b8a1c5..5165d24ca2e8 100644 --- a/lib_static/open_project/llm/features.rb +++ b/lib_static/open_project/llm/features.rb @@ -67,17 +67,11 @@ module Features KINDS = %i[chat embedding].freeze - # Deliberately small. Every capability is a probe, an admin-facing - # explanation and a maintenance burden, so one is only added when a - # shipping feature cannot work without it. - # - # :embeddings - the only hard gate. Cannot be emulated: it is a - # different endpoint answering with vectors. - # :structured_output - advisory only. Responses are validated and - # repaired regardless, so constrained decoding is - # an optimisation rather than a requirement. + # Mirrors Llm::Capabilities, which owns the vocabulary and knows how to + # read published values from the model registry. Duplicated as literals + # here because lib_static is autoloaded once, before app/ is available. CAPABILITIES = { - chat: %i[structured_output].freeze, + chat: %i[function_calling structured_output vision reasoning].freeze, embedding: %i[embeddings].freeze }.freeze From 95571c140d0eafd0f32ef0364997f2cf6f144688 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 10 Aug 2026 17:50:44 +0100 Subject: [PATCH 18/44] [#66020] Let an administrator choose the endpoint format The connection page now offers the API format alongside the URL and key. OpenAI-compatible remains the default and covers most gateways and self-hosted servers; the others are the providers RubyLLM speaks. Validated against the adapter list rather than a free string, so an unknown format is rejected at save time instead of failing at first use. https://community.openproject.org/work_packages/66020 --- app/contracts/llm_connections/base_contract.rb | 2 ++ app/controllers/admin/llm_connections_controller.rb | 2 +- app/forms/llm_connections/connection_form.rb | 13 +++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/app/contracts/llm_connections/base_contract.rb b/app/contracts/llm_connections/base_contract.rb index e0ef64789970..bb86534883b5 100644 --- a/app/contracts/llm_connections/base_contract.rb +++ b/app/contracts/llm_connections/base_contract.rb @@ -33,12 +33,14 @@ module LlmConnections # environment. Deliberately makes no network request -- see UpdateContract. class BaseContract < ModelContract attribute :enabled + attribute :api_format attribute :base_url attribute :api_key attribute :default_chat_model_id attribute :default_embedding_model_id validates :base_url, presence: true + validates :api_format, inclusion: { in: Llm::Adapters::FORMATS } # Resolves to the validate_url gem, which defaults to http and https. Plain # http is deliberately allowed: an on-premise LLM server on an internal # network commonly terminates TLS elsewhere, or not at all. diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb index 52cdaac7f13c..c5c1ef4222d2 100644 --- a/app/controllers/admin/llm_connections_controller.rb +++ b/app/controllers/admin/llm_connections_controller.rb @@ -90,7 +90,7 @@ def redirect_with_error(message) # saved value, so submitting it unchanged posts an empty string. def llm_connection_params permitted = params.expect( - llm_connection: %i[enabled base_url api_key default_chat_model_id default_embedding_model_id] + llm_connection: %i[enabled api_format base_url api_key default_chat_model_id default_embedding_model_id] ) permitted.delete(:api_key) if permitted[:api_key].blank? permitted.to_h.symbolize_keys diff --git a/app/forms/llm_connections/connection_form.rb b/app/forms/llm_connections/connection_form.rb index 30e061fd2581..24471b4134e8 100644 --- a/app/forms/llm_connections/connection_form.rb +++ b/app/forms/llm_connections/connection_form.rb @@ -38,6 +38,19 @@ class ConnectionForm < ApplicationForm disabled: read_only? ) + f.select_list( + name: :api_format, + label: LlmConnection.human_attribute_name(:api_format), + caption: I18n.t("admin.llm_connections.form.api_format_caption"), + include_blank: false, + input_width: :medium, + disabled: read_only? + ) do |select| + Llm::Adapters::FORMATS.each do |format| + select.option(value: format, label: I18n.t("llm.api_formats.#{format}")) + end + end + f.text_field( name: :base_url, label: LlmConnection.human_attribute_name(:base_url), From 424dddb0d2db7326314e7ae1b0004eb2aac82193 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 10 Aug 2026 20:14:21 +0100 Subject: [PATCH 19/44] [#66020] Let an administrator set model capabilities by hand Where nothing publishes a model's capabilities -- a self-hosted server naming its model "default", or a gateway exposing no model list at all -- the operator knows what it can do and OpenProject does not. Each model now has an edit screen where capabilities can be set, alongside a display name. Assertions are stored as admin-sourced verdicts and survive re-detection: an administrator looked at this deployment, and neither a published registry nor a probe outranks that. Setting a capability back to "not specified" clears the assertion rather than recording ignorance as fact, so later detection can still fill it in. A capability established by a probe or the registry is shown with its source rather than silently presented as the administrator's own. https://community.openproject.org/work_packages/66020 --- .../llm_connections/models_row_component.rb | 20 +++-- .../admin/llm_models_controller.rb | 41 +++++++++ app/views/admin/llm_models/edit.html.erb | 90 +++++++++++++++++++ config/locales/en.yml | 44 ++++++++- config/routes.rb | 2 +- spec/requests/admin/llm_models_spec.rb | 45 ++++++++++ 6 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 app/views/admin/llm_models/edit.html.erb diff --git a/app/components/llm_connections/models_row_component.rb b/app/components/llm_connections/models_row_component.rb index 6ee82abdff0f..79f79c749205 100644 --- a/app/components/llm_connections/models_row_component.rb +++ b/app/components/llm_connections/models_row_component.rb @@ -59,14 +59,20 @@ def source end def button_links - return [] unless llm_model.manual? + llm_model.manual? ? [edit_link, delete_link] : [edit_link] + end + + def edit_link + link_to(helpers.op_icon("icon-edit"), + url_helpers.edit_llm_model_path(llm_model), + title: I18n.t(:button_edit)) + end - [ - link_to(helpers.op_icon("icon-delete"), - url_helpers.llm_model_path(llm_model), - data: { turbo_method: :delete, turbo_confirm: I18n.t(:text_are_you_sure) }, - title: I18n.t(:button_delete)) - ] + def delete_link + link_to(helpers.op_icon("icon-delete"), + url_helpers.llm_model_path(llm_model), + data: { turbo_method: :delete, turbo_confirm: I18n.t(:text_are_you_sure) }, + title: I18n.t(:button_delete)) end end end diff --git a/app/controllers/admin/llm_models_controller.rb b/app/controllers/admin/llm_models_controller.rb index 41465a7d39fe..3c1f06ef37e9 100644 --- a/app/controllers/admin/llm_models_controller.rb +++ b/app/controllers/admin/llm_models_controller.rb @@ -41,6 +41,11 @@ class LlmModelsController < ApplicationController before_action :require_admin before_action :set_connection + def edit + @llm_model = @connection.models.find(params.expect(:id)) + @verdicts = @connection.capability_verdicts.for_model(@llm_model.external_id).index_by(&:capability) + end + def create llm_model = @connection.models.new(model_params.merge(manual: true)) @@ -53,6 +58,18 @@ def create redirect_to llm_connection_path, status: :see_other end + def update + @llm_model = @connection.models.find(params.expect(:id)) + + ActiveRecord::Base.transaction do + @llm_model.update!(display_name: params.dig(:llm_model, :display_name)) + apply_capabilities(@llm_model) + end + + flash[:notice] = t(".success", model: @llm_model.external_id) + redirect_to llm_connection_path, status: :see_other + end + def destroy llm_model = @connection.models.manual.find(params.expect(:id)) llm_model.destroy! @@ -70,5 +87,29 @@ def set_connection def model_params params.expect(llm_model: %i[external_id display_name]) end + + # Stored as admin-sourced verdicts, which survive re-detection: an + # administrator knows things about their deployment that neither a published + # registry nor a probe can determine. + def apply_capabilities(llm_model) + submitted = params.fetch(:capabilities, {}).permit!.to_h + + Llm::Capabilities::ALL.each do |capability| + assert(llm_model.external_id, capability, submitted[capability.to_s].presence) + end + end + + def assert(model_id, capability, state) + verdict = @connection.capability_verdicts + .find_or_initialize_by(model_id:, capability: capability.to_s) + + if state.blank? + # "Not specified" clears an assertion rather than recording ignorance as + # fact; detection may fill it in later. + verdict.destroy! if verdict.persisted? && verdict.source_admin? + else + verdict.update!(state:, source: "admin", checked_at: Time.current) + end + end end end diff --git a/app/views/admin/llm_models/edit.html.erb b/app/views/admin/llm_models/edit.html.erb new file mode 100644 index 000000000000..984efd8d2f42 --- /dev/null +++ b/app/views/admin/llm_models/edit.html.erb @@ -0,0 +1,90 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection"), @llm_model.external_id %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { @llm_model.external_id } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + { href: llm_connection_path, text: t("menus.admin.llm_connection") }, + @llm_model.external_id] + ) + end +%> + +<%= form_with(url: llm_model_path(@llm_model), method: :patch) do %> + <%= render(Primer::Box.new(mb: 3)) do %> + <%= label_tag("llm_model_display_name", LlmModel.human_attribute_name(:display_name), class: "FormControl-label") %> + <%= text_field_tag( + "llm_model[display_name]", @llm_model.display_name, + id: "llm_model_display_name", + autocomplete: "off", + class: "FormControl-input" + ) %> + <% end %> + + <%= render(Primer::Beta::Subhead.new(mt: 4)) do |component| %> + <% component.with_heading(tag: :h3) { t(".capabilities_heading") } %> + <% component.with_description { t(".capabilities_description") } %> + <% end %> + + <% Llm::Capabilities::ALL.each do |capability| %> + <% verdict = @verdicts[capability.to_s] %> + <%= render(Primer::Box.new(mb: 2)) do %> + <%= label_tag("capability_#{capability}", Llm::Capabilities.label(capability), class: "FormControl-label") %> + <%= select_tag( + "capabilities[#{capability}]", + options_for_select( + [[t(".state_unspecified"), ""], + [t(".state_supported"), "supported"], + [t(".state_unsupported"), "unsupported"]], + verdict&.source_admin? ? verdict.state : "" + ), + id: "capability_#{capability}", + class: "FormControl-select" + ) %> + <% if verdict.present? && !verdict.source_admin? %> + <%= render(Primer::Beta::Text.new(tag: :div, color: :muted, font_size: :small)) do %> + <%= t( + ".current_verdict", + state: t("llm.verdict_states.#{verdict.state}"), + source: t("llm.verdict_sources.#{verdict.source}") + ) %> + <% end %> + <% end %> + <% end %> + <% end %> + + <%= render(Primer::Beta::Button.new(type: :submit, scheme: :primary, mt: 3)) { t(:button_save) } %> + <%= render(Primer::Beta::Button.new(tag: :a, href: llm_connection_path, ml: 2, mt: 3)) { t(:button_cancel) } %> +<% end %> diff --git a/config/locales/en.yml b/config/locales/en.yml index cfc460527dae..13d2bdf0eed0 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -214,6 +214,7 @@ en: # ActiveRecord::Base.human_attribute_name strips the _id suffix. model: "Model" llm_connection: + api_format: "API format" api_key: "API key" base_url: "Host URL" # ActiveRecord::Base.human_attribute_name strips the _id suffix, so these @@ -1442,6 +1443,7 @@ en: success: "The model for %{feature} has been saved." llm_connections: form: + api_format_caption: "Which API the server speaks. Choose OpenAI-compatible for most gateways and self-hosted servers." api_key_caption: "The key OpenProject authenticates with. Leave blank if the server requires no authentication." api_key_caption_stored: "A key is stored. Leave blank to keep it, or enter a new one to replace it." base_url_caption: > @@ -1479,6 +1481,16 @@ en: success: "%{model} has been added." destroy: success: "%{model} has been removed." + edit: + capabilities_description: "Set what this model can do. Leave a capability unspecified to let OpenProject determine it." + capabilities_heading: "Capabilities" + current_verdict: "Currently %{state}, %{source}." + description: "Edit how OpenProject uses this model." + state_supported: "Supported" + state_unspecified: "Not specified" + state_unsupported: "Not supported" + update: + success: "%{model} has been updated." mcp_configurations: index: description: "The model context protocol allows AI agents to provide its users with tools and resources exposed by this OpenProject instance. This feature is still in beta." @@ -3953,14 +3965,40 @@ en: errors: page_not_found: "Cannot find the wiki page '%{name}'." llm: + api_formats: + anthropic: "Anthropic" + azure: "Azure OpenAI" + bedrock: "AWS Bedrock" + deepseek: "DeepSeek" + gemini: "Google Gemini" + gpustack: "GPUStack" + mistral: "Mistral" + ollama: "Ollama" + openai: "OpenAI-compatible" + openrouter: "OpenRouter" + perplexity: "Perplexity" + vertexai: "Google Vertex AI" + xai: "xAI" capabilities: embeddings: label: "Embeddings" - unsupported: "This model returned an error for embedding requests." - unknown: "The server does not report whether this model can produce embeddings." + function_calling: + label: "Tool calling" + reasoning: + label: "Reasoning" structured_output: label: "Structured output" - unknown: "The server does not report whether this model supports structured output." + vision: + label: "Vision" + verdict_sources: + admin: "set by an administrator" + metadata: "from the model registry" + observed: "observed in use" + probe: "verified against the server" + verdict_states: + supported: "Supported" + unknown: "Not verified" + unsupported: "Not supported" features: description_assistant: caption: "Rewrites and restructures work package text on request." diff --git a/config/routes.rb b/config/routes.rb index 7792bc2f7590..70c3b4089631 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -697,7 +697,7 @@ end # Manual entries only; discovered models are managed by the sync. - resources :llm_models, only: %i[create destroy], controller: "admin/llm_models" + resources :llm_models, only: %i[create edit update destroy], controller: "admin/llm_models" # Keyed by feature key rather than by record id: the binding is an attribute # of a registered feature, and a feature may not have a row yet. diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb index 4a4336ff36d8..b0dd057d4f2c 100644 --- a/spec/requests/admin/llm_models_spec.rb +++ b/spec/requests/admin/llm_models_spec.rb @@ -82,6 +82,51 @@ end end + describe "PATCH /admin/llm_models/:id" do + let!(:llm_model) { create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") } + + it "stores capabilities an administrator asserts" do + patch llm_model_path(llm_model), + params: { llm_model: { display_name: "Hand typed" }, + capabilities: { embeddings: "supported", vision: "unsupported" } } + + expect(response).to have_http_status(:see_other) + expect(llm_model.reload.display_name).to eq("Hand typed") + + verdicts = connection.capability_verdicts.for_model("hand-typed").pluck(:capability, :state, :source) + expect(verdicts).to include(["embeddings", "supported", "admin"], ["vision", "unsupported", "admin"]) + end + + it "makes an asserted capability satisfy a feature that requires it" do + patch llm_model_path(llm_model), params: { capabilities: { embeddings: "supported" } } + + patch llm_feature_binding_path("semantic_search"), + params: { llm_feature_binding: { model_id: "hand-typed" } } + + expect(connection.feature_bindings.find_by(feature_key: "semantic_search").model_id).to eq("hand-typed") + end + + # Clearing an assertion records nothing rather than recording ignorance as + # fact, so detection can still fill it in later. + it "clears an assertion when set back to unspecified" do + patch llm_model_path(llm_model), params: { capabilities: { embeddings: "supported" } } + patch llm_model_path(llm_model), params: { capabilities: { embeddings: "" } } + + expect(connection.capability_verdicts.for_model("hand-typed").for_capability(:embeddings)).to be_empty + end + + # An administrator looked at this deployment; a published registry did not. + it "is not overwritten by registry enrichment" do + patch llm_model_path(llm_model), params: { capabilities: { embeddings: "supported" } } + + LlmConnections::EnrichCapabilitiesService.new(connection).call + + verdict = connection.capability_verdicts.find_by(model_id: "hand-typed", capability: "embeddings") + expect(verdict.source).to eq("admin") + expect(verdict.state).to eq("supported") + end + end + describe "DELETE /admin/llm_models/:id" do it "removes a manual model" do llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") From 5e07ae977e50b49b22c8922ab2c4ac9f213b4927 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Mon, 10 Aug 2026 23:05:16 +0100 Subject: [PATCH 20/44] [#66020] Treat the model list as optional, not a precondition A connection could not be saved unless the server returned a model list, which made every server without one unusable -- including OpenProject's own hosted gateway, which routes /v1/chat/completions and answers 404 for /v1/models. Worse, it made the manual model entry unreachable on exactly the connections that need it: you could not save the connection, so you never reached the page that lets you name the models yourself. Reachability and credentials still gate the save. A connection that times out, is refused, resolves to a blocked address, or has its key rejected is still an error, because those say the connection does not work. A 404, 405 or 501 on the model list says only that the server does not publish one, so the save proceeds and the administrator is told to add models by hand. Formats whose model list comes from the registry rather than the server no longer probe the base URL at all; there was never anything there to find. One limitation worth knowing: on a server that exposes no model list there is nothing cheap left to authenticate against, so a wrong API key is not detected at save time. Validating it would mean spending a chat completion. https://community.openproject.org/work_packages/66020 --- .../admin/llm_connections_controller.rb | 15 +++++++++- app/validators/llm_server_validator.rb | 30 ++++++++++++++----- config/locales/en.yml | 2 +- .../llm_connections/update_contract_spec.rb | 22 +++++++++++--- spec/requests/admin/llm_connections_spec.rb | 26 ++++++++++++++++ 5 files changed, 81 insertions(+), 14 deletions(-) diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb index c5c1ef4222d2..f8a97cfa4ddb 100644 --- a/app/controllers/admin/llm_connections_controller.rb +++ b/app/controllers/admin/llm_connections_controller.rb @@ -45,7 +45,7 @@ def update .new(user: current_user, model: @connection) .call(**llm_connection_params) - result.on_success { redirect_with_notice(t(".success")) } + result.on_success { redirect_after_save } result.on_failure { render_form_with_errors } end @@ -71,6 +71,19 @@ def set_connection @connection = LlmConnection.instance end + # A connection can be perfectly usable without offering a model list, so the + # save succeeds either way; the administrator is told what to do next rather + # than being left with an empty table and no explanation. + def redirect_after_save + if @connection.reload.models.none? + flash[:warning] = t(".no_models") + else + flash[:notice] = t(".success") + end + + redirect_to llm_connection_path, status: :see_other + end + def render_form_with_errors update_via_turbo_stream(component: ::LlmConnections::FormComponent.new(@connection)) respond_with_turbo_streams { |format| format.html { render :show } } diff --git a/app/validators/llm_server_validator.rb b/app/validators/llm_server_validator.rb index cace33450ac2..a7e506bfaa83 100644 --- a/app/validators/llm_server_validator.rb +++ b/app/validators/llm_server_validator.rb @@ -38,8 +38,13 @@ class LlmServerValidator < ActiveModel::EachValidator CREDENTIAL_ATTRIBUTES = %w[base_url api_key].freeze + # Statuses that mean "this server has no model list here", as opposed to "this + # server is broken". A gateway may route chat completions and nothing else. + MODELS_ENDPOINT_ABSENT = [404, 405, 501].freeze + def validate_each(contract, attribute, value) return if value.blank? + return unless queries_the_server?(contract) return unless credentials_changed?(contract) return unless host_allowed?(contract, attribute, value) @@ -48,6 +53,13 @@ def validate_each(contract, attribute, value) private + # Only OpenAI-compatible connections discover models from the server. For the + # other formats the list comes from the model registry, so there is nothing at + # this URL to probe. + def queries_the_server?(contract) + contract.model.api_format == Llm::Adapters::OPENAI_COMPATIBLE + end + def credentials_changed?(contract) contract.model.changed_attributes.keys.intersect?(CREDENTIAL_ATTRIBUTES) end @@ -95,15 +107,17 @@ def add_error(contract, attribute, error) end end - # A 404 is worth separating: the server answered, so it is reachable and the - # credentials were not the problem. Either the URL is missing or carries the - # wrong version segment, or the server genuinely does not expose a model list. + # A missing model list does not block the save. + # + # The server answered, so it is reachable and the credentials were accepted; it + # simply does not offer a list here. OpenProject's own hosted gateway is exactly + # this case, and it will not be the only one. Blocking would leave an + # administrator unable to save a working connection -- and unable to reach the + # manual model entry that exists for precisely this situation. def add_api_error(contract, attribute, error) - if error.status == 404 - contract.errors.add(attribute, :models_endpoint_missing, path: "#{contract.model.base_url}/models") - else - contract.errors.add(attribute, :not_openai_compatible) - end + return if error.status.in?(MODELS_ENDPOINT_ABSENT) + + contract.errors.add(attribute, :not_openai_compatible) end def client(contract, base_url) diff --git a/config/locales/en.yml b/config/locales/en.yml index 13d2bdf0eed0..82fafa48a98b 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -693,7 +693,6 @@ en: singleton: "Only one LLM connection can be configured." base_url: cannot_be_connected_to: "could not be reached. Please ensure the LLM server is running and reachable from OpenProject." - models_endpoint_missing: "was reached, but has no model list at %{path}. Check that the URL includes the right API version segment, and that the server exposes a models endpoint." not_openai_compatible: "did not return a valid model list. Please ensure the URL points at an OpenAI-API-compatible endpoint, including the API version segment (for example https://example.com/v1)." request_timed_out: "did not respond in time. Please ensure the LLM server is reachable and not overloaded." ssrf_filtered: "resolves to a blocked address. If the LLM server runs on an internal network, allow its IP via the %{env_name} environment variable." @@ -1475,6 +1474,7 @@ en: models_heading: "Available models" refresh_models: "Refresh models" update: + no_models: "Connected, but the server did not return a model list. Add the models you want to use below." success: "Successfully connected to the LLM server." llm_models: create: diff --git a/spec/contracts/llm_connections/update_contract_spec.rb b/spec/contracts/llm_connections/update_contract_spec.rb index 80ee17a9d8e5..6ba731afc847 100644 --- a/spec/contracts/llm_connections/update_contract_spec.rb +++ b/spec/contracts/llm_connections/update_contract_spec.rb @@ -69,11 +69,25 @@ end # A server can speak the OpenAI API for chat and still not expose a model list: - # OpenProject's own hosted stack does exactly that while #77512 is unreleased. - context "when the server has no model list at that path" do - let!(:models_request) { mock_llm_models_response(base_url, response_code: 404) } + # OpenProject's own hosted gateway does exactly that. Blocking the save would + # leave the administrator unable to configure a working connection at all. + [404, 405, 501].each do |status| + context "when the server answers #{status} for the model list" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: status) } - include_examples "contract is invalid", base_url: :models_endpoint_missing + include_examples "contract is valid" + end + end + + # Formats whose model list comes from the registry have nothing to probe here. + context "with a format that does not discover models from the server" do + let(:connection) { create(:llm_connection, api_format: "anthropic", base_url: "https://previous.example") } + + it "does not contact the base URL" do + contract.validate + + expect(models_request).not_to have_been_made + end end context "when the endpoint is not OpenAI-compatible" do diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb index 760323015825..e520a070bc55 100644 --- a/spec/requests/admin/llm_connections_spec.rb +++ b/spec/requests/admin/llm_connections_spec.rb @@ -82,6 +82,20 @@ end end + # The case that matters for OpenProject's own gateway: chat completions are + # routed, the model list is not. + context "with a server that exposes no model list" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: 404) } + + it "still saves the connection and says models must be added by hand" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-test" } } + + expect(response).to have_http_status(:see_other) + expect(LlmConnection.first.base_url).to eq(base_url) + expect(flash[:warning]).to be_present + end + end + context "with an unreachable server" do let!(:models_request) { mock_llm_models_response(base_url, timeout: true) } @@ -92,6 +106,18 @@ end end + # Reachability and credentials still gate the save; only the model list is + # treated as optional. + context "with rejected credentials" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: 401) } + + it "persists nothing" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-wrong" } } + + expect(LlmConnection.count).to eq(0) + end + end + context "when an API key is already stored" do let!(:connection) { create(:llm_connection, base_url:, api_key: "sk-original") } let!(:models_request) { mock_llm_models_response(base_url) } From 55500e8473992fb619be0acf617c49e121c14060 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Tue, 11 Aug 2026 08:33:52 +0100 Subject: [PATCH 21/44] [#66020] Show and edit model type and context window Three gaps in the model list, all visible on a manually added model. The edit action was unreachable: rows render their button_links only when the table declares has_actions?, which defaults to false, so the link existed and was never drawn. Context window could not be set. A hand-entered model has nothing to report one, so the column showed a dash with no way to fill it. It is now editable, with precedence: an administrator's figure, then the server's (vLLM and SGLang publish the operator's actual --max-model-len), then whatever a registry says about the model in general. Clearing the field falls back rather than blanking. Model type was tracked but invisible. It stays derived from the embeddings verdict rather than stored twice -- a model that produces vectors is an embedding model, and that is the same fact -- but the list now has a Type column reading Chat, Embedding or Unknown, and the edit screen says that marking Embeddings as supported is what makes a model an embedding one. The type lookup is built once per table rather than per row. https://community.openproject.org/work_packages/66020 --- .../llm_connections/models_row_component.rb | 13 ++++++ .../llm_connections/models_table_component.rb | 19 +++++++- .../admin/llm_models_controller.rb | 8 +++- app/models/llm_model.rb | 27 +++++++++-- app/views/admin/llm_models/edit.html.erb | 22 +++++++++ config/locales/en.yml | 13 +++++- spec/requests/admin/llm_models_spec.rb | 45 +++++++++++++++++++ 7 files changed, 141 insertions(+), 6 deletions(-) diff --git a/app/components/llm_connections/models_row_component.rb b/app/components/llm_connections/models_row_component.rb index 79f79c749205..246a7ecfe8f4 100644 --- a/app/components/llm_connections/models_row_component.rb +++ b/app/components/llm_connections/models_row_component.rb @@ -48,6 +48,19 @@ def context_window number_with_delimiter(window) end + # Derived from the embeddings verdict rather than stored separately: a model + # that produces vectors is an embedding model, and that is the same fact. + def kind + case table.embeddings_states[llm_model.external_id] + when "supported" + render(Primer::Beta::Label.new(scheme: :success)) { I18n.t("llm.model_kinds.embedding") } + when "unsupported" + render(Primer::Beta::Label.new(scheme: :secondary)) { I18n.t("llm.model_kinds.chat") } + else + render(Primer::Beta::Label.new(scheme: :secondary, inline: true)) { I18n.t("llm.model_kinds.unknown") } + end + end + def source if llm_model.manual? render(Primer::Beta::Label.new(scheme: :accent)) { I18n.t("admin.llm_connections.models.source_manual") } diff --git a/app/components/llm_connections/models_table_component.rb b/app/components/llm_connections/models_table_component.rb index 9dba37bb7a3c..80679b3615a6 100644 --- a/app/components/llm_connections/models_table_component.rb +++ b/app/components/llm_connections/models_table_component.rb @@ -34,7 +34,7 @@ module LlmConnections # Rendering never issues an HTTP request: the catalogue is refreshed explicitly # through the "Refresh models" action. class ModelsTableComponent < OpPrimer::BorderBoxTableComponent - columns :identifier, :context_window, :source + columns :identifier, :kind, :context_window, :source mobile_columns :identifier @@ -42,6 +42,9 @@ def initial_sort = %i[identifier asc] def has_footer? = false + # Without this the row's button_links are never rendered. + def has_actions? = true + def mobile_title = I18n.t("admin.llm_connections.show.models_heading") # The row class is otherwise derived by convention as LlmConnections::RowComponent. @@ -50,11 +53,25 @@ def row_class = ModelsRowComponent def headers [ [:identifier, { caption: I18n.t("admin.llm_connections.models.identifier") }], + [:kind, { caption: I18n.t("admin.llm_connections.models.kind") }], [:context_window, { caption: I18n.t("admin.llm_connections.models.context_window") }], [:source, { caption: I18n.t("admin.llm_connections.models.source") }] ] end + # Built once for the whole table so each row does not query for its own + # verdict. Rows reach this through their +table+ accessor. + def embeddings_states + @embeddings_states ||= load_embeddings_states + end + + def load_embeddings_states + connection = rows.first&.llm_connection + return {} if connection.nil? + + connection.capability_verdicts.for_capability(:embeddings).pluck(:model_id, :state).to_h + end + def blank_title = I18n.t("admin.llm_connections.models.blank_title") def blank_description = I18n.t("admin.llm_connections.models.blank_description") diff --git a/app/controllers/admin/llm_models_controller.rb b/app/controllers/admin/llm_models_controller.rb index 3c1f06ef37e9..29f230cf8617 100644 --- a/app/controllers/admin/llm_models_controller.rb +++ b/app/controllers/admin/llm_models_controller.rb @@ -62,7 +62,7 @@ def update @llm_model = @connection.models.find(params.expect(:id)) ActiveRecord::Base.transaction do - @llm_model.update!(display_name: params.dig(:llm_model, :display_name)) + apply_attributes(@llm_model) apply_capabilities(@llm_model) end @@ -84,6 +84,12 @@ def set_connection @connection = LlmConnection.instance end + def apply_attributes(llm_model) + llm_model.assign_attributes(display_name: params.dig(:llm_model, :display_name)) + llm_model.admin_context_window = params.dig(:llm_model, :admin_context_window) + llm_model.save! + end + def model_params params.expect(llm_model: %i[external_id display_name]) end diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb index c5cf6b34b2ad..bff1cde9b11a 100644 --- a/app/models/llm_model.rb +++ b/app/models/llm_model.rb @@ -46,10 +46,31 @@ class LlmModel < ApplicationRecord def name = display_name.presence || external_id - # The server's own figure wins: vLLM and SGLang report the operator's actual - # --max-model-len, where a registry only knows what some vendor publishes. + # Precedence: what an administrator set, then what the server reported (vLLM + # and SGLang publish the operator's actual --max-model-len), then what a + # registry believes about the model in general. def context_window - raw_metadata["max_model_len"] || raw_metadata["context_window"] + raw_metadata["admin_context_window"] || + raw_metadata["max_model_len"] || + raw_metadata["context_window"] + end + + def admin_context_window = raw_metadata["admin_context_window"] + + def context_window_source + return :admin if raw_metadata["admin_context_window"].present? + return :server if raw_metadata["max_model_len"].present? + return :registry if raw_metadata["context_window"].present? + + nil + end + + def admin_context_window=(value) + self.raw_metadata = if value.blank? + raw_metadata.except("admin_context_window") + else + raw_metadata.merge("admin_context_window" => value.to_i) + end end # Discovered models that the server stopped offering are deactivated rather diff --git a/app/views/admin/llm_models/edit.html.erb b/app/views/admin/llm_models/edit.html.erb index 984efd8d2f42..b39a06c44500 100644 --- a/app/views/admin/llm_models/edit.html.erb +++ b/app/views/admin/llm_models/edit.html.erb @@ -53,6 +53,28 @@ See COPYRIGHT and LICENSE files for more details. ) %> <% end %> + <%= render(Primer::Box.new(mb: 3)) do %> + <%= label_tag("llm_model_admin_context_window", t(".context_window"), class: "FormControl-label") %> + <%= number_field_tag( + "llm_model[admin_context_window]", @llm_model.admin_context_window, + id: "llm_model_admin_context_window", + min: 1, + autocomplete: "off", + class: "FormControl-input" + ) %> + <%= render(Primer::Beta::Text.new(tag: :div, color: :muted, font_size: :small)) do %> + <% if @llm_model.context_window_source.present? && @llm_model.context_window_source != :admin %> + <%= t( + ".context_window_known", + value: number_with_delimiter(@llm_model.context_window), + source: t("llm.context_window_sources.#{@llm_model.context_window_source}") + ) %> + <% else %> + <%= t(".context_window_caption") %> + <% end %> + <% end %> + <% end %> + <%= render(Primer::Beta::Subhead.new(mt: 4)) do |component| %> <% component.with_heading(tag: :h3) { t(".capabilities_heading") } %> <% component.with_description { t(".capabilities_description") } %> diff --git a/config/locales/en.yml b/config/locales/en.yml index 82fafa48a98b..b936cee7704b 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1458,6 +1458,7 @@ en: blank_title: "No models available" context_window: "Context window" identifier: "Model" + kind: "Type" source: "Source" source_discovered: "Reported by server" source_manual: "Added manually" @@ -1482,8 +1483,11 @@ en: destroy: success: "%{model} has been removed." edit: - capabilities_description: "Set what this model can do. Leave a capability unspecified to let OpenProject determine it." + capabilities_description: "Set what this model can do. Leave a capability unspecified to let OpenProject determine it. Marking Embeddings as supported makes this an embedding model, and it will only be offered to features that need one." capabilities_heading: "Capabilities" + context_window: "Context window" + context_window_caption: "How many tokens this model accepts. Leave blank if you do not know." + context_window_known: "Leave blank to use %{value}, %{source}." current_verdict: "Currently %{state}, %{source}." description: "Edit how OpenProject uses this model." state_supported: "Supported" @@ -3990,6 +3994,13 @@ en: label: "Structured output" vision: label: "Vision" + context_window_sources: + registry: "the figure published for this model" + server: "reported by the server" + model_kinds: + chat: "Chat" + embedding: "Embedding" + unknown: "Unknown" verdict_sources: admin: "set by an administrator" metadata: "from the model registry" diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb index b0dd057d4f2c..ee559474e766 100644 --- a/spec/requests/admin/llm_models_spec.rb +++ b/spec/requests/admin/llm_models_spec.rb @@ -97,6 +97,31 @@ expect(verdicts).to include(["embeddings", "supported", "admin"], ["vision", "unsupported", "admin"]) end + it "stores a context window an administrator supplies" do + patch llm_model_path(llm_model), params: { llm_model: { admin_context_window: "32768" } } + + expect(llm_model.reload.context_window).to eq(32_768) + expect(llm_model.context_window_source).to eq(:admin) + end + + # The administrator's figure wins over whatever the server or a registry said. + it "prefers the administrator's context window over a reported one" do + llm_model.update!(raw_metadata: { "max_model_len" => 8192 }) + + patch llm_model_path(llm_model), params: { llm_model: { admin_context_window: "32768" } } + + expect(llm_model.reload.context_window).to eq(32_768) + end + + it "falls back to the reported figure when cleared" do + llm_model.update!(raw_metadata: { "max_model_len" => 8192, "admin_context_window" => 32_768 }) + + patch llm_model_path(llm_model), params: { llm_model: { admin_context_window: "" } } + + expect(llm_model.reload.context_window).to eq(8192) + expect(llm_model.context_window_source).to eq(:server) + end + it "makes an asserted capability satisfy a feature that requires it" do patch llm_model_path(llm_model), params: { capabilities: { embeddings: "supported" } } @@ -127,6 +152,26 @@ end end + describe "the model type shown in the list" do + it "reads as an embedding model once embeddings are supported" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "bge-m3") + patch llm_model_path(llm_model), params: { capabilities: { embeddings: "supported" } } + + get llm_connection_path + + expect(response.body).to include("Embedding") + end + + it "reads as a chat model when embeddings are not supported" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "qwen") + patch llm_model_path(llm_model), params: { capabilities: { embeddings: "unsupported" } } + + get llm_connection_path + + expect(response.body).to include("Chat") + end + end + describe "DELETE /admin/llm_models/:id" do it "removes a manual model" do llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") From 75c15dfe252a0add1e61ceb3b0442a9d998c47a5 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Tue, 11 Aug 2026 10:58:27 +0100 Subject: [PATCH 22/44] [#66020] Follow OpenProject's admin UI conventions A review against the rest of app/views/admin found four deviations. 28 admin views build forms with primer_form_with; the only two using raw select_tag and text_field_tag were these. Forms are now ApplicationForm subclasses like everything else. That is not only consistency: Primer inputs read builder.object.errors themselves, so validation failures render inline against the field that caused them. The hand-written versions reported them as a flash, or not at all. Capability assertions became virtual attributes on the model so they can be ordinary form fields, which lets the whole edit screen be one Primer form. A verdict from a probe or a registry is reported in the field's caption rather than loaded into it, so saving does not silently adopt someone else's finding as the administrator's. Adding a model followed no convention at all -- it was a form parked under the table. It is now a + action button leading to a new page, matching scim_clients and the Jira importer, and that page offers every field the edit screen does rather than just the name. Deleting a model asked for confirmation through the browser. It now opens a DangerDialog, which can say which features are bound to the model and will stop working. Test selectors added throughout, since OpenProject feature specs are built on within_test_selector and there were almost none. Still missing, and known: a feature spec, which is where be_axe_clean would check the accessibility of all this. https://community.openproject.org/work_packages/66020 --- .../delete_model_dialog_component.html.erb | 20 +++ .../delete_model_dialog_component.rb | 53 ++++++++ .../feature_binding_component.html.erb | 21 ++- .../feature_binding_component.rb | 46 ++----- .../llm_connections/models_row_component.rb | 7 +- .../admin/llm_models_controller.rb | 52 ++++++-- .../llm_connections/feature_binding_form.rb | 89 +++++++++++++ app/forms/llm_models/form.rb | 120 ++++++++++++++++++ app/models/llm_model.rb | 29 +++++ app/views/admin/llm_connections/show.html.erb | 57 +++++---- app/views/admin/llm_models/edit.html.erb | 74 +---------- app/views/admin/llm_models/new.html.erb | 54 ++++++++ config/locales/en.yml | 18 ++- config/routes.rb | 4 +- spec/requests/admin/llm_models_spec.rb | 65 ++++++++-- 15 files changed, 540 insertions(+), 169 deletions(-) create mode 100644 app/components/llm_connections/delete_model_dialog_component.html.erb create mode 100644 app/components/llm_connections/delete_model_dialog_component.rb create mode 100644 app/forms/llm_connections/feature_binding_form.rb create mode 100644 app/forms/llm_models/form.rb create mode 100644 app/views/admin/llm_models/new.html.erb diff --git a/app/components/llm_connections/delete_model_dialog_component.html.erb b/app/components/llm_connections/delete_model_dialog_component.html.erb new file mode 100644 index 000000000000..5fe5f2d5816c --- /dev/null +++ b/app/components/llm_connections/delete_model_dialog_component.html.erb @@ -0,0 +1,20 @@ +<%= + render( + Primer::OpenProject::DangerDialog.new( + title: t("admin.llm_models.destroy.title"), + form_arguments:, + test_selector: TEST_SELECTOR + ) + ) do |dialog| + dialog.with_confirmation_message do |message| + message.with_heading(tag: :h2) { t("admin.llm_models.destroy.heading", model: llm_model.external_id) } + message.with_description_content( + if bound_features.any? + t("admin.llm_models.destroy.description_bound", features: bound_features.to_sentence) + else + t("admin.llm_models.destroy.description") + end + ) + end + end +%> diff --git a/app/components/llm_connections/delete_model_dialog_component.rb b/app/components/llm_connections/delete_model_dialog_component.rb new file mode 100644 index 000000000000..d166425e2e3c --- /dev/null +++ b/app/components/llm_connections/delete_model_dialog_component.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class DeleteModelDialogComponent < ApplicationComponent + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + TEST_SELECTOR = "llm-model--delete-dialog" + + alias_method :llm_model, :model + + def form_arguments + { action: url_helpers.llm_model_path(llm_model), method: :delete } + end + + # Named so the message says what is actually at stake: features bound to this + # model stop resolving, rather than silently falling back to another one. + def bound_features + llm_model.llm_connection + .feature_bindings + .where(model_id: llm_model.external_id) + .filter_map { |binding| binding.feature&.label } + end + end +end diff --git a/app/components/llm_connections/feature_binding_component.html.erb b/app/components/llm_connections/feature_binding_component.html.erb index 034926139ca7..e3c28a593df1 100644 --- a/app/components/llm_connections/feature_binding_component.html.erb +++ b/app/components/llm_connections/feature_binding_component.html.erb @@ -17,18 +17,15 @@ <% end %> <% end %> - <%= form_with(url: form_url, method: :patch) do %> - <%= label_tag(select_id, LlmFeatureBinding.human_attribute_name(:model_id), class: "FormControl-label") %> - <%= select_tag( - "llm_feature_binding[model_id]", - options_for_select(select_choices, selected: selected_model_id.to_s, disabled: disabled_choices), - id: select_id, - disabled: locked?, - class: "FormControl-select" + <%= primer_form_with(model: form_model, url: form_url, method: :patch, scope: :llm_feature_binding) do |f| %> + <%= render( + LlmConnections::FeatureBindingForm.new( + f, + options: model_options, + inherit_label:, + feature_key: feature.key, + locked: locked? + ) ) %> - - <% unless locked? %> - <%= render(Primer::Beta::Button.new(type: :submit, scheme: :secondary, mt: 2)) { t(:button_save) } %> - <% end %> <% end %> <% end %> diff --git a/app/components/llm_connections/feature_binding_component.rb b/app/components/llm_connections/feature_binding_component.rb index f301177c7f1d..8f3e679b3002 100644 --- a/app/components/llm_connections/feature_binding_component.rb +++ b/app/components/llm_connections/feature_binding_component.rb @@ -41,9 +41,11 @@ def initialize(feature:, connection:, binding: nil) @binding = binding end - private - - attr_reader :feature, :connection, :binding + # The record the select binds to. A feature without a stored binding still + # needs one so the form has a model_id to read. + def form_model + binding || connection.feature_bindings.new(feature_key: feature.key.to_s) + end # Not named +options+: ApplicationComponent already owns that name and # initialises it to an empty hash, which silently swallowed the memoisation. @@ -51,12 +53,6 @@ def model_options @model_options ||= SelectableModelsQuery.new(connection, feature).call end - def selected_model_id = binding&.model_id - - def default_model_id - feature.embedding? ? connection.default_embedding_model_id : connection.default_chat_model_id - end - def inherit_label if default_model_id.present? I18n.t("admin.llm_feature_bindings.inherit_with_default", model: default_model_id) @@ -69,40 +65,16 @@ def locked? = binding&.locked? def dangling? = binding&.dangling? - # Built as label/value pairs rather than through a block, because the block - # form of Rails' select writes to the template's output buffer instead of - # into the select element. - def select_choices - [[inherit_label, ""]] + model_options.map { |option| [option_label(option), option.model_id] } - end - - # Listed but not choosable: a model whose required capability is known to be - # missing. It stays visible so the reason is visible with it. - def disabled_choices - model_options.reject(&:selectable?).map(&:model_id) - end + private - def option_label(option) - case option.state - when :unsupported - I18n.t("admin.llm_feature_bindings.option_unsupported", - model: option.model_id, - capability: capability_labels(option.reasons)) - when :unknown - I18n.t("admin.llm_feature_bindings.option_unknown", model: option.model_id) - else - option.model_id - end - end + attr_reader :feature, :connection, :binding - def capability_labels(capabilities) - capabilities.map { |capability| I18n.t("llm.capabilities.#{capability}.label") }.join(", ") + def default_model_id + feature.embedding? ? connection.default_embedding_model_id : connection.default_chat_model_id end def form_url url_helpers.llm_feature_binding_path(feature.key) end - - def select_id = "llm_feature_binding_model_id_#{feature.key}" end end diff --git a/app/components/llm_connections/models_row_component.rb b/app/components/llm_connections/models_row_component.rb index 246a7ecfe8f4..b2c79e74e5c8 100644 --- a/app/components/llm_connections/models_row_component.rb +++ b/app/components/llm_connections/models_row_component.rb @@ -78,13 +78,16 @@ def button_links def edit_link link_to(helpers.op_icon("icon-edit"), url_helpers.edit_llm_model_path(llm_model), + data: { test_selector: "llm-model--edit-#{llm_model.id}" }, title: I18n.t(:button_edit)) end + # Opens a DangerDialog rather than a browser confirm, so the message can say + # which features are bound to the model. def delete_link link_to(helpers.op_icon("icon-delete"), - url_helpers.llm_model_path(llm_model), - data: { turbo_method: :delete, turbo_confirm: I18n.t(:text_are_you_sure) }, + url_helpers.delete_dialog_llm_model_path(llm_model), + data: { controller: "async-dialog", test_selector: "llm-model--delete-#{llm_model.id}" }, title: I18n.t(:button_delete)) end end diff --git a/app/controllers/admin/llm_models_controller.rb b/app/controllers/admin/llm_models_controller.rb index 29f230cf8617..7b0c65fd8dce 100644 --- a/app/controllers/admin/llm_models_controller.rb +++ b/app/controllers/admin/llm_models_controller.rb @@ -35,27 +35,35 @@ module Admin # A gateway may route /v1/chat/completions and nothing else, in which case the # operator knows the model name and OpenProject cannot discover it. class LlmModelsController < ApplicationController + include OpTurbo::ComponentStream + layout "admin" menu_item :llm_connection before_action :require_admin before_action :set_connection + def new + @llm_model = @connection.models.new + end + def edit @llm_model = @connection.models.find(params.expect(:id)) @verdicts = @connection.capability_verdicts.for_model(@llm_model.external_id).index_by(&:capability) end def create - llm_model = @connection.models.new(model_params.merge(manual: true)) + llm_model = @connection.models.new(llm_model_params.except(*capability_param_names).merge(manual: true)) - if llm_model.save + if save_with_capabilities(llm_model) flash[:notice] = t(".success", model: llm_model.external_id) + redirect_to llm_connection_path, status: :see_other else - flash[:error] = llm_model.errors.full_messages.join(", ") + # Re-rendered rather than redirected so the Primer form shows the error + # inline against the field that caused it. + @llm_model = llm_model + render :new, status: :unprocessable_entity end - - redirect_to llm_connection_path, status: :see_other end def update @@ -70,6 +78,12 @@ def update redirect_to llm_connection_path, status: :see_other end + def delete_dialog + llm_model = @connection.models.manual.find(params.expect(:id)) + + respond_with_dialog LlmConnections::DeleteModelDialogComponent.new(llm_model) + end + def destroy llm_model = @connection.models.manual.find(params.expect(:id)) llm_model.destroy! @@ -85,23 +99,39 @@ def set_connection end def apply_attributes(llm_model) - llm_model.assign_attributes(display_name: params.dig(:llm_model, :display_name)) - llm_model.admin_context_window = params.dig(:llm_model, :admin_context_window) + llm_model.assign_attributes(llm_model_params.except(:external_id, *capability_param_names)) llm_model.save! end - def model_params - params.expect(llm_model: %i[external_id display_name]) + # external_id is only accepted when creating: verdicts and bindings reference + # a model by that string, so renaming one would orphan both. + def llm_model_params + params.expect(llm_model: [:external_id, :display_name, :admin_context_window, *capability_param_names]) + end + + def capability_param_names + Llm::Capabilities::ALL.map { |capability| :"capability_#{capability}" } + end + + def save_with_capabilities(llm_model) + ActiveRecord::Base.transaction do + llm_model.save! + apply_capabilities(llm_model) + end + + true + rescue ActiveRecord::RecordInvalid + false end # Stored as admin-sourced verdicts, which survive re-detection: an # administrator knows things about their deployment that neither a published # registry nor a probe can determine. def apply_capabilities(llm_model) - submitted = params.fetch(:capabilities, {}).permit!.to_h + submitted = llm_model_params Llm::Capabilities::ALL.each do |capability| - assert(llm_model.external_id, capability, submitted[capability.to_s].presence) + assert(llm_model.external_id, capability, submitted[:"capability_#{capability}"].presence) end end diff --git a/app/forms/llm_connections/feature_binding_form.rb b/app/forms/llm_connections/feature_binding_form.rb new file mode 100644 index 000000000000..bd890569b656 --- /dev/null +++ b/app/forms/llm_connections/feature_binding_form.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # The model select for one registered feature. + class FeatureBindingForm < ApplicationForm + # Primer::Forms::Base.new assigns the builder itself and calls this with the + # remaining keywords, so the builder must not appear in the signature. + def initialize(options:, inherit_label:, feature_key:, locked: false) + super() + @model_options = options + @inherit_label = inherit_label + @feature_key = feature_key + @locked = locked + end + + form do |f| + f.select_list( + name: :model_id, + label: LlmFeatureBinding.human_attribute_name(:model_id), + include_blank: false, + input_width: :large, + disabled: locked, + data: { test_selector: "llm-feature-binding--model-#{feature_key}" } + ) do |select| + select.option(value: "", label: inherit_label) + + model_options.each do |option| + # Listed but not choosable when a required capability is known to be + # missing: hiding it would leave the reason invisible too. + select.option(value: option.model_id, label: option_label(option), disabled: !option.selectable?) + end + end + + unless locked + f.submit( + name: :submit, + label: I18n.t(:button_save), + scheme: :secondary, + data: { test_selector: "llm-feature-binding--submit-#{feature_key}" } + ) + end + end + + private + + attr_reader :model_options, :inherit_label, :feature_key, :locked + + def option_label(option) + case option.state + when :unsupported + I18n.t("admin.llm_feature_bindings.option_unsupported", + model: option.model_id, + capability: option.reasons.map { |reason| Llm::Capabilities.label(reason) }.join(", ")) + when :unknown + I18n.t("admin.llm_feature_bindings.option_unknown", model: option.model_id) + else + option.model_id + end + end + end +end diff --git a/app/forms/llm_models/form.rb b/app/forms/llm_models/form.rb new file mode 100644 index 000000000000..b4245ca56af8 --- /dev/null +++ b/app/forms/llm_models/form.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmModels + class Form < ApplicationForm + form do |f| + # The identity of the model, and immutable once saved: verdicts and + # bindings reference it by this string. + if new_record? + f.text_field( + name: :external_id, + label: LlmModel.human_attribute_name(:external_id), + caption: I18n.t("admin.llm_models.form.external_id_caption"), + placeholder: "qwen3.6-35b-a3b", + required: true, + autocomplete: "off", + input_width: :large, + data: { test_selector: "llm-model--external-id" } + ) + end + + f.text_field( + name: :display_name, + label: LlmModel.human_attribute_name(:display_name), + caption: I18n.t("admin.llm_models.form.display_name_caption"), + autocomplete: "off", + input_width: :large, + data: { test_selector: "llm-model--display-name" } + ) + + f.text_field( + name: :admin_context_window, + label: I18n.t("admin.llm_models.form.context_window"), + caption: context_window_caption, + type: :number, + min: 1, + autocomplete: "off", + input_width: :medium, + data: { test_selector: "llm-model--context-window" } + ) + + Llm::Capabilities::ALL.each do |capability| + f.select_list( + name: :"capability_#{capability}", + label: Llm::Capabilities.label(capability), + caption: capability_caption(capability), + include_blank: false, + input_width: :medium, + data: { test_selector: "llm-model--capability-#{capability}" } + ) do |select| + select.option(value: "", label: I18n.t("admin.llm_models.form.state_unspecified")) + select.option(value: "supported", label: I18n.t("admin.llm_models.form.state_supported")) + select.option(value: "unsupported", label: I18n.t("admin.llm_models.form.state_unsupported")) + end + end + + f.submit( + name: :submit, + label: new_record? ? I18n.t("admin.llm_models.form.create_submit") : I18n.t(:button_save), + scheme: :primary, + data: { test_selector: "llm-model--submit" } + ) + end + + private + + def new_record? = model.new_record? + + def context_window_caption + source = model.context_window_source + + if source.present? && source != :admin + I18n.t("admin.llm_models.form.context_window_known", + value: model.context_window, + source: I18n.t("llm.context_window_sources.#{source}")) + else + I18n.t("admin.llm_models.form.context_window_caption") + end + end + + # A verdict established by a probe or a registry is reported rather than + # loaded into the field, so saving the form does not silently turn someone + # else's finding into the administrator's assertion. + def capability_caption(capability) + verdict = model.verdict_for(capability) + return if verdict.nil? || verdict.source_admin? + + I18n.t("admin.llm_models.form.current_verdict", + state: I18n.t("llm.verdict_states.#{verdict.state}"), + source: I18n.t("llm.verdict_sources.#{verdict.source}")) + end + end +end diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb index bff1cde9b11a..019f8efbe5ea 100644 --- a/app/models/llm_model.rb +++ b/app/models/llm_model.rb @@ -73,6 +73,35 @@ def admin_context_window=(value) end end + # Capability assertions are stored as verdicts, not columns. These virtual + # attributes let the edit form treat them as ordinary fields, so the whole + # screen can be a single Primer form rather than hand-written inputs. + Llm::Capabilities::ALL.each do |capability| + define_method(:"capability_#{capability}") do + capability_overrides.fetch(capability.to_s) { admin_capability_state(capability) } + end + + define_method(:"capability_#{capability}=") do |value| + capability_overrides[capability.to_s] = value.presence + end + end + + def capability_overrides = @capability_overrides ||= {} + + # Only an administrator's own assertion is shown as the field's value. A + # verdict from a probe or a registry is displayed alongside instead, so that + # saving the form does not silently adopt it as the administrator's. + def admin_capability_state(capability) + verdict_for(capability)&.then { |verdict| verdict.source_admin? ? verdict.state : nil } + end + + def verdict_for(capability) + llm_connection.capability_verdicts + .for_model(external_id) + .for_capability(capability) + .first + end + # Discovered models that the server stopped offering are deactivated rather # than deleted, so a binding or verdict pointing at one still has something to # name. Manual entries are never deactivated by a refresh: nothing confirms diff --git a/app/views/admin/llm_connections/show.html.erb b/app/views/admin/llm_connections/show.html.erb index 16998ca5867a..85bac3e8b8da 100644 --- a/app/views/admin/llm_connections/show.html.erb +++ b/app/views/admin/llm_connections/show.html.erb @@ -49,37 +49,40 @@ See COPYRIGHT and LICENSE files for more details. component.with_heading(tag: :h3) { t(".models_heading") } component.with_description { t(".models_description", fetched_at: format_time(@connection.catalogue_fetched_at)) } component.with_actions do - render( - Primer::Beta::Button.new( - tag: :a, - href: refresh_models_llm_connection_path, - data: { turbo_method: :post, controller: "disable-when-clicked" } - ) - ) do |button| - button.with_leading_visual_icon(icon: :sync) - t(".refresh_models") - end + concat( + render( + Primer::Beta::Button.new( + tag: :a, + href: new_llm_model_path, + scheme: :primary, + mr: 2, + data: { test_selector: "llm-model--add-button" } + ) + ) do |button| + button.with_leading_visual_icon(icon: :plus) + t(".add_model_submit") + end + ) + concat( + render( + Primer::Beta::Button.new( + tag: :a, + href: refresh_models_llm_connection_path, + data: { + turbo_method: :post, + controller: "disable-when-clicked", + test_selector: "llm-model--refresh-button" + } + ) + ) do |button| + button.with_leading_visual_icon(icon: :sync) + t(".refresh_models") + end + ) end end %> <%= render(LlmConnections::ModelsTableComponent.new(rows: @connection.models.by_identifier)) %> - <%= render(Primer::Box.new(mt: 3, p: 3, border: true, border_radius: 2)) do %> - <%= render(Primer::Beta::Text.new(tag: :h4, font_weight: :bold, mb: 1)) { t(".add_model_heading") } %> - <%= render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 2)) { t(".add_model_description") } %> - - <%= form_with(url: llm_models_path, method: :post) do %> - <%= label_tag("llm_model_external_id", LlmModel.human_attribute_name(:external_id), class: "FormControl-label") %> - <%= text_field_tag( - "llm_model[external_id]", nil, - id: "llm_model_external_id", - required: true, - autocomplete: "off", - placeholder: "qwen3.6-35b-a3b", - class: "FormControl-input" - ) %> - <%= render(Primer::Beta::Button.new(type: :submit, scheme: :secondary, ml: 2)) { t(".add_model_submit") } %> - <% end %> - <% end %> <% end %> diff --git a/app/views/admin/llm_models/edit.html.erb b/app/views/admin/llm_models/edit.html.erb index b39a06c44500..c6a5cfcbb6cd 100644 --- a/app/views/admin/llm_models/edit.html.erb +++ b/app/views/admin/llm_models/edit.html.erb @@ -42,71 +42,11 @@ See COPYRIGHT and LICENSE files for more details. end %> -<%= form_with(url: llm_model_path(@llm_model), method: :patch) do %> - <%= render(Primer::Box.new(mb: 3)) do %> - <%= label_tag("llm_model_display_name", LlmModel.human_attribute_name(:display_name), class: "FormControl-label") %> - <%= text_field_tag( - "llm_model[display_name]", @llm_model.display_name, - id: "llm_model_display_name", - autocomplete: "off", - class: "FormControl-input" - ) %> - <% end %> - - <%= render(Primer::Box.new(mb: 3)) do %> - <%= label_tag("llm_model_admin_context_window", t(".context_window"), class: "FormControl-label") %> - <%= number_field_tag( - "llm_model[admin_context_window]", @llm_model.admin_context_window, - id: "llm_model_admin_context_window", - min: 1, - autocomplete: "off", - class: "FormControl-input" - ) %> - <%= render(Primer::Beta::Text.new(tag: :div, color: :muted, font_size: :small)) do %> - <% if @llm_model.context_window_source.present? && @llm_model.context_window_source != :admin %> - <%= t( - ".context_window_known", - value: number_with_delimiter(@llm_model.context_window), - source: t("llm.context_window_sources.#{@llm_model.context_window_source}") - ) %> - <% else %> - <%= t(".context_window_caption") %> - <% end %> - <% end %> - <% end %> - - <%= render(Primer::Beta::Subhead.new(mt: 4)) do |component| %> - <% component.with_heading(tag: :h3) { t(".capabilities_heading") } %> - <% component.with_description { t(".capabilities_description") } %> - <% end %> - - <% Llm::Capabilities::ALL.each do |capability| %> - <% verdict = @verdicts[capability.to_s] %> - <%= render(Primer::Box.new(mb: 2)) do %> - <%= label_tag("capability_#{capability}", Llm::Capabilities.label(capability), class: "FormControl-label") %> - <%= select_tag( - "capabilities[#{capability}]", - options_for_select( - [[t(".state_unspecified"), ""], - [t(".state_supported"), "supported"], - [t(".state_unsupported"), "unsupported"]], - verdict&.source_admin? ? verdict.state : "" - ), - id: "capability_#{capability}", - class: "FormControl-select" - ) %> - <% if verdict.present? && !verdict.source_admin? %> - <%= render(Primer::Beta::Text.new(tag: :div, color: :muted, font_size: :small)) do %> - <%= t( - ".current_verdict", - state: t("llm.verdict_states.#{verdict.state}"), - source: t("llm.verdict_sources.#{verdict.source}") - ) %> - <% end %> - <% end %> - <% end %> - <% end %> - - <%= render(Primer::Beta::Button.new(type: :submit, scheme: :primary, mt: 3)) { t(:button_save) } %> - <%= render(Primer::Beta::Button.new(tag: :a, href: llm_connection_path, ml: 2, mt: 3)) { t(:button_cancel) } %> +<%= settings_primer_form_with( + model: @llm_model, + url: llm_model_path(@llm_model), + method: :patch, + data: { test_selector: "llm-model--edit-form" } + ) do |f| %> + <%= render(LlmModels::Form.new(f)) %> <% end %> diff --git a/app/views/admin/llm_models/new.html.erb b/app/views/admin/llm_models/new.html.erb new file mode 100644 index 000000000000..ebba7e58487c --- /dev/null +++ b/app/views/admin/llm_models/new.html.erb @@ -0,0 +1,54 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection"), t(".title") %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t(".title") } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + { href: llm_connection_path, text: t("menus.admin.llm_connection") }, + t(".title")] + ) + end +%> + +<%= + settings_primer_form_with( + model: @llm_model, + url: llm_models_path, + method: :post, + data: { test_selector: "llm-model--add-form" } + ) do |f| + render(LlmModels::Form.new(f)) + end +%> diff --git a/config/locales/en.yml b/config/locales/en.yml index b936cee7704b..dd87fe96bead 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1457,6 +1457,7 @@ en: blank_description: "The server reported no models. If it does not offer a model list, add the model names you want to use below." blank_title: "No models available" context_window: "Context window" + display_name_caption: "An optional friendly name shown instead of the model id." identifier: "Model" kind: "Type" source: "Source" @@ -1468,7 +1469,6 @@ en: success: "The model list has been refreshed." show: add_model_description: "If the server does not expose a model list, enter the model name exactly as the server expects it. Manually added models are kept when the list is refreshed." - add_model_heading: "Add a model manually" add_model_submit: "Add model" description: "Connect OpenProject to a server that speaks the OpenAI API, so AI features can use it." models_description: "Reported by the server on %{fetched_at}." @@ -1481,18 +1481,28 @@ en: create: success: "%{model} has been added." destroy: + description: "The model will no longer be offered to AI features." + description_bound: "This model is currently used by %{features}. Those features will stop working until another model is selected." + heading: "Remove %{model}?" success: "%{model} has been removed." + title: "Remove model" edit: - capabilities_description: "Set what this model can do. Leave a capability unspecified to let OpenProject determine it. Marking Embeddings as supported makes this an embedding model, and it will only be offered to features that need one." - capabilities_heading: "Capabilities" + description: "Edit how OpenProject uses this model." + form: + capabilities_description: "Leave a capability unspecified to let OpenProject determine it. Marking Embeddings as supported makes this an embedding model, offered only to features that need one." context_window: "Context window" context_window_caption: "How many tokens this model accepts. Leave blank if you do not know." context_window_known: "Leave blank to use %{value}, %{source}." + create_submit: "Add model" current_verdict: "Currently %{state}, %{source}." - description: "Edit how OpenProject uses this model." + display_name_caption: "An optional friendly name shown instead of the model id." + external_id_caption: "The model name exactly as the server expects it." state_supported: "Supported" state_unspecified: "Not specified" state_unsupported: "Not supported" + new: + description: "Name a model this server can use but does not advertise, and describe what it can do." + title: "Add a model" update: success: "%{model} has been updated." mcp_configurations: diff --git a/config/routes.rb b/config/routes.rb index 70c3b4089631..31ddbbccad5b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -697,7 +697,9 @@ end # Manual entries only; discovered models are managed by the sync. - resources :llm_models, only: %i[create edit update destroy], controller: "admin/llm_models" + resources :llm_models, only: %i[new create edit update destroy], controller: "admin/llm_models" do + member { get :delete_dialog } + end # Keyed by feature key rather than by record id: the binding is an attribute # of a registered feature, and a feature may not have a row yet. diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb index ee559474e766..ee286476dfba 100644 --- a/spec/requests/admin/llm_models_spec.rb +++ b/spec/requests/admin/llm_models_spec.rb @@ -40,6 +40,21 @@ before { login_as admin } describe "POST /admin/llm_models" do + it "accepts everything the edit screen accepts" do + post llm_models_path, params: { llm_model: { external_id: "bge-m3", + display_name: "BGE M3", + admin_context_window: "8192", + capability_embeddings: "supported", + capability_vision: "unsupported" } } + + llm_model = connection.models.find_by(external_id: "bge-m3") + expect(llm_model.display_name).to eq("BGE M3") + expect(llm_model.context_window).to eq(8192) + + verdicts = connection.capability_verdicts.for_model("bge-m3").pluck(:capability, :state, :source) + expect(verdicts).to include(["embeddings", "supported", "admin"], ["vision", "unsupported", "admin"]) + end + it "adds a model an administrator names" do post llm_models_path, params: { llm_model: { external_id: "qwen3.6-35b-a3b" } } @@ -87,8 +102,9 @@ it "stores capabilities an administrator asserts" do patch llm_model_path(llm_model), - params: { llm_model: { display_name: "Hand typed" }, - capabilities: { embeddings: "supported", vision: "unsupported" } } + params: { llm_model: { display_name: "Hand typed", + capability_embeddings: "supported", + capability_vision: "unsupported" } } expect(response).to have_http_status(:see_other) expect(llm_model.reload.display_name).to eq("Hand typed") @@ -123,7 +139,7 @@ end it "makes an asserted capability satisfy a feature that requires it" do - patch llm_model_path(llm_model), params: { capabilities: { embeddings: "supported" } } + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } patch llm_feature_binding_path("semantic_search"), params: { llm_feature_binding: { model_id: "hand-typed" } } @@ -134,15 +150,15 @@ # Clearing an assertion records nothing rather than recording ignorance as # fact, so detection can still fill it in later. it "clears an assertion when set back to unspecified" do - patch llm_model_path(llm_model), params: { capabilities: { embeddings: "supported" } } - patch llm_model_path(llm_model), params: { capabilities: { embeddings: "" } } + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "" } } expect(connection.capability_verdicts.for_model("hand-typed").for_capability(:embeddings)).to be_empty end # An administrator looked at this deployment; a published registry did not. it "is not overwritten by registry enrichment" do - patch llm_model_path(llm_model), params: { capabilities: { embeddings: "supported" } } + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } LlmConnections::EnrichCapabilitiesService.new(connection).call @@ -155,7 +171,7 @@ describe "the model type shown in the list" do it "reads as an embedding model once embeddings are supported" do llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "bge-m3") - patch llm_model_path(llm_model), params: { capabilities: { embeddings: "supported" } } + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } get llm_connection_path @@ -164,7 +180,7 @@ it "reads as a chat model when embeddings are not supported" do llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "qwen") - patch llm_model_path(llm_model), params: { capabilities: { embeddings: "unsupported" } } + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "unsupported" } } get llm_connection_path @@ -172,6 +188,39 @@ end end + describe "GET /admin/llm_models/new" do + it "renders the add-model form" do + get new_llm_model_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Model name") + end + + it "re-renders with the error inline when the name is taken" do + create(:llm_model, llm_connection: connection, external_id: "already-there") + + post llm_models_path, params: { llm_model: { external_id: "already-there" } } + + expect(response).to have_http_status(:unprocessable_entity) + expect(connection.models.where(external_id: "already-there").count).to eq(1) + end + end + + describe "GET /admin/llm_models/:id/delete_dialog" do + it "offers a confirmation naming the features that would break" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "hand-typed") + + # Requested by the async-dialog Stimulus controller, which asks for a + # turbo stream rather than HTML. + get delete_dialog_llm_model_path(llm_model), + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Description assistant") + end + end + describe "DELETE /admin/llm_models/:id" do it "removes a manual model" do llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") From 0a252ac81510c143395ff7f48ef2bb26d3dee717 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Tue, 11 Aug 2026 14:12:04 +0100 Subject: [PATCH 23/44] [#66020] Send LLM requests through RubyLLM Llm::Client could list models and ask for an embedding. Nothing could run a completion, so no registered feature had anything to call and a server that publishes no model list could not be verified at all. Llm::Session turns a connection into something that can issue requests, mapping api_format onto RubyLLM's per-provider configuration and applying the stored endpoint, credential and custom headers in one place. Three details are not obvious: It builds a per-call RubyLLM.context rather than calling RubyLLM.configure. The global configuration is process-wide, so writing an administrator's endpoint and credential into it would leak them across requests; a spec pins that the global config stays empty. It sets max_retries explicitly. RubyLLM retries POSTs three times by default, which would bill four completions for one call and multiply every timeout by four. It supplies a placeholder key when a provider demands one and the connection has none, because ensure_configured! otherwise raises -- breaking precisely the keyless self-hosted server this feature targets. The error taxonomy moves to Llm::Errors so both this path and model discovery report failures the same way, with Llm::Client keeping the old names as aliases. Llm::Errors.translate discards the incoming message deliberately: RubyLLM falls back to response.body, which routinely contains the submitted Authorization header and internal hostnames a gateway echoed back. Custom headers reach the wire through a prepended Provider#headers. RubyLLM's public API cannot express it -- Chat#with_headers reaches chat only, embeddings take no headers at all, and it merges additional headers under the provider's own so an override silently loses. Bedrock and Vertex AI are dropped from the format select and rejected by the contract: both need credentials beyond a single api_key. https://community.openproject.org/work_packages/66020 --- .../llm_connections/base_contract.rb | 5 + app/forms/llm_connections/connection_form.rb | 4 +- app/services/llm/client.rb | 38 ++--- app/services/llm/errors.rb | 161 ++++++++++++++++++ app/services/llm/session.rb | 153 +++++++++++++++++ config/locales/en.yml | 2 + .../patches/ruby_llm_provider_headers.rb | 74 ++++++++ .../patches/ruby_llm_provider_headers_spec.rb | 84 +++++++++ spec/services/llm/errors_spec.rb | 145 ++++++++++++++++ spec/services/llm/session_spec.rb | 132 ++++++++++++++ spec/support/llm_server_helpers.rb | 58 +++++++ 11 files changed, 827 insertions(+), 29 deletions(-) create mode 100644 app/services/llm/errors.rb create mode 100644 app/services/llm/session.rb create mode 100644 lib/open_project/patches/ruby_llm_provider_headers.rb create mode 100644 spec/lib/open_project/patches/ruby_llm_provider_headers_spec.rb create mode 100644 spec/services/llm/errors_spec.rb create mode 100644 spec/services/llm/session_spec.rb diff --git a/app/contracts/llm_connections/base_contract.rb b/app/contracts/llm_connections/base_contract.rb index bb86534883b5..a20755202a74 100644 --- a/app/contracts/llm_connections/base_contract.rb +++ b/app/contracts/llm_connections/base_contract.rb @@ -41,6 +41,11 @@ class BaseContract < ModelContract validates :base_url, presence: true validates :api_format, inclusion: { in: Llm::Adapters::FORMATS } + # Bedrock and Vertex AI can be discovered against, but not called: they need + # a secret key and region, or a project and location, and there is one + # api_key column. Offering them would only fail at request time. + validates :api_format, exclusion: { in: Llm::Session::UNSUPPORTED_FORMATS, message: :not_supported }, + unless: -> { api_format.blank? } # Resolves to the validate_url gem, which defaults to http and https. Plain # http is deliberately allowed: an on-premise LLM server on an internal # network commonly terminates TLS elsewhere, or not at all. diff --git a/app/forms/llm_connections/connection_form.rb b/app/forms/llm_connections/connection_form.rb index 24471b4134e8..55c8ab479213 100644 --- a/app/forms/llm_connections/connection_form.rb +++ b/app/forms/llm_connections/connection_form.rb @@ -46,7 +46,9 @@ class ConnectionForm < ApplicationForm input_width: :medium, disabled: read_only? ) do |select| - Llm::Adapters::FORMATS.each do |format| + # Only formats a request can actually be sent in. The contract rejects + # the rest as a backstop, but they should not be offered in the first place. + Llm::Adapters::FORMATS.select { |format| Llm::Session.supports?(format) }.each do |format| select.option(value: format, label: I18n.t("llm.api_formats.#{format}")) end end diff --git a/app/services/llm/client.rb b/app/services/llm/client.rb index d442f4bd44a4..c3b3aff98549 100644 --- a/app/services/llm/client.rb +++ b/app/services/llm/client.rb @@ -36,35 +36,17 @@ module Llm # and what every OpenAI client library expects. This client only appends the # endpoint path. # - # Errors are raised as a small taxonomy so that callers can map them onto - # per-attribute contract errors rather than leaking transport detail into the UI. - # Response bodies are never included in error messages: an OpenAI-compatible - # gateway routinely echoes the submitted Authorization header, upstream provider - # URLs and internal hostnames in its error payloads. + # Errors come from Llm::Errors, which is shared with the RubyLLM-backed + # inference path. They are aliased here because every existing caller rescues + # them by their Llm::Client:: name, and both names refer to the same classes. class Client - class Error < StandardError; end - - # The server could not be reached at all. - class ConnectionError < Error; end - # The host resolved to an address blocked by the SSRF policy. - class SsrfError < ConnectionError; end - # The server took too long to answer. - class TimeoutError < ConnectionError; end - # The server answered, but rejected our credentials. - class AuthenticationError < Error; end - - # The server answered with an unexpected status. - class ApiError < Error - attr_reader :status - - def initialize(message, status: nil) - super(message) - @status = status - end - end - - # The server answered successfully with something that is not an OpenAI model list. - class ParseError < Error; end + Error = Llm::Errors::Error + ConnectionError = Llm::Errors::ConnectionError + SsrfError = Llm::Errors::SsrfError + TimeoutError = Llm::Errors::TimeoutError + AuthenticationError = Llm::Errors::AuthenticationError + ApiError = Llm::Errors::ApiError + ParseError = Llm::Errors::ParseError # The global httpx defaults (connect 3s / read 3s / request 10s, all # writable: false) are tuned for storage and webhook calls and are far too diff --git a/app/services/llm/errors.rb b/app/services/llm/errors.rb new file mode 100644 index 000000000000..14d4c585bd6c --- /dev/null +++ b/app/services/llm/errors.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # The error taxonomy shared by every path that talks to an LLM server. + # + # Callers map these onto per-attribute contract errors or health check codes, + # so the set is deliberately small and describes *what the administrator has to + # fix*, not what the transport did. + # + # Response bodies never appear in a message. An OpenAI-compatible gateway + # routinely echoes the submitted Authorization header, upstream provider URLs + # and internal hostnames in its error payloads, and RubyLLM puts exactly that + # payload into its exception messages (+RubyLLM::Error#initialize+ falls back + # to +response.body+, and the middleware's own defaults come from parsing the + # body). #translate therefore discards the incoming message and logs it instead. + module Errors + class Error < StandardError; end + + # The server could not be reached at all. + class ConnectionError < Error; end + # The host resolved to an address blocked by the SSRF policy. + class SsrfError < ConnectionError; end + # The server took too long to answer. + class TimeoutError < ConnectionError; end + # The server answered, but rejected our credentials. + class AuthenticationError < Error; end + + # The server answered with an unexpected status. + class ApiError < Error + attr_reader :status + + def initialize(message, status: nil) + super(message) + @status = status + end + end + + # The prompt exceeded the model's context window. + class ContextLengthError < ApiError; end + # The server is throttling us. + class RateLimitedError < ApiError; end + + # The server answered successfully with something we cannot read. + class ParseError < Error; end + + # The connection cannot be expressed at all -- an api_format we cannot supply + # credentials for, or a model the provider refuses to accept. + class ConfigurationError < Error; end + + # A feature asked for a client before its model resolved. Carries the + # Llm::Runtime::Resolution status so a caller can tell "no server configured" + # apart from "this model cannot do that". + class NotReady < Error + attr_reader :status + + def initialize(status) + super("LLM is not ready to serve this feature (#{status})") + @status = status + end + end + + module_function + + # Maps anything RubyLLM or Faraday raised onto this taxonomy. + # + # @param error [StandardError] + # @return [Llm::Errors::Error] + def translate(error) + log(error) + + case error + when Llm::Errors::Error + error + when RubyLLM::Error, RubyLLM::ConfigurationError, RubyLLM::ModelNotFoundError + from_ruby_llm(error) + else + from_transport(error) + end + end + + def from_ruby_llm(error) + status = status_of(error) + + case error + when RubyLLM::UnauthorizedError, RubyLLM::ForbiddenError + AuthenticationError.new("Server rejected the API key (#{status})") + when RubyLLM::ContextLengthExceededError + ContextLengthError.new("Prompt exceeds the model's context window", status:) + when RubyLLM::RateLimitError + RateLimitedError.new("Server is rate limiting requests", status:) + when RubyLLM::ConfigurationError, RubyLLM::ModelNotFoundError + ConfigurationError.new("The connection is not usable as configured") + else + # Covers BadRequestError, PaymentRequiredError, ServerError, + # ServiceUnavailableError, OverloadedError and the middleware's catch-all + # -- the last of which is how 404/405/501 arrive, i.e. a server that does + # not implement the endpoint we asked for. + ApiError.new("Server responded with #{status || 'an error'}", status:) + end + end + + def from_transport(error) + case error + when Faraday::TimeoutError, Timeout::Error, Errno::ETIMEDOUT + TimeoutError.new("Request timed out") + when Faraday::ConnectionFailed, Faraday::SSLError, SocketError, Errno::ECONNREFUSED + ConnectionError.new(error.class.name) + when JSON::ParserError + ParseError.new("Response is not valid JSON") + else + Error.new(error.class.name) + end + end + + # Runs the block, re-raising any RubyLLM or Faraday failure as an Llm::Errors. + def wrap + yield + rescue StandardError => e + raise translate(e) + end + + def status_of(error) + error.respond_to?(:response) ? error.response&.status : nil + end + + # The original message is the only place the upstream detail exists, and an + # administrator debugging a gateway needs it -- it just must not travel back + # up to a view or a contract error. + def log(error) + Rails.logger.info { "LLM request failed: #{error.class}: #{error.message}" } + end + end +end diff --git a/app/services/llm/session.rb b/app/services/llm/session.rb new file mode 100644 index 000000000000..2a672852f962 --- /dev/null +++ b/app/services/llm/session.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Turns an LlmConnection into something that can issue requests. + # + # Every inference request in OpenProject goes through here, so that the + # administrator's stored settings -- endpoint, credential, custom headers -- + # are applied in exactly one place. + # + # Note that RubyLLM builds its own Faraday connection over net_http and so + # does *not* pass through OpenProject.httpx and its SSRF filter. Inference + # traffic to an administrator-supplied base URL is therefore not SSRF + # filtered. This is a deliberate, recorded decision. Model discovery + # deliberately stays on Llm::Client, which is filtered. + class Session + # RubyLLM exposes a single Faraday timeout rather than Llm::Client's + # connect/read/request triple, so those three values collapse to one. + # RubyLLM's own default is 300s, which is far too long to hold a Rails + # worker for a check. + PROBE_TIMEOUT = 20 + INFERENCE_TIMEOUT = 180 + + # Formats whose credentials an LlmConnection cannot express: Bedrock needs a + # secret key and a region as well as a key, Vertex AI a project and a + # location. The model has a single api_key column. + UNSUPPORTED_FORMATS = %w[bedrock vertexai].freeze + + # A self-hosted OpenAI-compatible server frequently needs no credential at + # all, but RubyLLM's ensure_configured! raises when a provider declares + # _api_key as required and none is set. A placeholder gets us past + # that check; a server that wants no credential ignores it. + PLACEHOLDER_API_KEY = "-" + + class << self + def for(connection, timeout: INFERENCE_TIMEOUT, max_retries: 1) + new(connection, timeout:, max_retries:) + end + + def supports?(api_format) + UNSUPPORTED_FORMATS.exclude?(api_format.to_s) + end + end + + def initialize(connection, timeout: INFERENCE_TIMEOUT, max_retries: 1) + @connection = connection + @timeout = timeout + @max_retries = max_retries + + unless self.class.supports?(connection.api_format) + raise Llm::Errors::ConfigurationError, + "#{connection.api_format} needs credentials an LLM connection cannot store" + end + end + + def provider + @provider ||= connection.api_format.to_sym + end + + # Translates failures raised by a chat's own request methods. + # + # RubyLLM::Chat is a builder: the request happens later, when the caller + # invokes #ask, long after this class has returned. Extending the instance + # is what makes the error taxonomy hold for that call too, rather than + # relying on every caller to remember to wrap it. The with_* builder methods + # return self, so the extension survives them. + module TranslatesErrors + def complete(...) + Llm::Errors.wrap { super } + end + end + + # @param model_id [String] + # @return [RubyLLM::Chat] + def chat(model_id) + Llm::Errors.wrap do + context.chat(model: model_id, provider:, assume_model_exists: true) + .extend(TranslatesErrors) + end + end + + # @param input [String, Array] + # @return [RubyLLM::Embedding] + def embed(input, model:, dimensions: nil) + Llm::Errors.wrap do + context.embed(input, model:, provider:, assume_model_exists: true, dimensions:) + end + end + + private + + attr_reader :connection, :timeout, :max_retries + + # A per-call context rather than RubyLLM.configure: the global configuration + # is process-wide, and writing an administrator's endpoint and credential + # into it would leak them across requests and across tenants. + def context + @context ||= RubyLLM.context do |config| + config.public_send(:"#{provider}_api_base=", connection.base_url) + apply_credentials(config) + + config.openproject_custom_headers = connection.custom_headers + config.request_timeout = timeout + # RubyLLM retries POSTs three times by default, so one completion can be + # billed four times. Callers state what they are willing to pay for. + config.max_retries = max_retries + config.logger = Rails.logger + end + end + + def apply_credentials(config) + key = connection.api_key.presence || (PLACEHOLDER_API_KEY if api_key_required?) + + config.public_send(:"#{provider}_api_key=", key) if key + end + + # Asked of the gem rather than hard-coded: which providers insist on a key, + # as opposed to accepting a bare base URL, is RubyLLM's business and changes + # between releases. + def api_key_required? + RubyLLM::Provider.resolve(provider) + .configuration_requirements + .include?(:"#{provider}_api_key") + end + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index dd87fe96bead..16b5a2aa7643 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -686,6 +686,8 @@ en: locked: "cannot be changed while data indexed with it still exists. Re-index to switch models." llm_connection: attributes: + api_format: + not_supported: "needs credentials that OpenProject cannot store, such as a separate secret key or region." api_key: invalid_api_key: "was rejected by the LLM server." unknown_error: "could not be validated with the LLM server. Please verify that the connection is functioning properly." diff --git a/lib/open_project/patches/ruby_llm_provider_headers.rb b/lib/open_project/patches/ruby_llm_provider_headers.rb new file mode 100644 index 000000000000..a3ed85160ab0 --- /dev/null +++ b/lib/open_project/patches/ruby_llm_provider_headers.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Lets an LlmConnection's custom headers reach every request RubyLLM makes. +# +# An OpenAI-compatible deployment is routinely fronted by a gateway that +# authenticates on its own terms -- APISIX's key-auth wants an +apikey+ header, +# not +Authorization+ -- so an administrator must be able to add, and override, +# whatever the provider would send. +# +# RubyLLM's public API cannot express that: +# +# * Chat#with_headers reaches chat only. Provider#embed takes no headers +# argument at all, so an embedding request could never carry them. +# * Even for chat it merges the wrong way round -- +additional_headers.merge( +# req.headers)+ -- so the provider's own headers win every collision and an +# administrator can never replace Authorization. +# +# Provider#headers is the one hook that reaches chat, embeddings and model +# listing alike, and merging our values last makes an override actually override. +# +# The module is prepended onto each registered provider class rather than onto +# RubyLLM::Provider, because all thirteen providers define #headers themselves +# without calling super -- a patch on the base class would never be reached. +# Azure inherits OpenAI, so it receives the module twice; the inner copy is +# simply never consulted. +module OpenProject + module Patches + module RubyLLMProviderHeaders + def headers + custom = config.openproject_custom_headers + + return super if custom.blank? + + super.merge(custom.stringify_keys) + end + end + end +end + +OpenProject::Patches.patch_gem_version("ruby_llm", "1.16.0") do + RubyLLM::Configuration.register_provider_options(%i[openproject_custom_headers]) + + RubyLLM::Provider.providers.each_value do |provider_class| + provider_class.prepend(OpenProject::Patches::RubyLLMProviderHeaders) + end +end diff --git a/spec/lib/open_project/patches/ruby_llm_provider_headers_spec.rb b/spec/lib/open_project/patches/ruby_llm_provider_headers_spec.rb new file mode 100644 index 000000000000..9373d5fdc729 --- /dev/null +++ b/spec/lib/open_project/patches/ruby_llm_provider_headers_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe OpenProject::Patches::RubyLLMProviderHeaders do + def provider_for(slug, custom_headers) + context = RubyLLM.context do |config| + config.public_send(:"#{slug}_api_key=", "test-key") + config.openproject_custom_headers = custom_headers + config.logger = Rails.logger + end + + RubyLLM::Provider.resolve(slug).new(context.config) + end + + it "adds the connection's custom headers to a provider's own" do + headers = provider_for(:openai, { "apikey" => "gateway-secret" }).headers + + expect(headers).to include("apikey" => "gateway-secret") + expect(headers).to include("Authorization" => "Bearer test-key") + end + + # The point of the patch: a gateway may authenticate on its own terms, so an + # administrator has to be able to replace what the provider would send. + # RubyLLM's own Chat#with_headers merges the other way round and cannot. + it "lets a custom header override the provider's" do + headers = provider_for(:openai, { "Authorization" => "Bearer override" }).headers + + expect(headers).to include("Authorization" => "Bearer override") + end + + it "accepts symbol keys" do + headers = provider_for(:openai, { apikey: "gateway-secret" }).headers + + expect(headers).to include("apikey" => "gateway-secret") + end + + it "leaves the provider's headers untouched when none are configured" do + expect(provider_for(:openai, {}).headers).to eq(provider_for(:openai, nil).headers) + expect(provider_for(:openai, nil).headers).to include("Authorization" => "Bearer test-key") + end + + # All thirteen providers define #headers without calling super, so the patch + # has to be prepended onto each one rather than onto the base class. + it "applies to every registered provider" do + RubyLLM::Provider.providers.each_value do |provider_class| + expect(provider_class.ancestors).to include(described_class) + end + end + + it "reaches a provider that is not openai" do + headers = provider_for(:anthropic, { "apikey" => "gateway-secret" }).headers + + expect(headers).to include("apikey" => "gateway-secret") + end +end diff --git a/spec/services/llm/errors_spec.rb b/spec/services/llm/errors_spec.rb new file mode 100644 index 000000000000..45c9dc4abd41 --- /dev/null +++ b/spec/services/llm/errors_spec.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Errors do + def rubyllm_response(status) + instance_double(Faraday::Env, status:) + end + + describe ".translate" do + it "maps an unauthorized response onto an authentication error" do + translated = described_class.translate(RubyLLM::UnauthorizedError.new(rubyllm_response(401), "nope")) + + expect(translated).to be_a(Llm::Errors::AuthenticationError) + end + + it "maps a forbidden response onto an authentication error" do + translated = described_class.translate(RubyLLM::ForbiddenError.new(rubyllm_response(403), "nope")) + + expect(translated).to be_a(Llm::Errors::AuthenticationError) + end + + it "maps an exceeded context window onto a context length error" do + translated = described_class.translate(RubyLLM::ContextLengthExceededError.new(rubyllm_response(400), "too long")) + + expect(translated).to be_a(Llm::Errors::ContextLengthError) + expect(translated.status).to eq(400) + end + + it "maps throttling onto a rate limited error" do + translated = described_class.translate(RubyLLM::RateLimitError.new(rubyllm_response(429), "slow down")) + + expect(translated).to be_a(Llm::Errors::RateLimitedError) + end + + it "maps a bad request onto an api error carrying the status" do + translated = described_class.translate(RubyLLM::BadRequestError.new(rubyllm_response(400), "bad")) + + expect(translated).to be_a(Llm::Errors::ApiError) + expect(translated.status).to eq(400) + end + + # A server that does not implement the endpoint answers 404/405/501, which + # RubyLLM's middleware raises as a bare Error. This is the path that tells a + # probe "this model cannot do that" rather than "the server is broken". + it "maps an unimplemented endpoint onto an api error carrying the status" do + translated = described_class.translate(RubyLLM::Error.new(rubyllm_response(404), "not found")) + + expect(translated).to be_a(Llm::Errors::ApiError) + expect(translated.status).to eq(404) + end + + it "maps a misconfiguration onto a configuration error" do + expect(described_class.translate(RubyLLM::ConfigurationError.new("missing key"))) + .to be_a(Llm::Errors::ConfigurationError) + expect(described_class.translate(RubyLLM::ModelNotFoundError.new("no such model"))) + .to be_a(Llm::Errors::ConfigurationError) + end + + it "maps transport failures onto connection errors" do + expect(described_class.translate(Faraday::TimeoutError.new("timeout"))) + .to be_a(Llm::Errors::TimeoutError) + expect(described_class.translate(Faraday::ConnectionFailed.new("refused"))) + .to be_a(Llm::Errors::ConnectionError) + expect(described_class.translate(JSON::ParserError.new("unexpected token"))) + .to be_a(Llm::Errors::ParseError) + end + + it "passes an already translated error through untouched" do + original = Llm::Errors::SsrfError.new("Host resolves to a blocked address") + + expect(described_class.translate(original)).to be(original) + end + + # An OpenAI-compatible gateway echoes the submitted Authorization header, + # upstream provider URLs and internal hostnames in its error payloads, and + # RubyLLM puts that payload straight into the exception message. + it "never carries the upstream message into the translated error" do + secret = "Bearer sk-super-secret upstream=http://10.0.0.5:8000" + + http_errors = [RubyLLM::Error, RubyLLM::BadRequestError, RubyLLM::ForbiddenError, + RubyLLM::ContextLengthExceededError, RubyLLM::OverloadedError, + RubyLLM::PaymentRequiredError, RubyLLM::RateLimitError, RubyLLM::ServerError, + RubyLLM::ServiceUnavailableError, RubyLLM::UnauthorizedError] + plain_errors = [RubyLLM::ConfigurationError, RubyLLM::ModelNotFoundError] + + errors = http_errors.map { |klass| klass.new(rubyllm_response(400), secret) } + + plain_errors.map { |klass| klass.new(secret) } + + errors.each do |error| + message = described_class.translate(error).message + + expect(message).not_to include("sk-super-secret") + expect(message).not_to include("10.0.0.5") + end + end + end + + describe ".wrap" do + it "returns the block's value when nothing is raised" do + expect(described_class.wrap { :fine }).to be(:fine) + end + + it "re-raises a RubyLLM failure as an Llm::Errors" do + expect { described_class.wrap { raise RubyLLM::UnauthorizedError.new(rubyllm_response(401), "nope") } } + .to raise_error(Llm::Errors::AuthenticationError) + end + end + + describe "the Llm::Client aliases" do + it "resolve to the same classes, so existing rescues keep working" do + expect(Llm::Client::Error).to be(Llm::Errors::Error) + expect(Llm::Client::ApiError).to be(Llm::Errors::ApiError) + expect(Llm::Client::SsrfError).to be(Llm::Errors::SsrfError) + end + end +end diff --git a/spec/services/llm/session_spec.rb b/spec/services/llm/session_spec.rb new file mode 100644 index 000000000000..3871da3264a3 --- /dev/null +++ b/spec/services/llm/session_spec.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Session, :llm_server_helpers, :webmock do + let(:base_url) { "https://example.com/v1" } + let(:connection) { create(:llm_connection, base_url:, api_key: "sk-test-key") } + + describe ".supports?" do + it "rejects the formats a connection cannot supply credentials for" do + expect(described_class.supports?("bedrock")).to be(false) + expect(described_class.supports?("vertexai")).to be(false) + end + + it "accepts the rest" do + expect(described_class.supports?("openai")).to be(true) + expect(described_class.supports?("anthropic")).to be(true) + end + end + + describe "#initialize" do + it "refuses a format whose credentials cannot be stored" do + connection.update_column(:api_format, "bedrock") + + expect { described_class.for(connection) } + .to raise_error(Llm::Errors::ConfigurationError, /bedrock/) + end + end + + describe "#chat" do + it "sends the completion to the configured base URL with the stored key" do + stub = mock_llm_chat_response(base_url, content: "pong") + + answer = described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") + + expect(answer.content).to eq("pong") + expect(stub).to have_been_requested.once + expect(WebMock).to have_requested(:post, "#{base_url}/chat/completions") + .with(headers: { "Authorization" => "Bearer sk-test-key" }) + end + + it "sends the connection's custom headers" do + connection.update!(custom_headers: { "apikey" => "gateway-secret" }) + mock_llm_chat_response(base_url) + + described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") + + expect(WebMock).to have_requested(:post, "#{base_url}/chat/completions") + .with(headers: { "apikey" => "gateway-secret" }) + end + + it "translates a rejected key into an authentication error" do + mock_llm_chat_response(base_url, response_code: 401) + + expect { described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") } + .to raise_error(Llm::Errors::AuthenticationError) + end + + # RubyLLM retries POSTs three times by default, so an unbounded retry would + # bill four completions for one call. + it "does not retry when told not to" do + stub = mock_llm_chat_response(base_url, response_code: 500) + + expect { described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") } + .to raise_error(Llm::Errors::ApiError) + expect(stub).to have_been_requested.once + end + + # A self-hosted OpenAI-compatible server commonly needs no credential, but + # RubyLLM's ensure_configured! raises unless one is set. + it "reaches a server that needs no API key" do + connection.update!(api_key: nil) + mock_llm_chat_response(base_url) + + expect { described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") } + .not_to raise_error + end + end + + describe "#embed" do + it "requests a vector and carries the custom headers" do + connection.update!(custom_headers: { "apikey" => "gateway-secret" }) + mock_llm_embeddings_response(base_url, dimensions: 8) + + embedding = described_class.for(connection, max_retries: 0).embed("hello", model: "bge-m3") + + expect(embedding.vectors.length).to eq(8) + expect(WebMock).to have_requested(:post, "#{base_url}/embeddings") + .with(headers: { "apikey" => "gateway-secret" }) + end + end + + describe "global configuration" do + it "never writes the connection's settings into RubyLLM's global config" do + mock_llm_chat_response(base_url) + + described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") + + expect(RubyLLM.config.openai_api_base).to be_nil + expect(RubyLLM.config.openai_api_key).to be_nil + expect(RubyLLM.config.openproject_custom_headers).to be_nil + end + end +end diff --git a/spec/support/llm_server_helpers.rb b/spec/support/llm_server_helpers.rb index b4562a7771cd..52fc9689e519 100644 --- a/spec/support/llm_server_helpers.rb +++ b/spec/support/llm_server_helpers.rb @@ -52,6 +52,64 @@ def mock_llm_models_response(base_url, ) end + DEFAULT_CHAT_BODY = { + id: "chatcmpl-test", + object: "chat.completion", + model: "qwen3.6-27b", + choices: [{ index: 0, message: { role: "assistant", content: "pong" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } + }.freeze + + # Stubs POST /chat/completions. + # + # Note that RubyLLM retries POSTs up to max_retries times on 429 and 5xx, so a + # spec stubbing one of those against a session built with the default + # max_retries sees more than one request. Build the session with an explicit + # max_retries when the request count matters. + def mock_llm_chat_response(base_url, + content: "pong", + response_code: 200, + body: nil, + timeout: false) + stub = stub_request(:post, "#{base_url.chomp('/')}/chat/completions") + + return stub.to_timeout if timeout + + payload = body || DEFAULT_CHAT_BODY.merge( + choices: [{ index: 0, message: { role: "assistant", content: }, finish_reason: "stop" }] + ).to_json + + stub.to_return( + status: response_code, + headers: { "Content-Type" => "application/json" }, + body: payload.is_a?(String) ? payload : payload.to_json + ) + end + + # Stubs POST /embeddings, returning a vector of the requested size. + def mock_llm_embeddings_response(base_url, + dimensions: 4, + response_code: 200, + body: nil, + timeout: false) + stub = stub_request(:post, "#{base_url.chomp('/')}/embeddings") + + return stub.to_timeout if timeout + + payload = body || { + object: "list", + model: "bge-m3", + data: [{ object: "embedding", index: 0, embedding: Array.new(dimensions) { 0.1 } }], + usage: { prompt_tokens: 1, total_tokens: 1 } + } + + stub.to_return( + status: response_code, + headers: { "Content-Type" => "application/json" }, + body: payload.is_a?(String) ? payload : payload.to_json + ) + end + # example.com resolves publicly, but a spec that needs a literal or private # host has to say so explicitly rather than opening the allowlist to 0.0.0.0/0. def allow_llm_host(*hosts) From 098b8a7f7c9bdabda9fe5250fe0805581af32166 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Tue, 11 Aug 2026 16:35:38 +0100 Subject: [PATCH 24/44] [#66020] Let a resolved feature run a request Llm::Runtime could say which model a feature should use, but a feature then had no way to use it. Resolution gains #chat, #embed and #session, so model resolution and request construction stay in one place. They refuse unless the resolution is ready, raising NotReady with the status so a caller can tell "no server configured" from "this model cannot do that", and refuse a kind mismatch outright -- asking a chat feature to embed is a bug, and should not reach the server to find that out. A comment records that RubyLLM enforces none of a feature's declared requirements: with_schema performs no capability check, and an assumed model is described by Model::Info.default, which claims structured output, vision and function calling for everything. The capability verdicts are the only real gate. StructuredOutput.parse! exists for the same reason -- RubyLLM rescues a JSON parse failure and leaves the content a String, so a feature asking for a schema can silently receive prose. That is likeliest on a self-hosted server, exactly where the structured_output verdict is unknown rather than supported. The embeddings probe moves onto Llm::Session too, so every request that reaches an LLM server goes through one place and the probe inherits the connection's custom headers -- which it never had, so a probe against a gateway that authenticates on its own terms would have been recorded as "unsupported" when it was merely unauthenticated. Llm::Client keeps model discovery and loses everything else. RubyLLM's model parsing discards max_model_len and root, the only trustworthy statement of a self-hosted deployment's context window, and discovery stays on OpenProject.httpx so it remains covered by the SSRF filter RubyLLM's Faraday stack bypasses. https://community.openproject.org/work_packages/66020 --- app/services/llm/client.rb | 33 +++--------- app/services/llm/probes/embeddings_probe.rb | 28 +++++----- app/services/llm/runtime.rb | 36 +++++++++++++ app/services/llm/structured_output.rb | 55 +++++++++++++++++++ spec/services/llm/runtime_spec.rb | 42 +++++++++++++++ spec/services/llm/structured_output_spec.rb | 58 +++++++++++++++++++++ 6 files changed, 215 insertions(+), 37 deletions(-) create mode 100644 app/services/llm/structured_output.rb create mode 100644 spec/services/llm/structured_output_spec.rb diff --git a/app/services/llm/client.rb b/app/services/llm/client.rb index c3b3aff98549..6709f0e145e7 100644 --- a/app/services/llm/client.rb +++ b/app/services/llm/client.rb @@ -29,7 +29,14 @@ #++ module Llm - # A thin client for an OpenAI-API-compatible server. + # A thin model-discovery client for an OpenAI-API-compatible server. + # + # Inference goes through Llm::Session and RubyLLM. Discovery deliberately + # stays here, for two reasons. RubyLLM's model parsing discards max_model_len + # and root, which are the only trustworthy statement of a self-hosted + # deployment's real context window, and it substitutes OpenAI's capability + # heuristics for arbitrary model ids. And this path runs on OpenProject.httpx, + # so it is covered by the SSRF filter that RubyLLM's own Faraday stack bypasses. # # The configured base URL is expected to already contain the API version segment # (for example +https://example.com/v1+), matching what every provider documents @@ -55,10 +62,6 @@ class Client timeout: { connect_timeout: 5, read_timeout: 15, request_timeout: 20 } }.freeze - INFERENCE_TIMEOUT = { - timeout: { connect_timeout: 5, read_timeout: 120, request_timeout: 180 } - }.freeze - def initialize(base_url:, api_key: nil, timeout: PROBE_TIMEOUT, headers: {}) @base_url = base_url.to_s.chomp("/") @api_key = api_key @@ -80,30 +83,10 @@ def models body end - # Requests an embedding vector for a single short input. - # - # Used to determine whether a model can serve embeddings at all: the model - # list says nothing about it, and posting a chat completion to an embedding - # model (or the reverse) is the only reliable way to find out. - # - # @return [Hash] the parsed +POST /embeddings+ body - def embeddings(model:, input:) - post("/embeddings", { model:, input: }) - end - private attr_reader :base_url, :api_key, :timeout, :headers - def post(path, payload) - response = session.post(uri_for(path), json: payload) - handle_transport_error(response) if response.is_a?(HTTPX::ErrorResponse) - handle_status(response) - parse(response) - rescue OpenProject::HttpxSsrfFilter::ServerSideRequestForgeryError - raise SsrfError, "Host resolves to a blocked address" - end - def get(path) response = session.get(uri_for(path)) # A connection-level failure yields an HTTPX::ErrorResponse. A real response diff --git a/app/services/llm/probes/embeddings_probe.rb b/app/services/llm/probes/embeddings_probe.rb index f362798e3ff0..4bf2fc225606 100644 --- a/app/services/llm/probes/embeddings_probe.rb +++ b/app/services/llm/probes/embeddings_probe.rb @@ -45,6 +45,10 @@ module Probes class EmbeddingsProbe PROBE_INPUT = "openproject" + # The server understood the request and refused it for this model. Anything + # else -- 5xx, throttling -- says something about the server, not the model. + REFUSED_STATUSES = [400, 404, 405, 501].freeze + Result = Data.define(:state, :detail) do def supported? = state == :supported end @@ -54,17 +58,14 @@ def initialize(connection) end def call(model_id) - body = client.embeddings(model: model_id, input: PROBE_INPUT) - classify(body) - rescue Llm::Client::ApiError => e - # The server understood the request and refused it for this model. - return unsupported(e.status) if e.status.in?([400, 404, 405, 501]) + classify(session.embed(PROBE_INPUT, model: model_id)) + rescue Llm::Errors::ApiError => e + return unsupported(e.status) if e.status.in?(REFUSED_STATUSES) - # 5xx and anything else says something about the server, not the model. unknown("http_#{e.status}") - rescue Llm::Client::AuthenticationError + rescue Llm::Errors::AuthenticationError unknown("unauthorized") - rescue Llm::Client::Error => e + rescue Llm::Errors::Error => e unknown(e.class.name.demodulize.underscore) end @@ -72,12 +73,15 @@ def call(model_id) attr_reader :connection - def client - @client ||= Llm::Client.new(base_url: connection.base_url, api_key: connection.api_key) + # Never retried: a refusal is the answer we are looking for, and repeating + # it would only slow the probe down. + def session + @session ||= Llm::Session.for(connection, timeout: Llm::Session::PROBE_TIMEOUT, max_retries: 0) end - def classify(body) - vector = Array(body["data"]).first&.dig("embedding") + def classify(embedding) + vector = embedding.vectors + vector = vector.first if vector.is_a?(Array) && vector.first.is_a?(Array) if vector.is_a?(Array) && vector.any? && vector.all?(Numeric) Result.new(state: :supported, detail: { "dimensions" => vector.length }) diff --git a/app/services/llm/runtime.rb b/app/services/llm/runtime.rb index 11835b68cfd2..8acfcb0aa763 100644 --- a/app/services/llm/runtime.rb +++ b/app/services/llm/runtime.rb @@ -46,6 +46,42 @@ class Runtime # :incapable - the chosen model is known not to support what is needed Resolution = Data.define(:feature, :connection, :model_id, :status, :missing_capabilities) do def ready? = status == :ready + + # A chat builder for the resolved model. + # + # Note that RubyLLM enforces none of the feature's declared requirements: + # Chat#with_schema performs no capability check, and a model absent from + # RubyLLM's registry is described by Model::Info.default, which claims + # structured output, vision and function calling for everything. The + # capability verdicts consulted in #call above are the only real gate. + # + # @return [RubyLLM::Chat] + def chat(**) + ensure_usable!(:chat) + session(**).chat(model_id) + end + + # @return [RubyLLM::Embedding] + def embed(input, dimensions: nil, **) + ensure_usable!(:embedding) + session(**).embed(input, model: model_id, dimensions:) + end + + # @return [Llm::Session] + def session(**) + Llm::Session.for(connection, **) + end + + private + + # Features are resolved by kind, so asking a chat feature to embed means a + # caller has confused two features -- a bug, not a configuration problem. + def ensure_usable!(kind) + raise Llm::Errors::NotReady, status unless ready? + return if feature.public_send(:"#{kind}?") + + raise Llm::Errors::NotReady, :wrong_kind + end end class << self diff --git a/app/services/llm/structured_output.rb b/app/services/llm/structured_output.rb new file mode 100644 index 000000000000..5aa0af7f0dd9 --- /dev/null +++ b/app/services/llm/structured_output.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Turns a schema-constrained answer into a Hash, or fails loudly. + # + # RubyLLM's Chat#with_schema fails soft: when the answer does not parse as + # JSON it rescues and leaves the content as a String, so a caller expecting a + # Hash gets a String and only notices further downstream. + # + # That matters most on a self-hosted server, which is exactly where the + # structured_output capability verdict is :unknown rather than :supported, and + # so exactly where the model is most likely to answer in prose. + module StructuredOutput + module_function + + # @param message [RubyLLM::Message] the answer from Chat#ask + # @raise [Llm::Errors::ParseError] when the model did not honour the schema + # @return [Hash] + def parse!(message) + content = message.respond_to?(:content) ? message.content : message + + return content.deep_symbolize_keys if content.is_a?(Hash) + + raise Llm::Errors::ParseError, "Model did not answer with the requested structure" + end + end +end diff --git a/spec/services/llm/runtime_spec.rb b/spec/services/llm/runtime_spec.rb index dca4948f5a87..b578b75bc93f 100644 --- a/spec/services/llm/runtime_spec.rb +++ b/spec/services/llm/runtime_spec.rb @@ -119,4 +119,46 @@ expect(resolution).to be_ready end end + + describe "running a request", :llm_server_helpers, :webmock do + let!(:connection) { create(:llm_connection, :with_models, :enabled, default_chat_model_id: "qwen3.6-27b") } + + it "sends a completion for the resolved model" do + mock_llm_chat_response("https://example.com/v1", content: "pong") + + expect(resolution.chat(max_retries: 0).ask("ping").content).to eq("pong") + expect(WebMock).to have_requested(:post, "https://example.com/v1/chat/completions") + .with(body: hash_including("model" => "qwen3.6-27b")) + end + + it "refuses when the feature is not ready" do + connection.update!(enabled: false) + + expect { resolution.chat }.to raise_error(Llm::Errors::NotReady) { |e| expect(e.status).to eq(:no_connection) } + end + + # Features are resolved by kind, so asking a chat feature to embed means a + # caller has confused two features. + it "refuses to embed through a chat feature" do + expect { resolution.embed("hello") } + .to raise_error(Llm::Errors::NotReady) { |e| expect(e.status).to eq(:wrong_kind) } + end + + context "with an embedding feature" do + let(:feature_key) { :semantic_search } + + before { connection.update!(default_embedding_model_id: "bge-m3") } + + it "requests a vector for the resolved model" do + mock_llm_embeddings_response("https://example.com/v1", dimensions: 8) + + expect(resolution.embed("hello", max_retries: 0).vectors.length).to eq(8) + end + + it "refuses to chat through an embedding feature" do + expect { resolution.chat } + .to raise_error(Llm::Errors::NotReady) { |e| expect(e.status).to eq(:wrong_kind) } + end + end + end end diff --git a/spec/services/llm/structured_output_spec.rb b/spec/services/llm/structured_output_spec.rb new file mode 100644 index 000000000000..b7ee9a7d4357 --- /dev/null +++ b/spec/services/llm/structured_output_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::StructuredOutput do + def message(content) + instance_double(RubyLLM::Message, content:) + end + + it "returns the structure with symbol keys" do + expect(described_class.parse!(message({ "summary" => "hi", "tags" => %w[a b] }))) + .to eq(summary: "hi", tags: %w[a b]) + end + + it "symbolizes nested keys" do + expect(described_class.parse!(message({ "a" => { "b" => 1 } }))).to eq(a: { b: 1 }) + end + + # RubyLLM rescues a JSON parse failure and leaves the content a String, so + # without this the caller gets a String where it expected a Hash and only + # notices much further downstream. + it "raises when the model answered in prose instead" do + expect { described_class.parse!(message("Sure! Here is the summary you asked for.")) } + .to raise_error(Llm::Errors::ParseError) + end + + it "accepts a bare structure as well as a message" do + expect(described_class.parse!({ "a" => 1 })).to eq(a: 1) + end +end From f20c25c77286e20d53ec2ed09c6756501d590d3b Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Tue, 11 Aug 2026 19:20:49 +0100 Subject: [PATCH 25/44] [#66020] Prepare the health report framework for a third consumer Two small changes to shared code, kept separate so they can be reviewed on their own. HealthReports::ResultComponent hard-coded every "More information" link to the file storages troubleshooting page. That was accurate while storages was the only consumer, became wrong when wikis adopted the framework, and would be wrong again for the LLM connection. Both components now take an optional docs_href, defaulting to the storages link so the existing consumers are untouched. Every consumer reads the newest report for a subject, and health_reports carries only a [subject_type, subject_id] index, so that read sorts. It has not mattered because the table only grew when somebody clicked "Run checks". The LLM connection adds a scheduled check and a pruning delete that filters on age, so the index is added now -- concurrently, since the table exists in production. Storages and wikis get the faster read for free. https://community.openproject.org/work_packages/66020 --- .../health_reports/report_component.html.erb | 2 +- .../health_reports/report_component.rb | 8 +++- .../health_reports/result_component.rb | 10 +++- config/static_links.yml | 2 + ..._add_created_at_index_to_health_reports.rb | 47 +++++++++++++++++++ 5 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 db/migrate/20260813090000_add_created_at_index_to_health_reports.rb diff --git a/app/components/health_reports/report_component.html.erb b/app/components/health_reports/report_component.html.erb index f177400bbf27..16a4b745c38c 100644 --- a/app/components/health_reports/report_component.html.erb +++ b/app/components/health_reports/report_component.html.erb @@ -65,7 +65,7 @@ See COPYRIGHT and LICENSE files for more details. result_group.results.each do |value| box.with_row do - render(HealthReports::ResultComponent.new(group: result_group.key, result: value, i18n_scope:)) + render(HealthReports::ResultComponent.new(group: result_group.key, result: value, i18n_scope:, docs_href:)) end end end diff --git a/app/components/health_reports/report_component.rb b/app/components/health_reports/report_component.rb index f270b323c41f..1eb5f3b184e7 100644 --- a/app/components/health_reports/report_component.rb +++ b/app/components/health_reports/report_component.rb @@ -37,14 +37,18 @@ class ReportComponent < ApplicationComponent # The i18n_scope parameter defines the I18n scope that should be used to resolve # names of groups, checks and error messages indicated by the results. - def initialize(*, i18n_scope:, **) + # + # docs_href overrides where each result's "More information" link points; + # without it, results link to the file storages troubleshooting page. + def initialize(*, i18n_scope:, docs_href: nil, **) super(*, **) @i18n_scope = i18n_scope + @docs_href = docs_href end private - attr_reader :i18n_scope + attr_reader :i18n_scope, :docs_href def summary_icon(check_tally) case check_tally diff --git a/app/components/health_reports/result_component.rb b/app/components/health_reports/result_component.rb index 5a57739f09f7..f96057dfc423 100644 --- a/app/components/health_reports/result_component.rb +++ b/app/components/health_reports/result_component.rb @@ -32,10 +32,16 @@ module HealthReports class ResultComponent < ApplicationComponent include OpPrimer::ComponentHelpers - def initialize(group:, result:, i18n_scope:) + # Where "More information" points. Defaults to the file storages + # documentation because that was this component's only consumer for a long + # time; a subject with its own troubleshooting page passes its own. + DEFAULT_DOCS_HREF = -> { ::OpenProject::Static::Links.url_for(:storage_docs, :health_status) } + + def initialize(group:, result:, i18n_scope:, docs_href: nil) super(result) @group = group @i18n_scope = i18n_scope + @docs_href = docs_href end private @@ -49,7 +55,7 @@ def error_text I18n.t("errors.#{model.code}", scope: @i18n_scope, **model.context&.symbolize_keys) end - def docs_href = ::OpenProject::Static::Links.url_for(:storage_docs, :health_status) + def docs_href = @docs_href || DEFAULT_DOCS_HREF.call def error_code if model.failure? diff --git a/config/static_links.yml b/config/static_links.yml index 2dccc1fb9540..7f2f02891022 100644 --- a/config/static_links.yml +++ b/config/static_links.yml @@ -186,6 +186,8 @@ sysadmin_docs: href: https://www.openproject.org/docs/system-admin-guide/authentication/ldap-connections/ ldap_group_sync: href: https://www.openproject.org/docs/system-admin-guide/authentication/ldap-connections/ldap-group-synchronization/ + llm_connection: + href: https://www.openproject.org/docs/system-admin-guide/ai/llm-connection/ mcp_resources: href: https://www.openproject.org/docs/system-admin-guide/integrations/mcp-server/#resources mcp_tools: diff --git a/db/migrate/20260813090000_add_created_at_index_to_health_reports.rb b/db/migrate/20260813090000_add_created_at_index_to_health_reports.rb new file mode 100644 index 000000000000..dfcba13aa170 --- /dev/null +++ b/db/migrate/20260813090000_add_created_at_index_to_health_reports.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Every consumer of a health report reads the newest one for a subject +# (`health_reports.order(created_at: :asc).last`), and the table has only a +# [subject_type, subject_id] index, so that read sorts. Until now the table grew +# a row per manual "Run checks" click; the LLM connection adds a scheduled check, +# which makes it grow unattended and adds a pruning delete that filters on age. +# +# Storages and wikis benefit from this too. +class AddCreatedAtIndexToHealthReports < ActiveRecord::Migration[8.1] + disable_ddl_transaction! + + def change + add_index :health_reports, + %i[subject_type subject_id created_at], + algorithm: :concurrently, + if_not_exists: true + end +end From e6ef88c8b157f5c8b7ffaeb0b6fd0810cd60a23a Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Tue, 11 Aug 2026 22:47:11 +0100 Subject: [PATCH 26/44] [#66020] Check and show an LLM connection's health The branch could configure a connection but never verify one. Saving proved the model list was readable, which says nothing about a server that publishes no model list -- a state that became supported when manual model entry landed. Llm::Validators::ConnectionValidator answers the question on the same HealthReports framework storages and wikis use. Five groups: configuration, what is knowable without asking the server; server, the free model-list call where the endpoint offers one; inference, a real completion; models, what the stored catalogue says; and features, whether each registered feature can actually run. The features group is driven by Llm::Runtime, so the health report and the runtime can never disagree about what is usable. It answers "will the AI features work", which is what an administrator actually wants to know. The inference group is separated because it is the only one that costs money. It runs when an administrator asks, gated on a non-persisted deep_health_check accessor -- a property of the run, not of the connection. Two things are deliberately not probed. Tool calling and structured output: several current vLLM releases answer a tool call with plain text on a fully capable server, so a probe would launder a wrong answer into a confident one. And a missing model list is never a failure, only a warning, because a gateway exposing chat alone is a supported deployment. The UI adds the side panel, the report page and the downloadable report, so "Test connection" -- which the ticket asked for and the branch never had -- is now the "Run checks" action, and the answer persists instead of vanishing with a flash. The download is built from non_confidential_configuration: it excludes the API key and the custom headers, since a gateway header routinely carries a second credential, and a request spec asserts both are absent. A spec walks the validator across four server states and asserts every check key and error code resolves; without it a missing key renders as "translation missing" and nothing else would notice. https://community.openproject.org/work_packages/66020 --- .../health_status_component.html.erb | 58 ++++++ .../side_panel/health_status_component.rb | 75 ++++++++ .../admin/llm_health_status_controller.rb | 105 ++++++++++ app/models/llm_connection.rb | 41 ++++ .../llm/validators/configuration_validator.rb | 87 +++++++++ .../llm/validators/connection_validator.rb | 54 ++++++ .../llm/validators/feature_validator.rb | 105 ++++++++++ .../llm/validators/inference_validator.rb | 91 +++++++++ .../llm/validators/model_validator.rb | 94 +++++++++ .../llm/validators/server_validator.rb | 99 ++++++++++ app/views/admin/llm_connections/show.html.erb | 8 + .../admin/llm_health_status/show.html.erb | 98 ++++++++++ config/locales/en.yml | 66 +++++++ config/routes.rb | 4 + spec/requests/admin/llm_health_status_spec.rb | 129 +++++++++++++ .../validators/connection_validator_spec.rb | 179 ++++++++++++++++++ 16 files changed, 1293 insertions(+) create mode 100644 app/components/llm_connections/side_panel/health_status_component.html.erb create mode 100644 app/components/llm_connections/side_panel/health_status_component.rb create mode 100644 app/controllers/admin/llm_health_status_controller.rb create mode 100644 app/services/llm/validators/configuration_validator.rb create mode 100644 app/services/llm/validators/connection_validator.rb create mode 100644 app/services/llm/validators/feature_validator.rb create mode 100644 app/services/llm/validators/inference_validator.rb create mode 100644 app/services/llm/validators/model_validator.rb create mode 100644 app/services/llm/validators/server_validator.rb create mode 100644 app/views/admin/llm_health_status/show.html.erb create mode 100644 spec/requests/admin/llm_health_status_spec.rb create mode 100644 spec/services/llm/validators/connection_validator_spec.rb diff --git a/app/components/llm_connections/side_panel/health_status_component.html.erb b/app/components/llm_connections/side_panel/health_status_component.html.erb new file mode 100644 index 000000000000..dba7bc54848e --- /dev/null +++ b/app/components/llm_connections/side_panel/health_status_component.html.erb @@ -0,0 +1,58 @@ +<%= + component_wrapper(tag: :turbo_frame, refresh: :morph) do + render(Primer::OpenProject::SidePanel::Section.new) do |section| + section.with_title { t(".title") } + + flex_layout do |container| + if report.present? + container.with_row do + header = summary_header + concat(render(Primer::Beta::Octicon.new(icon: header[:icon], color: header[:icon_color], mr: 2))) + concat(render(Primer::Beta::Text.new(font_weight: :bold)) { header[:text] }) + end + + container.with_row(mt: 2) do + render( + Primer::Beta::Text.new(color: :muted, test_selector: "llm-connection--health-summary") + ) { summary_description } + end + + container.with_row(mt: 2) do + render( + Primer::Beta::Button.new( + scheme: :link, color: :default, font_weight: :bold, + tag: :a, href: llm_connection_health_status_report_path, + test_selector: "llm-connection--open-health-report" + ) + ) do |button| + button.with_leading_visual_icon(icon: :meter) + t(".open_report") + end + end + else + container.with_row do + render(Primer::Beta::Text.new(color: :muted)) { t(".never_checked") } + end + + container.with_row(mt: 2) do + primer_form_with( + url: create_health_status_report_llm_connection_health_status_report_path, + method: :post, + data: { turbo: true } + ) do + render( + Primer::Beta::Button.new( + scheme: :link, color: :default, font_weight: :bold, type: :submit, + test_selector: "llm-connection--run-health-checks" + ) + ) do |button| + button.with_leading_visual_icon(icon: :meter) + t(".run_checks") + end + end + end + end + end + end + end +%> diff --git a/app/components/llm_connections/side_panel/health_status_component.rb b/app/components/llm_connections/side_panel/health_status_component.rb new file mode 100644 index 000000000000..e7145a678387 --- /dev/null +++ b/app/components/llm_connections/side_panel/health_status_component.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + module SidePanel + # "Has this connection ever actually worked?", answered on the settings page. + class HealthStatusComponent < ApplicationComponent + include ApplicationHelper + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + alias_method :connection, :model + + private + + def report + connection.latest_health_report + end + + def summary_header + tally = report.tally + + case tally + in { failure: 1.. } + { icon: :alert, icon_color: :danger, + text: I18n.t("health_reports.common.checks.failures", count: tally[:failure]) } + in { warning: 1.. } + { icon: :alert, icon_color: :attention, + text: I18n.t("health_reports.common.checks.warnings", count: tally[:warning]) } + else + { icon: :"check-circle", icon_color: :success, text: I18n.t("health_reports.common.checks.success") } + end + end + + def summary_description + text = if report.healthy? + I18n.t("health_reports.common.summary.success") + elsif report.unhealthy? + I18n.t("health_reports.common.summary.failure") + else + I18n.t("health_reports.common.summary.warning") + end + + "#{text} #{t('.last_check', datetime: helpers.format_time(report.created_at))}" + end + end + end +end diff --git a/app/controllers/admin/llm_health_status_controller.rb b/app/controllers/admin/llm_health_status_controller.rb new file mode 100644 index 000000000000..3ddb70f926d6 --- /dev/null +++ b/app/controllers/admin/llm_health_status_controller.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + class LlmHealthStatusController < ApplicationController + include OpTurbo::ComponentStream + + layout :admin_or_frame_layout + + before_action :require_admin + before_action :find_connection + + menu_item :llm_connection + + def show + @report = @connection.latest_health_report + + respond_to do |format| + format.html + format.text do + return head :not_found if @report.nil? + + timestamp = @report.created_at.iso8601 + send_data text_report(timestamp), + filename: "llm_connection_health_report_#{timestamp}.txt", + type: "text/plain", + disposition: :attachment + end + end + end + + # A full run, including the billed completion: an administrator clicking + # "Run checks" is asking whether the connection actually works. + def create + run_checks + redirect_to llm_connection_health_status_report_path, status: :see_other + end + + def create_health_status_report + run_checks + update_via_turbo_stream(component: LlmConnections::SidePanel::HealthStatusComponent.new(@connection)) + respond_with_turbo_streams + end + + private + + def admin_or_frame_layout + turbo_frame_request? ? "turbo_rails/frame" : "admin" + end + + def run_checks + @connection.deep_health_check = true + report = Llm::Validators::ConnectionValidator.new(@connection).call + report.save! + report + end + + # Downloaded and pasted into support tickets, so it must not contain the API + # key or the custom headers -- a gateway header routinely carries a second + # credential. + def text_report(timestamp) + { + connection: @connection.name, + configuration: @connection.non_confidential_configuration, + ran_at: timestamp, + results: @report ? @report.results.map(&:to_h) : [] + }.to_yaml(stringify_names: true) + end + + # HealthReports::Validator builds the report through the association, which + # would insert an unpersisted singleton along with it. + def find_connection + @connection = LlmConnection.instance + + redirect_to llm_connection_path unless @connection.persisted? + end + end +end diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb index 6e393c890664..b7e845cc3699 100644 --- a/app/models/llm_connection.rb +++ b/app/models/llm_connection.rb @@ -99,6 +99,47 @@ def server_flavour options["server_flavour"].presence&.to_sym end + # Whether a health check run may spend a real completion. + # + # Not a column: it describes the run, not the connection. An administrator + # asking to test the connection wants it; the scheduled re-check leaves it off + # so that a connection to a paid provider is not billed four times a day. + attr_accessor :deep_health_check + + def deep_health_check? = ActiveModel::Type::Boolean.new.cast(deep_health_check).present? + + def latest_health_report + health_reports.order(created_at: :asc).last + end + + # Derived rather than stored, for the same reason LlmFeatureBinding#dangling? + # is: a status column would be a cache with no invalidation trigger, and would + # be stale exactly when it matters. + def health_state + report = latest_health_report + + return :unknown if report.nil? + return :unhealthy if report.unhealthy? + return :warning if report.warning? + + :healthy + end + + # Feeds the downloadable health report. Deliberately excludes api_key *and* + # custom_headers: a gateway header routinely carries a second credential. + def non_confidential_configuration + { + base_url:, + api_format:, + enabled:, + server_flavour:, + catalogue_fetched_at:, + last_connected_at:, + model_count: models.count, + manual_model_count: models.where(manual: true).count + } + end + private def only_one_connection diff --git a/app/services/llm/validators/configuration_validator.rb b/app/services/llm/validators/configuration_validator.rb new file mode 100644 index 000000000000..0170d38bb62c --- /dev/null +++ b/app/services/llm/validators/configuration_validator.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # What can be said about the connection without asking the server anything. + class ConfigurationValidator < HealthReports::ValidatorGroup + def self.key = :configuration + + private + + def validate + register_checks(:feature_flag, :base_url_present, :api_format_supported, :credentials_present, :enabled) + + feature_flag + base_url_present + api_format_supported + credentials_present + connection_enabled + end + + def feature_flag + if OpenProject::FeatureDecisions.llm_connection_active? + pass_check(:feature_flag) + else + # Not a failure: an administrator may reasonably configure and test the + # connection before switching the feature on. + warn_check(:feature_flag, :feature_flag_off) + end + end + + def base_url_present + return pass_check(:base_url_present) if subject.base_url.present? + + fail_check(:base_url_present, :not_configured) + end + + def api_format_supported + return pass_check(:api_format_supported) if Llm::Session.supports?(subject.api_format) + + fail_check(:api_format_supported, :unsupported_api_format, context: { api_format: subject.api_format.to_s }) + end + + # Only ever a warning. A self-hosted server on a trusted network legitimately + # needs no credential, and a key that is genuinely required but wrong is + # reported precisely by the server group rather than guessed at here. + def credentials_present + return pass_check(:credentials_present) if subject.api_key.present? + + warn_check(:credentials_present, :api_key_missing) + end + + def connection_enabled + return pass_check(:enabled) if subject.enabled? + + warn_check(:enabled, :connection_disabled) + end + end + end +end diff --git a/app/services/llm/validators/connection_validator.rb b/app/services/llm/validators/connection_validator.rb new file mode 100644 index 000000000000..77ee9858b7d7 --- /dev/null +++ b/app/services/llm/validators/connection_validator.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # The health report for an LLM connection. + # + # Nothing here reads User.current: the whole report has to be reproducible + # from a background job, which is what lets the same code answer both the + # administrator's "Run checks" and the scheduled re-check. + class ConnectionValidator < HealthReports::Validator + register_group ConfigurationValidator + + register_group ServerValidator, + precondition: ->(_, report) { report.group(:configuration).non_failure? } + + # Costs a real completion, so it is not part of the scheduled run. + register_group InferenceValidator, + precondition: lambda { |connection, report| + connection.deep_health_check? && report.group(:configuration).non_failure? + } + + register_group ModelValidator + register_group FeatureValidator + end + end +end diff --git a/app/services/llm/validators/feature_validator.rb b/app/services/llm/validators/feature_validator.rb new file mode 100644 index 000000000000..0d587a7e0443 --- /dev/null +++ b/app/services/llm/validators/feature_validator.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # Whether each registered feature can actually run. + # + # Driven by Llm::Runtime, so the health report and the runtime can never + # disagree about what is usable. That makes this the most informative group + # in the report: it answers "will the AI features work" rather than "is the + # server up". + # + # The check keys are fixed and the feature names travel in the result + # context. Per-feature keys would need a translation per feature and would + # break HealthReports::ResultComponent#text, which resolves a check's label + # from its key. + class FeatureValidator < HealthReports::ValidatorGroup + def self.key = :features + + private + + def validate + register_checks(:bindings_resolvable, :locked_bindings_intact) + + bindings_resolvable + locked_bindings_intact + end + + def bindings_resolvable + by_status = resolutions.group_by(&:status) + + if (incapable = by_status[:incapable]).present? + fail_check(:bindings_resolvable, :features_incapable, + context: { features: labels(incapable), + capabilities: capability_labels(incapable) }) + end + + if (missing = by_status[:model_missing]).present? + fail_check(:bindings_resolvable, :features_model_missing, context: { features: labels(missing) }) + end + + if (unbound = by_status[:unbound]).present? + warn_check(:bindings_resolvable, :features_unbound, context: { features: labels(unbound) }) + end + + pass_check(:bindings_resolvable) + end + + # A locked binding is the record that a vector index exists and which model + # and dimension it was written under. If that model has left the catalogue + # the index can no longer be extended or queried consistently, which is a + # data problem rather than a configuration one. + def locked_bindings_intact + broken = subject.feature_bindings.select { |binding| binding.locked? && binding.dangling? } + + return pass_check(:locked_bindings_intact) if broken.empty? + + fail_check(:locked_bindings_intact, :locked_model_missing, + context: { features: broken.filter_map { |b| b.feature&.label }.join(", ") }) + end + + def resolutions + @resolutions ||= OpenProject::Llm::Features.available.map do |feature| + Llm::Runtime.for(feature.key) + end + end + + def labels(resolutions) = resolutions.map { |resolution| resolution.feature.label }.join(", ") + + def capability_labels(resolutions) + resolutions.flat_map(&:missing_capabilities) + .uniq + .map { |capability| Llm::Capabilities.label(capability) } + .join(", ") + end + end + end +end diff --git a/app/services/llm/validators/inference_validator.rb b/app/services/llm/validators/inference_validator.rb new file mode 100644 index 000000000000..fe7d2c39aa1e --- /dev/null +++ b/app/services/llm/validators/inference_validator.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # Whether the server will actually answer a request. + # + # This is the only check that proves a connection works, and the only one + # available at all for a deployment that publishes no model list. It is also + # the only one that costs money, so it runs on demand rather than on the + # schedule -- see LlmConnection#deep_health_check. + # + # Deliberately absent: probes for tool calling and structured output. Several + # current vLLM releases answer a tool call with plain text on a fully capable + # server, deterministically, so a probe would turn a wrong answer into a + # confident one rather than discovering anything. + class InferenceValidator < HealthReports::ValidatorGroup + PROMPT = "ping" + + def self.key = :inference + + private + + def validate + register_checks(:chat_round_trip) + + model_id = chat_model_id + return warn_check(:chat_round_trip, :no_model_to_test) if model_id.blank? + + round_trip(model_id) + end + + def round_trip(model_id) + answer = session.chat(model_id).with_temperature(0).ask(PROMPT) + + if answer.content.to_s.strip.empty? + warn_check(:chat_round_trip, :empty_completion, context: { model: model_id }) + else + pass_check(:chat_round_trip) + end + rescue Llm::Errors::AuthenticationError + fail_check(:chat_round_trip, :invalid_api_key) + rescue Llm::Errors::TimeoutError + fail_check(:chat_round_trip, :request_timed_out) + rescue Llm::Errors::ApiError => e + fail_check(:chat_round_trip, :chat_failed, context: { model: model_id, status: e.status.to_s }) + rescue Llm::Errors::Error + fail_check(:chat_round_trip, :connection_error) + end + + # The configured default first, so the check exercises what features will + # actually use rather than an arbitrary entry in the catalogue. + def chat_model_id + subject.default_chat_model_id.presence || subject.available_model_ids.first + end + + # Never retried: a failing check should report the failure, not pay for it + # three more times. + def session + Llm::Session.for(subject, timeout: Llm::Session::PROBE_TIMEOUT, max_retries: 0) + end + end + end +end diff --git a/app/services/llm/validators/model_validator.rb b/app/services/llm/validators/model_validator.rb new file mode 100644 index 000000000000..e02f0f6ad809 --- /dev/null +++ b/app/services/llm/validators/model_validator.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # What the stored catalogue says, without asking the server anything. + class ModelValidator < HealthReports::ValidatorGroup + STALE_AFTER = 30.days + + def self.key = :models + + private + + def validate + register_checks(:catalogue_present, :catalogue_fresh, :default_chat_model, :default_embedding_model) + + catalogue_present + catalogue_fresh + default_chat_model + default_embedding_model + end + + # Never a failure. A gateway may expose chat and no catalogue at all, which + # is why models can be entered by hand in the first place. + def catalogue_present + return pass_check(:catalogue_present) if subject.available_model_ids.any? + + warn_check(:catalogue_present, :no_models) + end + + def catalogue_fresh + fetched_at = subject.catalogue_fetched_at + + if fetched_at.blank? + warn_check(:catalogue_fresh, :catalogue_never_fetched) + elsif fetched_at < STALE_AFTER.ago + # Context is serialised to jsonb, so a Time has to be formatted here. + warn_check(:catalogue_fresh, :catalogue_stale, context: { fetched_at: I18n.l(fetched_at.to_date) }) + else + pass_check(:catalogue_fresh) + end + end + + def default_chat_model + check_default(:default_chat_model, subject.default_chat_model_id) + end + + # Only meaningful once something wants embeddings; otherwise an unset + # default is not a defect. + def default_embedding_model + return pass_check(:default_embedding_model) if OpenProject::Llm::Features.for_kind(:embedding).empty? + + check_default(:default_embedding_model, subject.default_embedding_model_id) + end + + def check_default(key, model_id) + if model_id.blank? + warn_check(key, :default_model_unset) + elsif subject.available_model_ids.exclude?(model_id) + warn_check(key, :default_model_missing, context: { model: model_id }) + else + pass_check(key) + end + end + end + end +end diff --git a/app/services/llm/validators/server_validator.rb b/app/services/llm/validators/server_validator.rb new file mode 100644 index 000000000000..13d15dd763c0 --- /dev/null +++ b/app/services/llm/validators/server_validator.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # Whether the server answers, and whether it accepts our credentials. + # + # Uses the model list, which costs nothing on any provider. That only exists + # for OpenAI-compatible endpoints -- for the others the catalogue comes from + # RubyLLM's registry, so there is nothing free to ask, and reachability can + # only be established by the inference group's billed request. + class ServerValidator < HealthReports::ValidatorGroup + # A gateway may implement chat and nothing else. That is a supported + # deployment, not a broken one, so it must not read as unreachable. + MODELS_ENDPOINT_ABSENT = LlmServerValidator::MODELS_ENDPOINT_ABSENT + + def self.key = :server + + private + + def validate + register_checks(:reachable, :credentials_accepted) + + return unless queries_the_server? + + list_models + end + + def queries_the_server? + subject.api_format == Llm::Adapters::OPENAI_COMPATIBLE + end + + def list_models + client.models + pass_check(:reachable) + pass_check(:credentials_accepted) + rescue Llm::Errors::AuthenticationError + pass_check(:reachable) + fail_check(:credentials_accepted, :invalid_api_key) + rescue Llm::Errors::TimeoutError + fail_check(:reachable, :request_timed_out) + rescue Llm::Errors::SsrfError + fail_check(:reachable, :ssrf_filtered) + rescue Llm::Errors::ApiError => e + answered_with_error(e) + rescue Llm::Errors::ParseError + pass_check(:reachable) + fail_check(:credentials_accepted, :not_openai_compatible) + rescue Llm::Errors::Error + fail_check(:reachable, :connection_error) + end + + # The server answered, so it is reachable and did not reject us. + def answered_with_error(error) + pass_check(:reachable) + + if error.status.in?(MODELS_ENDPOINT_ABSENT) + # It simply does not publish a catalogue. A supported deployment. + warn_check(:credentials_accepted, :no_models_endpoint) + else + fail_check(:credentials_accepted, :server_error, context: { status: error.status.to_s }) + end + end + + def client + Llm::Client.new(base_url: subject.base_url, + api_key: subject.api_key, + headers: subject.custom_headers) + end + end + end +end diff --git a/app/views/admin/llm_connections/show.html.erb b/app/views/admin/llm_connections/show.html.erb index 85bac3e8b8da..3eeffd74968d 100644 --- a/app/views/admin/llm_connections/show.html.erb +++ b/app/views/admin/llm_connections/show.html.erb @@ -41,6 +41,14 @@ See COPYRIGHT and LICENSE files for more details. end %> +<% if @connection.persisted? %> + <%= + render(Primer::OpenProject::SidePanel.new(mb: 3)) do |panel| + panel.with_section(LlmConnections::SidePanel::HealthStatusComponent.new(@connection)) + end + %> +<% end %> + <%= render(LlmConnections::FormComponent.new(@connection)) %> <% if @connection.persisted? %> diff --git a/app/views/admin/llm_health_status/show.html.erb b/app/views/admin/llm_health_status/show.html.erb new file mode 100644 index 000000000000..6b2d8f339a2f --- /dev/null +++ b/app/views/admin/llm_health_status/show.html.erb @@ -0,0 +1,98 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection"), t(".title") %> + +<%= turbo_frame_tag "llm_health_status_report", refresh: :morph do %> + <%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t(".title") } + if @report.present? + header.with_description { t(".last_check", datetime: format_time(@report.created_at)) } + end + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + { href: llm_connection_path, text: t("menus.admin.llm_connection") }, + t(".title")] + ) + end + %> + + <% if @report.present? %> + <%= + render(Primer::OpenProject::SubHeader.new) do |subheader| + subheader.with_action_button( + scheme: :secondary, + label: t(".download"), + mobile_label: t(".download"), + mobile_icon: :download, + leading_icon: :download, + tag: :a, + href: llm_connection_health_status_report_path(format: :txt), + data: { turbo: false } + ) { t(".download") } + + subheader.with_action_button( + scheme: :primary, + label: t(".run_checks"), + mobile_label: t(".run_checks"), + mobile_icon: "op-reload", + leading_icon: "op-reload", + tag: :a, + href: llm_connection_health_status_report_path, + data: { turbo_method: :post, turbo: true }, + test_selector: "llm-connection--rerun-health-checks" + ) { t(".run_checks") } + end + %> + + <%= + render( + HealthReports::ReportComponent.new( + @report, + i18n_scope: "llm.health_checks", + docs_href: OpenProject::Static::Links.url_for(:sysadmin_docs, :llm_connection) + ) + ) + %> + <% else %> + <%= + render(Primer::Beta::Blankslate.new(test_selector: "llm-connection--health-blankslate")) do |blankslate| + blankslate.with_visual_icon(icon: :meter) + blankslate.with_heading(tag: :h2) { t(".blankslate_heading") } + blankslate.with_description { t(".blankslate_description") } + blankslate.with_primary_action( + href: llm_connection_health_status_report_path, + data: { turbo_method: :post, turbo: true } + ) { t(".run_checks") } + end + %> + <% end %> +<% end %> diff --git a/config/locales/en.yml b/config/locales/en.yml index 16b5a2aa7643..f005e8e00b39 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1507,6 +1507,14 @@ en: title: "Add a model" update: success: "%{model} has been updated." + llm_health_status: + show: + blankslate_description: "Run the checks to find out whether OpenProject can reach the LLM server, whether it accepts the API key, and whether every AI feature has a usable model." + blankslate_heading: "This connection has not been checked yet" + download: "Download report" + last_check: "Last checked %{datetime}." + run_checks: "Run checks" + title: "Health status" mcp_configurations: index: description: "The model context protocol allows AI agents to provide its users with tools and resources exposed by this OpenProject instance. This feature is still in beta." @@ -3981,6 +3989,56 @@ en: errors: page_not_found: "Cannot find the wiki page '%{name}'." llm: + health_checks: + configuration: + header: "Configuration" + api_format_supported: "Endpoint format is usable" + base_url_present: "Endpoint is configured" + credentials_present: "API key is stored" + enabled: "Connection is switched on" + feature_flag: "AI features are switched on" + features: + header: "AI features" + bindings_resolvable: "Every feature has a usable model" + locked_bindings_intact: "Indexed data still has its model" + inference: + header: "Inference" + chat_round_trip: "The server answers a request" + models: + header: "Models" + catalogue_fresh: "Model list is up to date" + catalogue_present: "Models are available" + default_chat_model: "Default chat model" + default_embedding_model: "Default embedding model" + server: + header: "Server" + credentials_accepted: "The server accepts the API key" + reachable: "The server can be reached" + errors: + api_key_missing: "No API key is stored. This is only correct if the LLM server requires no authentication." + catalogue_never_fetched: "The model list has never been retrieved from the server." + catalogue_stale: "The model list was last retrieved on %{fetched_at} and may no longer match what the server offers." + chat_failed: "The server refused a request for %{model} with status %{status}." + connection_disabled: "The connection is configured but switched off, so no feature will use it." + connection_error: "The server could not be reached. Please ensure it is running and reachable from OpenProject." + default_model_missing: "%{model} is no longer offered by the server." + default_model_unset: "No default model has been chosen, so features fall back to their own selection." + empty_completion: "%{model} answered with an empty message." + feature_flag_off: "The LLM connection feature is switched off, so the connection is not used yet." + features_incapable: "%{features} cannot run, because the chosen model does not provide: %{capabilities}." + features_model_missing: "%{features} point at a model the server no longer offers." + features_unbound: "%{features} have no model chosen yet." + invalid_api_key: "The server rejected the stored API key." + locked_model_missing: "%{features} indexed data with a model the server no longer offers. The index cannot be extended until it is rebuilt." + no_model_to_test: "No model is available to test with. Add a model or choose a default first." + no_models: "The server does not publish a model list. Models can be added manually instead." + no_models_endpoint: "The server does not offer a model list, so the API key could not be verified. Run the inference check to verify it." + not_configured: "No endpoint has been configured." + not_openai_compatible: "The endpoint did not return a valid model list. Please ensure it points at an OpenAI-API-compatible endpoint, including the API version segment." + request_timed_out: "The server did not respond in time." + server_error: "The server responded with status %{status}." + ssrf_filtered: "The endpoint resolves to a blocked address." + unsupported_api_format: "%{api_format} needs credentials that OpenProject cannot store." api_formats: anthropic: "Anthropic" azure: "Azure OpenAI" @@ -4029,6 +4087,14 @@ en: semantic_search: caption: "Indexes work packages so they can be found by meaning rather than by keyword." label: "Semantic search" + llm_connections: + side_panel: + health_status_component: + last_check: "Last checked %{datetime}." + never_checked: "This connection has not been checked yet." + open_report: "Open full health report" + run_checks: "Run checks now" + title: "Health status" mail: actions: "Actions" digests: diff --git a/config/routes.rb b/config/routes.rb index 31ddbbccad5b..32cf4ceacf1b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -694,6 +694,10 @@ resource :llm_connection, only: %i[show update], controller: "admin/llm_connections" do post :refresh_models delete :api_key, action: :delete_api_key + + resource :health_status_report, only: %i[show create], controller: "admin/llm_health_status" do + post :create_health_status_report + end end # Manual entries only; discovered models are managed by the sync. diff --git a/spec/requests/admin/llm_health_status_spec.rb b/spec/requests/admin/llm_health_status_spec.rb new file mode 100644 index 000000000000..ad04b17ffa8f --- /dev/null +++ b/spec/requests/admin/llm_health_status_spec.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe "LLM connection health status", :llm_server_helpers, :skip_csrf, :webmock, + type: :rails_request, with_flag: { llm_connection: true } do + shared_let(:admin) { create(:admin) } + + let(:base_url) { "https://example.com/v1" } + + before { login_as(admin) } + + describe "GET /admin/llm_connection/health_status_report" do + it "redirects to the settings page when nothing is configured yet" do + get llm_connection_health_status_report_path + + expect(response).to redirect_to(llm_connection_path) + end + + context "with a connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + it "offers to run the checks when none have run" do + get llm_connection_health_status_report_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("has not been checked yet") + end + + it "renders the report once it exists" do + mock_llm_models_response(base_url) + mock_llm_chat_response(base_url) + post llm_connection_health_status_report_path + + get llm_connection_health_status_report_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Configuration") + end + end + end + + describe "POST /admin/llm_connection/health_status_report" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:, default_chat_model_id: "qwen3.6-27b") } + + before do + mock_llm_models_response(base_url) + mock_llm_chat_response(base_url) + end + + it "stores a report and sends the completion an administrator asked for" do + expect { post llm_connection_health_status_report_path } + .to change(connection.health_reports, :count).by(1) + + expect(response).to redirect_to(llm_connection_health_status_report_path) + expect(WebMock).to have_requested(:post, "#{base_url}/chat/completions").once + end + end + + describe "GET /admin/llm_connection/health_status_report.txt" do + let!(:connection) do + create(:llm_connection, :with_models, :enabled, base_url:, + api_key: "sk-super-secret", + custom_headers: { "apikey" => "gateway-secret" }) + end + + before do + mock_llm_models_response(base_url) + mock_llm_chat_response(base_url) + post llm_connection_health_status_report_path + end + + it "downloads the report" do + get llm_connection_health_status_report_path(format: :txt) + + expect(response).to have_http_status(:ok) + expect(response.body).to include("base_url") + end + + # The report is pasted into support tickets. A gateway header routinely + # carries a second credential, so neither it nor the API key may appear. + it "contains no credentials" do + get llm_connection_health_status_report_path(format: :txt) + + expect(response.body).not_to include("sk-super-secret") + expect(response.body).not_to include("gateway-secret") + end + end + + describe "authorisation" do + let!(:connection) { create(:llm_connection, :enabled, base_url:) } + + it "is refused to a non-admin" do + login_as(create(:user)) + + get llm_connection_health_status_report_path + + expect(response).not_to have_http_status(:ok) + end + end +end diff --git a/spec/services/llm/validators/connection_validator_spec.rb b/spec/services/llm/validators/connection_validator_spec.rb new file mode 100644 index 000000000000..43ba395a2d8e --- /dev/null +++ b/spec/services/llm/validators/connection_validator_spec.rb @@ -0,0 +1,179 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Validators::ConnectionValidator, :llm_server_helpers, :webmock, + with_flag: { llm_connection: true } do + subject(:report) { described_class.new(connection).call } + + let(:base_url) { "https://example.com/v1" } + let(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + def result_for(group, key) + report.group(group)&.result_for(key) + end + + context "with a healthy connection" do + before do + mock_llm_models_response(base_url) + connection.update!(default_chat_model_id: "qwen3.6-27b") + end + + it "passes the configuration and server groups" do + expect(result_for(:configuration, :base_url_present).state).to eq(:success) + expect(result_for(:server, :reachable).state).to eq(:success) + expect(result_for(:server, :credentials_accepted).state).to eq(:success) + end + + # The billed request only runs when an administrator asks for it. + it "skips inference unless a deep check was asked for" do + expect(report.group(:inference)).to be_nil + expect(WebMock).not_to have_requested(:post, "#{base_url}/chat/completions") + end + + context "when a deep check is asked for" do + before do + connection.deep_health_check = true + mock_llm_chat_response(base_url, content: "pong") + end + + it "sends a completion and reports the round trip" do + expect(result_for(:inference, :chat_round_trip).state).to eq(:success) + expect(WebMock).to have_requested(:post, "#{base_url}/chat/completions").once + end + end + end + + context "when the server publishes no model list" do + before { mock_llm_models_response(base_url, response_code: 404) } + + # The case that motivated manual model entry: a gateway exposing only chat. + it "is still reachable, and says the key could not be verified" do + expect(result_for(:server, :reachable).state).to eq(:success) + expect(result_for(:server, :credentials_accepted).state).to eq(:warning) + expect(result_for(:server, :credentials_accepted).code).to eq(:no_models_endpoint) + end + + it "can still prove the connection works through inference" do + connection.deep_health_check = true + connection.update!(default_chat_model_id: "qwen3.6-27b") + mock_llm_chat_response(base_url) + + expect(result_for(:inference, :chat_round_trip).state).to eq(:success) + end + end + + context "when the key is rejected" do + before { mock_llm_models_response(base_url, response_code: 401) } + + it "separates reachability from authentication" do + expect(result_for(:server, :reachable).state).to eq(:success) + expect(result_for(:server, :credentials_accepted).state).to eq(:failure) + expect(report).to be_unhealthy + end + end + + context "when the server cannot be reached" do + before { mock_llm_models_response(base_url, timeout: true) } + + it "fails reachability and skips the credential check" do + expect(result_for(:server, :reachable).state).to eq(:failure) + expect(result_for(:server, :credentials_accepted).state).to eq(:skipped) + end + end + + context "without a base URL" do + let(:connection) { create(:llm_connection, :enabled).tap { |c| c.update_column(:base_url, "") } } + + it "fails and does not ask the server anything" do + expect(result_for(:configuration, :base_url_present).state).to eq(:failure) + expect(report.group(:server)).to be_nil + expect(WebMock).not_to have_requested(:get, "#{base_url}/models") + end + end + + describe "the features group" do + before { mock_llm_models_response(base_url) } + + it "warns about features with no model chosen" do + result = result_for(:features, :bindings_resolvable) + + expect(result.state).to eq(:warning) + expect(result.code).to eq(:features_unbound) + end + + it "fails when a binding points at a model the server no longer offers" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "vanished") + + result = result_for(:features, :bindings_resolvable) + + expect(result.state).to eq(:failure) + expect(result.code).to eq(:features_model_missing) + end + end + + # HealthReports::ResultComponent resolves a check's label from its key and its + # explanation from its code, so a key with no translation renders as + # "translation missing" rather than failing anywhere a spec would notice. + describe "translations" do + let(:scenarios) do + [ + -> { mock_llm_models_response(base_url) }, + -> { mock_llm_models_response(base_url, response_code: 401) }, + -> { mock_llm_models_response(base_url, response_code: 404) }, + -> { mock_llm_models_response(base_url, timeout: true) } + ] + end + + it "exist for every check and every code the validator can emit" do + # A dangling binding, so the features group emits its failure codes too. + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "vanished") + + scenarios.each do |setup| + WebMock.reset! + setup.call + + described_class.new(connection).call.results.each do |group| + expect(I18n.t("#{group.key}.header", scope: "llm.health_checks", raise: true)).to be_present + + group.results.each do |result| + expect(I18n.t("#{group.key}.#{result.key}", scope: "llm.health_checks", raise: true)).to be_present + next if result.code.nil? + + context_vars = result.context&.symbolize_keys || {} + expect(I18n.t("errors.#{result.code}", scope: "llm.health_checks", raise: true, **context_vars)) + .to be_present + end + end + end + end + end +end From f2621fd9ec045221af0ed4d5f68e2d3a34823852 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Wed, 12 Aug 2026 09:07:33 +0100 Subject: [PATCH 27/44] [#66020] Re-check the connection on a schedule, and prune old reports A connection that dies after setup stayed green until somebody happened to look at the page. Llm::HealthCheckJob re-runs the checks every six hours. It deliberately does not set deep_health_check. The inference group spends a real completion, which is billed on a hosted provider, and an unattended job must not run up a bill four times a day. The schedule therefore covers everything free -- an expired key, a withdrawn model, a binding that stopped resolving -- and the billed round trip stays behind "Run checks". A spec asserts no completion is requested. The cron key is enabled and disabled from UpdateService, so it idles rather than waking every six hours to find no connection. Llm::PruneHealthReportsJob is the first pruner health_reports has ever had, which is only now necessary: until this commit the table grew a row per button click. It keeps the 50 newest reports and anything under 90 days, and both conditions matter -- age alone would erase the only check a rarely-touched connection ever had. It is scoped to LLM connections, since storages and wikis share the table. https://community.openproject.org/work_packages/66020 --- .../llm_connections/update_service.rb | 7 +- app/workers/llm/health_check_job.rb | 75 +++++++++++++ app/workers/llm/prune_health_reports_job.rb | 59 +++++++++++ config/initializers/cronjobs.rb | 8 ++ spec/workers/llm/health_check_job_spec.rb | 100 ++++++++++++++++++ .../llm/prune_health_reports_job_spec.rb | 78 ++++++++++++++ 6 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 app/workers/llm/health_check_job.rb create mode 100644 app/workers/llm/prune_health_reports_job.rb create mode 100644 spec/workers/llm/health_check_job_spec.rb create mode 100644 spec/workers/llm/prune_health_reports_job_spec.rb diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb index a9b0e94a2226..069d62fd6233 100644 --- a/app/services/llm_connections/update_service.rb +++ b/app/services/llm_connections/update_service.rb @@ -46,8 +46,13 @@ def initialize(*, sync_models: true, **) # the save. A sync failure is therefore logged, not surfaced. def after_perform(service_call) super.tap do - next unless @sync_models && service_call.success? + next unless service_call.success? + # Enabling or disabling the connection decides whether the scheduled + # health check has anything to do. + Llm::HealthCheckJob.toggle_cron_job + + next unless @sync_models next unless credentials_changed?(service_call.result) SyncModelsService.new(service_call.result).call diff --git a/app/workers/llm/health_check_job.rb b/app/workers/llm/health_check_job.rb new file mode 100644 index 000000000000..059b68ac5bf8 --- /dev/null +++ b/app/workers/llm/health_check_job.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Re-checks the configured connection, so a server that dies after setup does + # not stay green until somebody happens to look. + # + # Deliberately does *not* set deep_health_check: the inference group spends a + # real completion, which is billed on a hosted provider. An unattended job must + # not run up a bill, so the schedule covers everything that is free -- an + # expired key, a withdrawn model, a binding that stopped resolving -- and the + # billed round trip stays behind the administrator's "Run checks". + class HealthCheckJob < ApplicationJob + CRON_JOB_KEY = :"Llm::HealthCheckJob" + + queue_with_priority :low + + class << self + # Keeps the cron idle while there is nothing to check, rather than waking + # every six hours to find no connection. + def toggle_cron_job + if runnable? + GoodJob::Setting.cron_key_enable(CRON_JOB_KEY) unless GoodJob::Setting.cron_key_enabled?(CRON_JOB_KEY) + elsif GoodJob::Setting.cron_key_enabled?(CRON_JOB_KEY) + GoodJob::Setting.cron_key_disable(CRON_JOB_KEY) + end + end + + def runnable? + return false unless OpenProject::FeatureDecisions.llm_connection_active? + + connection = LlmConnection.first + connection.present? && connection.configured? && connection.enabled? + end + end + + def perform + return unless self.class.runnable? + + # .first, not .instance: the latter builds an unsaved record, and the + # validator writes through the health_reports association. + connection = LlmConnection.first + report = Llm::Validators::ConnectionValidator.new(connection).call + report.save! + report + end + end +end diff --git a/app/workers/llm/prune_health_reports_job.rb b/app/workers/llm/prune_health_reports_job.rb new file mode 100644 index 000000000000..aae37cc65ae3 --- /dev/null +++ b/app/workers/llm/prune_health_reports_job.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Keeps the health report history bounded. + # + # Until the scheduled check existed, health_reports only grew when somebody + # clicked "Run checks", so nothing anywhere pruned the table. A four-times-daily + # writer changes that, and this is the first pruner it has had. + # + # Only ever prunes reports belonging to an LLM connection: storages and wikis + # share the table and still write a row only on demand. + class PruneHealthReportsJob < ApplicationJob + # Enough history to see a pattern in when a flaky server fails, without + # keeping a year of identical green reports. + KEEP = 50 + MAX_AGE = 90.days + + queue_with_priority :low + + def perform + LlmConnection.find_each do |connection| + recent = connection.health_reports.order(created_at: :desc).limit(KEEP).pluck(:id) + + connection.health_reports + .where.not(id: recent) + .where(created_at: ...MAX_AGE.ago) + .delete_all + end + end + end +end diff --git a/config/initializers/cronjobs.rb b/config/initializers/cronjobs.rb index 240116c5ef93..6b9d35af3029 100644 --- a/config/initializers/cronjobs.rb +++ b/config/initializers/cronjobs.rb @@ -68,6 +68,14 @@ cron: "30 23 * * *", class: Ldap::SynchronizationJob.name }, + "Llm::HealthCheckJob": { + cron: "7 */6 * * *", # every six hours at xx:07 + class: Llm::HealthCheckJob.name + }, + "Llm::PruneHealthReportsJob": { + cron: "25 3 * * *", # runs at 3:25 nightly + class: Llm::PruneHealthReportsJob.name + }, "RecurringMeetings::InitNextOccurrenceWatchdogJob": { cron: "11 05 * * *", class: RecurringMeetings::InitNextOccurrenceWatchdogJob.name diff --git a/spec/workers/llm/health_check_job_spec.rb b/spec/workers/llm/health_check_job_spec.rb new file mode 100644 index 000000000000..92603af70ff2 --- /dev/null +++ b/spec/workers/llm/health_check_job_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::HealthCheckJob, :llm_server_helpers, :webmock, with_flag: { llm_connection: true } do + let(:base_url) { "https://example.com/v1" } + + describe "#perform" do + context "with an enabled connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before { mock_llm_models_response(base_url) } + + it "stores a report" do + expect { described_class.perform_now }.to change(connection.health_reports, :count).by(1) + end + + # The whole reason the inference group is gated: an unattended job must not + # run up a bill on a hosted provider four times a day. + # Group keys come back from jsonb as strings, so a stored report is + # queried by string where an in-memory one is queried by symbol. + it "does not spend a completion" do + described_class.perform_now + + expect(WebMock).not_to have_requested(:post, "#{base_url}/chat/completions") + expect(connection.latest_health_report.results.map(&:key)).not_to include("inference") + end + + it "still reports what the free checks found" do + described_class.perform_now + + expect(connection.latest_health_report.group("server").result_for("reachable").state).to eq(:success) + end + end + + it "does nothing without a connection" do + expect { described_class.perform_now }.not_to change(HealthReport, :count) + end + + it "does nothing while the connection is switched off" do + create(:llm_connection, :with_models, enabled: false, base_url:) + + expect { described_class.perform_now }.not_to change(HealthReport, :count) + end + + context "with the feature flag off", with_flag: { llm_connection: false } do + it "does nothing" do + create(:llm_connection, :with_models, :enabled, base_url:) + + expect { described_class.perform_now }.not_to change(HealthReport, :count) + end + end + end + + describe ".toggle_cron_job" do + it "enables the cron once a connection is usable" do + create(:llm_connection, :with_models, :enabled, base_url:) + + described_class.toggle_cron_job + + expect(GoodJob::Setting.cron_key_enabled?(described_class::CRON_JOB_KEY)).to be(true) + end + + it "disables the cron while there is nothing to check" do + GoodJob::Setting.cron_key_enable(described_class::CRON_JOB_KEY) + + described_class.toggle_cron_job + + expect(GoodJob::Setting.cron_key_enabled?(described_class::CRON_JOB_KEY)).to be(false) + end + end +end diff --git a/spec/workers/llm/prune_health_reports_job_spec.rb b/spec/workers/llm/prune_health_reports_job_spec.rb new file mode 100644 index 000000000000..7d3af3097978 --- /dev/null +++ b/spec/workers/llm/prune_health_reports_job_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::PruneHealthReportsJob do + shared_let(:connection) { create(:llm_connection) } + + def report(created_at:, subject: connection) + subject.health_reports.create!( + results: [HealthReport::ResultGroup.new(key: :configuration, results: [])], + created_at: + ) + end + + it "deletes reports that are both old and beyond the retention count" do + stub_const("#{described_class}::KEEP", 2) + oldest = report(created_at: 300.days.ago) + report(created_at: 200.days.ago) + report(created_at: 100.days.ago) + + expect { described_class.perform_now }.to change { HealthReport.exists?(oldest.id) }.from(true).to(false) + end + + it "keeps recent reports however many there are" do + recent = report(created_at: 1.day.ago) + + described_class.perform_now + + expect(HealthReport.exists?(recent.id)).to be(true) + end + + # Age alone is not enough: a connection checked once a year ago should still + # show that check rather than nothing at all. + it "keeps the newest reports even when they are old" do + oldest = report(created_at: 200.days.ago) + + described_class.perform_now + + expect(HealthReport.exists?(oldest.id)).to be(true) + end + + it "leaves other subjects' reports alone" do + storage = create(:nextcloud_storage) + foreign = report(created_at: 200.days.ago, subject: storage) + + described_class.perform_now + + expect(HealthReport.exists?(foreign.id)).to be(true) + end +end From 23f8cbc5b49766906de60647e21f5ae91fe1592a Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Wed, 12 Aug 2026 11:41:20 +0100 Subject: [PATCH 28/44] [#66020] Let an administrator curate which models may be used A gateway can report hundreds of models -- OpenRouter returns 341 -- and an administrator had no way to say which of them the organisation actually wants used. The choice cannot live in `active`. The catalogue sync owns that column: it sets it on every model on every refresh and clears it for models the server stopped reporting. A choice stored there would be undone by the next "Refresh models", and the model would be labelled "no longer reported" when it was merely hidden. So deactivated_at is its own column, and a spec runs a sync that still reports a deactivated model to prove the deactivation survives. available_model_ids deliberately does not change. It is what Llm::Runtime resolves against, and hiding a model must never silently break a feature already bound to it -- deactivation curates the pickers, it does not enforce anything. A bound feature keeps working and says so with a banner. Both pickers keep offering the model already selected, since dropping it would blank the field on the next save. A withdrawn model renders a disabled toggle rather than none, so the column stays aligned and the row explains itself. The toggle carries an explicit aria-label: Primer's ToggleSwitch has no accessible name of its own. https://community.openproject.org/work_packages/66020 --- .../feature_binding_component.html.erb | 6 ++ .../feature_binding_component.rb | 9 ++ .../llm_connections/models_row_component.rb | 49 +++++++++-- .../llm_connections/models_table_component.rb | 5 +- .../admin/llm_models_controller.rb | 14 ++++ app/forms/llm_connections/connection_form.rb | 8 +- app/models/llm_connection.rb | 10 +++ app/models/llm_model.rb | 13 +++ .../selectable_models_query.rb | 13 ++- config/locales/en.yml | 4 + config/routes.rb | 5 +- ...100000_add_deactivated_at_to_llm_models.rb | 42 ++++++++++ spec/factories/llm_model_factory.rb | 5 ++ spec/models/llm_model_deactivation_spec.rb | 84 +++++++++++++++++++ spec/requests/admin/llm_models_spec.rb | 44 ++++++++++ 15 files changed, 299 insertions(+), 12 deletions(-) create mode 100644 db/migrate/20260813100000_add_deactivated_at_to_llm_models.rb create mode 100644 spec/models/llm_model_deactivation_spec.rb diff --git a/app/components/llm_connections/feature_binding_component.html.erb b/app/components/llm_connections/feature_binding_component.html.erb index e3c28a593df1..b36d3a358923 100644 --- a/app/components/llm_connections/feature_binding_component.html.erb +++ b/app/components/llm_connections/feature_binding_component.html.erb @@ -11,6 +11,12 @@ <% end %> <% end %> + <% if deactivated? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :alert)) do %> + <%= t("admin.llm_feature_bindings.deactivated", model: binding.resolved_model_id) %> + <% end %> + <% end %> + <% if locked? %> <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :lock)) do %> <%= t("admin.llm_feature_bindings.locked", model: binding.model_id) %> diff --git a/app/components/llm_connections/feature_binding_component.rb b/app/components/llm_connections/feature_binding_component.rb index 8f3e679b3002..88585e5aef01 100644 --- a/app/components/llm_connections/feature_binding_component.rb +++ b/app/components/llm_connections/feature_binding_component.rb @@ -65,6 +65,15 @@ def locked? = binding&.locked? def dangling? = binding&.dangling? + # Still resolvable, so not dangling -- but an administrator has hidden it + # from the pickers, so say so rather than let the choice look unremarkable. + def deactivated? + model_id = binding&.resolved_model_id + return false if model_id.blank? + + connection.models.deactivated.exists?(external_id: model_id) + end + private attr_reader :feature, :connection, :binding diff --git a/app/components/llm_connections/models_row_component.rb b/app/components/llm_connections/models_row_component.rb index b2c79e74e5c8..1c0ba2abc2ed 100644 --- a/app/components/llm_connections/models_row_component.rb +++ b/app/components/llm_connections/models_row_component.rb @@ -62,15 +62,50 @@ def kind end def source - if llm_model.manual? - render(Primer::Beta::Label.new(scheme: :accent)) { I18n.t("admin.llm_connections.models.source_manual") } - elsif llm_model.withdrawn? - render(Primer::Beta::Label.new(scheme: :attention)) { I18n.t("admin.llm_connections.models.source_withdrawn") } - else - render(Primer::Beta::Label.new(scheme: :secondary)) { I18n.t("admin.llm_connections.models.source_discovered") } - end + scheme, key = source_label + + render(Primer::Beta::Label.new(scheme:)) { I18n.t("admin.llm_connections.models.#{key}") } + end + + def source_label + return %i[attention source_deactivated] if llm_model.deactivated? + return %i[accent source_manual] if llm_model.manual? + return %i[attention source_withdrawn] if llm_model.withdrawn? + + %i[secondary source_discovered] + end + + # Whether a feature may choose this model. A withdrawn model has nothing to + # switch on -- the server stopped offering it -- so its toggle is inert + # rather than absent, which keeps the column aligned and says why. + def status + render(Primer::Alpha::ToggleSwitch.new(**toggle_options)) + end + + def toggle_options + options = { + checked: llm_model.selectable?, + enabled: togglable?, + size: :small, + # A bare ToggleSwitch has no accessible name, and axe fails without one. + aria: { label: I18n.t("admin.llm_connections.models.toggle_aria_label", model: llm_model.name) }, + test_selector: "llm-model--toggle-#{llm_model.id}" + } + + togglable? ? options.merge(mutation_options) : options end + def mutation_options + { + src: url_helpers.toggle_llm_model_path(llm_model), + csrf_token: helpers.form_authenticity_token, + data: { "turbo-method": :post, "turbo-stream": true }, + classes: "op-primer-adjustments__toggle-switch--hidden-loading-indicator" + } + end + + def togglable? = llm_model.active? + def button_links llm_model.manual? ? [edit_link, delete_link] : [edit_link] end diff --git a/app/components/llm_connections/models_table_component.rb b/app/components/llm_connections/models_table_component.rb index 80679b3615a6..a0aa5869f8ab 100644 --- a/app/components/llm_connections/models_table_component.rb +++ b/app/components/llm_connections/models_table_component.rb @@ -34,7 +34,7 @@ module LlmConnections # Rendering never issues an HTTP request: the catalogue is refreshed explicitly # through the "Refresh models" action. class ModelsTableComponent < OpPrimer::BorderBoxTableComponent - columns :identifier, :kind, :context_window, :source + columns :identifier, :kind, :context_window, :source, :status mobile_columns :identifier @@ -55,7 +55,8 @@ def headers [:identifier, { caption: I18n.t("admin.llm_connections.models.identifier") }], [:kind, { caption: I18n.t("admin.llm_connections.models.kind") }], [:context_window, { caption: I18n.t("admin.llm_connections.models.context_window") }], - [:source, { caption: I18n.t("admin.llm_connections.models.source") }] + [:source, { caption: I18n.t("admin.llm_connections.models.source") }], + [:status, { caption: I18n.t("admin.llm_connections.models.status") }] ] end diff --git a/app/controllers/admin/llm_models_controller.rb b/app/controllers/admin/llm_models_controller.rb index 7b0c65fd8dce..a95c47cc388a 100644 --- a/app/controllers/admin/llm_models_controller.rb +++ b/app/controllers/admin/llm_models_controller.rb @@ -92,6 +92,20 @@ def destroy redirect_to llm_connection_path, status: :see_other end + # Hides a model from the pickers, or puts it back. Deliberately does not + # touch +active+, which the catalogue sync owns and would overwrite. + def toggle + llm_model = @connection.models.find(params.expect(:id)) + + # A withdrawn model has nothing to switch on; its toggle is rendered + # disabled, and this refuses a request that got here anyway. + return render(json: {}, status: :unprocessable_entity) unless llm_model.active? + + llm_model.update!(deactivated_at: llm_model.deactivated? ? nil : Time.current) + + render json: {}, status: :ok + end + private def set_connection diff --git a/app/forms/llm_connections/connection_form.rb b/app/forms/llm_connections/connection_form.rb index 55c8ab479213..e1a6730980d9 100644 --- a/app/forms/llm_connections/connection_form.rb +++ b/app/forms/llm_connections/connection_form.rb @@ -88,7 +88,7 @@ class ConnectionForm < ApplicationForm input_width: :large, disabled: read_only? ) do |select| - model.available_model_ids.each do |model_id| + default_chat_model_options.each do |model_id| select.option(value: model_id, label: model_id) end end @@ -114,6 +114,12 @@ def models_available? model.available_model_ids.any? end + # Deactivated models are hidden, except the one already chosen -- dropping + # that would silently blank the field on the next save. + def default_chat_model_options + (model.selectable_model_ids + [model.default_chat_model_id]).compact_blank.uniq + end + def submit_label model.persisted? ? I18n.t(:button_save) : I18n.t("admin.llm_connections.form.button_connect") end diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb index b7e845cc3699..2ec8cda20877 100644 --- a/app/models/llm_connection.rb +++ b/app/models/llm_connection.rb @@ -91,10 +91,20 @@ def configured_from_env? # Every model that can be addressed today: discovered and still offered, plus # anything an administrator entered by hand. + # + # Deliberately includes models an administrator has deactivated. This is what + # Llm::Runtime resolves against, and hiding a model from the pickers must not + # break a feature that is already bound to it. def available_model_ids models.active.by_identifier.pluck(:external_id) end + # What a picker should offer: the above, minus what an administrator has + # switched off. + def selectable_model_ids + models.selectable.by_identifier.pluck(:external_id) + end + def server_flavour options["server_flavour"].presence&.to_sym end diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb index 019f8efbe5ea..6e81db82b597 100644 --- a/app/models/llm_model.rb +++ b/app/models/llm_model.rb @@ -44,6 +44,19 @@ class LlmModel < ApplicationRecord scope :manual, -> { where(manual: true) } scope :by_identifier, -> { order(:external_id) } + # What an administrator is willing to have chosen. Distinct from +active+, + # which the catalogue sync owns and rewrites on every refresh. + scope :deactivated, -> { where.not(deactivated_at: nil) } + scope :selectable, -> { active.where(deactivated_at: nil) } + + def deactivated? = deactivated_at.present? + + # Offerable in a picker. Note that this is *not* what decides whether a model + # still resolves: a feature already bound to a deactivated model keeps working, + # and is surfaced as a warning instead. Switching a row off must never silently + # break a running feature. + def selectable? = active? && !deactivated? + def name = display_name.presence || external_id # Precedence: what an administrator set, then what the server reported (vLLM diff --git a/app/services/llm_connections/selectable_models_query.rb b/app/services/llm_connections/selectable_models_query.rb index 0855892ba13a..1ca7d8a4a974 100644 --- a/app/services/llm_connections/selectable_models_query.rb +++ b/app/services/llm_connections/selectable_models_query.rb @@ -49,13 +49,24 @@ def initialize(connection, feature) end def call - connection.available_model_ids.map { |model_id| option_for(model_id) } + offerable_model_ids.map { |model_id| option_for(model_id) } end private attr_reader :connection, :feature + # Models an administrator has switched off are not offered, but the one this + # feature is already bound to stays listed -- otherwise the select silently + # shows nothing where a working binding exists. + def offerable_model_ids + (connection.selectable_model_ids + [bound_model_id]).compact_blank.uniq + end + + def bound_model_id + connection.feature_bindings.find_by(feature_key: feature.key.to_s)&.model_id + end + def option_for(model_id) states = feature.requires.index_with { |capability| verdict_state(model_id, capability) } diff --git a/config/locales/en.yml b/config/locales/en.yml index f005e8e00b39..50c6e885c6d1 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1431,6 +1431,7 @@ en: llm_feature_bindings: dangling: "%{model} is no longer offered by the LLM server. This feature will not run until another model is selected." + deactivated: "%{model} has been hidden by an administrator. This feature keeps using it, but it can no longer be chosen elsewhere." index: blank_description: "Connect OpenProject to an LLM server first. Models can be assigned once the server reports which ones it offers." blank_title: "No LLM server configured" @@ -1463,8 +1464,11 @@ en: identifier: "Model" kind: "Type" source: "Source" + source_deactivated: "Hidden by administrator" source_discovered: "Reported by server" source_manual: "Added manually" + status: "Available for use" + toggle_aria_label: "Make %{model} available to AI features" source_withdrawn: "No longer reported" refresh_models: failure: "The model list could not be refreshed. Please check that the LLM server is still reachable." diff --git a/config/routes.rb b/config/routes.rb index 32cf4ceacf1b..c27af828cbb8 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -702,7 +702,10 @@ # Manual entries only; discovered models are managed by the sync. resources :llm_models, only: %i[new create edit update destroy], controller: "admin/llm_models" do - member { get :delete_dialog } + member do + get :delete_dialog + post :toggle + end end # Keyed by feature key rather than by record id: the binding is an attribute diff --git a/db/migrate/20260813100000_add_deactivated_at_to_llm_models.rb b/db/migrate/20260813100000_add_deactivated_at_to_llm_models.rb new file mode 100644 index 000000000000..8f59a81367c9 --- /dev/null +++ b/db/migrate/20260813100000_add_deactivated_at_to_llm_models.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Separates "the server stopped offering this" from "an administrator does not +# want this used". +# +# +active+ is owned by the catalogue sync, which sets it on every refresh and +# clears it for models the server no longer reports. An administrator's choice +# cannot live there: the next "Refresh models" would silently undo it, and the +# model would be labelled "no longer reported" when it is merely hidden. +class AddDeactivatedAtToLlmModels < ActiveRecord::Migration[8.1] + def change + add_column :llm_models, :deactivated_at, :datetime + end +end diff --git a/spec/factories/llm_model_factory.rb b/spec/factories/llm_model_factory.rb index 2082eb3c0bbb..59527ef65278 100644 --- a/spec/factories/llm_model_factory.rb +++ b/spec/factories/llm_model_factory.rb @@ -44,5 +44,10 @@ trait :withdrawn do active { false } end + + # Still offered by the server; hidden by an administrator. + trait :deactivated do + deactivated_at { Time.current } + end end end diff --git a/spec/models/llm_model_deactivation_spec.rb b/spec/models/llm_model_deactivation_spec.rb new file mode 100644 index 000000000000..be415f977b5c --- /dev/null +++ b/spec/models/llm_model_deactivation_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe LlmModel, "deactivation", :llm_server_helpers, :webmock, + with_flag: { llm_connection: true } do + let(:base_url) { "https://example.com/v1" } + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + let(:model) { connection.models.find_by(external_id: "qwen3.6-27b") } + + before { model.update!(deactivated_at: Time.current) } + + it "hides the model from the pickers" do + expect(connection.selectable_model_ids).not_to include("qwen3.6-27b") + expect(connection.selectable_model_ids).to include("bge-m3") + end + + # The decision that makes the toggle safe: curation, not enforcement. A row an + # administrator switches off must never silently break a running feature. + it "keeps a feature already bound to it working" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + expect(connection.available_model_ids).to include("qwen3.6-27b") + expect(Llm::Runtime.for(:description_assistant)).to be_ready + end + + it "still offers it to the feature that is bound to it" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + options = LlmConnections::SelectableModelsQuery + .new(connection, OpenProject::Llm::Features[:description_assistant]) + .call + + expect(options.map(&:model_id)).to include("qwen3.6-27b") + end + + # The reason deactivated_at exists rather than reusing active: the sync writes + # active on every refresh, so an administrator's choice stored there would be + # undone by the next "Refresh models". + it "survives a catalogue sync that still reports the model" do + mock_llm_models_response(base_url) + + LlmConnections::SyncModelsService.new(connection).call + + expect(model.reload).to be_deactivated + expect(model).to be_active + expect(model).not_to be_selectable + end + + it "is distinct from a model the server withdrew" do + withdrawn = create(:llm_model, :withdrawn, llm_connection: connection, external_id: "gone") + + expect(withdrawn).to be_withdrawn + expect(model).not_to be_withdrawn + end +end diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb index ee286476dfba..4d4a926e63c0 100644 --- a/spec/requests/admin/llm_models_spec.rb +++ b/spec/requests/admin/llm_models_spec.rb @@ -241,4 +241,48 @@ expect(LlmModel.where(id: llm_model.id)).to exist end end + + describe "POST /admin/llm_models/:id/toggle" do + let!(:llm_model) { create(:llm_model, llm_connection: connection, external_id: "qwen3.6-27b") } + + it "hides the model from the pickers and puts it back" do + post toggle_llm_model_path(llm_model) + + expect(response).to have_http_status(:ok) + expect(llm_model.reload).to be_deactivated + expect(connection.selectable_model_ids).not_to include("qwen3.6-27b") + + post toggle_llm_model_path(llm_model) + + expect(llm_model.reload).not_to be_deactivated + expect(connection.selectable_model_ids).to include("qwen3.6-27b") + end + + # Curation, not enforcement: a feature already pointing at the model keeps + # resolving, so switching a row off cannot silently break anything. + it "leaves an existing binding working" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + post toggle_llm_model_path(llm_model) + + expect(connection.available_model_ids).to include("qwen3.6-27b") + end + + it "refuses a model the server has withdrawn" do + withdrawn = create(:llm_model, :withdrawn, llm_connection: connection, external_id: "gone") + + post toggle_llm_model_path(withdrawn) + + expect(response).to have_http_status(:unprocessable_entity) + expect(withdrawn.reload).not_to be_deactivated + end + + it "is refused to a non-admin" do + login_as create(:user) + + post toggle_llm_model_path(llm_model) + + expect(llm_model.reload).not_to be_deactivated + end + end end From 21c650884848d7c6fcda75a0d086d14a0fd273e6 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Wed, 12 Aug 2026 13:26:55 +0100 Subject: [PATCH 29/44] [#66020] Let an administrator remove the API key or disconnect Two actions the ticket asked for that the branch never had. delete_api_key had existed since the first commit here -- a route, an action and a passing spec -- with nothing anywhere linking to it. The mechanism was built and never rendered, so a key could be replaced but not cleared. Both actions now sit in a kebab menu on the page header. Neither dialog carries a confirmation checkbox, because nothing either does is irreversible. The API key dialog exists for what is not recoverable: the catalogue sync fingerprints base_url and api_key together, so the next refresh after the key changes discards every capability verdict, including the ones an administrator asserted by hand that every other code path preserves. The dialog says so, but only when such verdicts exist. Disconnecting clears the credential and switches the connection off, keeping the endpoint, the catalogue and every binding. A destroying variant was rejected: dependent: :delete_all skips callbacks, so the cascade would take the locked embedding bindings with it -- the only record that a vector index exists and which model and dimension it was written under. Removing the key keeps updating the record directly rather than going through UpdateService, and now says why. The contract probes the server whenever credentials change, and a server that requires authentication rejects the now-keyless probe -- so routing it through the service would let the server refuse an administrator permission to remove a credential. The environment guard the contract did usefully provide is checked explicitly instead. https://community.openproject.org/work_packages/66020 --- .../delete_api_key_dialog_component.html.erb | 20 ++++ .../delete_api_key_dialog_component.rb | 57 ++++++++++ .../disconnect_dialog_component.html.erb | 35 ++++++ .../disconnect_dialog_component.rb | 60 ++++++++++ .../admin/llm_connections_controller.rb | 32 ++++++ app/views/admin/llm_connections/show.html.erb | 41 +++++++ config/locales/en.yml | 16 +++ config/routes.rb | 3 + spec/requests/admin/llm_connections_spec.rb | 106 ++++++++++++++++++ 9 files changed, 370 insertions(+) create mode 100644 app/components/llm_connections/delete_api_key_dialog_component.html.erb create mode 100644 app/components/llm_connections/delete_api_key_dialog_component.rb create mode 100644 app/components/llm_connections/disconnect_dialog_component.html.erb create mode 100644 app/components/llm_connections/disconnect_dialog_component.rb diff --git a/app/components/llm_connections/delete_api_key_dialog_component.html.erb b/app/components/llm_connections/delete_api_key_dialog_component.html.erb new file mode 100644 index 000000000000..175b36d3303c --- /dev/null +++ b/app/components/llm_connections/delete_api_key_dialog_component.html.erb @@ -0,0 +1,20 @@ +<%= + render( + Primer::OpenProject::DangerDialog.new( + title: t("admin.llm_connections.delete_api_key.title"), + form_arguments:, + test_selector: TEST_SELECTOR + ) + ) do |dialog| + dialog.with_confirmation_message do |message| + message.with_heading(tag: :h2) { t("admin.llm_connections.delete_api_key.heading") } + message.with_description_content( + if loses_admin_verdicts? + t("admin.llm_connections.delete_api_key.description_verdicts") + else + t("admin.llm_connections.delete_api_key.description") + end + ) + end + end +%> diff --git a/app/components/llm_connections/delete_api_key_dialog_component.rb b/app/components/llm_connections/delete_api_key_dialog_component.rb new file mode 100644 index 000000000000..6c0aa7280fee --- /dev/null +++ b/app/components/llm_connections/delete_api_key_dialog_component.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Confirms removing the stored API key. + # + # No confirmation checkbox: the key itself can simply be pasted again. The + # dialog exists for what is *not* recoverable -- see #loses_admin_verdicts?. + class DeleteApiKeyDialogComponent < ApplicationComponent + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + TEST_SELECTOR = "llm-connection--delete-api-key-dialog" + + alias_method :connection, :model + + def form_arguments + { action: url_helpers.api_key_llm_connection_path, method: :delete } + end + + # The catalogue sync fingerprints base_url and api_key together, so the next + # refresh after the key changes treats the endpoint as a different deployment + # and discards every capability verdict -- including the ones an + # administrator asserted by hand, which nothing else in the system throws + # away. Worth saying out loud before the key goes. + def loses_admin_verdicts? + connection.capability_verdicts.exists?(source: "admin") + end + end +end diff --git a/app/components/llm_connections/disconnect_dialog_component.html.erb b/app/components/llm_connections/disconnect_dialog_component.html.erb new file mode 100644 index 000000000000..6f57b393557b --- /dev/null +++ b/app/components/llm_connections/disconnect_dialog_component.html.erb @@ -0,0 +1,35 @@ +<%= + render( + Primer::OpenProject::DangerDialog.new( + title: t("admin.llm_connections.disconnect.title"), + form_arguments:, + test_selector: TEST_SELECTOR + ) + ) do |dialog| + dialog.with_confirmation_message do |message| + message.with_heading(tag: :h2) { t("admin.llm_connections.disconnect.heading") } + message.with_description_content(t("admin.llm_connections.disconnect.description")) + end + + dialog.with_additional_details do + render(Primer::OpenProject::FlexLayout.new) do |flex| + flex.with_row do + content_tag(:ul) do + safe_join( + [ + content_tag(:li, t("admin.llm_connections.disconnect.keeps_settings")), + content_tag(:li, t("admin.llm_connections.disconnect.keeps_models")), + if bound_features.any? + content_tag( + :li, + t("admin.llm_connections.disconnect.keeps_bindings", features: bound_features.to_sentence) + ) + end + ].compact + ) + end + end + end + end + end +%> diff --git a/app/components/llm_connections/disconnect_dialog_component.rb b/app/components/llm_connections/disconnect_dialog_component.rb new file mode 100644 index 000000000000..4df992cf05da --- /dev/null +++ b/app/components/llm_connections/disconnect_dialog_component.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Confirms disconnecting from the LLM server. + # + # Disconnecting clears the credential and switches the connection off. It does + # not delete anything: the endpoint, the model catalogue, the capability + # verdicts and every feature binding are kept, so reconnecting is a matter of + # entering the key again. + # + # That is also why there is no confirmation checkbox -- nothing here is + # irreversible. A destroying variant would have been, and would have taken the + # locked embedding bindings with it: those are the only record that a vector + # index exists and which model and dimension it was written under, and + # dependent: :delete_all bypasses the guard that protects them. + class DisconnectDialogComponent < ApplicationComponent + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + TEST_SELECTOR = "llm-connection--disconnect-dialog" + + alias_method :connection, :model + + def form_arguments + { action: url_helpers.disconnect_llm_connection_path, method: :post } + end + + def bound_features + connection.feature_bindings.filter_map { |binding| binding.feature&.label if binding.model_id.present? } + end + end +end diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb index f8a97cfa4ddb..772fe0817b38 100644 --- a/app/controllers/admin/llm_connections_controller.rb +++ b/app/controllers/admin/llm_connections_controller.rb @@ -59,7 +59,39 @@ def refresh_models end end + def disconnect_dialog + respond_with_dialog LlmConnections::DisconnectDialogComponent.new(@connection) + end + + # Clears the credential and switches the connection off, keeping the endpoint, + # the catalogue and every feature binding. Deliberately not a destroy: the + # cascade would take the locked embedding bindings with it, and those are the + # only record that a vector index exists and what it was written with. + def disconnect + return redirect_with_error(t(".configured_from_env")) if @connection.configured_from_env? + + @connection.update!(api_key: nil, enabled: false) + Llm::HealthCheckJob.toggle_cron_job + + redirect_with_notice(t(".success")) + end + + def delete_api_key_dialog + respond_with_dialog LlmConnections::DeleteApiKeyDialogComponent.new(@connection) + end + + # Deliberately not routed through UpdateService. Its contract probes the + # server whenever credentials change, and a server that requires + # authentication would reject the now-keyless probe -- so going through the + # service would let a server refuse an administrator permission to remove a + # credential. Removing one must always be possible. + # + # The one thing the contract did give us is the environment guard, which is + # therefore checked explicitly here: without it a hand-crafted request could + # wipe a key the UI correctly refuses to touch. def delete_api_key + return redirect_with_error(t(".configured_from_env")) if @connection.configured_from_env? + @connection.update!(api_key: nil) redirect_with_notice(t(".success")) diff --git a/app/views/admin/llm_connections/show.html.erb b/app/views/admin/llm_connections/show.html.erb index 3eeffd74968d..4887ee7278e2 100644 --- a/app/views/admin/llm_connections/show.html.erb +++ b/app/views/admin/llm_connections/show.html.erb @@ -38,6 +38,47 @@ See COPYRIGHT and LICENSE files for more details. { href: mcp_configurations_path, text: t("menus.admin.ai") }, t("menus.admin.llm_connection")] ) + + # Nothing here applies to a connection provisioned from the environment: + # those settings belong to the deployment, not to the administrator. + if @connection.persisted? && !@connection.configured_from_env? + header.with_action_menu( + menu_arguments: { anchor_align: :end }, + button_arguments: { + icon: "kebab-horizontal", + "aria-label": t(:label_actions), + test_selector: "llm-connection--actions" + } + ) do |menu| + if @connection.api_key.present? + menu.with_item( + label: t("admin.llm_connections.delete_api_key.menu_label"), + scheme: :danger, + tag: :a, + href: delete_api_key_dialog_llm_connection_path, + content_arguments: { + data: { controller: "async-dialog" }, + test_selector: "llm-connection--delete-api-key" + } + ) do |item| + item.with_leading_visual_icon(icon: :key) + end + end + + menu.with_item( + label: t("admin.llm_connections.disconnect.menu_label"), + scheme: :danger, + tag: :a, + href: disconnect_dialog_llm_connection_path, + content_arguments: { + data: { controller: "async-dialog" }, + test_selector: "llm-connection--disconnect" + } + ) do |item| + item.with_leading_visual_icon(icon: :"circle-slash") + end + end + end end %> diff --git a/config/locales/en.yml b/config/locales/en.yml index 50c6e885c6d1..f122f8b5b9b3 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1454,8 +1454,24 @@ en: default_chat_model_caption: "Used by AI features that do not select a model themselves." enabled_caption: "When turned off, all LLM-backed AI features stop working. The connection settings are kept." label_connecting: "Contacting the LLM server…" + disconnect: + configured_from_env: "This connection is configured through the environment and cannot be changed here." + description: "OpenProject will stop using the LLM server. AI features will be unavailable until you connect again." + heading: "Disconnect from the LLM server?" + keeps_bindings: "The model chosen for each feature is kept: %{features}." + keeps_models: "The model list, including any models you added manually, is kept." + keeps_settings: "The endpoint and API format are kept. Only the stored API key is removed." + menu_label: "Disconnect" + success: "OpenProject has disconnected from the LLM server." + title: "Disconnect" delete_api_key: + configured_from_env: "This connection is configured through the environment and cannot be changed here." + description: "OpenProject will stop sending an API key with its requests. Features will keep working only if the server requires no authentication. You can enter a new key at any time." + description_verdicts: "OpenProject will stop sending an API key with its requests, and the next model refresh will discard what is known about each model's capabilities, including the assertions you made yourself. You can enter a new key at any time." + heading: "Remove the stored API key?" + menu_label: "Remove API key" success: "The API key has been removed." + title: "Remove API key" models: blank_description: "The server reported no models. If it does not offer a model list, add the model names you want to use below." blank_title: "No models available" diff --git a/config/routes.rb b/config/routes.rb index c27af828cbb8..e428cb1ed270 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -694,6 +694,9 @@ resource :llm_connection, only: %i[show update], controller: "admin/llm_connections" do post :refresh_models delete :api_key, action: :delete_api_key + get :delete_api_key_dialog + get :disconnect_dialog + post :disconnect resource :health_status_report, only: %i[show create], controller: "admin/llm_health_status" do post :create_health_status_report diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb index e520a070bc55..cc9027ebd730 100644 --- a/spec/requests/admin/llm_connections_spec.rb +++ b/spec/requests/admin/llm_connections_spec.rb @@ -163,4 +163,110 @@ expect(LlmConnection.first.available_model_ids).to include("bge-m3") end end + + describe "GET /admin/llm_connection/delete_api_key_dialog" do + let!(:connection) { create(:llm_connection, base_url: "https://example.com/v1", api_key: "sk-test") } + + before { login_as admin } + + it "offers the confirmation" do + # Requested by the async-dialog Stimulus controller, which asks for a + # turbo stream rather than HTML. + get delete_api_key_dialog_llm_connection_path, + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Remove the stored API key?") + end + + # The catalogue sync fingerprints base_url and api_key together, so changing + # the key discards every verdict -- including hand-made ones, which nothing + # else throws away. + it "warns when hand-made capability assertions would be lost" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "supported", source: "admin", + checked_at: Time.current) + + get delete_api_key_dialog_llm_connection_path, + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response.body).to include("assertions you made yourself") + end + end + + describe "DELETE /admin/llm_connection/api_key when provisioned" do + before { login_as admin } + + it "clears the stored key and keeps everything else" do + connection = create(:llm_connection, base_url: "https://example.com/v1", api_key: "sk-test") + + delete api_key_llm_connection_path + + expect(connection.reload.api_key).to be_blank + expect(connection.base_url).to eq("https://example.com/v1") + end + + # Previously update! bypassed the contract, so this could be wiped by a + # hand-crafted request even though the UI never offers it. + it "refuses when the connection comes from the environment" do + connection = create(:llm_connection, base_url: "https://example.com/v1", api_key: "sk-test") + allow(Setting).to receive(:llm_connection).and_return({ "base_url" => "https://example.com/v1" }) + + delete api_key_llm_connection_path + + expect(connection.reload.api_key).to eq("sk-test") + end + end + + describe "disconnecting" do + let!(:connection) do + create(:llm_connection, :with_models, :enabled, + base_url: "https://example.com/v1", api_key: "sk-test") + end + + before { login_as admin } + + it "offers the confirmation, naming what is kept" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + get disconnect_dialog_llm_connection_path, + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Disconnect from the LLM server?") + expect(response.body).to include("Description assistant") + end + + # Disconnecting is reversible on purpose: destroying the connection would + # cascade to the models, the verdicts and every binding. + it "clears the credential and switches the connection off, keeping everything else" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + post disconnect_llm_connection_path + + connection.reload + expect(connection.api_key).to be_blank + expect(connection).not_to be_enabled + expect(connection.base_url).to eq("https://example.com/v1") + expect(connection.models.count).to eq(2) + expect(connection.feature_bindings.first.model_id).to eq("qwen3.6-27b") + end + + it "refuses when the connection comes from the environment" do + allow(Setting).to receive(:llm_connection).and_return({ "base_url" => "https://example.com/v1" }) + + post disconnect_llm_connection_path + + expect(connection.reload.api_key).to eq("sk-test") + expect(connection).to be_enabled + end + + it "is refused to a non-admin" do + login_as create(:user) + + post disconnect_llm_connection_path + + expect(connection.reload.api_key).to eq("sk-test") + end + end end From 1b4c0348af95c01d37527fcb7e70dccaefb08951 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Wed, 12 Aug 2026 15:58:14 +0100 Subject: [PATCH 30/44] [#66020] Configure the vector settings an embedding feature indexes with llm_feature_bindings has carried dimensions, input_prefix, query_prefix and locked_at since the table was created, with no way to set any of them. Semantic search needs all four. The prefixes are stored exactly as typed and deliberately not stripped: the trailing space in "passage: " is load-bearing for the E5 and BGE families, and the captions quote the examples so it is visible. Dimensions defaults to blank, because the server decides the vector size and baking in a number it may contradict helps nobody. Where the embeddings probe has already seen a vector, its size is reported in the caption as information rather than filled into the field. The lock now freezes everything the stored index depends on, not just the model. A prefix mismatch is worse than a model mismatch: an index built with one prefix and queried under another does not error, it quietly returns worse results. It also only constrains later edits. Enforcing it on create refused to record a lock at all, since every attribute reads as changed from nil -- a latent bug in the model-only version of this guard that nothing exercised until now. A locked binding renders its values as text rather than disabled inputs. A disabled input submits nothing, so the values would arrive blank and wipe the columns the lock exists to protect. https://community.openproject.org/work_packages/66020 --- .../feature_binding_component.html.erb | 17 +++++- .../feature_binding_component.rb | 22 +++++++ .../admin/llm_feature_bindings_controller.rb | 25 +++++++- .../llm_connections/feature_binding_form.rb | 46 ++++++++++++++- app/models/llm_feature_binding.rb | 43 ++++++++++++-- config/locales/en.yml | 17 ++++++ .../admin/llm_feature_bindings_spec.rb | 57 +++++++++++++++++++ 7 files changed, 218 insertions(+), 9 deletions(-) diff --git a/app/components/llm_connections/feature_binding_component.html.erb b/app/components/llm_connections/feature_binding_component.html.erb index b36d3a358923..65fe55090d43 100644 --- a/app/components/llm_connections/feature_binding_component.html.erb +++ b/app/components/llm_connections/feature_binding_component.html.erb @@ -30,8 +30,23 @@ options: model_options, inherit_label:, feature_key: feature.key, - locked: locked? + locked: locked?, + embedding: feature.embedding?, + dimensions_hint: probed_dimensions ) ) %> <% end %> + + <% if locked? && feature.embedding? %> + <%# Rendered as text rather than disabled inputs: a disabled input submits + nothing, so the values would arrive blank and wipe the columns. %> + <%= render(Primer::Beta::Text.new(tag: :p, font_weight: :bold, mt: 2, mb: 1)) do %> + <%= t("admin.llm_feature_bindings.locked_values_heading") %> + <% end %> + <% locked_values.each do |label, value| %> + <%= render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 0)) do %> + <%= "#{label}: #{value}" %> + <% end %> + <% end %> + <% end %> <% end %> diff --git a/app/components/llm_connections/feature_binding_component.rb b/app/components/llm_connections/feature_binding_component.rb index 88585e5aef01..9d5d16e5a555 100644 --- a/app/components/llm_connections/feature_binding_component.rb +++ b/app/components/llm_connections/feature_binding_component.rb @@ -65,6 +65,28 @@ def locked? = binding&.locked? def dangling? = binding&.dangling? + # What the embeddings probe last saw, offered as information. Never filled + # into the field: the server decides the vector size at index time. + def probed_dimensions + return unless feature.embedding? + + model_id = binding&.resolved_model_id + return if model_id.blank? + + connection.capability_verdicts.for_model(model_id).for_capability(:embeddings).first&.dimensions + end + + # Quoted so a trailing space -- load-bearing for the E5 and BGE families -- + # is visible rather than invisible. + def locked_values + [ + [LlmFeatureBinding.human_attribute_name(:model_id), binding.model_id], + [LlmFeatureBinding.human_attribute_name(:dimensions), binding.dimensions || "—"], + [LlmFeatureBinding.human_attribute_name(:input_prefix), binding.input_prefix.to_s.inspect], + [LlmFeatureBinding.human_attribute_name(:query_prefix), binding.query_prefix.to_s.inspect] + ] + end + # Still resolvable, so not dangling -- but an administrator has hidden it # from the pickers, so say so rather than let the choice look unremarkable. def deactivated? diff --git a/app/controllers/admin/llm_feature_bindings_controller.rb b/app/controllers/admin/llm_feature_bindings_controller.rb index 5824c871405f..a4c65390a064 100644 --- a/app/controllers/admin/llm_feature_bindings_controller.rb +++ b/app/controllers/admin/llm_feature_bindings_controller.rb @@ -66,8 +66,7 @@ def binding_for(feature) end def assign(feature) - binding = binding_for(feature) - binding.model_id = params.dig(:llm_feature_binding, :model_id).presence + binding = build_binding(feature) if binding.save probe_capabilities(feature, binding) @@ -77,6 +76,28 @@ def assign(feature) end end + def build_binding(feature) + binding = binding_for(feature) + binding.model_id = params.dig(:llm_feature_binding, :model_id).presence + + # Only ever accepted for the kind of feature they describe; the model + # rejects them elsewhere, and they are not read at all for a chat feature. + assign_embedding_settings(binding) if feature.embedding? + + binding + end + + # The prefixes are stored exactly as typed. The trailing space in "passage: " + # is load-bearing for the E5 and BGE families, so stripping would silently + # degrade retrieval. + def assign_embedding_settings(binding) + settings = params.fetch(:llm_feature_binding, {}) + + binding.dimensions = settings[:dimensions].presence + binding.input_prefix = settings[:input_prefix] + binding.query_prefix = settings[:query_prefix] + end + # The verdict that actually matters is the one for the model an administrator # just chose, so it is fetched now rather than left unknown until first use. def probe_capabilities(feature, binding) diff --git a/app/forms/llm_connections/feature_binding_form.rb b/app/forms/llm_connections/feature_binding_form.rb index bd890569b656..050f2fa6ad24 100644 --- a/app/forms/llm_connections/feature_binding_form.rb +++ b/app/forms/llm_connections/feature_binding_form.rb @@ -33,12 +33,14 @@ module LlmConnections class FeatureBindingForm < ApplicationForm # Primer::Forms::Base.new assigns the builder itself and calls this with the # remaining keywords, so the builder must not appear in the signature. - def initialize(options:, inherit_label:, feature_key:, locked: false) + def initialize(options:, inherit_label:, feature_key:, locked: false, embedding: false, dimensions_hint: nil) super() @model_options = options @inherit_label = inherit_label @feature_key = feature_key @locked = locked + @embedding = embedding + @dimensions_hint = dimensions_hint end form do |f| @@ -59,6 +61,37 @@ def initialize(options:, inherit_label:, feature_key:, locked: false) end end + # Only for an embedding feature, and only while unlocked. A locked binding + # renders these as text instead: a disabled input submits nothing, so the + # values would arrive blank and wipe the columns. + if embedding && !locked + f.text_field( + name: :dimensions, + type: :number, + min: 1, + label: LlmFeatureBinding.human_attribute_name(:dimensions), + caption: dimensions_caption, + input_width: :small, + data: { test_selector: "llm-feature-binding--dimensions-#{feature_key}" } + ) + + f.text_field( + name: :input_prefix, + label: LlmFeatureBinding.human_attribute_name(:input_prefix), + caption: I18n.t("admin.llm_feature_bindings.form.input_prefix_caption"), + input_width: :medium, + data: { test_selector: "llm-feature-binding--input-prefix-#{feature_key}" } + ) + + f.text_field( + name: :query_prefix, + label: LlmFeatureBinding.human_attribute_name(:query_prefix), + caption: I18n.t("admin.llm_feature_bindings.form.query_prefix_caption"), + input_width: :medium, + data: { test_selector: "llm-feature-binding--query-prefix-#{feature_key}" } + ) + end + unless locked f.submit( name: :submit, @@ -71,7 +104,16 @@ def initialize(options:, inherit_label:, feature_key:, locked: false) private - attr_reader :model_options, :inherit_label, :feature_key, :locked + attr_reader :model_options, :inherit_label, :feature_key, :locked, :embedding, :dimensions_hint + + # Blank is the right default: the server decides the vector size, and baking + # in a number it may contradict helps nobody. Where the probe has already + # seen a vector, its size is offered as information rather than filled in. + def dimensions_caption + return I18n.t("admin.llm_feature_bindings.form.dimensions_caption") if dimensions_hint.blank? + + I18n.t("admin.llm_feature_bindings.form.dimensions_caption_probed", dimensions: dimensions_hint) + end def option_label(option) case option.state diff --git a/app/models/llm_feature_binding.rb b/app/models/llm_feature_binding.rb index 5ff1214d200d..8595c24a3f1b 100644 --- a/app/models/llm_feature_binding.rb +++ b/app/models/llm_feature_binding.rb @@ -36,9 +36,19 @@ class LlmFeatureBinding < ApplicationRecord belongs_to :llm_connection + # Settings that describe how vectors are written, and so only mean anything + # for an embedding feature. + EMBEDDING_SETTINGS = %i[dimensions input_prefix query_prefix].freeze + + # Everything a stored index depends on. Changing any of it invalidates the + # vectors already written, not just the model. + LOCKED_SETTINGS = ([:model_id] + EMBEDDING_SETTINGS).freeze + validates :feature_key, presence: true, uniqueness: { scope: :llm_connection_id } + validates :dimensions, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true validate :feature_registered - validate :pinned_model_unchanged + validate :embedding_settings_only_for_embedding_features + validate :locked_settings_unchanged def feature OpenProject::Llm::Features[feature_key] @@ -77,12 +87,37 @@ def feature_registered errors.add(:feature_key, :not_registered) end + def embedding_settings_only_for_embedding_features + return if feature.nil? || feature.embedding? + + EMBEDDING_SETTINGS.each do |attribute| + next if public_send(attribute).blank? + + errors.add(attribute, :not_for_chat_feature) + end + end + # Vectors written under one embedding model are meaningless under another, and # the dimension count is baked into the index, so a locked binding can only be # changed by an explicit re-index. - def pinned_model_unchanged - return unless locked? && model_id_changed? + # + # The prefixes are locked for the same reason and matter just as much: an index + # built with "passage: " but queried under a different prefix does not error, + # it quietly returns worse results, which is the hardest kind of failure to + # notice. + # + # TODO(#69620): re-indexing is what clears locked_at. Until that job exists a + # locked binding can only be changed in the database. + def locked_settings_unchanged + # Only constrains later edits. On the save that records the lock -- and on + # create -- every attribute reads as changed from nil, and there is nothing + # indexed yet for them to contradict. + return unless locked? && locked_at_was.present? + + LOCKED_SETTINGS.each do |attribute| + next unless public_send(:"#{attribute}_changed?") - errors.add(:model_id, :locked) + errors.add(attribute, :locked) + end end end diff --git a/config/locales/en.yml b/config/locales/en.yml index f122f8b5b9b3..ee1e2b4afebd 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -211,8 +211,10 @@ en: llm_feature_binding: dimensions: "Dimensions" feature_key: "Feature" + input_prefix: "Document prefix" # ActiveRecord::Base.human_attribute_name strips the _id suffix. model: "Model" + query_prefix: "Query prefix" llm_connection: api_format: "API format" api_key: "API key" @@ -684,6 +686,15 @@ en: not_registered: "does not belong to a known AI feature." model_id: locked: "cannot be changed while data indexed with it still exists. Re-index to switch models." + dimensions: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." + input_prefix: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." + query_prefix: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." llm_connection: attributes: api_format: @@ -1431,6 +1442,12 @@ en: llm_feature_bindings: dangling: "%{model} is no longer offered by the LLM server. This feature will not run until another model is selected." + form: + dimensions_caption: "How many numbers each vector has. Leave blank to use whatever the server returns." + dimensions_caption_probed: "The server returned %{dimensions}-dimension vectors for this model. Leave blank to use whatever it returns at index time." + input_prefix_caption: "Prepended to each document before it is indexed. Some models expect one, for example \"passage: \" including the trailing space." + query_prefix_caption: "Prepended to each search query. Some models expect one, for example \"query: \" including the trailing space." + locked_values_heading: "Values fixed by the existing index" deactivated: "%{model} has been hidden by an administrator. This feature keeps using it, but it can no longer be chosen elsewhere." index: blank_description: "Connect OpenProject to an LLM server first. Models can be assigned once the server reports which ones it offers." diff --git a/spec/requests/admin/llm_feature_bindings_spec.rb b/spec/requests/admin/llm_feature_bindings_spec.rb index bddf4cd72bf0..4b58a11eda8e 100644 --- a/spec/requests/admin/llm_feature_bindings_spec.rb +++ b/spec/requests/admin/llm_feature_bindings_spec.rb @@ -125,4 +125,61 @@ expect(response).to have_http_status(:not_found) end end + + describe "embedding settings" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before do + login_as admin + # Binding an embedding feature probes the model for a vector. + mock_llm_embeddings_response(base_url) + end + + it "stores the vector settings, keeping the prefixes exactly as typed" do + patch llm_feature_binding_path(:semantic_search), + params: { llm_feature_binding: { model_id: "bge-m3", + dimensions: "1024", + input_prefix: "passage: ", + query_prefix: "query: " } } + + binding = connection.feature_bindings.find_by(feature_key: "semantic_search") + + expect(binding.dimensions).to eq(1024) + # The trailing space is load-bearing for the E5 and BGE families. + expect(binding.input_prefix).to eq("passage: ") + expect(binding.query_prefix).to eq("query: ") + end + + it "rejects a dimension count that is not a positive integer" do + patch llm_feature_binding_path(:semantic_search), + params: { llm_feature_binding: { model_id: "bge-m3", dimensions: "0" } } + + expect(connection.feature_bindings.find_by(feature_key: "semantic_search")&.dimensions).to be_nil + end + + it "ignores vector settings sent to a chat feature" do + patch llm_feature_binding_path(:description_assistant), + params: { llm_feature_binding: { model_id: "qwen3.6-27b", dimensions: "1024" } } + + binding = connection.feature_bindings.find_by(feature_key: "description_assistant") + + expect(binding.model_id).to eq("qwen3.6-27b") + expect(binding.dimensions).to be_nil + end + + # A locked binding is the record that a vector index exists. Everything the + # index depends on is frozen, not just the model. + it "refuses to change anything a locked index depends on" do + binding = connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "bge-m3", + dimensions: 1024, input_prefix: "passage: ", + locked_at: Time.current) + + patch llm_feature_binding_path(:semantic_search), + params: { llm_feature_binding: { model_id: "bge-m3", dimensions: "512", input_prefix: "other: " } } + + binding.reload + expect(binding.dimensions).to eq(1024) + expect(binding.input_prefix).to eq("passage: ") + end + end end From b244baca8281fcff35c55d3acd387b36554d8432 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Wed, 12 Aug 2026 19:12:47 +0100 Subject: [PATCH 31/44] [#66020] Make the model list usable at gateway scale The table rendered every model a server reported on every page load -- 341 of them for OpenRouter, which is roughly two thousand component renders, as many CSRF tokens for the row toggles, and an accessibility tree of the same size. Finding one model among them meant scrolling a dropdown with no way to search. BorderBoxTableComponent already paginates whatever it is given, so that half only had to hand it a paginated relation. Filtering is server-side on the same Queries stack the reserved identifiers admin uses, so it composes with pagination and a filtered URL can be shared; it matches the identifier the server uses and the friendly name an administrator may have given it, since either is what somebody would type. The connection is now passed to the table explicitly rather than derived from the first row: with a paginated, filtered list a page can legitimately be empty, and deriving it silently degraded the kind column to "Unknown" for every row. The model pickers become autocompleters so a model can be found by typing. To be clear about what that does and does not fix: the list is still inlined in the page, as a JSON attribute rather than option elements. The gain is that it is searchable, not that the page got smaller. default_embedding_model_id is rendered at last. It had a column, a contract attribute, a validation, an error key, an attribute translation and a permitted parameter -- everything except an input, so the value could not be set at all. https://community.openproject.org/work_packages/66020 --- .../feature_binding_component.html.erb | 1 + .../models/index_component.html.erb | 3 + .../llm_connections/models/index_component.rb | 49 ++++++++++ .../models/sub_header_component.html.erb | 13 +++ .../models/sub_header_component.rb | 65 +++++++++++++ .../llm_connections/models_table_component.rb | 13 ++- .../admin/llm_connections_controller.rb | 20 +++- app/forms/llm_connections/connection_form.rb | 56 +++++++++-- .../llm_connections/feature_binding_form.rb | 28 ++++-- app/models/queries/llm_models.rb | 35 +++++++ .../llm_models/filters/llm_model_filter.rb | 33 +++++++ .../queries/llm_models/filters/name_filter.rb | 59 ++++++++++++ .../queries/llm_models/llm_model_query.rb | 43 +++++++++ app/views/admin/llm_connections/show.html.erb | 3 +- config/locales/en.yml | 2 + config/routes.rb | 1 + spec/requests/admin/llm_connections_spec.rb | 92 +++++++++++++++++++ 17 files changed, 497 insertions(+), 19 deletions(-) create mode 100644 app/components/llm_connections/models/index_component.html.erb create mode 100644 app/components/llm_connections/models/index_component.rb create mode 100644 app/components/llm_connections/models/sub_header_component.html.erb create mode 100644 app/components/llm_connections/models/sub_header_component.rb create mode 100644 app/models/queries/llm_models.rb create mode 100644 app/models/queries/llm_models/filters/llm_model_filter.rb create mode 100644 app/models/queries/llm_models/filters/name_filter.rb create mode 100644 app/models/queries/llm_models/llm_model_query.rb diff --git a/app/components/llm_connections/feature_binding_component.html.erb b/app/components/llm_connections/feature_binding_component.html.erb index 65fe55090d43..762e4a7b5705 100644 --- a/app/components/llm_connections/feature_binding_component.html.erb +++ b/app/components/llm_connections/feature_binding_component.html.erb @@ -32,6 +32,7 @@ feature_key: feature.key, locked: locked?, embedding: feature.embedding?, + selected_model_id: binding&.model_id, dimensions_hint: probed_dimensions ) ) %> diff --git a/app/components/llm_connections/models/index_component.html.erb b/app/components/llm_connections/models/index_component.html.erb new file mode 100644 index 000000000000..84d1c5941804 --- /dev/null +++ b/app/components/llm_connections/models/index_component.html.erb @@ -0,0 +1,3 @@ +<%= component_wrapper do %> + <%= render(LlmConnections::ModelsTableComponent.new(rows:, connection:)) %> +<% end %> diff --git a/app/components/llm_connections/models/index_component.rb b/app/components/llm_connections/models/index_component.rb new file mode 100644 index 000000000000..844b94fe199a --- /dev/null +++ b/app/components/llm_connections/models/index_component.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + module Models + # A turbo-frame around the model table, so filtering can replace just the + # table rather than reloading the settings page. + class IndexComponent < ApplicationComponent + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + alias_method :rows, :model + + def initialize(rows, connection:) + super(rows) + @connection = connection + end + + attr_reader :connection + end + end +end diff --git a/app/components/llm_connections/models/sub_header_component.html.erb b/app/components/llm_connections/models/sub_header_component.html.erb new file mode 100644 index 000000000000..cdb1a730619a --- /dev/null +++ b/app/components/llm_connections/models/sub_header_component.html.erb @@ -0,0 +1,13 @@ +<%= render(Primer::OpenProject::SubHeader.new(collapsed_search: false, data: sub_header_data_attributes)) do |subheader| %> + <% subheader.with_filter_input( + name: "name", + label: t("admin.llm_connections.models.filter_label"), + visually_hide_label: true, + value: filter_input_value, + placeholder: t("admin.llm_connections.models.filter_label"), + leading_visual: { icon: :search, size: :small }, + show_clear_button: true, + clear_button_id:, + data: filter_input_data_attributes + ) %> +<% end %> diff --git a/app/components/llm_connections/models/sub_header_component.rb b/app/components/llm_connections/models/sub_header_component.rb new file mode 100644 index 000000000000..57cb69e49ec6 --- /dev/null +++ b/app/components/llm_connections/models/sub_header_component.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + module Models + # The filter bar above the model list. + class SubHeaderComponent < ApplicationComponent + include OpPrimer::ComponentHelpers + + alias_method :query, :model + + def filter_input_value + query.find_active_filter(:name)&.values&.first + end + + def clear_button_id = "llm-models-filter-clear" + + def sub_header_data_attributes + { + controller: "filter--filters-form", + "filter--filters-form-perform-turbo-requests-value": true, + "filter--filters-form-output-format-value": "json", + "filter--filters-form-url-path-name-value": helpers.search_models_llm_connection_path, + "filter--filters-form-clear-button-id-value": clear_button_id + } + end + + def filter_input_data_attributes + { + "filter-name": "name", + "filter-type": "string", + "filter-operator": "~", + "filter--filters-form-target": "simpleFilter filterValueContainer simpleValue" + } + end + end + end +end diff --git a/app/components/llm_connections/models_table_component.rb b/app/components/llm_connections/models_table_component.rb index a0aa5869f8ab..e57ca958cada 100644 --- a/app/components/llm_connections/models_table_component.rb +++ b/app/components/llm_connections/models_table_component.rb @@ -38,6 +38,16 @@ class ModelsTableComponent < OpPrimer::BorderBoxTableComponent mobile_columns :identifier + # The connection is passed in rather than derived from the first row: with a + # paginated, filtered list a page can legitimately be empty, and deriving it + # would silently degrade the kind column to "Unknown". + def initialize(connection:, **) + super(**) + @connection = connection + end + + attr_reader :connection + def initial_sort = %i[identifier asc] def has_footer? = false @@ -67,9 +77,6 @@ def embeddings_states end def load_embeddings_states - connection = rows.first&.llm_connection - return {} if connection.nil? - connection.capability_verdicts.for_capability(:embeddings).pluck(:model_id, :state).to_h end diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb index 772fe0817b38..463487ba8d79 100644 --- a/app/controllers/admin/llm_connections_controller.rb +++ b/app/controllers/admin/llm_connections_controller.rb @@ -31,6 +31,7 @@ module Admin class LlmConnectionsController < ApplicationController include OpTurbo::ComponentStream + include PaginationHelper layout "admin" menu_item :llm_connection @@ -38,7 +39,24 @@ class LlmConnectionsController < ApplicationController before_action :require_admin before_action :set_connection - def show; end + def show + @query = ParamsToQueryService + .new(LlmModel, current_user, query_class: Queries::LlmModels::LlmModelQuery) + .call(params) + @models = @query.results.paginate(page: page_param, per_page: per_page_param) + end + + # Answers the sub-header's filter input, replacing just the table. + def search_models + show + + replace_via_turbo_stream( + component: LlmConnections::Models::IndexComponent.new(@models, connection: @connection) + ) + turbo_streams << turbo_stream.push_state(llm_connection_path(params.permit(:filters))) + + respond_with_turbo_streams + end def update result = ::LlmConnections::UpdateService diff --git a/app/forms/llm_connections/connection_form.rb b/app/forms/llm_connections/connection_form.rb index e1a6730980d9..806dee1e42d5 100644 --- a/app/forms/llm_connections/connection_form.rb +++ b/app/forms/llm_connections/connection_form.rb @@ -80,16 +80,52 @@ class ConnectionForm < ApplicationForm ) if models_available? - f.select_list( + # An autocompleter rather than a select: a gateway reports hundreds of + # models, and every one of them would otherwise be inlined as an option + # in the page body. decorated: true serialises the list into the element, + # so this needs no endpoint of its own. + f.autocompleter( name: :default_chat_model_id, label: LlmConnection.human_attribute_name(:default_chat_model_id), caption: I18n.t("admin.llm_connections.form.default_chat_model_caption"), - include_blank: true, - input_width: :large, - disabled: read_only? - ) do |select| + disabled: read_only?, + autocomplete_options: { + decorated: true, + inputValue: model.default_chat_model_id, + placeholder: I18n.t("label_none_parentheses") + } + ) do |list| + list.option(label: I18n.t("label_none_parentheses"), value: "", + selected: model.default_chat_model_id.blank?) + default_chat_model_options.each do |model_id| - select.option(value: model_id, label: model_id) + list.option(label: model_id, value: model_id, selected: model.default_chat_model_id == model_id) + end + end + + # Only worth asking for once something embeds. The column, the contract + # attribute, its validation and every translation for this field already + # existed; the input was simply never rendered, so the value could not be + # set through the UI at all. + if embedding_features? + f.autocompleter( + name: :default_embedding_model_id, + label: LlmConnection.human_attribute_name(:default_embedding_model_id), + caption: I18n.t("admin.llm_connections.form.default_embedding_model_caption"), + disabled: read_only?, + autocomplete_options: { + decorated: true, + inputValue: model.default_embedding_model_id, + placeholder: I18n.t("label_none_parentheses") + } + ) do |list| + list.option(label: I18n.t("label_none_parentheses"), value: "", + selected: model.default_embedding_model_id.blank?) + + default_embedding_model_options.each do |model_id| + list.option(label: model_id, value: model_id, + selected: model.default_embedding_model_id == model_id) + end end end end @@ -120,6 +156,14 @@ def default_chat_model_options (model.selectable_model_ids + [model.default_chat_model_id]).compact_blank.uniq end + def default_embedding_model_options + (model.selectable_model_ids + [model.default_embedding_model_id]).compact_blank.uniq + end + + def embedding_features? + OpenProject::Llm::Features.for_kind(:embedding).any? + end + def submit_label model.persisted? ? I18n.t(:button_save) : I18n.t("admin.llm_connections.form.button_connect") end diff --git a/app/forms/llm_connections/feature_binding_form.rb b/app/forms/llm_connections/feature_binding_form.rb index 050f2fa6ad24..cf7a50fbd5f7 100644 --- a/app/forms/llm_connections/feature_binding_form.rb +++ b/app/forms/llm_connections/feature_binding_form.rb @@ -33,8 +33,10 @@ module LlmConnections class FeatureBindingForm < ApplicationForm # Primer::Forms::Base.new assigns the builder itself and calls this with the # remaining keywords, so the builder must not appear in the signature. - def initialize(options:, inherit_label:, feature_key:, locked: false, embedding: false, dimensions_hint: nil) + def initialize(options:, inherit_label:, feature_key:, locked: false, embedding: false, dimensions_hint: nil, + selected_model_id: nil) super() + @selected_model_id = selected_model_id @model_options = options @inherit_label = inherit_label @feature_key = feature_key @@ -44,20 +46,29 @@ def initialize(options:, inherit_label:, feature_key:, locked: false, embedding: end form do |f| - f.select_list( + # An autocompleter rather than a select, so a model can be found by typing + # among the hundreds a gateway reports. decorated: true serialises the list + # into the element, so no endpoint is needed. + f.autocompleter( name: :model_id, label: LlmFeatureBinding.human_attribute_name(:model_id), - include_blank: false, - input_width: :large, disabled: locked, + autocomplete_options: { + decorated: true, + inputValue: selected_model_id, + placeholder: inherit_label + }, data: { test_selector: "llm-feature-binding--model-#{feature_key}" } - ) do |select| - select.option(value: "", label: inherit_label) + ) do |list| + list.option(label: inherit_label, value: "", selected: selected_model_id.blank?) model_options.each do |option| # Listed but not choosable when a required capability is known to be # missing: hiding it would leave the reason invisible too. - select.option(value: option.model_id, label: option_label(option), disabled: !option.selectable?) + list.option(label: option_label(option), + value: option.model_id, + selected: selected_model_id == option.model_id, + disabled: !option.selectable?) end end @@ -104,7 +115,8 @@ def initialize(options:, inherit_label:, feature_key:, locked: false, embedding: private - attr_reader :model_options, :inherit_label, :feature_key, :locked, :embedding, :dimensions_hint + attr_reader :model_options, :inherit_label, :feature_key, :locked, :embedding, :dimensions_hint, + :selected_model_id # Blank is the right default: the server decides the vector size, and baking # in a number it may contradict helps nobody. Where the probe has already diff --git a/app/models/queries/llm_models.rb b/app/models/queries/llm_models.rb new file mode 100644 index 000000000000..50492ff3b215 --- /dev/null +++ b/app/models/queries/llm_models.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Queries::LlmModels + ::Queries::Register.register(LlmModelQuery) do + filter Filters::NameFilter + end +end diff --git a/app/models/queries/llm_models/filters/llm_model_filter.rb b/app/models/queries/llm_models/filters/llm_model_filter.rb new file mode 100644 index 000000000000..bf0be12e74b5 --- /dev/null +++ b/app/models/queries/llm_models/filters/llm_model_filter.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class Queries::LlmModels::Filters::LlmModelFilter < Queries::Filters::Base + self.model = LlmModel +end diff --git a/app/models/queries/llm_models/filters/name_filter.rb b/app/models/queries/llm_models/filters/name_filter.rb new file mode 100644 index 000000000000..4fe27abbaa63 --- /dev/null +++ b/app/models/queries/llm_models/filters/name_filter.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class Queries::LlmModels::Filters::NameFilter < Queries::LlmModels::Filters::LlmModelFilter + def self.key + :name + end + + def type + :string + end + + def human_name + I18n.t("admin.llm_connections.models.filter_label") + end + + # Matches the identifier the server uses and the friendly name an + # administrator may have given it, since either is what someone types. + def where + escaped = ActiveRecord::Base.sanitize_sql_like(values.first) + + case operator + when "~", "**" + ["llm_models.external_id ILIKE :q OR llm_models.display_name ILIKE :q", { q: "%#{escaped}%" }] + when "!~" + ["llm_models.external_id NOT ILIKE :q AND (llm_models.display_name IS NULL OR llm_models.display_name NOT ILIKE :q)", + { q: "%#{escaped}%" }] + else + raise "Unsupported operator #{operator}" + end + end +end diff --git a/app/models/queries/llm_models/llm_model_query.rb b/app/models/queries/llm_models/llm_model_query.rb new file mode 100644 index 000000000000..e6d096a22749 --- /dev/null +++ b/app/models/queries/llm_models/llm_model_query.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class Queries::LlmModels::LlmModelQuery + include Queries::BaseQuery + include Queries::UnpersistedQuery + + def self.model + LlmModel + end + + # There is exactly one LLM connection, so every model belongs to it. + def default_scope + LlmModel.by_identifier + end +end diff --git a/app/views/admin/llm_connections/show.html.erb b/app/views/admin/llm_connections/show.html.erb index 4887ee7278e2..fd41f26a38db 100644 --- a/app/views/admin/llm_connections/show.html.erb +++ b/app/views/admin/llm_connections/show.html.erb @@ -132,6 +132,7 @@ See COPYRIGHT and LICENSE files for more details. end %> - <%= render(LlmConnections::ModelsTableComponent.new(rows: @connection.models.by_identifier)) %> + <%= render(LlmConnections::Models::SubHeaderComponent.new(@query)) %> + <%= render(LlmConnections::Models::IndexComponent.new(@models, connection: @connection)) %> <% end %> diff --git a/config/locales/en.yml b/config/locales/en.yml index ee1e2b4afebd..26216d82c821 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1469,6 +1469,7 @@ en: The full base URL of the server, including the API version segment, exactly as your provider documents it (for example https://example.com/v1). OpenProject appends only the endpoint path. button_connect: "Connect" default_chat_model_caption: "Used by AI features that do not select a model themselves." + default_embedding_model_caption: "Used by AI features that index text for search and do not select a model themselves." enabled_caption: "When turned off, all LLM-backed AI features stop working. The connection settings are kept." label_connecting: "Contacting the LLM server…" disconnect: @@ -1497,6 +1498,7 @@ en: identifier: "Model" kind: "Type" source: "Source" + filter_label: "Filter models" source_deactivated: "Hidden by administrator" source_discovered: "Reported by server" source_manual: "Added manually" diff --git a/config/routes.rb b/config/routes.rb index e428cb1ed270..0e7c10867486 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -697,6 +697,7 @@ get :delete_api_key_dialog get :disconnect_dialog post :disconnect + get :search_models, defaults: { format: :turbo_stream } resource :health_status_report, only: %i[show create], controller: "admin/llm_health_status" do post :create_health_status_report diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb index cc9027ebd730..c11e971c184a 100644 --- a/spec/requests/admin/llm_connections_spec.rb +++ b/spec/requests/admin/llm_connections_spec.rb @@ -269,4 +269,96 @@ expect(connection.reload.api_key).to eq("sk-test") end end + + describe "the default embedding model field" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url: "https://example.com/v1") } + + before { login_as admin } + + # Column, contract attribute, validation, error key, locale key and permitted + # param all existed; the input was never rendered, so the value could not be + # set through the UI at all. + it "is rendered once something embeds" do + get llm_connection_path + + expect(response.body).to include("llm_connection[default_embedding_model_id]") + end + + it "is saved" do + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "bge-m3" } } + + expect(connection.reload.default_embedding_model_id).to eq("bge-m3") + end + end + + describe "paginating the model list" do + let!(:connection) { create(:llm_connection, :enabled, base_url: "https://example.com/v1") } + + before do + login_as admin + # A gateway can report hundreds; OpenRouter returns 341. + 25.times { |n| create(:llm_model, llm_connection: connection, external_id: format("model-%03d", n)) } + end + + # Counted by row, not by model id: the default-model select still lists + # every model, so the ids appear in the body regardless of the table. + def rendered_rows(body) = body.scan("llm-model--toggle-").size + + it "shows one page of rows at a time rather than every model" do + get llm_connection_path, params: { per_page: 20 } + + expect(rendered_rows(response.body)).to eq(20) + expect(response.body).to include("op-pagination") + end + + it "serves the remainder on the next page" do + get llm_connection_path, params: { per_page: 20, page: 2 } + + expect(rendered_rows(response.body)).to eq(5) + end + end + + describe "filtering the model list" do + let!(:connection) { create(:llm_connection, :enabled, base_url: "https://example.com/v1") } + let(:filters) { [{ name: { operator: "~", values: ["bge"] } }].to_json } + + before do + login_as admin + create(:llm_model, llm_connection: connection, external_id: "qwen3.6-27b") + create(:llm_model, llm_connection: connection, external_id: "bge-m3") + create(:llm_model, llm_connection: connection, external_id: "e5-large", display_name: "BGE compatible") + end + + def rendered_rows(body) = body.scan("llm-model--toggle-").size + + it "narrows the table to matching models" do + get search_models_llm_connection_path, params: { filters: } + + expect(response).to have_http_status(:ok) + # The identifier and the friendly name both match, since either is what + # somebody would type. + expect(rendered_rows(response.body)).to eq(2) + expect(response.body).to include("bge-m3") + end + + it "answers with a turbo stream so only the table is replaced" do + get search_models_llm_connection_path, params: { filters: } + + expect(response.media_type).to eq("text/vnd.turbo-stream.html") + end + + it "applies the filter to the full page too, so a shared link works" do + get llm_connection_path, params: { filters: } + + expect(rendered_rows(response.body)).to eq(2) + end + + it "shows everything without a filter" do + get llm_connection_path + + expect(rendered_rows(response.body)).to eq(3) + end + end end From 13db6c1dea3ee44b7dd76bea0d8f496c4029cd26 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Wed, 12 Aug 2026 23:44:29 +0100 Subject: [PATCH 32/44] [#66020] Add a feature spec for the AI administration pages These pages are hand-written markup rather than generated, and nothing had ever put them through axe. The spec covers the settings page, the model list, both danger dialogs, the health report and the AI models page. Writing and running it turned up three defects. The danger dialogs' confirm buttons both read "Delete" -- Primer's default -- which is wrong for actions that delete nothing; they now say "Remove API key" and "Disconnect". The deactivation check expected the source label to change in place. The toggle acknowledges with JSON rather than re-rendering the row, so the label only catches up on the next load. Both dialogs were opened by clicking the menu item's test selector directly, which raced the anchored popover and failed intermittently. They now wait for the link and click it by label. The spec is tagged :selenium deliberately, not incidentally: axe-core-api drives the browser through Selenium's #manage API, so be_axe_clean does not work under cuprite. Every other axe assertion in this repository is tagged the same way. Every example was additionally run under cuprite with the axe assertions removed -- all seven pass, so the interactions are proven against a real browser. The be_axe_clean calls themselves are executed first by CI. https://community.openproject.org/work_packages/66020 --- .../delete_api_key_dialog_component.html.erb | 2 + .../disconnect_dialog_component.html.erb | 2 + spec/features/admin/llm_connection_spec.rb | 152 ++++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 spec/features/admin/llm_connection_spec.rb diff --git a/app/components/llm_connections/delete_api_key_dialog_component.html.erb b/app/components/llm_connections/delete_api_key_dialog_component.html.erb index 175b36d3303c..0c317a9534ec 100644 --- a/app/components/llm_connections/delete_api_key_dialog_component.html.erb +++ b/app/components/llm_connections/delete_api_key_dialog_component.html.erb @@ -3,6 +3,8 @@ Primer::OpenProject::DangerDialog.new( title: t("admin.llm_connections.delete_api_key.title"), form_arguments:, + # Defaults to "Delete", which is wrong here: neither action deletes anything. + confirm_button_text: t("admin.llm_connections.delete_api_key.title"), test_selector: TEST_SELECTOR ) ) do |dialog| diff --git a/app/components/llm_connections/disconnect_dialog_component.html.erb b/app/components/llm_connections/disconnect_dialog_component.html.erb index 6f57b393557b..b5affa99625d 100644 --- a/app/components/llm_connections/disconnect_dialog_component.html.erb +++ b/app/components/llm_connections/disconnect_dialog_component.html.erb @@ -3,6 +3,8 @@ Primer::OpenProject::DangerDialog.new( title: t("admin.llm_connections.disconnect.title"), form_arguments:, + # Defaults to "Delete", which is wrong here: neither action deletes anything. + confirm_button_text: t("admin.llm_connections.disconnect.title"), test_selector: TEST_SELECTOR ) ) do |dialog| diff --git a/spec/features/admin/llm_connection_spec.rb b/spec/features/admin/llm_connection_spec.rb new file mode 100644 index 000000000000..f7f9713fb3ab --- /dev/null +++ b/spec/features/admin/llm_connection_spec.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +# The markup on these pages is hand-written rather than generated, so this spec +# exists mainly to put it through axe. The row toggle in particular has no +# accessible name of its own and depends on an explicit aria-label. +# :selenium is required, not incidental: axe-core-api drives the browser through +# Selenium's #manage API, so be_axe_clean does not work under cuprite. Every other +# axe spec in this repository is tagged the same way for the same reason. +RSpec.describe "LLM connection administration", + :js, :llm_server_helpers, :selenium, :webmock, + driver: :firefox_de, + with_flag: { llm_connection: true } do + shared_let(:admin) { create(:admin) } + + let(:base_url) { "https://example.com/v1" } + + current_user { admin } + + context "when nothing is configured yet" do + it "renders an accessible, empty settings page" do + visit llm_connection_path + + expect(page).to have_field("Host URL") + expect(page).to be_axe_clean.within("#content") + end + end + + context "with a configured connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before { mock_llm_models_response(base_url) } + + it "renders the model list accessibly" do + visit llm_connection_path + + expect(page).to have_test_selector("llm-model--toggle-#{connection.models.first.id}") + expect(page).to be_axe_clean.within("#content") + end + + it "hides a model from the feature pickers when it is switched off" do + llm_model = connection.models.find_by(external_id: "qwen3.6-27b") + + visit llm_connection_path + find_test_selector("llm-model--toggle-#{llm_model.id}").click + + wait_for { llm_model.reload.deactivated_at }.not_to be_nil + + # The toggle acknowledges with JSON rather than re-rendering the row, so + # the source label only catches up on the next load. + visit llm_connection_path + expect(page).to have_text("Hidden by administrator") + end + + it "removes the stored API key" do + visit llm_connection_path + + find_test_selector("llm-connection--actions").click + # The menu renders into an anchored popover, so wait for it rather than + # racing the click. + expect(page).to have_link("Remove API key") + click_link "Remove API key" + + within_test_selector("llm-connection--delete-api-key-dialog") do + expect(page).to be_axe_clean + click_on "Remove API key" + end + + wait_for { connection.reload.api_key }.to be_blank + expect(connection.base_url).to eq(base_url) + end + + it "disconnects without losing the configuration" do + visit llm_connection_path + + find_test_selector("llm-connection--actions").click + expect(page).to have_link("Disconnect") + click_link "Disconnect" + + within_test_selector("llm-connection--disconnect-dialog") do + expect(page).to be_axe_clean + click_on "Disconnect" + end + + wait_for { connection.reload.enabled? }.to be(false) + expect(connection.api_key).to be_blank + # The point of disconnecting rather than deleting. + expect(connection.models.count).to eq(2) + end + + describe "the health report" do + before { mock_llm_chat_response(base_url) } + + it "runs the checks and renders the report accessibly" do + visit llm_connection_path + + find_test_selector("llm-connection--run-health-checks").click + + wait_for { connection.health_reports.count }.to eq(1) + + find_test_selector("llm-connection--open-health-report").click + + expect(page).to have_current_path(llm_connection_health_status_report_path) + expect(page).to have_text("Configuration") + expect(page).to be_axe_clean.within("#content") + end + end + end + + describe "the AI models page" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before { mock_llm_embeddings_response(base_url) } + + it "offers the vector settings only for features that embed" do + visit llm_feature_bindings_path + + expect(page).to have_test_selector("llm-feature-binding--dimensions-semantic_search") + expect(page).to have_no_test_selector("llm-feature-binding--dimensions-description_assistant") + expect(page).to be_axe_clean.within("#content") + end + end +end From fbce0c7ca282cbad5f28e747ed7dec7b4eb12cad Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 13 Aug 2026 10:25:59 +0100 Subject: [PATCH 33/44] [#66020] Fix three inconsistencies found in review A model with no embedding capability could be chosen as the default embedding model. The per-feature picker had always refused such a model and said why; the connection-level default did not. It now follows the same rule -- never hide a model, but disable one the server has told us cannot embed, and label it. The contract enforces the same thing, so a crafted request cannot set what the form refuses. An unknown verdict still does not block: that is the normal state for a server that publishes nothing about its models. The health report's "More information" links pointed at an admin guide page that does not exist. Linking an error message to a 404 is worse than not linking, and falling back to the file storages troubleshooting page -- the component's default -- would have been worse still. The link is now suppressible and suppressed here; each check already explains itself in its own message. The static_links entry that pointed at the missing page is removed with it. The settings page did not follow the shape other admin pages of its kind use. The side panel sat full-width above the form instead of in a sidebar, and the model list carried two stacked bars: a Subhead with the actions and a SubHeader with the filter. It now uses Primer::Alpha::Layout with the panel in the sidebar, exactly as the storages admin page does, and the model actions moved into the filter sub-header so there is one bar. The layout is only used once a connection exists, since Primer::Alpha::Layout renders nothing without a sidebar slot and there is nothing to show beside the form before then. Fixing the last of these exposed a 500: on a failed save the controller rendered :show without running it, so the sub-header dereferenced a nil query. Any validation error on this page was an error page. https://community.openproject.org/work_packages/66020 --- .../health_reports/result_component.html.erb | 12 +-- .../health_reports/result_component.rb | 10 ++- .../models/sub_header_component.html.erb | 23 ++++++ .../llm_connections/base_contract.rb | 17 +++++ .../admin/llm_connections_controller.rb | 9 ++- app/forms/llm_connections/connection_form.rb | 33 +++++++- app/views/admin/llm_connections/show.html.erb | 76 ++++++------------- .../admin/llm_health_status/show.html.erb | 5 +- config/locales/en.yml | 3 + config/static_links.yml | 2 - spec/features/admin/llm_connection_spec.rb | 22 ++++-- spec/requests/admin/llm_connections_spec.rb | 41 ++++++++++ 12 files changed, 180 insertions(+), 73 deletions(-) diff --git a/app/components/health_reports/result_component.html.erb b/app/components/health_reports/result_component.html.erb index 67b17ee95405..8335ceba3e65 100644 --- a/app/components/health_reports/result_component.html.erb +++ b/app/components/health_reports/result_component.html.erb @@ -38,18 +38,20 @@ See COPYRIGHT and LICENSE files for more details. line.with_column(mr: 2) do render(Primer::Beta::Text.new(font_size: :small, color: status_color)) { status_text } end - if error_code.present? + if error_code.present? && docs_href.present? line.with_column do render(Primer::Beta::Label.new(scheme: status_color)) { error_code } end end end - if error_code.present? + if error_code.present? && docs_href.present? row.with_column do - helpers.static_link_to(href: docs_href, - label: I18n.t(:label_more_information), - underline: true) + helpers.static_link_to( + href: docs_href, + label: I18n.t(:label_more_information), + underline: true + ) end end end diff --git a/app/components/health_reports/result_component.rb b/app/components/health_reports/result_component.rb index f96057dfc423..24cccb0d8d6d 100644 --- a/app/components/health_reports/result_component.rb +++ b/app/components/health_reports/result_component.rb @@ -34,7 +34,9 @@ class ResultComponent < ApplicationComponent # Where "More information" points. Defaults to the file storages # documentation because that was this component's only consumer for a long - # time; a subject with its own troubleshooting page passes its own. + # time; a subject with its own troubleshooting page passes its own, and one + # with no page yet passes false to suppress the link rather than send an + # administrator somewhere unrelated. DEFAULT_DOCS_HREF = -> { ::OpenProject::Static::Links.url_for(:storage_docs, :health_status) } def initialize(group:, result:, i18n_scope:, docs_href: nil) @@ -55,7 +57,11 @@ def error_text I18n.t("errors.#{model.code}", scope: @i18n_scope, **model.context&.symbolize_keys) end - def docs_href = @docs_href || DEFAULT_DOCS_HREF.call + def docs_href + return @docs_href if @docs_href == false + + @docs_href || DEFAULT_DOCS_HREF.call + end def error_code if model.failure? diff --git a/app/components/llm_connections/models/sub_header_component.html.erb b/app/components/llm_connections/models/sub_header_component.html.erb index cdb1a730619a..6ecd35edc0f9 100644 --- a/app/components/llm_connections/models/sub_header_component.html.erb +++ b/app/components/llm_connections/models/sub_header_component.html.erb @@ -10,4 +10,27 @@ clear_button_id:, data: filter_input_data_attributes ) %> + + <% subheader.with_action_button( + scheme: :primary, + label: t("admin.llm_connections.show.add_model_submit"), + mobile_label: t("admin.llm_connections.show.add_model_submit"), + mobile_icon: :plus, + leading_icon: :plus, + tag: :a, + href: helpers.new_llm_model_path, + test_selector: "llm-model--add-button" + ) { t("admin.llm_connections.show.add_model_submit") } %> + + <% subheader.with_action_button( + scheme: :default, + label: t("admin.llm_connections.show.refresh_models"), + mobile_label: t("admin.llm_connections.show.refresh_models"), + mobile_icon: :sync, + leading_icon: :sync, + tag: :a, + href: helpers.refresh_models_llm_connection_path, + data: { turbo_method: :post, controller: "disable-when-clicked" }, + test_selector: "llm-model--refresh-button" + ) { t("admin.llm_connections.show.refresh_models") } %> <% end %> diff --git a/app/contracts/llm_connections/base_contract.rb b/app/contracts/llm_connections/base_contract.rb index a20755202a74..c1881e680a10 100644 --- a/app/contracts/llm_connections/base_contract.rb +++ b/app/contracts/llm_connections/base_contract.rb @@ -53,6 +53,7 @@ class BaseContract < ModelContract validate :enabled_requires_connection validate :default_models_offered_by_server + validate :default_embedding_model_can_embed validate :not_configured_from_env def not_configured_from_env @@ -63,6 +64,22 @@ def not_configured_from_env private + # A model the server has positively told us cannot embed is not a candidate + # for the embedding default, however it got submitted. An unknown verdict + # does not block: that is the normal state for a server that publishes + # nothing about its models. + def default_embedding_model_can_embed + model_id = model.default_embedding_model_id + return if model_id.blank? + return unless model.changed_attributes.include?("default_embedding_model_id") + + unsupported = model.capability_verdicts + .for_capability(:embeddings) + .exists?(model_id:, state: "unsupported") + + errors.add(:default_embedding_model_id, :cannot_embed) if unsupported + end + def enabled_requires_connection return unless model.enabled? return if model.base_url.present? diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb index 463487ba8d79..341150136e8d 100644 --- a/app/controllers/admin/llm_connections_controller.rb +++ b/app/controllers/admin/llm_connections_controller.rb @@ -136,7 +136,14 @@ def redirect_after_save def render_form_with_errors update_via_turbo_stream(component: ::LlmConnections::FormComponent.new(@connection)) - respond_with_turbo_streams { |format| format.html { render :show } } + # The HTML fallback re-renders the whole page, which needs everything the + # show action assigns -- not just the form that failed. + respond_with_turbo_streams do |format| + format.html do + show + render :show + end + end end def redirect_with_notice(message) diff --git a/app/forms/llm_connections/connection_form.rb b/app/forms/llm_connections/connection_form.rb index 806dee1e42d5..7182c791633b 100644 --- a/app/forms/llm_connections/connection_form.rb +++ b/app/forms/llm_connections/connection_form.rb @@ -123,8 +123,14 @@ class ConnectionForm < ApplicationForm selected: model.default_embedding_model_id.blank?) default_embedding_model_options.each do |model_id| - list.option(label: model_id, value: model_id, - selected: model.default_embedding_model_id == model_id) + # Same rule the per-feature picker follows: never hide a model, + # but refuse one the server has told us cannot embed, and say so. + state = embeddings_state(model_id) + + list.option(label: embedding_option_label(model_id, state), + value: model_id, + selected: model.default_embedding_model_id == model_id, + disabled: state == :unsupported) end end end @@ -164,6 +170,29 @@ def embedding_features? OpenProject::Llm::Features.for_kind(:embedding).any? end + # No verdict at all is the same as an inconclusive one: we do not know. + def embeddings_state(model_id) + embeddings_verdicts[model_id]&.to_sym || :unknown + end + + def embeddings_verdicts + @embeddings_verdicts ||= model.capability_verdicts + .for_capability(:embeddings) + .pluck(:model_id, :state) + .to_h + end + + def embedding_option_label(model_id, state) + case state + when :unsupported + I18n.t("admin.llm_connections.form.embedding_option_unsupported", model: model_id) + when :unknown + I18n.t("admin.llm_connections.form.embedding_option_unknown", model: model_id) + else + model_id + end + end + def submit_label model.persisted? ? I18n.t(:button_save) : I18n.t("admin.llm_connections.form.button_connect") end diff --git a/app/views/admin/llm_connections/show.html.erb b/app/views/admin/llm_connections/show.html.erb index fd41f26a38db..38a8e2702ee5 100644 --- a/app/views/admin/llm_connections/show.html.erb +++ b/app/views/admin/llm_connections/show.html.erb @@ -83,56 +83,28 @@ See COPYRIGHT and LICENSE files for more details. %> <% if @connection.persisted? %> - <%= - render(Primer::OpenProject::SidePanel.new(mb: 3)) do |panel| - panel.with_section(LlmConnections::SidePanel::HealthStatusComponent.new(@connection)) - end - %> -<% end %> - -<%= render(LlmConnections::FormComponent.new(@connection)) %> - -<% if @connection.persisted? %> - <%= - render(Primer::Beta::Subhead.new(mt: 4)) do |component| - component.with_heading(tag: :h3) { t(".models_heading") } - component.with_description { t(".models_description", fetched_at: format_time(@connection.catalogue_fetched_at)) } - component.with_actions do - concat( - render( - Primer::Beta::Button.new( - tag: :a, - href: new_llm_model_path, - scheme: :primary, - mr: 2, - data: { test_selector: "llm-model--add-button" } - ) - ) do |button| - button.with_leading_visual_icon(icon: :plus) - t(".add_model_submit") - end - ) - concat( - render( - Primer::Beta::Button.new( - tag: :a, - href: refresh_models_llm_connection_path, - data: { - turbo_method: :post, - controller: "disable-when-clicked", - test_selector: "llm-model--refresh-button" - } - ) - ) do |button| - button.with_leading_visual_icon(icon: :sync) - t(".refresh_models") - end - ) - end - end - %> - - <%= render(LlmConnections::Models::SubHeaderComponent.new(@query)) %> - <%= render(LlmConnections::Models::IndexComponent.new(@models, connection: @connection)) %> - + <%= render(Primer::Alpha::Layout.new(stacking_breakpoint: :lg)) do |layout| %> + <% layout.with_main do %> + <%= render(LlmConnections::FormComponent.new(@connection)) %> + + <%= + render(Primer::Beta::Subhead.new(mt: 4)) do |component| + component.with_heading(tag: :h3) { t(".models_heading") } + component.with_description { t(".models_description", fetched_at: format_time(@connection.catalogue_fetched_at)) } + end + %> + + <%= render(LlmConnections::Models::SubHeaderComponent.new(@query)) %> + <%= render(LlmConnections::Models::IndexComponent.new(@models, connection: @connection)) %> + <% end %> + + <% layout.with_sidebar(col_placement: :end, row_placement: :end) do %> + <%= render(Primer::OpenProject::SidePanel.new) do |panel| %> + <% panel.with_section(LlmConnections::SidePanel::HealthStatusComponent.new(@connection)) %> + <% end %> + <% end %> + <% end %> +<% else %> + <%# Nothing to show alongside the form until a connection exists. %> + <%= render(LlmConnections::FormComponent.new(@connection)) %> <% end %> diff --git a/app/views/admin/llm_health_status/show.html.erb b/app/views/admin/llm_health_status/show.html.erb index 6b2d8f339a2f..5af9df1f4d86 100644 --- a/app/views/admin/llm_health_status/show.html.erb +++ b/app/views/admin/llm_health_status/show.html.erb @@ -78,7 +78,10 @@ See COPYRIGHT and LICENSE files for more details. HealthReports::ReportComponent.new( @report, i18n_scope: "llm.health_checks", - docs_href: OpenProject::Static::Links.url_for(:sysadmin_docs, :llm_connection) + # No admin guide page exists for this yet. Every check explains itself + # in its own message, so a "More information" link would only lead + # somewhere unrelated -- suppressed until that page is written. + docs_href: false ) ) %> diff --git a/config/locales/en.yml b/config/locales/en.yml index 26216d82c821..7547cc41cf61 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -712,6 +712,7 @@ en: default_chat_model_id: not_available: "is not offered by the configured LLM server." default_embedding_model_id: + cannot_embed: "cannot create embeddings, according to the LLM server." not_available: "is not offered by the configured LLM server." enabled: requires_connection: "cannot be turned on before a connection has been configured." @@ -1469,6 +1470,8 @@ en: The full base URL of the server, including the API version segment, exactly as your provider documents it (for example https://example.com/v1). OpenProject appends only the endpoint path. button_connect: "Connect" default_chat_model_caption: "Used by AI features that do not select a model themselves." + embedding_option_unknown: "%{model} (not verified as an embedding model)" + embedding_option_unsupported: "%{model} (cannot create embeddings)" default_embedding_model_caption: "Used by AI features that index text for search and do not select a model themselves." enabled_caption: "When turned off, all LLM-backed AI features stop working. The connection settings are kept." label_connecting: "Contacting the LLM server…" diff --git a/config/static_links.yml b/config/static_links.yml index 7f2f02891022..2dccc1fb9540 100644 --- a/config/static_links.yml +++ b/config/static_links.yml @@ -186,8 +186,6 @@ sysadmin_docs: href: https://www.openproject.org/docs/system-admin-guide/authentication/ldap-connections/ ldap_group_sync: href: https://www.openproject.org/docs/system-admin-guide/authentication/ldap-connections/ldap-group-synchronization/ - llm_connection: - href: https://www.openproject.org/docs/system-admin-guide/ai/llm-connection/ mcp_resources: href: https://www.openproject.org/docs/system-admin-guide/integrations/mcp-server/#resources mcp_tools: diff --git a/spec/features/admin/llm_connection_spec.rb b/spec/features/admin/llm_connection_spec.rb index f7f9713fb3ab..79dc4100d1d1 100644 --- a/spec/features/admin/llm_connection_spec.rb +++ b/spec/features/admin/llm_connection_spec.rb @@ -46,6 +46,16 @@ current_user { admin } + # The kebab is a Primer ActionMenu: clicking it before its behaviour is + # attached silently does nothing, so wait for the page to settle first and + # for the item itself to become visible. + def choose_action(item) + expect(page).to have_test_selector("llm-connection--actions") + find_test_selector("llm-connection--actions").click + expect(page).to have_test_selector(item) + find_test_selector(item).click + end + context "when nothing is configured yet" do it "renders an accessible, empty settings page" do visit llm_connection_path @@ -83,12 +93,9 @@ it "removes the stored API key" do visit llm_connection_path + expect(page).to have_test_selector("llm-model--refresh-button") - find_test_selector("llm-connection--actions").click - # The menu renders into an anchored popover, so wait for it rather than - # racing the click. - expect(page).to have_link("Remove API key") - click_link "Remove API key" + choose_action("llm-connection--delete-api-key") within_test_selector("llm-connection--delete-api-key-dialog") do expect(page).to be_axe_clean @@ -101,10 +108,9 @@ it "disconnects without losing the configuration" do visit llm_connection_path + expect(page).to have_test_selector("llm-model--refresh-button") - find_test_selector("llm-connection--actions").click - expect(page).to have_link("Disconnect") - click_link "Disconnect" + choose_action("llm-connection--disconnect") within_test_selector("llm-connection--disconnect-dialog") do expect(page).to be_axe_clean diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb index c11e971c184a..0dcc70614f6b 100644 --- a/spec/requests/admin/llm_connections_spec.rb +++ b/spec/requests/admin/llm_connections_spec.rb @@ -361,4 +361,45 @@ def rendered_rows(body) = body.scan("llm-model--toggle-").size expect(rendered_rows(response.body)).to eq(3) end end + + describe "choosing a default embedding model" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url: "https://example.com/v1") } + + before { login_as admin } + + def verdict(model_id, state) + connection.capability_verdicts.create!(model_id:, capability: "embeddings", state:, + source: "probe", checked_at: Time.current) + end + + it "refuses a model the server says cannot embed" do + verdict("qwen3.6-27b", "unsupported") + + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "qwen3.6-27b" } } + + expect(connection.reload.default_embedding_model_id).to be_nil + end + + it "accepts one that can" do + verdict("bge-m3", "supported") + + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "bge-m3" } } + + expect(connection.reload.default_embedding_model_id).to eq("bge-m3") + end + + # Unknown is the normal state for a server that publishes nothing, so it + # must not block the choice. + it "allows one with no verdict at all" do + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "bge-m3" } } + + expect(connection.reload.default_embedding_model_id).to eq("bge-m3") + end + end end From d657ebe3b28984648db743dd42d60e531c59468f Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Fri, 14 Aug 2026 17:33:36 +0100 Subject: [PATCH 34/44] [#66020] Stop claiming a connection we have not proven Saving a connection whose model list is absent reported "Connected, but the server did not return a model list". We had not connected. A 404 on /models is indistinguishable from a 404 caused by an endpoint that is simply wrong -- most often a base URL missing its API version segment, since this client appends only the endpoint path. That is not hypothetical: it is exactly what a base URL of https://llm-stack.openproject-edge.eu (no /v1) does. Every request 404s, while the save reports success, because the tolerance that lets a gateway without a model list be configured also swallows the evidence that the URL is wrong. Both messages now name the two possibilities and point at the health check, which sends a real completion and is the only thing that can tell them apart. https://community.openproject.org/work_packages/66020 --- config/locales/en.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 7547cc41cf61..ea7e2acdf08f 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1519,7 +1519,7 @@ en: models_heading: "Available models" refresh_models: "Refresh models" update: - no_models: "Connected, but the server did not return a model list. Add the models you want to use below." + no_models: "Saved, but the server did not return a model list. Either it does not offer one, or the endpoint is wrong — a URL missing its API version segment (for example /v1) looks exactly the same from here. Add the models you want to use below, then run the health checks to confirm the server answers." success: "Successfully connected to the LLM server." llm_models: create: @@ -4073,7 +4073,7 @@ en: invalid_api_key: "The server rejected the stored API key." locked_model_missing: "%{features} indexed data with a model the server no longer offers. The index cannot be extended until it is rebuilt." no_model_to_test: "No model is available to test with. Add a model or choose a default first." - no_models: "The server does not publish a model list. Models can be added manually instead." + no_models: "The server does not publish a model list. Either it does not offer one, or the endpoint is missing its API version segment. Models can be added manually; the inference check is what proves the endpoint is right." no_models_endpoint: "The server does not offer a model list, so the API key could not be verified. Run the inference check to verify it." not_configured: "No endpoint has been configured." not_openai_compatible: "The endpoint did not return a valid model list. Please ensure it points at an OpenAI-API-compatible endpoint, including the API version segment." From 8cf92bcf2c3428d8c3bc324a8b9df64241eec856 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Fri, 14 Aug 2026 17:40:34 +0100 Subject: [PATCH 35/44] [#66020] Let a manually added model be renamed The identifier field was only rendered when creating a model, on the reasoning that verdicts and bindings reference a model by that string and renaming would orphan them. The reasoning was right and the conclusion was wrong. A typo in a hand-typed identifier could only be fixed by deleting the model and entering it again, which threw away its capability assertions, its feature bindings and its place as a connection default -- a worse outcome than the orphaning it avoided. And hand-typing is exactly where typos come from: models are entered by hand only because the server publishes no list to pick from. Renaming now cascades. Capability verdicts, feature bindings and the connection defaults all follow the new identifier, so a feature bound to the model keeps resolving across the rename. It writes directly rather than through validation, deliberately: a locked binding must not refuse to follow the very model it is locked to, since this changes the name of that model rather than the model in use. Discovered models stay read-only. The server names those, and the next refresh would only put the old name back. https://community.openproject.org/work_packages/66020 --- .../admin/llm_models_controller.rb | 12 +++- app/forms/llm_models/form.rb | 9 ++- app/models/llm_model.rb | 23 +++++++ spec/requests/admin/llm_models_spec.rb | 68 +++++++++++++++++++ 4 files changed, 106 insertions(+), 6 deletions(-) diff --git a/app/controllers/admin/llm_models_controller.rb b/app/controllers/admin/llm_models_controller.rb index a95c47cc388a..05c52bc6d493 100644 --- a/app/controllers/admin/llm_models_controller.rb +++ b/app/controllers/admin/llm_models_controller.rb @@ -113,12 +113,18 @@ def set_connection end def apply_attributes(llm_model) - llm_model.assign_attributes(llm_model_params.except(:external_id, *capability_param_names)) + attributes = llm_model_params.except(*capability_param_names) + # A discovered model is named by the server; only a hand-entered one may be + # renamed here, and everything referencing the old name follows it. + attributes = attributes.except(:external_id) unless llm_model.new_record? || llm_model.manual? + + previous_external_id = llm_model.external_id + llm_model.assign_attributes(attributes) llm_model.save! + llm_model.cascade_rename!(previous_external_id) end - # external_id is only accepted when creating: verdicts and bindings reference - # a model by that string, so renaming one would orphan both. + # external_id is accepted on create, and on update for manually added models. def llm_model_params params.expect(llm_model: [:external_id, :display_name, :admin_context_window, *capability_param_names]) end diff --git a/app/forms/llm_models/form.rb b/app/forms/llm_models/form.rb index b4245ca56af8..294e71b8432e 100644 --- a/app/forms/llm_models/form.rb +++ b/app/forms/llm_models/form.rb @@ -31,9 +31,12 @@ module LlmModels class Form < ApplicationForm form do |f| - # The identity of the model, and immutable once saved: verdicts and - # bindings reference it by this string. - if new_record? + # A discovered model is named by the server, so its identifier is not ours + # to change -- the next refresh would only put it back. One entered by hand + # is editable, because a typo in it is otherwise unfixable except by + # deleting the model and losing everything asserted about it. Renaming + # cascades; see LlmModel#cascade_rename!. + if new_record? || model.manual? f.text_field( name: :external_id, label: LlmModel.human_attribute_name(:external_id), diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb index 6e81db82b597..f7fbd34630c1 100644 --- a/app/models/llm_model.rb +++ b/app/models/llm_model.rb @@ -51,6 +51,21 @@ class LlmModel < ApplicationRecord def deactivated? = deactivated_at.present? + # Everything that points at a model does so by its identifier string, so a + # rename has to carry them along or it silently orphans them. + # + # Renaming is a correction of the name, not a change of model, which is why + # this writes directly: a locked binding must not refuse to follow the model + # it is locked to, and the connection defaults are pointing at this very row. + def cascade_rename!(previous_external_id) + return if previous_external_id.blank? || previous_external_id == external_id + + llm_connection.capability_verdicts.where(model_id: previous_external_id).update_all(model_id: external_id) + llm_connection.feature_bindings.where(model_id: previous_external_id).update_all(model_id: external_id) + + rename_connection_defaults(previous_external_id) + end + # Offerable in a picker. Note that this is *not* what decides whether a model # still resolves: a feature already bound to a deactivated model keeps working, # and is surfaced as a warning instead. Switching a row off must never silently @@ -59,6 +74,14 @@ def selectable? = active? && !deactivated? def name = display_name.presence || external_id + def rename_connection_defaults(previous_external_id) + defaults = %i[default_chat_model_id default_embedding_model_id] + .select { |attribute| llm_connection.public_send(attribute) == previous_external_id } + .index_with { external_id } + + llm_connection.update_columns(defaults) if defaults.any? + end + # Precedence: what an administrator set, then what the server reported (vLLM # and SGLang publish the operator's actual --max-model-len), then what a # registry believes about the model in general. diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb index 4d4a926e63c0..656bfc2edc1d 100644 --- a/spec/requests/admin/llm_models_spec.rb +++ b/spec/requests/admin/llm_models_spec.rb @@ -285,4 +285,72 @@ expect(llm_model.reload).not_to be_deactivated end end + + describe "renaming a manually added model" do + let!(:llm_model) do + create(:llm_model, :manual, llm_connection: connection, external_id: "qwen/qwen3.6-35b-a3b") + end + + before do + connection.update!(default_chat_model_id: "qwen/qwen3.6-35b-a3b") + connection.feature_bindings.create!(feature_key: "description_assistant", + model_id: "qwen/qwen3.6-35b-a3b") + connection.capability_verdicts.create!(model_id: "qwen/qwen3.6-35b-a3b", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: Time.current) + end + + it "offers the identifier field on the edit page" do + get edit_llm_model_path(llm_model) + + expect(response).to have_http_status(:ok) + expect(response.body).to include("llm_model[external_id]") + end + + it "does not offer it for a discovered model" do + discovered = create(:llm_model, llm_connection: connection, external_id: "server-named") + + get edit_llm_model_path(discovered) + + expect(response).to have_http_status(:ok) + expect(response.body).not_to include("llm_model[external_id]") + end + + # A typo in a hand-typed identifier was previously only fixable by deleting + # the model, which threw away everything asserted about it. + it "renames it and carries every reference along" do + patch llm_model_path(llm_model), params: { llm_model: { external_id: "qwen/qwen3.6-35b-a3b:bf16" } } + + expect(llm_model.reload.external_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + expect(connection.reload.default_chat_model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + expect(connection.feature_bindings.first.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + expect(connection.capability_verdicts.first.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + end + + it "keeps the feature resolving afterwards", with_flag: { llm_connection: true } do + connection.update!(enabled: true) + + patch llm_model_path(llm_model), params: { llm_model: { external_id: "qwen/qwen3.6-35b-a3b:bf16" } } + + expect(Llm::Runtime.for(:description_assistant).model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + end + + it "follows a model a locked binding depends on" do + binding = connection.feature_bindings.first + binding.update!(locked_at: Time.current) + + patch llm_model_path(llm_model), params: { llm_model: { external_id: "qwen/qwen3.6-35b-a3b:bf16" } } + + expect(binding.reload.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + end + + # The server names its own models; renaming one here would only be undone by + # the next refresh. + it "refuses to rename a discovered model" do + discovered = create(:llm_model, llm_connection: connection, external_id: "server-named") + + patch llm_model_path(discovered), params: { llm_model: { external_id: "renamed" } } + + expect(discovered.reload.external_id).to eq("server-named") + end + end end From 89a512f07a6625a1eba0c98c5555a558bec56529 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Fri, 14 Aug 2026 17:45:41 +0100 Subject: [PATCH 36/44] [#66020] Name the inherited capability value in the option itself A capability field showed "Not specified" in the select with "Currently Supported, from the model registry." underneath it. Both described the same field and they contradicted each other. The blank option now says what actually applies while nothing is asserted here -- "Supported (from the model registry)" -- and the caption is reduced to what choosing something else would mean. The behaviour behind it is unchanged and deliberate: a verdict from a probe or a registry is never loaded into the field, so saving the form cannot turn someone else's finding into the administrator's own assertion. Only the wording was wrong. An administrator's own assertion is loaded into the field, so it is never presented as inherited. https://community.openproject.org/work_packages/66020 --- app/forms/llm_models/form.rb | 25 +++++++++++++------ config/locales/en.yml | 3 ++- spec/requests/admin/llm_models_spec.rb | 34 ++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/app/forms/llm_models/form.rb b/app/forms/llm_models/form.rb index 294e71b8432e..444767ca85bf 100644 --- a/app/forms/llm_models/form.rb +++ b/app/forms/llm_models/form.rb @@ -78,7 +78,7 @@ class Form < ApplicationForm input_width: :medium, data: { test_selector: "llm-model--capability-#{capability}" } ) do |select| - select.option(value: "", label: I18n.t("admin.llm_models.form.state_unspecified")) + select.option(value: "", label: inherited_state_label(capability)) select.option(value: "supported", label: I18n.t("admin.llm_models.form.state_supported")) select.option(value: "unsupported", label: I18n.t("admin.llm_models.form.state_unsupported")) end @@ -108,16 +108,27 @@ def context_window_caption end end - # A verdict established by a probe or a registry is reported rather than - # loaded into the field, so saving the form does not silently turn someone - # else's finding into the administrator's assertion. - def capability_caption(capability) + # A verdict established by a probe or a registry is never loaded into the + # field -- saving must not turn someone else's finding into the + # administrator's assertion -- so the blank option names what applies while + # nothing is asserted here. Labelling it "Not specified" next to a caption + # reading "Currently Supported" stated two contradictory things at once. + def inherited_state_label(capability) verdict = model.verdict_for(capability) - return if verdict.nil? || verdict.source_admin? + return I18n.t("admin.llm_models.form.state_unspecified") if verdict.nil? || verdict.source_admin? - I18n.t("admin.llm_models.form.current_verdict", + I18n.t("admin.llm_models.form.state_inherited", state: I18n.t("llm.verdict_states.#{verdict.state}"), source: I18n.t("llm.verdict_sources.#{verdict.source}")) end + + # The state is now carried by the option itself; the caption only has to say + # what choosing something else here means. + def capability_caption(capability) + verdict = model.verdict_for(capability) + return if verdict.nil? || verdict.source_admin? + + I18n.t("admin.llm_models.form.capability_override_caption") + end end end diff --git a/config/locales/en.yml b/config/locales/en.yml index ea7e2acdf08f..dfaeef279a73 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1538,7 +1538,8 @@ en: context_window_caption: "How many tokens this model accepts. Leave blank if you do not know." context_window_known: "Leave blank to use %{value}, %{source}." create_submit: "Add model" - current_verdict: "Currently %{state}, %{source}." + capability_override_caption: "Choose a value to override this." + state_inherited: "%{state} (%{source})" display_name_caption: "An optional friendly name shown instead of the model id." external_id_caption: "The model name exactly as the server expects it." state_supported: "Supported" diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb index 656bfc2edc1d..cd2addf612ad 100644 --- a/spec/requests/admin/llm_models_spec.rb +++ b/spec/requests/admin/llm_models_spec.rb @@ -353,4 +353,38 @@ expect(discovered.reload.external_id).to eq("server-named") end end + + describe "how an inherited capability verdict is shown" do + let!(:llm_model) { create(:llm_model, :manual, llm_connection: connection, external_id: "qwen3.6-27b") } + + # The blank option used to read "Not specified" while the caption underneath + # read "Currently Supported, from the model registry" -- two contradictory + # statements about the same field. + it "names the inherited value in the option itself" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "function_calling", + state: "supported", source: "metadata", checked_at: Time.current) + + get edit_llm_model_path(llm_model) + + expect(response.body).to include("Supported (from the model registry)") + expect(response.body).not_to include("Currently Supported") + end + + it "falls back to Not specified when nothing is known" do + get edit_llm_model_path(llm_model) + + expect(response.body).to include("Not specified") + end + + # An administrator's own assertion is loaded into the field, so the blank + # option must not claim it as inherited. + it "does not present an administrator's own assertion as inherited" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "function_calling", + state: "supported", source: "admin", checked_at: Time.current) + + get edit_llm_model_path(llm_model) + + expect(response.body).not_to include("Supported (set by an administrator)") + end + end end From 3018c7c423a716f4d18de2ecf2f8a60b41d59bea Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Fri, 14 Aug 2026 18:06:53 +0100 Subject: [PATCH 37/44] [#66020] Offer only models known to create embeddings The default embedding model picker listed the whole catalogue and annotated what was wrong with each entry. Against a registry-backed provider that meant 132 entries, 3 of which can embed: 122 marked as unable, 7 merely unverified. An unconfirmed capability is not a capability. Offering a model because nothing has ruled it out invites a choice that fails much later, at index time, when the vectors are already being written. Only models known to embed are offered now. That leaves nothing to choose when nothing is known to embed, which is honest rather than a dead end: an administrator who knows better than the registry says so on the model itself, by setting its embeddings capability. The caption says that, rather than leaving an empty select unexplained. The model already chosen is kept listed regardless, so a save cannot silently blank a working configuration. https://community.openproject.org/work_packages/66020 --- app/forms/llm_connections/connection_form.rb | 39 +++++++++++++---- config/locales/en.yml | 3 +- spec/requests/admin/llm_connections_spec.rb | 44 ++++++++++++++++++++ 3 files changed, 76 insertions(+), 10 deletions(-) diff --git a/app/forms/llm_connections/connection_form.rb b/app/forms/llm_connections/connection_form.rb index 7182c791633b..e00b936c7985 100644 --- a/app/forms/llm_connections/connection_form.rb +++ b/app/forms/llm_connections/connection_form.rb @@ -111,7 +111,7 @@ class ConnectionForm < ApplicationForm f.autocompleter( name: :default_embedding_model_id, label: LlmConnection.human_attribute_name(:default_embedding_model_id), - caption: I18n.t("admin.llm_connections.form.default_embedding_model_caption"), + caption: default_embedding_model_caption, disabled: read_only?, autocomplete_options: { decorated: true, @@ -123,14 +123,9 @@ class ConnectionForm < ApplicationForm selected: model.default_embedding_model_id.blank?) default_embedding_model_options.each do |model_id| - # Same rule the per-feature picker follows: never hide a model, - # but refuse one the server has told us cannot embed, and say so. - state = embeddings_state(model_id) - - list.option(label: embedding_option_label(model_id, state), + list.option(label: embedding_option_label(model_id, embeddings_state(model_id)), value: model_id, - selected: model.default_embedding_model_id == model_id, - disabled: state == :unsupported) + selected: model.default_embedding_model_id == model_id) end end end @@ -162,8 +157,34 @@ def default_chat_model_options (model.selectable_model_ids + [model.default_chat_model_id]).compact_blank.uniq end + # Only models actually known to embed. + # + # An unconfirmed capability is not a capability: offering a model here on the + # grounds that nothing has ruled it out invites an administrator to pick one + # that cannot embed, and the failure would surface much later, at index time. + # A catalogue from a registry-backed provider makes that vivid -- 132 models, + # 3 of which embed. + # + # This leaves nothing to choose when no model is known to embed, and that is + # the honest state rather than a dead end: an administrator who knows better + # than the registry says so on the model itself, by setting its embeddings + # capability, which is what default_embedding_model_hint points at. + # + # The model already chosen is kept regardless, so a save cannot silently + # blank a working configuration. def default_embedding_model_options - (model.selectable_model_ids + [model.default_embedding_model_id]).compact_blank.uniq + capable = model.selectable_model_ids.select { |id| embeddings_state(id) == :supported } + + (capable + [model.default_embedding_model_id]).compact_blank.uniq + end + + # Says how to make a model eligible when none is, rather than leaving an + # empty select with no explanation. + def default_embedding_model_caption + return I18n.t("admin.llm_connections.form.default_embedding_model_caption") if + default_embedding_model_options.any? + + I18n.t("admin.llm_connections.form.default_embedding_model_none") end def embedding_features? diff --git a/config/locales/en.yml b/config/locales/en.yml index dfaeef279a73..b4aa9a4db7aa 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1472,7 +1472,8 @@ en: default_chat_model_caption: "Used by AI features that do not select a model themselves." embedding_option_unknown: "%{model} (not verified as an embedding model)" embedding_option_unsupported: "%{model} (cannot create embeddings)" - default_embedding_model_caption: "Used by AI features that index text for search and do not select a model themselves." + default_embedding_model_caption: "Used by AI features that index text for search and do not select a model themselves. Only models known to create embeddings are offered." + default_embedding_model_none: "No model is known to create embeddings. If you know one does, open it under Available models and set its embeddings capability." enabled_caption: "When turned off, all LLM-backed AI features stop working. The connection settings are kept." label_connecting: "Contacting the LLM server…" disconnect: diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb index 0dcc70614f6b..6f3c5df2c874 100644 --- a/spec/requests/admin/llm_connections_spec.rb +++ b/spec/requests/admin/llm_connections_spec.rb @@ -372,6 +372,50 @@ def verdict(model_id, state) source: "probe", checked_at: Time.current) end + it "does not offer a model the server says cannot embed" do + verdict("qwen3.6-27b", "unsupported") + verdict("bge-m3", "supported") + + get llm_connection_path + + expect(response.body).to include("bge-m3") + expect(response.body).not_to include("cannot create embeddings") + end + + # An unconfirmed capability is not a capability: offering such a model + # invites a choice that fails much later, at index time. + it "does not offer a model whose capability is merely unconfirmed" do + get llm_connection_path + + expect(response.body).not_to include("not verified as an embedding model") + end + + it "says how to make a model eligible when none is" do + get llm_connection_path + + expect(response.body).to include("set its embeddings capability") + end + + it "offers one an administrator has asserted can embed" do + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "supported", source: "admin", checked_at: Time.current) + + get llm_connection_path + + expect(response.body).to include("bge-m3") + expect(response.body).not_to include("set its embeddings capability") + end + + # Otherwise a save would silently blank a working configuration. + it "keeps the chosen model listed even once it is ruled out" do + connection.update_column(:default_embedding_model_id, "qwen3.6-27b") + verdict("qwen3.6-27b", "unsupported") + + get llm_connection_path + + expect(response.body).to include("qwen3.6-27b") + end + it "refuses a model the server says cannot embed" do verdict("qwen3.6-27b", "unsupported") From 75dbfa8c1173e2129e7c9482285dc91266f32274 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 20 Aug 2026 11:14:15 +0200 Subject: [PATCH 38/44] [#66020] Follow the SSRF error rename from dev The merge of dev brought a refactoring that replaced OpenProject::HttpxSsrfFilter with OpenProject::ServerSideRequestForgeryError. Git merged both sides cleanly because no line conflicted, but the client still rescued the old constant, so every probe raised NameError instead of reporting a blocked host. Caught by the spec suite, not by the merge. https://community.openproject.org/work_packages/66020 --- app/services/llm/client.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/services/llm/client.rb b/app/services/llm/client.rb index 6709f0e145e7..00641c991d48 100644 --- a/app/services/llm/client.rb +++ b/app/services/llm/client.rb @@ -96,9 +96,9 @@ def get(path) handle_transport_error(response) if response.is_a?(HTTPX::ErrorResponse) handle_status(response) parse(response) - rescue OpenProject::HttpxSsrfFilter::ServerSideRequestForgeryError - # Raised from HttpxSsrfFilter#addresses=; the throw/catch path surfaces as - # response.error instead and is handled in #handle_transport_error. + rescue OpenProject::ServerSideRequestForgeryError + # The SSRF plugin either raises directly or surfaces the error as + # response.error, which #handle_transport_error covers. raise SsrfError, "Host resolves to a blocked address" end @@ -118,7 +118,7 @@ def handle_transport_error(response) error = response.error case error - when OpenProject::HttpxSsrfFilter::ServerSideRequestForgeryError + when OpenProject::ServerSideRequestForgeryError raise SsrfError, "Host resolves to a blocked address" when HTTPX::TimeoutError raise TimeoutError, "Request timed out" From 0f35ec151427a34ed706955510a3631ea34a87d8 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 20 Aug 2026 11:14:25 +0200 Subject: [PATCH 39/44] [#66020] Treat an API format change as a connection change Changing the format means talking to a different server, or to the same server in a different dialect, but only base URL and API key counted as connection changes. Switching the format alone therefore neither re-proved the connection nor refreshed the model list, leaving the previous provider's models and defaults active while requests went to the new one. The format is now part of the connection identity: it triggers the probe when switching to an OpenAI-compatible endpoint, triggers a model resync in both directions, and is part of the verdict fingerprint, so capability verdicts from the previous provider are discarded rather than carried over. Found by Codex review on the PR. https://community.openproject.org/work_packages/66020 --- .../llm_connections/sync_models_service.rb | 2 +- app/services/llm_connections/update_service.rb | 6 +++--- app/validators/llm_server_validator.rb | 17 ++++++++++------- .../llm_connections/update_contract_spec.rb | 11 +++++++++++ 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb index 8421757cde71..0453b49a6c3c 100644 --- a/app/services/llm_connections/sync_models_service.rb +++ b/app/services/llm_connections/sync_models_service.rb @@ -79,7 +79,7 @@ def connection_attributes end def fingerprint - @fingerprint ||= Digest::SHA256.hexdigest("#{connection.base_url}\0#{connection.api_key}") + @fingerprint ||= Digest::SHA256.hexdigest("#{connection.api_format}\0#{connection.base_url}\0#{connection.api_key}") end def upsert(cards) diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb index 069d62fd6233..790f53b1bd04 100644 --- a/app/services/llm_connections/update_service.rb +++ b/app/services/llm_connections/update_service.rb @@ -53,15 +53,15 @@ def after_perform(service_call) Llm::HealthCheckJob.toggle_cron_job next unless @sync_models - next unless credentials_changed?(service_call.result) + next unless connection_changed?(service_call.result) SyncModelsService.new(service_call.result).call Llm::DetectCapabilitiesJob.perform_later end end - def credentials_changed?(connection) - connection.saved_changes.keys.intersect?(LlmServerValidator::CREDENTIAL_ATTRIBUTES) + def connection_changed?(connection) + connection.saved_changes.keys.intersect?(LlmServerValidator::CONNECTION_ATTRIBUTES) end end end diff --git a/app/validators/llm_server_validator.rb b/app/validators/llm_server_validator.rb index a7e506bfaa83..2bd1924547fc 100644 --- a/app/validators/llm_server_validator.rb +++ b/app/validators/llm_server_validator.rb @@ -32,11 +32,14 @@ # accepts the configured credentials, by fetching its model list. # # Runs inside the contract so that a server which cannot be reached never gets -# persisted. It is guarded on the credential attributes: without that guard every -# unrelated save -- and every form render that builds a model through -# SetAttributesService -- would fire an outbound HTTP request. +# persisted. It is guarded on the attributes that identify the connection: +# without that guard every unrelated save -- and every form render that builds a +# model through SetAttributesService -- would fire an outbound HTTP request. class LlmServerValidator < ActiveModel::EachValidator - CREDENTIAL_ATTRIBUTES = %w[base_url api_key].freeze + # Changing any of these means talking to a different server, or to the same + # server in a different dialect, so the connection must be proven again and + # its models resynchronised. + CONNECTION_ATTRIBUTES = %w[base_url api_key api_format].freeze # Statuses that mean "this server has no model list here", as opposed to "this # server is broken". A gateway may route chat completions and nothing else. @@ -45,7 +48,7 @@ class LlmServerValidator < ActiveModel::EachValidator def validate_each(contract, attribute, value) return if value.blank? return unless queries_the_server?(contract) - return unless credentials_changed?(contract) + return unless connection_changed?(contract) return unless host_allowed?(contract, attribute, value) probe(contract, attribute, value) @@ -60,8 +63,8 @@ def queries_the_server?(contract) contract.model.api_format == Llm::Adapters::OPENAI_COMPATIBLE end - def credentials_changed?(contract) - contract.model.changed_attributes.keys.intersect?(CREDENTIAL_ATTRIBUTES) + def connection_changed?(contract) + contract.model.changed_attributes.keys.intersect?(CONNECTION_ATTRIBUTES) end # A pre-flight check purely so the administrator gets an actionable message diff --git a/spec/contracts/llm_connections/update_contract_spec.rb b/spec/contracts/llm_connections/update_contract_spec.rb index 6ba731afc847..96ed9c75bdfb 100644 --- a/spec/contracts/llm_connections/update_contract_spec.rb +++ b/spec/contracts/llm_connections/update_contract_spec.rb @@ -147,6 +147,17 @@ expect(models_request).to have_been_made.once end + # Switching the dialect is switching the server, even at the same URL: a + # connection previously talking to a registry-backed provider must be proven + # again when it becomes OpenAI-compatible. + it "probes when only the API format changed to an OpenAI-compatible one" do + connection.update_columns(api_format: "anthropic", base_url: base_url) + connection.reload.api_format = "openai" + + expect(contract.validate).to be(true) + expect(models_request).to have_been_made.once + end + it "does not probe when only the enabled flag changed" do connection.save! WebMock.reset_executed_requests! From 1af327fc1f9cd87172da8a94cf336b95a1d09730 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 20 Aug 2026 11:14:34 +0200 Subject: [PATCH 40/44] [#66020] Make environment provisioning complete and reversible Two defects in the environment path, both found by Codex review on the PR. A default model configured through the environment failed provisioning on a fresh installation. The seed runs before any model synchronisation, so the catalogue the default was validated against was necessarily empty and the seeder raised. The environment contract now skips that validation; a wrong id surfaces afterwards the same way as a model that vanished from the server. Values removed from the environment stayed in the database. Absent keys were compacted away instead of written as nil, and since the form is read-only while the connection is environment-provisioned, an operator moving to an unauthenticated endpoint had no supported way to clear the stale API key. Absent keys now clear their values: the environment is the source of truth while it is in charge. https://community.openproject.org/work_packages/66020 --- .../environment_update_contract.rb | 6 ++ .../llm_connections/env_sync_service.rb | 6 +- .../env_data/llm_connection_seeder_spec.rb | 81 +++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 spec/seeders/env_data/llm_connection_seeder_spec.rb diff --git a/app/contracts/llm_connections/environment_update_contract.rb b/app/contracts/llm_connections/environment_update_contract.rb index a393c655c5b8..72529132bef7 100644 --- a/app/contracts/llm_connections/environment_update_contract.rb +++ b/app/contracts/llm_connections/environment_update_contract.rb @@ -37,5 +37,11 @@ module LlmConnections # guard, since this is the code path that legitimately writes those values. class EnvironmentUpdateContract < BaseContract def not_configured_from_env = nil + + # On a fresh installation the seed runs before any model synchronisation, so + # there is no catalogue to validate a default model against. A wrong id is + # surfaced afterwards, the same way as a model that vanished: the binding + # shows as no longer offered. + def default_models_offered_by_server = nil end end diff --git a/app/services/llm_connections/env_sync_service.rb b/app/services/llm_connections/env_sync_service.rb index f86ace88ea0f..9b7c0184bfa8 100644 --- a/app/services/llm_connections/env_sync_service.rb +++ b/app/services/llm_connections/env_sync_service.rb @@ -52,6 +52,10 @@ def call attr_reader :config + # Absent keys are written as nil on purpose: the environment is the source + # of truth here, and the form is read-only while it is. Keeping a stored + # value that was removed from the environment would leave, for example, an + # obsolete API key in use with no supported way to clear it. def attributes { base_url: config.fetch(:base_url), @@ -59,7 +63,7 @@ def attributes default_chat_model_id: config[:default_chat_model], default_embedding_model_id: config[:default_embedding_model], enabled: ActiveRecord::Type::Boolean.new.deserialize(config.fetch(:enabled, true)) - }.compact + } end end end diff --git a/spec/seeders/env_data/llm_connection_seeder_spec.rb b/spec/seeders/env_data/llm_connection_seeder_spec.rb new file mode 100644 index 000000000000..f1bc7a834e12 --- /dev/null +++ b/spec/seeders/env_data/llm_connection_seeder_spec.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe EnvData::LlmConnectionSeeder do + subject(:seed) { described_class.new(seed_data).seed! } + + let(:seed_data) { Source::SeedData.new({}) } + + it "does not seed a connection without configuration" do + expect { seed }.not_to change(LlmConnection, :count) + end + + # On a fresh installation the seed runs before any model synchronisation, so + # a configured default model cannot be validated against a catalogue yet. + # Provisioning must still complete; a wrong id surfaces later as dangling. + context "with a default model configured on a fresh installation", with_settings: { + llm_connection: { + "base_url" => "https://example.com/v1", + "api_key" => "sk-from-env", + "default_chat_model" => "qwen3.6-35b-a3b" + } + } do + it "seeds the connection without contacting the server" do + expect { seed }.to change(LlmConnection, :count).from(0).to(1) + + connection = LlmConnection.first + expect(connection.default_chat_model_id).to eq("qwen3.6-35b-a3b") + expect(connection.api_key).to eq("sk-from-env") + end + end + + # The environment is the source of truth while the form is read-only under it, + # so a value removed from the environment must not linger in the database. + context "when a previously set key is removed from the environment", with_settings: { + llm_connection: { "base_url" => "https://example.com/v1" } + } do + before do + create(:llm_connection, base_url: "https://example.com/v1", + api_key: "sk-stale", + default_chat_model_id: "old-default") + end + + it "clears the values the environment no longer provides" do + expect { seed }.not_to change(LlmConnection, :count) + + connection = LlmConnection.first + expect(connection.api_key).to be_nil + expect(connection.default_chat_model_id).to be_nil + expect(connection.base_url).to eq("https://example.com/v1") + end + end +end From 36b5e290f0e42b69d68793c2292dd32ffc3e6441 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 20 Aug 2026 11:14:42 +0200 Subject: [PATCH 41/44] [#66020] Show the error when renaming a model to a taken id Renaming a manual model to an identifier already used by this connection raised through save! and produced a 500. The update now follows the create path: on a failed save the edit form is re-rendered with the uniqueness error against the field, and neither the rename cascade nor the capability changes are applied. Found by Codex review on the PR. https://community.openproject.org/work_packages/66020 --- .../admin/llm_models_controller.rb | 34 ++++++++++++------- spec/requests/admin/llm_models_spec.rb | 12 +++++++ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/app/controllers/admin/llm_models_controller.rb b/app/controllers/admin/llm_models_controller.rb index 05c52bc6d493..374425ca040e 100644 --- a/app/controllers/admin/llm_models_controller.rb +++ b/app/controllers/admin/llm_models_controller.rb @@ -69,13 +69,16 @@ def create def update @llm_model = @connection.models.find(params.expect(:id)) - ActiveRecord::Base.transaction do - apply_attributes(@llm_model) - apply_capabilities(@llm_model) + if update_with_capabilities(@llm_model) + flash[:notice] = t(".success", model: @llm_model.external_id) + redirect_to llm_connection_path, status: :see_other + else + # Re-rendered rather than redirected so the Primer form shows the error + # inline against the field that caused it, e.g. a rename that collides + # with an existing model id. + @verdicts = @connection.capability_verdicts.for_model(@llm_model.external_id_was).index_by(&:capability) + render :edit, status: :unprocessable_entity end - - flash[:notice] = t(".success", model: @llm_model.external_id) - redirect_to llm_connection_path, status: :see_other end def delete_dialog @@ -112,16 +115,23 @@ def set_connection @connection = LlmConnection.instance end - def apply_attributes(llm_model) + def update_with_capabilities(llm_model) attributes = llm_model_params.except(*capability_param_names) # A discovered model is named by the server; only a hand-entered one may be # renamed here, and everything referencing the old name follows it. - attributes = attributes.except(:external_id) unless llm_model.new_record? || llm_model.manual? + attributes = attributes.except(:external_id) unless llm_model.manual? + + saved = false + ActiveRecord::Base.transaction do + previous_external_id = llm_model.external_id + llm_model.assign_attributes(attributes) + raise ActiveRecord::Rollback unless llm_model.save - previous_external_id = llm_model.external_id - llm_model.assign_attributes(attributes) - llm_model.save! - llm_model.cascade_rename!(previous_external_id) + llm_model.cascade_rename!(previous_external_id) + apply_capabilities(llm_model) + saved = true + end + saved end # external_id is accepted on create, and on update for manually added models. diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb index cd2addf612ad..51f1dd3a41a5 100644 --- a/spec/requests/admin/llm_models_spec.rb +++ b/spec/requests/admin/llm_models_spec.rb @@ -221,6 +221,18 @@ end end + describe "renaming to a taken id" do + it "re-renders the form with the error instead of failing" do + create(:llm_model, llm_connection: connection, external_id: "taken") + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "mine") + + patch llm_model_path(llm_model), params: { llm_model: { external_id: "taken" } } + + expect(response).to have_http_status(:unprocessable_entity) + expect(llm_model.reload.external_id).to eq("mine") + end + end + describe "DELETE /admin/llm_models/:id" do it "removes a manual model" do llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") From 58243fda971eae4715d8230a5ddbf84869062c47 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 20 Aug 2026 11:56:42 +0200 Subject: [PATCH 42/44] [#66020] Satisfy the linters and the environment docs guard Three CI follow-ups, none of them behavioural. The llm_* locale keys were inserted out of alphabetical order in two mappings, which yamllint's key-ordering rule rejects. Pure moves, no string changed. The Stimulus controller carried a block-comment copyright header instead of the canonical line-comment form the headers/header-format ESLint rule expects. Note for the next person: rake copyright:update_typescript corrupted the file when converting from the block form, so the header was applied by hand from COPYRIGHT_short. The generated environment variable list now includes the settings this branch introduces, regenerated with the incantation the docs:env_vars task itself prints for an instance with hocuspocus configured. https://community.openproject.org/work_packages/66020 --- config/locales/en.yml | 52 ++++++++--------- .../configuration/environment/README.md | 3 + .../admin/llm-connection-form.controller.ts | 56 +++++++++---------- 3 files changed, 56 insertions(+), 55 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 062999263ba9..67b841cb9e58 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -201,21 +201,10 @@ en: onthefly: "Automatic user creation" port: "Port" tls_certificate_string: "LDAP server SSL certificate" - llm_model: - display_name: "Display name" - # human_attribute_name strips the _id suffix, so the key omits it. - external: "Model name" llm_capability_verdict: capability: "Capability" model_id: "Model" state: "State" - llm_feature_binding: - dimensions: "Dimensions" - feature_key: "Feature" - input_prefix: "Document prefix" - # ActiveRecord::Base.human_attribute_name strips the _id suffix. - model: "Model" - query_prefix: "Query prefix" llm_connection: api_format: "API format" api_key: "API key" @@ -226,6 +215,17 @@ en: default_embedding_model: "Default embedding model" enabled: "Enable LLMs for this instance" last_connected_at: "Last connected" + llm_feature_binding: + dimensions: "Dimensions" + feature_key: "Feature" + input_prefix: "Document prefix" + # ActiveRecord::Base.human_attribute_name strips the _id suffix. + model: "Model" + query_prefix: "Query prefix" + llm_model: + display_name: "Display name" + # human_attribute_name strips the _id suffix, so the key omits it. + external: "Model name" mcp_configuration: description: Description enabled: Enabled @@ -696,21 +696,6 @@ en: tls_certificate_string: format: "%{message}" invalid_certificate: "The provided SSL certificate is invalid: %{additional_message}" - llm_feature_binding: - attributes: - feature_key: - not_registered: "does not belong to a known AI feature." - model_id: - locked: "cannot be changed while data indexed with it still exists. Re-index to switch models." - dimensions: - locked: "cannot be changed while data indexed with it still exists. Re-index to change it." - not_for_chat_feature: "only applies to features that create embeddings." - input_prefix: - locked: "cannot be changed while data indexed with it still exists. Re-index to change it." - not_for_chat_feature: "only applies to features that create embeddings." - query_prefix: - locked: "cannot be changed while data indexed with it still exists. Re-index to change it." - not_for_chat_feature: "only applies to features that create embeddings." llm_connection: attributes: api_format: @@ -732,6 +717,21 @@ en: not_available: "is not offered by the configured LLM server." enabled: requires_connection: "cannot be turned on before a connection has been configured." + llm_feature_binding: + attributes: + feature_key: + not_registered: "does not belong to a known AI feature." + model_id: + locked: "cannot be changed while data indexed with it still exists. Re-index to switch models." + dimensions: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." + input_prefix: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." + query_prefix: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." meeting: error_conflict: "Unable to save because the meeting was updated by someone else in the meantime. Please reload the page." member: diff --git a/docs/installation-and-operations/configuration/environment/README.md b/docs/installation-and-operations/configuration/environment/README.md index 60405bd8994a..693a98f380d4 100644 --- a/docs/installation-and-operations/configuration/environment/README.md +++ b/docs/installation-and-operations/configuration/environment/README.md @@ -234,11 +234,13 @@ OPENPROJECT_ENTERPRISE__PLAN (default="enterprise-on-premises---basic---euro---1 OPENPROJECT_ENTERPRISE__TRIAL__CREATION__HOST (default="https://start.openproject.com") Host for EE trial service OPENPROJECT_FEATURE__BUILT__IN__OAUTH__APPLICATIONS__ACTIVE (default=true) Allows the display and use of built-in OAuth applications. OPENPROJECT_FEATURE__DEPLOY__TARGETS__ACTIVE (default=false) +OPENPROJECT_FEATURE__LLM__CONNECTION__ACTIVE (default=false) Enables the administration page connecting OpenProject to an OpenAI-API-compatible LLM server, and the AI features built on it. OPENPROJECT_FEATURE__MINUTES__STYLING__MEETING__PDF__ACTIVE (default=false) Allow exporting a meeting with FITKO styling. See #65124 for details. OPENPROJECT_FEATURE__SPRINT__REPORTS__ACTIVE (default=false) Enables sprint reporting within the backlogs module. It shows a dashboard with various widgets regarding the sprint progress. OPENPROJECT_FEATURE__STORAGE__FILE__PICKING__SELECT__ALL__ACTIVE (default=false) OPENPROJECT_FEATURE__TYPE__VARIANTS__ACTIVE (default=false) Enables work package type variants. OPENPROJECT_FEATURE__WIKI__ENHANCEMENTS__ACTIVE (default=true) Enables Wiki enhancements, such as the Wikis tab and XWiki integration. +OPENPROJECT_FEATURE__WORK__PACKAGE__MULTIPLE__VERSIONS__ACTIVE (default=false) Enables assigning multiple (target) versions to a work package. Experimental; the user-facing setting and admin switch follow later. OPENPROJECT_FEEDS__ENABLED (default=true) Enable Feeds OPENPROJECT_FEEDS__LIMIT (default=15) Feed content limit OPENPROJECT_FILE__MAX__SIZE__DISPLAYED (default=512) Max size of text files displayed inline @@ -285,6 +287,7 @@ OPENPROJECT_LDAP__FORCE__NO__PAGE (default=nil) Force LDAP to respond as a singl OPENPROJECT_LDAP__GROUPS__DISABLE__SYNC__JOB (default=false) Deactivate regular synchronization job for groups in case scheduled as a separate cronjob OPENPROJECT_LDAP__USERS__DISABLE__SYNC__JOB (default=false) Deactivate user attributes synchronization from LDAP OPENPROJECT_LDAP__USERS__SYNC__STATUS (default=false) Enable user status (locked/unlocked) synchronization from LDAP +OPENPROJECT_LLM__CONNECTION (default={}) Configure the connection to an OpenAI-API-compatible LLM server through environment variables OPENPROJECT_LOG__LEVEL (default="info") Set the OpenProject logger level OPENPROJECT_LOG__REQUESTING__USER (default=false) Log user login, name, and mail address for all requests OPENPROJECT_LOGIN__REQUIRED (default=true) Authentication required diff --git a/frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts b/frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts index a6fc4e9173d7..1ee8cc63ced1 100644 --- a/frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts +++ b/frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts @@ -1,32 +1,30 @@ -/* - * -- copyright - * OpenProject is an open source project management software. - * Copyright (C) the OpenProject GmbH - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 3. - * - * OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: - * Copyright (C) 2006-2013 Jean-Philippe Lang - * Copyright (C) 2010-2013 the ChiliProject Team - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - * See COPYRIGHT and LICENSE files for more details. - * ++ - */ +//-- copyright +// OpenProject is an open source project management software. +// Copyright (C) the OpenProject GmbH +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License version 3. +// +// OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +// Copyright (C) 2006-2013 Jean-Philippe Lang +// Copyright (C) 2010-2013 the ChiliProject Team +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +// +// See COPYRIGHT and LICENSE files for more details. +//++ import { Controller } from '@hotwired/stimulus'; From f6e13e36882e39f14fe2322e1c56aafbcb94e0dd Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 20 Aug 2026 12:01:45 +0200 Subject: [PATCH 43/44] [#66020] Order every llm locale key the way yamllint demands The first Yamllint run only reported the first two misordered mappings, so the previous fix stopped there. This one ran yamllint with the repository configuration over the whole file and sorted every flagged mapping until it came back clean, which also swept the sections later commits had inserted unsorted. Verified as pure reordering: the parsed YAML is identical to the previous commit, and every changed line reappears unchanged elsewhere in the diff. The line-count shrink is de-duplicated blank lines between moved blocks. https://community.openproject.org/work_packages/66020 --- config/locales/en.yml | 368 +++++++++++++++--------------------------- 1 file changed, 128 insertions(+), 240 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 67b841cb9e58..60ec9f84e42a 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -46,7 +46,6 @@ en: version: "A dynamic link to a version placed using a macro." view: "A dynamic link to a view placed using a macro." work_package: "A dynamic link to a work package placed using a macro." - account: auth_source_login_html: Please login as %{login} to activate your account. delete: "Delete account" @@ -77,14 +76,11 @@ en: omniauth_login: Please login to activate your account. signup_title: "Create an account in %{app_title}" signup_with_external_account: "Sign up with an external account" - actionview_instancetag_blank_option: "Please select" - activemodel: attributes: projects/copy_options: dependencies: "Dependencies" - activerecord: attributes: announcements: @@ -719,16 +715,16 @@ en: requires_connection: "cannot be turned on before a connection has been configured." llm_feature_binding: attributes: - feature_key: - not_registered: "does not belong to a known AI feature." - model_id: - locked: "cannot be changed while data indexed with it still exists. Re-index to switch models." dimensions: locked: "cannot be changed while data indexed with it still exists. Re-index to change it." not_for_chat_feature: "only applies to features that create embeddings." + feature_key: + not_registered: "does not belong to a known AI feature." input_prefix: locked: "cannot be changed while data indexed with it still exists. Re-index to change it." not_for_chat_feature: "only applies to features that create embeddings." + model_id: + locked: "cannot be changed while data indexed with it still exists. Re-index to switch models." query_prefix: locked: "cannot be changed while data indexed with it still exists. Re-index to change it." not_for_chat_feature: "only applies to features that create embeddings." @@ -1114,7 +1110,6 @@ en: wiki_page: "Wiki page" work_package: "Work package" workflow: "Workflow" - activities: index: no_results_title_text: There has not been any activity for the project within this time frame. @@ -1145,7 +1140,6 @@ en: no_results_description_text: 'Choose "Show everything" to show all activity and comments' no_results_title_text: No activity to display unsaved_changes_confirmation_message: "Are you sure you want to dismiss your comment? The content that you have written will be lost." - activity: filter: changeset: "Changesets" @@ -1191,7 +1185,6 @@ en: phase_and_both_gates: "%{phase_message}. %{start_gate_message}, and %{finish_gate_message}" phase_and_one_gate: "%{phase_message}. %{gate_message}" removed_date: "date deleted %{date}" - admin: banners: environment_configured_readonly: "These values are configured via environment variables and cannot be edited here." @@ -1258,7 +1251,6 @@ en:
We have prepared [upgrade guides for all installation methods](upgrade_guide). You can perform the upgrade ahead of the next release at any time by following the guides. - jemalloc_allocator: Jemalloc memory allocator jira: actions: @@ -1501,28 +1493,25 @@ en: journal_aggregation: caption_with_maximum: > User actions on a work package (changing description, status, values, or writing comments) are grouped if performed within this period. It also controls notification and [webhook](webhook_link) delays. The maximum is %{max} minutes. - - llm_feature_bindings: - dangling: "%{model} is no longer offered by the LLM server. This feature will not run until another model is selected." - form: - dimensions_caption: "How many numbers each vector has. Leave blank to use whatever the server returns." - dimensions_caption_probed: "The server returned %{dimensions}-dimension vectors for this model. Leave blank to use whatever it returns at index time." - input_prefix_caption: "Prepended to each document before it is indexed. Some models expect one, for example \"passage: \" including the trailing space." - query_prefix_caption: "Prepended to each search query. Some models expect one, for example \"query: \" including the trailing space." - locked_values_heading: "Values fixed by the existing index" - deactivated: "%{model} has been hidden by an administrator. This feature keeps using it, but it can no longer be chosen elsewhere." - index: - blank_description: "Connect OpenProject to an LLM server first. Models can be assigned once the server reports which ones it offers." - blank_title: "No LLM server configured" - description: "Choose which model each AI feature uses. Features without a choice use the instance default." - inherit_with_default: "Use the default (%{model})" - inherit_without_default: "Use the default (none set)" - locked: "%{model} is in use by indexed data and cannot be changed here. Re-index to switch models." - option_unknown: "%{model} — not verified" - option_unsupported: "%{model} — no %{capability} support" - update: - success: "The model for %{feature} has been saved." llm_connections: + delete_api_key: + configured_from_env: "This connection is configured through the environment and cannot be changed here." + description: "OpenProject will stop sending an API key with its requests. Features will keep working only if the server requires no authentication. You can enter a new key at any time." + description_verdicts: "OpenProject will stop sending an API key with its requests, and the next model refresh will discard what is known about each model's capabilities, including the assertions you made yourself. You can enter a new key at any time." + heading: "Remove the stored API key?" + menu_label: "Remove API key" + success: "The API key has been removed." + title: "Remove API key" + disconnect: + configured_from_env: "This connection is configured through the environment and cannot be changed here." + description: "OpenProject will stop using the LLM server. AI features will be unavailable until you connect again." + heading: "Disconnect from the LLM server?" + keeps_bindings: "The model chosen for each feature is kept: %{features}." + keeps_models: "The model list, including any models you added manually, is kept." + keeps_settings: "The endpoint and API format are kept. Only the stored API key is removed." + menu_label: "Disconnect" + success: "OpenProject has disconnected from the LLM server." + title: "Disconnect" form: api_format_caption: "Which API the server speaks. Choose OpenAI-compatible for most gateways and self-hosted servers." api_key_caption: "The key OpenProject authenticates with. Leave blank if the server requires no authentication." @@ -1531,45 +1520,27 @@ en: The full base URL of the server, including the API version segment, exactly as your provider documents it (for example https://example.com/v1). OpenProject appends only the endpoint path. button_connect: "Connect" default_chat_model_caption: "Used by AI features that do not select a model themselves." - embedding_option_unknown: "%{model} (not verified as an embedding model)" - embedding_option_unsupported: "%{model} (cannot create embeddings)" default_embedding_model_caption: "Used by AI features that index text for search and do not select a model themselves. Only models known to create embeddings are offered." default_embedding_model_none: "No model is known to create embeddings. If you know one does, open it under Available models and set its embeddings capability." + embedding_option_unknown: "%{model} (not verified as an embedding model)" + embedding_option_unsupported: "%{model} (cannot create embeddings)" enabled_caption: "When turned off, all LLM-backed AI features stop working. The connection settings are kept." label_connecting: "Contacting the LLM server…" - disconnect: - configured_from_env: "This connection is configured through the environment and cannot be changed here." - description: "OpenProject will stop using the LLM server. AI features will be unavailable until you connect again." - heading: "Disconnect from the LLM server?" - keeps_bindings: "The model chosen for each feature is kept: %{features}." - keeps_models: "The model list, including any models you added manually, is kept." - keeps_settings: "The endpoint and API format are kept. Only the stored API key is removed." - menu_label: "Disconnect" - success: "OpenProject has disconnected from the LLM server." - title: "Disconnect" - delete_api_key: - configured_from_env: "This connection is configured through the environment and cannot be changed here." - description: "OpenProject will stop sending an API key with its requests. Features will keep working only if the server requires no authentication. You can enter a new key at any time." - description_verdicts: "OpenProject will stop sending an API key with its requests, and the next model refresh will discard what is known about each model's capabilities, including the assertions you made yourself. You can enter a new key at any time." - heading: "Remove the stored API key?" - menu_label: "Remove API key" - success: "The API key has been removed." - title: "Remove API key" models: blank_description: "The server reported no models. If it does not offer a model list, add the model names you want to use below." blank_title: "No models available" context_window: "Context window" display_name_caption: "An optional friendly name shown instead of the model id." + filter_label: "Filter models" identifier: "Model" kind: "Type" source: "Source" - filter_label: "Filter models" source_deactivated: "Hidden by administrator" source_discovered: "Reported by server" source_manual: "Added manually" + source_withdrawn: "No longer reported" status: "Available for use" toggle_aria_label: "Make %{model} available to AI features" - source_withdrawn: "No longer reported" refresh_models: failure: "The model list could not be refreshed. Please check that the LLM server is still reachable." success: "The model list has been refreshed." @@ -1583,6 +1554,34 @@ en: update: no_models: "Saved, but the server did not return a model list. Either it does not offer one, or the endpoint is wrong — a URL missing its API version segment (for example /v1) looks exactly the same from here. Add the models you want to use below, then run the health checks to confirm the server answers." success: "Successfully connected to the LLM server." + llm_feature_bindings: + dangling: "%{model} is no longer offered by the LLM server. This feature will not run until another model is selected." + deactivated: "%{model} has been hidden by an administrator. This feature keeps using it, but it can no longer be chosen elsewhere." + form: + dimensions_caption: "How many numbers each vector has. Leave blank to use whatever the server returns." + dimensions_caption_probed: "The server returned %{dimensions}-dimension vectors for this model. Leave blank to use whatever it returns at index time." + input_prefix_caption: "Prepended to each document before it is indexed. Some models expect one, for example \"passage: \" including the trailing space." + query_prefix_caption: "Prepended to each search query. Some models expect one, for example \"query: \" including the trailing space." + index: + blank_description: "Connect OpenProject to an LLM server first. Models can be assigned once the server reports which ones it offers." + blank_title: "No LLM server configured" + description: "Choose which model each AI feature uses. Features without a choice use the instance default." + inherit_with_default: "Use the default (%{model})" + inherit_without_default: "Use the default (none set)" + locked: "%{model} is in use by indexed data and cannot be changed here. Re-index to switch models." + locked_values_heading: "Values fixed by the existing index" + option_unknown: "%{model} — not verified" + option_unsupported: "%{model} — no %{capability} support" + update: + success: "The model for %{feature} has been saved." + llm_health_status: + show: + blankslate_description: "Run the checks to find out whether OpenProject can reach the LLM server, whether it accepts the API key, and whether every AI feature has a usable model." + blankslate_heading: "This connection has not been checked yet" + download: "Download report" + last_check: "Last checked %{datetime}." + run_checks: "Run checks" + title: "Health status" llm_models: create: success: "%{model} has been added." @@ -1596,14 +1595,14 @@ en: description: "Edit how OpenProject uses this model." form: capabilities_description: "Leave a capability unspecified to let OpenProject determine it. Marking Embeddings as supported makes this an embedding model, offered only to features that need one." + capability_override_caption: "Choose a value to override this." context_window: "Context window" context_window_caption: "How many tokens this model accepts. Leave blank if you do not know." context_window_known: "Leave blank to use %{value}, %{source}." create_submit: "Add model" - capability_override_caption: "Choose a value to override this." - state_inherited: "%{state} (%{source})" display_name_caption: "An optional friendly name shown instead of the model id." external_id_caption: "The model name exactly as the server expects it." + state_inherited: "%{state} (%{source})" state_supported: "Supported" state_unspecified: "Not specified" state_unsupported: "Not supported" @@ -1612,14 +1611,6 @@ en: title: "Add a model" update: success: "%{model} has been updated." - llm_health_status: - show: - blankslate_description: "Run the checks to find out whether OpenProject can reach the LLM server, whether it accepts the API key, and whether every AI feature has a usable model." - blankslate_heading: "This connection has not been checked yet" - download: "Download report" - last_check: "Last checked %{datetime}." - run_checks: "Run checks" - title: "Health status" mcp_configurations: index: description: "The model context protocol allows AI agents to provide its users with tools and resources exposed by this OpenProject instance. This feature is still in beta." @@ -1871,7 +1862,6 @@ en: is_active: currently displayed is_inactive: currently not displayed show_until: Show until - antivirus_scan: deleted_by_admin: "The quarantined file '%{filename}' has been deleted by an administrator." deleted_message: "A virus was detected in file '%{filename}'. The file has been deleted." @@ -1958,7 +1948,6 @@ en: parent: Undisclosed - The parent is invisible because of lacking permissions. project: Undisclosed - The project is invisible because of lacking permissions. workPackage: Undisclosed - The work package is invisible because of lacking permissions. - attribute_help_texts: add_new: "Add help text" caption: "This short version will be displayed as caption of the attribute." @@ -2073,12 +2062,10 @@ en: work_package_id: "Work package" authentication: login_and_registration: "Login and registration" - background_jobs: status: cancelled_due_to: "Job was cancelled due to error: %{message}" error_requeue: "Job experienced an error but is retrying. The error was: %{message}" - backup: error: backup_pending: There is already a backup pending. @@ -2114,7 +2101,6 @@ en: When you create a new token you will only be allowed to request a backup after 24 hours. This is a safety measure. After that you can request a backup any time using that token. text_token_deleted: Backup token deleted. Backups are now disabled. - button_actions: "Actions" button_add: "Add" button_add_comment: "Add comment" @@ -2224,7 +2210,6 @@ en: mathematical: "The mathematical formula leads to an error. Please review the project calculation attribute and try again." missing_value: The attribute "%{custom_field_name}" is required by this Calculated value, but is empty. unknown: "An unknown error occurred. Please review the formula for this Calculated value." - colors: edit: label_edit_color: "Edit Color" @@ -2243,10 +2228,8 @@ en: new: label_new_color: "New Color" - concatenation: single: "or" - consent: checkbox_label: I have noted and do consent to the above. contact_mail_instructions: Define the mail address that users can reach a data controller to perform data change or removal requests. @@ -2259,7 +2242,6 @@ en: title: User Consent update_consent_last_time: "Last update of consent: %{update_time}" user_has_consented: The user gave their consent to your [configured consent information text](consent_settings). - copy_project: copy_options: dependencies_label: "Copy from project" @@ -2283,13 +2265,11 @@ en: description: Start from scratch. Manually add portfolio attributes, members and modules. label: "Blank portfolio" template_heading: "Select a portfolio template to work with the most common project management methods, or create a portfolio from scratch." - create_program: blank_template: description: Start from scratch. Manually add program attributes, members and modules. label: "Blank program" template_heading: "Select a program template to work with the most common project management methods, or create a program from scratch." - create_project: attributes_heading: "Fill in this mandatory information to work on your projects." blank_description: No description provided. @@ -2300,7 +2280,6 @@ en: dependencies_label: "Copy from template" template_heading: "Select a project template to work with the most common project management methods, or create a project from scratch." template_label: "Use template" - create_wiki_page: "Create new wiki page" create_wiki_page_button: "Wiki page" custom_actions: @@ -2314,7 +2293,6 @@ en: execute: "Execute %{name}" new: "New custom action" plural: "Custom actions" - custom_fields: admin: custom_field_projects: @@ -2420,11 +2398,9 @@ en: text_add_new_custom_field: > To add new custom fields to a project you first need to create them before you can add them to this project. - danger_dialog: confirmation_live_message_checked: "The button to proceed is now active." confirmation_live_message_unchecked: "The button to proceed is now inactive. You need to tick the checkbox to continue." - date: abbr_day_names: ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"] abbr_month_names: @@ -2483,7 +2459,6 @@ en: - :year - :month - :day - datetime: distance_in_words: about_x_hours: @@ -2548,7 +2523,6 @@ en: minute_abbreviated: one: "min" other: "mins" - departments: add_department: "Add department" add_department_form: @@ -2585,7 +2559,6 @@ en: managed_heading: "This user cannot be moved" managed_title: "User managed by LDAP" title: "User already in a department" - description_active: "Active?" description_attachment_toggle: "Show/Hide attachments" description_autocomplete: > @@ -2621,7 +2594,6 @@ en: # Text direction: Left-to-Right (ltr) or Right-to-Left (rtl) direction: ltr - doorkeeper: access_token_url: "Access token URL" auth_url: "Auth URL" @@ -2657,7 +2629,6 @@ en: unsupported_response_type: "The authorization server does not support this response type." pre_authorization: status: "Pre-authorization" - ee: features: baseline_comparison: Baseline Comparisons @@ -2809,7 +2780,6 @@ en: work_package_subject_generation: description: "Create automatically generated subjects using referenced attributes and text." - email_calendar_updates: button: disabled: "Enable" @@ -2819,11 +2789,9 @@ en: enabled: "Enabled." enterprise_plans: legacy_enterprise: "Enterprise Plan" - enterprise_trials: dialog_component: title: Enterprise Trial - enumeration_activities: "Time tracking activities" enumeration_caption_order_changed: "Order successfully changed." enumeration_could_not_be_moved: "Enumeration could not be moved." @@ -2831,7 +2799,6 @@ en: enumeration_reported_project_statuses: "Reported status" enumeration_work_package_priorities: "Work package priorities" error_attribute_not_highlightable: "Attribute(s) not highlightable: %{attributes}" - error_auth_source_sso_failed: "Single Sign-On (SSO) for user '%{value}' failed" error_can_not_archive_project: "This project cannot be archived: %{errors}" error_can_not_deactivate_type_invisible_projects: "There are also work packages in projects you cannot see." @@ -2888,12 +2855,10 @@ en: error_wiki_root_menu_item_conflict: > Can't rename "%{old_name}" to "%{new_name}" due to a conflict in the resulting menu item with the existing menu item "%{existing_caption}" (%{existing_identifier}). - error_work_package_id_not_found: "The work package was not found." error_work_package_not_found_in_project: "The work package was not found or does not belong to this project" error_workflow_copy_source: "Please select a source type or role" error_workflow_copy_target: "Please select target type(s) and role(s)" - errors: field_erroneous_label: "This field is invalid: %{full_errors}\nPlease enter a valid value." header_additional_invalid_fields: @@ -2908,7 +2873,6 @@ en: must_be_template: "must be template" storage_error: "There was an error with the storage connection." unsupported_storage_type: "is not a supported storage type." - events: changeset: "Changeset edited" message: Message edited @@ -2925,7 +2889,6 @@ en: work_package_closed: "Work Package closed" work_package_edit: "Work Package edited" work_package_note: "Work Package note added" - export: demo: button_text: Generate Demo PDF @@ -3026,7 +2989,6 @@ en: continue_message: "Are you sure you want to proceed to the following external link?" title: "Leaving OpenProject" warning_message: "You are about to leave OpenProject and visit an external website. Please be aware that external websites are not under our control and may have different privacy and security policies." - extraction: available: catdoc: "Catdoc available (optional)" @@ -3035,7 +2997,6 @@ en: tesseract: "Tesseract available (optional)" unrtf: "Unrtf available (optional)" xls2csv: "Xls2csv available (optional)" - filterable_tree_view: filter_mode: all: "All" @@ -3047,7 +3008,6 @@ en: show: no_results_title_text: There are currently no posts for the forum. general_csv_decimal_separator: "." - general_csv_encoding: "UTF-8" general_csv_separator: "," general_first_day_of_week: "7" @@ -3068,7 +3028,6 @@ en: all_projects: 'Search for "%{search_term}" in all projects' current_project: 'Search for "%{search_term}" in %{project_name}' project_and_subprojects: 'Search for "%{search_term}" in %{project_name} and all subprojects' - groups: edit: synchronized_groups: "Synchronized groups" @@ -3088,10 +3047,8 @@ en: user_count: User count users: no_results_title_text: There are currently no users part of this group. - gui_validation_error: "1 error" gui_validation_error_plural: "%{count} errors" - header: project_select_component: all_projects: "All projects" @@ -3099,7 +3056,6 @@ en: leave_project: "Leave project" no_favorite_projects: "You have no favorite projects." title: "Projects" - health_reports: common: checks: @@ -3128,7 +3084,6 @@ en: You can only select projects here where the %{plural} module is active. After creating a %{singular} you can add work packages from other projects to it. public: "Publish this view, allowing other users to access your view. Users with the 'Manage public views' permission can modify or remove public query. This does not affect the visibility of work package results in that view and depending on their permissions, users may see different results." - homescreen: additional: favorite_projects: @@ -3175,19 +3130,15 @@ en: missing_authorization: "The server side request failed due to missing authorization information." response: unexpected: "Unexpected response received." - image_conversion: imagemagick: "Imagemagick" - incoming_mails: ignore_filenames: > Specify a list of names to ignore when processing attachments for incoming mails (e.g., signatures or icons). Enter one filename per line. - instructions_after_error_link: "You can try to sign in again by clicking [here](signin_url). If the error persists, ask your admin for help." instructions_after_logout_link: "You can sign in again by clicking [here](signin_url)." instructions_after_registration_link: "You can sign in as soon as your account has been activated by clicking [here](signin_url)." - journals: cause_descriptions: budget_deleted: Budget has been deleted @@ -3277,7 +3228,6 @@ en: version: "a non-visible version" work_package: "a non-visible work package" label_accessibility: "Accessibility" - label_account: "Account" label_actions: "Actions" label_activate_user: "Activate user" @@ -3586,7 +3536,6 @@ en: label_internal: "Internal" label_introduction_video: "Getting started video" label_invitation: Invitation - label_invite_user: "Invite user" label_item: "Item" label_item_plural: "Items" @@ -4057,7 +4006,6 @@ en: other: "Time off: %{count} working days" label_yesterday: "yesterday" label_zen_mode: "Zen mode" - ldap_auth_sources: attribute_texts: admin_map_html: "Optional: The attribute key in LDAP that if present marks the OpenProject user an admin. Leave empty when in doubt." @@ -4140,51 +4088,50 @@ en: update: failure: "The modified list cannot be saved: %{errors}" success: "The modified list has been saved" - macro_execution_error: "Error executing the macro %{macro_name}" - macro_unavailable: "Macro %{macro_name} cannot be displayed." - macro_unknown: "Unknown or unsupported macro." - macros: - create_work_package_link: - errors: - invalid_type: "No type found with name '%{type}' in project '%{project}'." - no_project_context: "Calling create_work_package_link macro from outside project context." - link_name: "New work package" - errors: - missing_or_invalid_parameter: "Missing or invalid macro parameter." - include_wiki_page: - removed: "The macro does no longer exist." - legacy_warning: - timeline: "This legacy timeline macro has been removed and is no longer available. You can replace the functionality with an embedded table macro." - placeholder: "[Placeholder] Macro %{macro_name}" - wiki_child_pages: - errors: - page_not_found: "Cannot find the wiki page '%{name}'." llm: + api_formats: + anthropic: "Anthropic" + azure: "Azure OpenAI" + bedrock: "AWS Bedrock" + deepseek: "DeepSeek" + gemini: "Google Gemini" + gpustack: "GPUStack" + mistral: "Mistral" + ollama: "Ollama" + openai: "OpenAI-compatible" + openrouter: "OpenRouter" + perplexity: "Perplexity" + vertexai: "Google Vertex AI" + xai: "xAI" + capabilities: + embeddings: + label: "Embeddings" + function_calling: + label: "Tool calling" + reasoning: + label: "Reasoning" + structured_output: + label: "Structured output" + vision: + label: "Vision" + context_window_sources: + registry: "the figure published for this model" + server: "reported by the server" + features: + description_assistant: + caption: "Rewrites and restructures work package text on request." + label: "Description assistant" + semantic_search: + caption: "Indexes work packages so they can be found by meaning rather than by keyword." + label: "Semantic search" health_checks: configuration: - header: "Configuration" api_format_supported: "Endpoint format is usable" base_url_present: "Endpoint is configured" credentials_present: "API key is stored" enabled: "Connection is switched on" feature_flag: "AI features are switched on" - features: - header: "AI features" - bindings_resolvable: "Every feature has a usable model" - locked_bindings_intact: "Indexed data still has its model" - inference: - header: "Inference" - chat_round_trip: "The server answers a request" - models: - header: "Models" - catalogue_fresh: "Model list is up to date" - catalogue_present: "Models are available" - default_chat_model: "Default chat model" - default_embedding_model: "Default embedding model" - server: - header: "Server" - credentials_accepted: "The server accepts the API key" - reachable: "The server can be reached" + header: "Configuration" errors: api_key_missing: "No API key is stored. This is only correct if the LLM server requires no authentication." catalogue_never_fetched: "The model list has never been retrieved from the server." @@ -4210,34 +4157,23 @@ en: server_error: "The server responded with status %{status}." ssrf_filtered: "The endpoint resolves to a blocked address." unsupported_api_format: "%{api_format} needs credentials that OpenProject cannot store." - api_formats: - anthropic: "Anthropic" - azure: "Azure OpenAI" - bedrock: "AWS Bedrock" - deepseek: "DeepSeek" - gemini: "Google Gemini" - gpustack: "GPUStack" - mistral: "Mistral" - ollama: "Ollama" - openai: "OpenAI-compatible" - openrouter: "OpenRouter" - perplexity: "Perplexity" - vertexai: "Google Vertex AI" - xai: "xAI" - capabilities: - embeddings: - label: "Embeddings" - function_calling: - label: "Tool calling" - reasoning: - label: "Reasoning" - structured_output: - label: "Structured output" - vision: - label: "Vision" - context_window_sources: - registry: "the figure published for this model" - server: "reported by the server" + features: + bindings_resolvable: "Every feature has a usable model" + header: "AI features" + locked_bindings_intact: "Indexed data still has its model" + inference: + chat_round_trip: "The server answers a request" + header: "Inference" + models: + catalogue_fresh: "Model list is up to date" + catalogue_present: "Models are available" + default_chat_model: "Default chat model" + default_embedding_model: "Default embedding model" + header: "Models" + server: + credentials_accepted: "The server accepts the API key" + header: "Server" + reachable: "The server can be reached" model_kinds: chat: "Chat" embedding: "Embedding" @@ -4251,13 +4187,6 @@ en: supported: "Supported" unknown: "Not verified" unsupported: "Not supported" - features: - description_assistant: - caption: "Rewrites and restructures work package text on request." - label: "Description assistant" - semantic_search: - caption: "Indexes work packages so they can be found by meaning rather than by keyword." - label: "Semantic search" llm_connections: side_panel: health_status_component: @@ -4266,6 +4195,25 @@ en: open_report: "Open full health report" run_checks: "Run checks now" title: "Health status" + macro_execution_error: "Error executing the macro %{macro_name}" + macro_unavailable: "Macro %{macro_name} cannot be displayed." + macro_unknown: "Unknown or unsupported macro." + macros: + create_work_package_link: + errors: + invalid_type: "No type found with name '%{type}' in project '%{project}'." + no_project_context: "Calling create_work_package_link macro from outside project context." + link_name: "New work package" + errors: + missing_or_invalid_parameter: "Missing or invalid macro parameter." + include_wiki_page: + removed: "The macro does no longer exist." + legacy_warning: + timeline: "This legacy timeline macro has been removed and is no longer available. You can replace the functionality with an embedded table macro." + placeholder: "[Placeholder] Macro %{macro_name}" + wiki_child_pages: + errors: + page_not_found: "Cannot find the wiki page '%{name}'." mail: actions: "Actions" digests: @@ -4382,7 +4330,6 @@ en: without_message: "%{user} added you as a member to the project '%{project}'." roles: "You have the following roles:" subject: "%{project} - You have been added as a member" - mail_member_updated_global: body: roles: "You now have the following roles:" @@ -4390,7 +4337,6 @@ en: with_message: "%{user} updated the roles you have globally writing:" without_message: "%{user} updated the roles you have globally." subject: "Your global permissions have been updated" - mail_member_updated_project: body: roles: "You now have the following roles:" @@ -4410,7 +4356,6 @@ en: mail_subject_register: "Your %{value} account activation" mail_subject_wiki_content_added: "'%{id}' wiki page has been added" mail_subject_wiki_content_updated: "'%{id}' wiki page has been updated" - mail_user_activation_limit_reached: message_html: | A new user (%{email}) tried to create an account on an OpenProject environment that you manage (%{host}). @@ -4424,7 +4369,6 @@ en: server_url_component: caption: "The URL at which the OpenProject MCP server will be reachable. Required for setting up MCP clients." label: "Server URL" - members: columns: shared: "Shared" @@ -4486,7 +4430,6 @@ en: send_invite_to: "Send invite to" menu_item: "Menu item" menu_item_setting: "Visibility" - menus: admin: aggregation: "Aggregation" @@ -4499,9 +4442,7 @@ en: mcp_configurations: "Model Context Protocol (MCP)" quick_add: label: "Add…" - more_actions: "More functions" - my: access_token: created_dialog: @@ -4529,7 +4470,6 @@ en: password_confirmation_dialog: confirmation_required: "You need to enter your account password to confirm this change." title: "Confirm your password to continue" - my_account: access_tokens: access_tokens: "Access tokens" @@ -4669,21 +4609,17 @@ en: email_reminders: "Email reminders" notifications: "Notification settings" title: "Notification and email" - news: index: no_results_content_text: Add a news item no_results_title_text: There is currently no news to report. no_results_title_text: There is currently nothing to display. - noscript_description: "You need to activate JavaScript in order to use OpenProject!" noscript_heading: "JavaScript disabled" noscript_learn_more: "Learn more" note: Note - note_password_login_disabled_link: "Password login has been disabled through a [configuration setting](configuration_url)." nothing_to_preview: "Nothing to preview" - notice_accessibility_mode: The accessibility mode can be enabled in your [account settings](url). notice_account_activated: "Your account has been activated. You can now log in." notice_account_already_activated: The account has already been activated. @@ -4706,7 +4642,6 @@ en: notice_attachment_migration_wiki_page: > This page was generated automatically during the update of OpenProject. It contains all attachments previously associated with the %{container_type} "%{container_name}". - notice_auth_stage_error: "Authentication stage '%{stage}' failed." notice_auth_stage_verification_error: "Could not verify stage '%{stage}'." notice_auth_stage_wrong_stage: "Expected to finish authentication stage '%{expected}', but '%{actual}' returned." @@ -4718,7 +4653,6 @@ en: notice_email_sent: "An email was sent to %{value}" notice_failed_to_save_members: "Failed to save member(s): %{errors}." notice_failed_to_save_work_packages: "Failed to save %{count} work package(s) on %{total} selected: %{ids}." - notice_file_not_found: "The page you were trying to access doesn't exist or has been removed." notice_forced_logout: "You have been automatically logged out after %{ttl_time} minutes of inactivity." notice_internal_server_error: "An error occurred on the page you were trying to access. If you continue to experience problems please contact your %{app_title} administrator for assistance." @@ -4761,7 +4695,6 @@ en: notice_user_invitation_resent: An invitation has been sent to %{email}. notice_user_missing_authentication_method: User has yet to choose a password or another way to sign in. notice_wont_delete_auth_source: The LDAP connection cannot be deleted as long as there are still users using it. - notifications: facets: all: "All" @@ -4876,7 +4809,6 @@ en: api_v3_text: "Application will receive full read & write access to the OpenProject API v3 to perform actions on your behalf." mcp: "Access to MCP" mcp_text: "Application will receive access to the OpenProject MCP endpoints. They are a limited subset of APIv3, that's tailored for usage with AI agents." - oauth_client: errors: oauth_authorization_code_grant_had_errors: "OAuth2 Authorization grant unsuccessful" @@ -4924,13 +4856,11 @@ en: error: "Error" failed_authorization: "Authorization failed" not_connected: "Not connected" - onboarding: heading_getting_started: "Get an overview" select_language: "Please select your language" text_getting_started_description: "Get a quick overview of project management and team collaboration with OpenProject. You can restart this video from the help menu." welcome: "Welcome to %{app_title}" - op_dry_validation: errors: array?: "must be an array." @@ -4981,7 +4911,6 @@ en: short: "Short name" weight: "Weight" open_link_in_a_new_tab: "Open link in a new tab" - open_project: common: work_package_card_component: @@ -4993,7 +4922,6 @@ en: undisclosed: "Undisclosed" page: text: "Text" - pagination: label: "Pagination" next: "Next" @@ -5047,7 +4975,6 @@ en: permission_add_subprojects: "Create subprojects" permission_add_work_package_attachments: "Add attachments" permission_add_work_package_attachments_explanation: "Allows adding attachments without Edit work packages permission" - permission_add_work_package_comments: "Add comments" permission_add_work_package_watchers: "Add watchers" permission_add_work_packages: "Add work packages" @@ -5174,18 +5101,15 @@ en: right_to_manage_members_missing: > You are not allowed to delete the placeholder user. You do not have the right to manage members for all projects that the placeholder user is a member of. - placeholders: default: "-" templated_hint: Automatically generated through type %{type} - plugin_openproject_auth_plugins: description: "Integration of OmniAuth strategy providers for authentication in OpenProject." name: "OpenProject Auth Plugins" plugin_openproject_auth_saml: description: "Adds the OmniAuth SAML provider to OpenProject" name: "OmniAuth SAML / Single-Sign On" - portfolio: count: one: "1 portfolio" @@ -5212,7 +5136,6 @@ en: priority_color_text: | Click to assign or change the color of this priority. It can be used for highlighting work packages in the table. - program: count: one: "1 program" @@ -5220,7 +5143,6 @@ en: zero: "0 programs" filters: name: "Part of Program" - project: archive: archived: "Archived" @@ -5258,14 +5180,12 @@ en: make_template: "Set as template" remove_from_templates: "Remove from templates" use_template: "Use template" - project_module_activity: "Activity" project_module_forums: "Forums" project_module_news: "News" project_module_repository: "Repository" project_module_wiki: "Wiki" project_module_work_package_tracking: "Work packages" - projects: # Contains custom strings for options when copying a project that cannot be found elsewhere. copy: @@ -5475,7 +5395,6 @@ en: automatic: description: "Order the %{plural} by one or more sorting criteria. You will lose the previous sorting." heading: "Automatic" - query: attribute_and_direction: "%{attribute} (%{direction})" @@ -5500,12 +5419,10 @@ en: and_user: "and %{user}" react_with: "React with %{reaction}" reaction_by: "%{reaction} by" - reportings: index: no_results_content_text: Add a status reporting no_results_title_text: There are currently no status reportings. - repositories: at_identifier: "at %{identifier}" atom_revision_feed: "Atom revision feed" @@ -5613,7 +5530,6 @@ en: url: "URL to repository" warnings: cannot_annotate: "This file cannot be annotated." - roles: edit: default_for_new_projects_warning: >- @@ -5630,11 +5546,9 @@ en: matrix_check_uncheck_all_in_row_label_html: "Toggle %{permission} permission for all roles" matrix_checkbox_label: "Assign %{permission} permission to %{role} role" matrix_uncheck_all_label: "Unassign all %{module} permissions from all roles" - scheduling: automatic: "set to Automatic" manual: "set to Manual" - search_input_placeholder: "Search ..." setting_accessibility_mode_for_anonymous: "Enable accessibility mode for anonymous users" setting_activity_days_default: "Days displayed on project activity" @@ -5648,7 +5562,6 @@ en: Set a default path to redirect users after login, if no back link was provided. Redirects to home page if not set.
Example: %{example_code} - setting_allowed_link_protocols: "Allowed link protocols" setting_allowed_link_protocols_text_html: >- Allow these protocols to be rendered as links in work package descriptions, long text fields and comments. @@ -5897,7 +5810,6 @@ en: Maximum number of work packages or projects that can be included in a single export. Larger exports are truncated to this limit. setting_working_days: "Working days" - settings: antivirus: actions: @@ -6211,14 +6123,12 @@ en: Adding additional users will exceed the current limit. Please [upgrade your plan](upgrade_url) to be able to ensure external users are able to access this %{entity}. status_active: "active" - status_archived: "archived" status_blocked: "blocked" status_deleted: deleted status_invited: invited status_locked: locked status_registered: registered - statuses: edit: status_color_text: | @@ -6253,7 +6163,6 @@ en: sentence_connector: "and" skip_last_comma: "false" text_access_token_hint: "Access tokens allow you to grant external applications access to resources in OpenProject." - text_accessibility_hint: "The accessibility mode is designed for users who are blind, motorically handicaped or have a bad eyesight. For the latter focused elements are specially highlighted. Please notice, that the Backlogs module is not available in this mode." text_analyze: "Further analyze: %{subject}" text_are_you_sure: "Are you sure?" @@ -6414,7 +6323,6 @@ en: text_wrote: "wrote" text_zoom_in: "Zoom in" text_zoom_out: "Zoom out" - themes: dark: "Dark" light: "Light" @@ -6427,19 +6335,15 @@ en: short: "%d %b %H:%M" time: "%I:%M %p" pm: "pm" - timeframe: end: "to" show: "Show timeframe" start: "from" title_enterprise_upgrade: "Upgrade to unlock more users." - title_remove_and_delete_user: Remove the invited user from the project and delete him/her. - toggle_switch: label_off: "Off" label_on: "On" - token: hashed_token: display_value_placeholder: "***" @@ -6448,23 +6352,18 @@ en: If enabled a user will be unable to chose a login during registration. Instead their given email address will serve as the login. An administrator may still change the login separately. - tooltip_resend_invitation: > Sends another invitation email with a fresh token in case the old one expired or the user did not get the original email. Can also be used for active users to choose a new authentication method. When used with active users their status will be changed to 'invited'. - tooltip_user_default_timezone: > The default time zone for new users. Can be changed in a user's settings. - top_menu: additional_resources: "Additional resources" getting_started: "Getting started" help_and_support: "Help and support" - total_progress: "Total progress" - types: creation_wizard: add_variant: "Add variant to %{name}" @@ -6694,13 +6593,11 @@ en: one: "1 variant" other: "%{count} variants" milestone_indicator: "Milestone" - unsupported_browser: close_warning: "Ignore this warning." message: "You may run into errors and degraded experience on this page." title: "Your browser is outdated and unsupported." update_message: "Please update your browser." - user: activate: "Activate" activate_and_reset_failed_logins: "Activate and reset failed logins" @@ -6741,7 +6638,6 @@ en: user_preferences: disable_keyboard_shortcuts_caption: > You can choose to disable default [keyboard shortcuts](docs_url) if you use a screen reader or want to avoid accidentally triggering an action with a shortcut. - users: autologins: prompt: "Stay logged in for %{num_days}" @@ -6882,11 +6778,9 @@ en: one: "1 working day" other: "%{count} working days" work_hours: "Work hours" - version_status_closed: "closed" version_status_locked: "locked" version_status_open: "open" - versions: overview: no_results_title_text: There are currently no work packages assigned to this version. @@ -6914,7 +6808,6 @@ en: Please [upgrade your plan](upgrade_url) or block existing users in order to allow invited and registered users to join. warning_protocol_mismatch_html: > - warning_registration_token_expired: | The activation email has expired. We sent you a new one to %{email}. Please click the link inside of it to activate your account. @@ -6927,7 +6820,6 @@ en: warning_user_limit_reached_instructions: > You reached your user limit (%{current}/%{max} active users). Please contact sales@openproject.com to upgrade your Enterprise edition plan and add additional users. - wiki: index: no_results_content_text: Add a new wiki page @@ -6936,7 +6828,6 @@ en: page_not_editable_index: The requested page does not (yet) exist. You have been redirected to the index of all wiki pages. print_hint: This will print the content of this wiki page without any navigation bars. wiki_menu_item_delete_not_permitted: The wiki menu item of the only wiki page cannot be deleted. - wiki_menu_item_for: 'Menu item for wikipage "%{title}"' wiki_menu_item_new_main_item_explanation: > You are deleting the only main wiki menu item. You now have to choose a wiki page for which a new main item will be generated. @@ -7068,7 +6959,6 @@ en: relates_to_description: "Creates a visible link between the two work packages with no additional effect" required_description: "Marks this work package as being a requirement to the related one" requires_description: "Marks the related work package as a requirement to this one" - work_packages: bulk: copy_failed: "The work packages could not be copied." @@ -7236,7 +7126,6 @@ en: x_descendants: one: "One descendant work package" other: "%{count} work package descendants" - workflows: copies: form: @@ -7309,5 +7198,4 @@ en: warning: > Changing which days of the week are considered working days or non-working days can affect the start and finish days of all work packages and life cycles in all projects in this instance. - you: you From 7d8de2cb0f19e1a9e6125335f7b1042ffca86400 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Thu, 20 Aug 2026 12:30:56 +0200 Subject: [PATCH 44/44] [#66020] Keep the heading order intact on the connection page The side panel section renders its title as h4 by default, and depending on which parts of the page are present that h4 can follow the page header directly, skipping a level. axe flags this as a heading-order violation, which is why the accessibility feature spec failed intermittently: the violation only appears in some page states. The section title is h3 now, which is correct in every state the page can be in. https://community.openproject.org/work_packages/66020 --- .../llm_connections/side_panel/health_status_component.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/components/llm_connections/side_panel/health_status_component.html.erb b/app/components/llm_connections/side_panel/health_status_component.html.erb index dba7bc54848e..88ca7f6e6924 100644 --- a/app/components/llm_connections/side_panel/health_status_component.html.erb +++ b/app/components/llm_connections/side_panel/health_status_component.html.erb @@ -1,7 +1,7 @@ <%= component_wrapper(tag: :turbo_frame, refresh: :morph) do render(Primer::OpenProject::SidePanel::Section.new) do |section| - section.with_title { t(".title") } + section.with_title(tag: :h3) { t(".title") } flex_layout do |container| if report.present?