From 75ff06eb657492d0f24947d7f35d403287370b40 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Sat, 22 Aug 2026 18:12:20 +0200 Subject: [PATCH] [AI-3] Let administrators add, edit and delete models by hand Not every OpenAI-compatible server exposes a model list: a gateway may route /v1/chat/completions and nothing else, in which case the operator knows the model name and OpenProject cannot discover it. This adds hand-entered LlmModel rows with their own form, including capability assertions stored as admin-sourced verdicts that survive re-detection, and an administrator-supplied context window that takes precedence over reported figures. A manual model's identifier may be corrected after the fact; the rename carries every verdict and the connection defaults along, because everything references a model by its identifier string. Discovered models are named by the server and cannot be renamed or deleted here, and a catalogue refresh never withdraws a manual entry. Part 6 of the AI-3 stack. https://community.openproject.org/work_packages/66020 --- .../delete_model_dialog_component.html.erb | 20 ++ .../delete_model_dialog_component.rb | 57 ++++ .../models/sub_header_component.html.erb | 11 + .../llm_connections/models_row_component.rb | 20 ++ .../llm_connections/models_table_component.rb | 3 + .../admin/llm_models_controller.rb | 183 +++++++++++ app/forms/llm_models/form.rb | 134 ++++++++ app/models/llm_model.rb | 53 +++ app/views/admin/llm_models/edit.html.erb | 52 +++ app/views/admin/llm_models/new.html.erb | 54 ++++ config/locales/en.yml | 49 +++ config/routes.rb | 7 + spec/requests/admin/llm_models_spec.rb | 305 ++++++++++++++++++ 13 files changed, 948 insertions(+) create mode 100644 app/components/llm_connections/delete_model_dialog_component.html.erb create mode 100644 app/components/llm_connections/delete_model_dialog_component.rb create mode 100644 app/controllers/admin/llm_models_controller.rb create mode 100644 app/forms/llm_models/form.rb create mode 100644 app/views/admin/llm_models/edit.html.erb create mode 100644 app/views/admin/llm_models/new.html.erb create mode 100644 spec/requests/admin/llm_models_spec.rb diff --git a/app/components/llm_connections/delete_model_dialog_component.html.erb b/app/components/llm_connections/delete_model_dialog_component.html.erb new file mode 100644 index 000000000000..5fe5f2d5816c --- /dev/null +++ b/app/components/llm_connections/delete_model_dialog_component.html.erb @@ -0,0 +1,20 @@ +<%= + render( + Primer::OpenProject::DangerDialog.new( + title: t("admin.llm_models.destroy.title"), + form_arguments:, + test_selector: TEST_SELECTOR + ) + ) do |dialog| + dialog.with_confirmation_message do |message| + message.with_heading(tag: :h2) { t("admin.llm_models.destroy.heading", model: llm_model.external_id) } + message.with_description_content( + if bound_features.any? + t("admin.llm_models.destroy.description_bound", features: bound_features.to_sentence) + else + t("admin.llm_models.destroy.description") + end + ) + end + end +%> diff --git a/app/components/llm_connections/delete_model_dialog_component.rb b/app/components/llm_connections/delete_model_dialog_component.rb new file mode 100644 index 000000000000..970d5efdf961 --- /dev/null +++ b/app/components/llm_connections/delete_model_dialog_component.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class DeleteModelDialogComponent < ApplicationComponent + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + TEST_SELECTOR = "llm-model--delete-dialog" + + alias_method :llm_model, :model + + def form_arguments + { action: url_helpers.llm_model_path(llm_model), method: :delete } + end + + # Named so the message says what is actually at stake. The connection + # defaults count as bindings here -- deleting their model breaks every + # feature that inherits them. + def bound_features + affected_defaults + end + + def affected_defaults + %i[default_chat_model_id default_embedding_model_id] + .select { |attribute| llm_model.llm_connection.public_send(attribute) == llm_model.external_id } + .map { |attribute| LlmConnection.human_attribute_name(attribute) } + end + end +end diff --git a/app/components/llm_connections/models/sub_header_component.html.erb b/app/components/llm_connections/models/sub_header_component.html.erb index 0107aed88da3..6ecd35edc0f9 100644 --- a/app/components/llm_connections/models/sub_header_component.html.erb +++ b/app/components/llm_connections/models/sub_header_component.html.erb @@ -11,6 +11,17 @@ data: filter_input_data_attributes ) %> + <% subheader.with_action_button( + scheme: :primary, + label: t("admin.llm_connections.show.add_model_submit"), + mobile_label: t("admin.llm_connections.show.add_model_submit"), + mobile_icon: :plus, + leading_icon: :plus, + tag: :a, + href: helpers.new_llm_model_path, + test_selector: "llm-model--add-button" + ) { t("admin.llm_connections.show.add_model_submit") } %> + <% subheader.with_action_button( scheme: :default, label: t("admin.llm_connections.show.refresh_models"), diff --git a/app/components/llm_connections/models_row_component.rb b/app/components/llm_connections/models_row_component.rb index c638fec8fea7..cfc511511220 100644 --- a/app/components/llm_connections/models_row_component.rb +++ b/app/components/llm_connections/models_row_component.rb @@ -73,5 +73,25 @@ def source_label %i[secondary source_discovered] end + + def button_links + llm_model.manual? ? [edit_link, delete_link] : [edit_link] + end + + def edit_link + link_to(helpers.op_icon("icon-edit"), + url_helpers.edit_llm_model_path(llm_model), + data: { test_selector: "llm-model--edit-#{llm_model.id}" }, + title: I18n.t(:button_edit)) + end + + # Opens a DangerDialog rather than a browser confirm, so the message can say + # which features are bound to the model. + def delete_link + link_to(helpers.op_icon("icon-delete"), + url_helpers.delete_dialog_llm_model_path(llm_model), + data: { controller: "async-dialog", test_selector: "llm-model--delete-#{llm_model.id}" }, + title: I18n.t(:button_delete)) + end end end diff --git a/app/components/llm_connections/models_table_component.rb b/app/components/llm_connections/models_table_component.rb index a1fdb3f2f298..6974f57f469e 100644 --- a/app/components/llm_connections/models_table_component.rb +++ b/app/components/llm_connections/models_table_component.rb @@ -52,6 +52,9 @@ def initial_sort = %i[identifier asc] def has_footer? = false + # Without this the row's button_links are never rendered. + def has_actions? = true + def mobile_title = I18n.t("admin.llm_connections.show.models_heading") # The row class is otherwise derived by convention as LlmConnections::RowComponent. diff --git a/app/controllers/admin/llm_models_controller.rb b/app/controllers/admin/llm_models_controller.rb new file mode 100644 index 000000000000..7614f868f8e9 --- /dev/null +++ b/app/controllers/admin/llm_models_controller.rb @@ -0,0 +1,183 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + # Models an administrator enters by hand. + # + # Necessary because not every OpenAI-compatible server exposes a model list. + # A gateway may route /v1/chat/completions and nothing else, in which case the + # operator knows the model name and OpenProject cannot discover it. + class LlmModelsController < ApplicationController + include OpTurbo::ComponentStream + + layout "admin" + menu_item :llm_connection + + before_action :require_feature + before_action :require_admin + before_action :set_connection + + def new + @llm_model = @connection.models.new + end + + def edit + @llm_model = @connection.models.find(params.expect(:id)) + @verdicts = @connection.capability_verdicts.for_model(@llm_model.external_id).index_by(&:capability) + end + + def create + llm_model = @connection.models.new(llm_model_params.except(*capability_param_names).merge(manual: true)) + + if save_with_capabilities(llm_model) + flash[:notice] = t(".success", model: llm_model.external_id) + redirect_to llm_connection_path, status: :see_other + else + # Re-rendered rather than redirected so the Primer form shows the error + # inline against the field that caused it. + @llm_model = llm_model + render :new, status: :unprocessable_entity + end + end + + def update + @llm_model = @connection.models.find(params.expect(:id)) + + if update_with_capabilities(@llm_model) + flash[:notice] = t(".success", model: @llm_model.external_id) + redirect_to llm_connection_path, status: :see_other + else + # Re-rendered rather than redirected so the Primer form shows the error + # inline against the field that caused it, e.g. a rename that collides + # with an existing model id. + @verdicts = @connection.capability_verdicts.for_model(@llm_model.external_id_was).index_by(&:capability) + render :edit, status: :unprocessable_entity + end + end + + def delete_dialog + llm_model = @connection.models.manual.find(params.expect(:id)) + + respond_with_dialog LlmConnections::DeleteModelDialogComponent.new(llm_model) + end + + def destroy + llm_model = @connection.models.manual.find(params.expect(:id)) + destroy_with_verdicts(llm_model) + + flash[:notice] = t(".success", model: llm_model.external_id) + redirect_to llm_connection_path, status: :see_other + end + + private + + def set_connection + @connection = LlmConnection.instance + end + + # Verdicts are keyed by the identifier string, not by foreign key, so they + # would silently apply to a future model re-added under the same name. + def destroy_with_verdicts(llm_model) + ActiveRecord::Base.transaction do + llm_model.destroy! + @connection.capability_verdicts.for_model(llm_model.external_id).delete_all + end + end + + # The flag gates the endpoints, not only the menu entry: an unfinished page + # must not accept writes just because somebody knows the URL. + def require_feature + render_404 unless OpenProject::FeatureDecisions.llm_connection_active? + end + + def update_with_capabilities(llm_model) + attributes = llm_model_params.except(*capability_param_names) + # A discovered model is named by the server; only a hand-entered one may be + # renamed here, and everything referencing the old name follows it. + attributes = attributes.except(:external_id) unless llm_model.manual? + + saved = false + ActiveRecord::Base.transaction do + previous_external_id = llm_model.external_id + llm_model.assign_attributes(attributes) + raise ActiveRecord::Rollback unless llm_model.save + + llm_model.cascade_rename!(previous_external_id) + apply_capabilities(llm_model) + saved = true + end + saved + end + + # external_id is accepted on create, and on update for manually added models. + def llm_model_params + params.expect(llm_model: [:external_id, :display_name, :admin_context_window, *capability_param_names]) + end + + def capability_param_names + Llm::Capabilities::ALL.map { |capability| :"capability_#{capability}" } + end + + def save_with_capabilities(llm_model) + ActiveRecord::Base.transaction do + llm_model.save! + apply_capabilities(llm_model) + end + + true + rescue ActiveRecord::RecordInvalid + false + end + + # Stored as admin-sourced verdicts, which survive re-detection: an + # administrator knows things about their deployment that neither a published + # registry nor a probe can determine. + def apply_capabilities(llm_model) + submitted = llm_model_params + + Llm::Capabilities::ALL.each do |capability| + assert(llm_model.external_id, capability, submitted[:"capability_#{capability}"].presence) + end + end + + def assert(model_id, capability, state) + verdict = @connection.capability_verdicts + .find_or_initialize_by(model_id:, capability: capability.to_s) + + if state.blank? + # "Not specified" clears an assertion rather than recording ignorance as + # fact; detection may fill it in later. + verdict.destroy! if verdict.persisted? && verdict.source_admin? + else + verdict.update!(state:, source: "admin", checked_at: Time.current) + end + end + end +end diff --git a/app/forms/llm_models/form.rb b/app/forms/llm_models/form.rb new file mode 100644 index 000000000000..444767ca85bf --- /dev/null +++ b/app/forms/llm_models/form.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmModels + class Form < ApplicationForm + form do |f| + # A discovered model is named by the server, so its identifier is not ours + # to change -- the next refresh would only put it back. One entered by hand + # is editable, because a typo in it is otherwise unfixable except by + # deleting the model and losing everything asserted about it. Renaming + # cascades; see LlmModel#cascade_rename!. + if new_record? || model.manual? + f.text_field( + name: :external_id, + label: LlmModel.human_attribute_name(:external_id), + caption: I18n.t("admin.llm_models.form.external_id_caption"), + placeholder: "qwen3.6-35b-a3b", + required: true, + autocomplete: "off", + input_width: :large, + data: { test_selector: "llm-model--external-id" } + ) + end + + f.text_field( + name: :display_name, + label: LlmModel.human_attribute_name(:display_name), + caption: I18n.t("admin.llm_models.form.display_name_caption"), + autocomplete: "off", + input_width: :large, + data: { test_selector: "llm-model--display-name" } + ) + + f.text_field( + name: :admin_context_window, + label: I18n.t("admin.llm_models.form.context_window"), + caption: context_window_caption, + type: :number, + min: 1, + autocomplete: "off", + input_width: :medium, + data: { test_selector: "llm-model--context-window" } + ) + + Llm::Capabilities::ALL.each do |capability| + f.select_list( + name: :"capability_#{capability}", + label: Llm::Capabilities.label(capability), + caption: capability_caption(capability), + include_blank: false, + input_width: :medium, + data: { test_selector: "llm-model--capability-#{capability}" } + ) do |select| + select.option(value: "", label: inherited_state_label(capability)) + select.option(value: "supported", label: I18n.t("admin.llm_models.form.state_supported")) + select.option(value: "unsupported", label: I18n.t("admin.llm_models.form.state_unsupported")) + end + end + + f.submit( + name: :submit, + label: new_record? ? I18n.t("admin.llm_models.form.create_submit") : I18n.t(:button_save), + scheme: :primary, + data: { test_selector: "llm-model--submit" } + ) + end + + private + + def new_record? = model.new_record? + + def context_window_caption + source = model.context_window_source + + if source.present? && source != :admin + I18n.t("admin.llm_models.form.context_window_known", + value: model.context_window, + source: I18n.t("llm.context_window_sources.#{source}")) + else + I18n.t("admin.llm_models.form.context_window_caption") + end + end + + # A verdict established by a probe or a registry is never loaded into the + # field -- saving must not turn someone else's finding into the + # administrator's assertion -- so the blank option names what applies while + # nothing is asserted here. Labelling it "Not specified" next to a caption + # reading "Currently Supported" stated two contradictory things at once. + def inherited_state_label(capability) + verdict = model.verdict_for(capability) + return I18n.t("admin.llm_models.form.state_unspecified") if verdict.nil? || verdict.source_admin? + + I18n.t("admin.llm_models.form.state_inherited", + state: I18n.t("llm.verdict_states.#{verdict.state}"), + source: I18n.t("llm.verdict_sources.#{verdict.source}")) + end + + # The state is now carried by the option itself; the caption only has to say + # what choosing something else here means. + def capability_caption(capability) + verdict = model.verdict_for(capability) + return if verdict.nil? || verdict.source_admin? + + I18n.t("admin.llm_models.form.capability_override_caption") + end + end +end diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb index eff2b53c2ae9..b573e0cb41c8 100644 --- a/app/models/llm_model.rb +++ b/app/models/llm_model.rb @@ -44,8 +44,29 @@ class LlmModel < ApplicationRecord scope :manual, -> { where(manual: true) } scope :by_identifier, -> { order(:external_id) } + # Everything that points at a model does so by its identifier string, so a + # rename has to carry them along or it silently orphans them. + # + # Renaming is a correction of the name, not a change of model, which is why + # this writes directly: a locked binding must not refuse to follow the model + # it is locked to, and the connection defaults are pointing at this very row. + def cascade_rename!(previous_external_id) + return if previous_external_id.blank? || previous_external_id == external_id + + llm_connection.capability_verdicts.where(model_id: previous_external_id).update_all(model_id: external_id) + rename_connection_defaults(previous_external_id) + end + def name = display_name.presence || external_id + def rename_connection_defaults(previous_external_id) + defaults = %i[default_chat_model_id default_embedding_model_id] + .select { |attribute| llm_connection.public_send(attribute) == previous_external_id } + .index_with { external_id } + + llm_connection.update_columns(defaults) if defaults.any? + end + # Precedence: what an administrator set, then what the server reported (vLLM # and SGLang publish the operator's actual --max-model-len), then what a # registry believes about the model in general. @@ -55,6 +76,8 @@ def context_window raw_metadata["context_window"] end + def admin_context_window = raw_metadata["admin_context_window"] + def context_window_source return :admin if raw_metadata["admin_context_window"].present? return :server if raw_metadata["max_model_len"].present? @@ -63,6 +86,36 @@ def context_window_source nil end + def admin_context_window=(value) + self.raw_metadata = if value.blank? + raw_metadata.except("admin_context_window") + else + raw_metadata.merge("admin_context_window" => value.to_i) + end + end + + # Capability assertions are stored as verdicts, not columns. These virtual + # attributes let the edit form treat them as ordinary fields, so the whole + # screen can be a single Primer form rather than hand-written inputs. + Llm::Capabilities::ALL.each do |capability| + define_method(:"capability_#{capability}") do + capability_overrides.fetch(capability.to_s) { admin_capability_state(capability) } + end + + define_method(:"capability_#{capability}=") do |value| + capability_overrides[capability.to_s] = value.presence + end + end + + def capability_overrides = @capability_overrides ||= {} + + # Only an administrator's own assertion is shown as the field's value. A + # verdict from a probe or a registry is displayed alongside instead, so that + # saving the form does not silently adopt it as the administrator's. + def admin_capability_state(capability) + verdict_for(capability)&.then { |verdict| verdict.source_admin? ? verdict.state : nil } + end + def verdict_for(capability) llm_connection.capability_verdicts .for_model(external_id) diff --git a/app/views/admin/llm_models/edit.html.erb b/app/views/admin/llm_models/edit.html.erb new file mode 100644 index 000000000000..c6a5cfcbb6cd --- /dev/null +++ b/app/views/admin/llm_models/edit.html.erb @@ -0,0 +1,52 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection"), @llm_model.external_id %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { @llm_model.external_id } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + { href: llm_connection_path, text: t("menus.admin.llm_connection") }, + @llm_model.external_id] + ) + end +%> + +<%= settings_primer_form_with( + model: @llm_model, + url: llm_model_path(@llm_model), + method: :patch, + data: { test_selector: "llm-model--edit-form" } + ) do |f| %> + <%= render(LlmModels::Form.new(f)) %> +<% end %> diff --git a/app/views/admin/llm_models/new.html.erb b/app/views/admin/llm_models/new.html.erb new file mode 100644 index 000000000000..ebba7e58487c --- /dev/null +++ b/app/views/admin/llm_models/new.html.erb @@ -0,0 +1,54 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection"), t(".title") %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t(".title") } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + { href: llm_connection_path, text: t("menus.admin.llm_connection") }, + t(".title")] + ) + end +%> + +<%= + settings_primer_form_with( + model: @llm_model, + url: llm_models_path, + method: :post, + data: { test_selector: "llm-model--add-form" } + ) do |f| + render(LlmModels::Form.new(f)) + end +%> diff --git a/config/locales/en.yml b/config/locales/en.yml index 6a7bbabba0fa..77f49be763a0 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1754,6 +1754,35 @@ en: update: no_models: "Saved, but the server did not return a model list. Either it does not offer one, or the endpoint is wrong — a URL missing its API version segment (for example /v1) looks exactly the same from here. Add the models you want to use below, then run the health checks to confirm the server answers." success: "Successfully connected to the LLM server." + llm_models: + create: + success: "%{model} has been added." + destroy: + description: "The model will no longer be offered to AI features." + description_bound: "This model is currently used by %{features}. Those features will stop working until another model is selected." + heading: "Remove %{model}?" + success: "%{model} has been removed." + title: "Remove model" + edit: + description: "Edit how OpenProject uses this model." + form: + capabilities_description: "Leave a capability unspecified to let OpenProject determine it. Marking Embeddings as supported makes this an embedding model, offered only to features that need one." + capability_override_caption: "Choose a value to override this." + context_window: "Context window" + context_window_caption: "How many tokens this model accepts. Leave blank if you do not know." + context_window_known: "Leave blank to use %{value}, %{source}." + create_submit: "Add model" + display_name_caption: "An optional friendly name shown instead of the model id." + external_id_caption: "The model name exactly as the server expects it." + state_inherited: "%{state} (%{source})" + state_supported: "Supported" + state_unspecified: "Not specified" + state_unsupported: "Not supported" + new: + description: "Name a model this server can use but does not advertise, and describe what it can do." + title: "Add a model" + update: + success: "%{model} has been updated." mcp_configurations: index: description: "The model context protocol allows AI agents to provide its users with tools and resources exposed by this OpenProject instance. This feature is still in beta." @@ -4239,6 +4268,17 @@ en: perplexity: "Perplexity" vertexai: "Google Vertex AI" xai: "xAI" + capabilities: + embeddings: + label: "Embeddings" + function_calling: + label: "Tool calling" + reasoning: + label: "Reasoning" + structured_output: + label: "Structured output" + vision: + label: "Vision" context_window_sources: registry: "the figure published for this model" server: "reported by the server" @@ -4246,6 +4286,15 @@ en: chat: "Chat" embedding: "Embedding" unknown: "Unknown" + verdict_sources: + admin: "set by an administrator" + metadata: "from the model registry" + observed: "observed in use" + probe: "verified against the server" + verdict_states: + supported: "Supported" + unknown: "Not verified" + unsupported: "Not supported" llm_connections: macro_execution_error: "Error executing the macro %{macro_name}" macro_unavailable: "Macro %{macro_name} cannot be displayed." diff --git a/config/routes.rb b/config/routes.rb index de2c81e44942..86ef3b0ffe80 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -751,6 +751,13 @@ get :search_models, defaults: { format: :turbo_stream } end + # Manual entries only; discovered models are managed by the sync. + resources :llm_models, only: %i[new create edit update destroy], controller: "admin/llm_models" do + member do + get :delete_dialog + end + end + resources :mcp_configurations, only: %i[index update], controller: "admin/mcp_configurations" do collection do post :multi_update diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb new file mode 100644 index 000000000000..b9a794034726 --- /dev/null +++ b/spec/requests/admin/llm_models_spec.rb @@ -0,0 +1,305 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +# Manual entry exists for servers that route /v1/chat/completions but expose no +# model list -- OpenProject's own hosted gateway does exactly that today. +RSpec.describe "Admin manual LLM models", :llm_server_helpers, :skip_csrf, :webmock, + type: :rails_request, with_flag: { llm_connection: true } do + let(:admin) { create(:admin) } + let(:base_url) { "https://example.com/v1" } + let!(:connection) { create(:llm_connection, base_url:) } + + before { login_as admin } + + describe "POST /admin/llm_models" do + it "accepts everything the edit screen accepts" do + post llm_models_path, params: { llm_model: { external_id: "bge-m3", + display_name: "BGE M3", + admin_context_window: "8192", + capability_embeddings: "supported", + capability_vision: "unsupported" } } + + llm_model = connection.models.find_by(external_id: "bge-m3") + expect(llm_model.display_name).to eq("BGE M3") + expect(llm_model.context_window).to eq(8192) + + verdicts = connection.capability_verdicts.for_model("bge-m3").pluck(:capability, :state, :source) + expect(verdicts).to include(["embeddings", "supported", "admin"], ["vision", "unsupported", "admin"]) + end + + it "adds a model an administrator names" do + post llm_models_path, params: { llm_model: { external_id: "qwen3.6-35b-a3b" } } + + expect(response).to have_http_status(:see_other) + llm_model = connection.models.find_by(external_id: "qwen3.6-35b-a3b") + expect(llm_model).to be_manual + expect(connection.available_model_ids).to include("qwen3.6-35b-a3b") + end + + it "rejects a duplicate" do + create(:llm_model, llm_connection: connection, external_id: "already-there") + + post llm_models_path, params: { llm_model: { external_id: "already-there" } } + + expect(connection.models.where(external_id: "already-there").count).to eq(1) + end + end + + describe "a refresh that cannot see the manual model" do + it "keeps it, and withdraws discovered models instead" do + create(:llm_model, llm_connection: connection, external_id: "was-discovered") + post llm_models_path, params: { llm_model: { external_id: "hand-typed" } } + mock_llm_models_response(base_url) + + post refresh_models_llm_connection_path + + expect(connection.models.find_by(external_id: "hand-typed")).to be_active + expect(connection.models.find_by(external_id: "was-discovered")).not_to be_active + expect(connection.available_model_ids).to include("hand-typed", "qwen3.6-27b") + end + end + + describe "PATCH /admin/llm_models/:id" do + let!(:llm_model) { create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") } + + it "stores capabilities an administrator asserts" do + patch llm_model_path(llm_model), + params: { llm_model: { display_name: "Hand typed", + capability_embeddings: "supported", + capability_vision: "unsupported" } } + + expect(response).to have_http_status(:see_other) + expect(llm_model.reload.display_name).to eq("Hand typed") + + verdicts = connection.capability_verdicts.for_model("hand-typed").pluck(:capability, :state, :source) + expect(verdicts).to include(["embeddings", "supported", "admin"], ["vision", "unsupported", "admin"]) + end + + it "stores a context window an administrator supplies" do + patch llm_model_path(llm_model), params: { llm_model: { admin_context_window: "32768" } } + + expect(llm_model.reload.context_window).to eq(32_768) + expect(llm_model.context_window_source).to eq(:admin) + end + + # The administrator's figure wins over whatever the server or a registry said. + it "prefers the administrator's context window over a reported one" do + llm_model.update!(raw_metadata: { "max_model_len" => 8192 }) + + patch llm_model_path(llm_model), params: { llm_model: { admin_context_window: "32768" } } + + expect(llm_model.reload.context_window).to eq(32_768) + end + + it "falls back to the reported figure when cleared" do + llm_model.update!(raw_metadata: { "max_model_len" => 8192, "admin_context_window" => 32_768 }) + + patch llm_model_path(llm_model), params: { llm_model: { admin_context_window: "" } } + + expect(llm_model.reload.context_window).to eq(8192) + expect(llm_model.context_window_source).to eq(:server) + end + + # Clearing an assertion records nothing rather than recording ignorance as + # fact, so detection can still fill it in later. + it "clears an assertion when set back to unspecified" do + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "" } } + + expect(connection.capability_verdicts.for_model("hand-typed").for_capability(:embeddings)).to be_empty + end + + # An administrator looked at this deployment; a published registry did not. + it "is not overwritten by registry enrichment" do + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } + + LlmConnections::EnrichCapabilitiesService.new(connection).call + + verdict = connection.capability_verdicts.find_by(model_id: "hand-typed", capability: "embeddings") + expect(verdict.source).to eq("admin") + expect(verdict.state).to eq("supported") + end + end + + describe "the model type shown in the list" do + it "reads as an embedding model once embeddings are supported" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "bge-m3") + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } + + get llm_connection_path + + expect(response.body).to include("Embedding") + end + + it "reads as a chat model when embeddings are not supported" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "qwen") + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "unsupported" } } + + get llm_connection_path + + expect(response.body).to include("Chat") + end + end + + describe "GET /admin/llm_models/new" do + it "renders the add-model form" do + get new_llm_model_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Model name") + end + + it "re-renders with the error inline when the name is taken" do + create(:llm_model, llm_connection: connection, external_id: "already-there") + + post llm_models_path, params: { llm_model: { external_id: "already-there" } } + + expect(response).to have_http_status(:unprocessable_entity) + expect(connection.models.where(external_id: "already-there").count).to eq(1) + end + end + + describe "renaming to a taken id" do + it "re-renders the form with the error instead of failing" do + create(:llm_model, llm_connection: connection, external_id: "taken") + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "mine") + + patch llm_model_path(llm_model), params: { llm_model: { external_id: "taken" } } + + expect(response).to have_http_status(:unprocessable_entity) + expect(llm_model.reload.external_id).to eq("mine") + end + end + + describe "DELETE /admin/llm_models/:id" do + it "removes a manual model" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") + + delete llm_model_path(llm_model) + + expect(response).to have_http_status(:see_other) + expect(LlmModel.where(id: llm_model.id)).to be_empty + end + + # Discovered models are the server's to add and remove, not the administrator's. + it "refuses to remove a discovered model" do + llm_model = create(:llm_model, llm_connection: connection, external_id: "from-server") + + delete llm_model_path(llm_model) + + expect(response).to have_http_status(:not_found) + expect(LlmModel.where(id: llm_model.id)).to exist + end + end + + describe "renaming a manually added model" do + let!(:llm_model) do + create(:llm_model, :manual, llm_connection: connection, external_id: "qwen/qwen3.6-35b-a3b") + end + + before do + connection.update!(default_chat_model_id: "qwen/qwen3.6-35b-a3b") + connection.capability_verdicts.create!(model_id: "qwen/qwen3.6-35b-a3b", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: Time.current) + end + + it "offers the identifier field on the edit page" do + get edit_llm_model_path(llm_model) + + expect(response).to have_http_status(:ok) + expect(response.body).to include("llm_model[external_id]") + end + + it "does not offer it for a discovered model" do + discovered = create(:llm_model, llm_connection: connection, external_id: "server-named") + + get edit_llm_model_path(discovered) + + expect(response).to have_http_status(:ok) + expect(response.body).not_to include("llm_model[external_id]") + end + + # A typo in a hand-typed identifier was previously only fixable by deleting + # the model, which threw away everything asserted about it. + it "renames it and carries every reference along" do + patch llm_model_path(llm_model), params: { llm_model: { external_id: "qwen/qwen3.6-35b-a3b:bf16" } } + + expect(llm_model.reload.external_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + expect(connection.reload.default_chat_model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + expect(connection.capability_verdicts.first.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + end + + # The server names its own models; renaming one here would only be undone by + # the next refresh. + it "refuses to rename a discovered model" do + discovered = create(:llm_model, llm_connection: connection, external_id: "server-named") + + patch llm_model_path(discovered), params: { llm_model: { external_id: "renamed" } } + + expect(discovered.reload.external_id).to eq("server-named") + end + end + + describe "how an inherited capability verdict is shown" do + let!(:llm_model) { create(:llm_model, :manual, llm_connection: connection, external_id: "qwen3.6-27b") } + + # The blank option used to read "Not specified" while the caption underneath + # read "Currently Supported, from the model registry" -- two contradictory + # statements about the same field. + it "names the inherited value in the option itself" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "function_calling", + state: "supported", source: "metadata", checked_at: Time.current) + + get edit_llm_model_path(llm_model) + + expect(response.body).to include("Supported (from the model registry)") + expect(response.body).not_to include("Currently Supported") + end + + it "falls back to Not specified when nothing is known" do + get edit_llm_model_path(llm_model) + + expect(response.body).to include("Not specified") + end + + # An administrator's own assertion is loaded into the field, so the blank + # option must not claim it as inherited. + it "does not present an administrator's own assertion as inherited" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "function_calling", + state: "supported", source: "admin", checked_at: Time.current) + + get edit_llm_model_path(llm_model) + + expect(response.body).not_to include("Supported (set by an administrator)") + end + end +end