diff --git a/Gemfile b/Gemfile index 6f420fd300c7..2c502bdccea3 100644 --- a/Gemfile +++ b/Gemfile @@ -256,6 +256,12 @@ gem "turbo-rails", "~> 2.0.20" gem "httpx", "~> 1.8.1" +# Provider adapters and a model metadata registry for the AI features. Used as +# transport and as a source of published model capabilities; what a given +# connection actually offers is tracked in llm_models / llm_capability_verdicts, +# never in RubyLLM's application-wide registry. +gem "ruby_llm", "~> 1.16" + # Brings actual deep-freezing to most ruby objects gem "ice_nine" diff --git a/Gemfile.lock b/Gemfile.lock index 56c6d1800c59..6b7503cd3349 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -571,6 +571,7 @@ GEM escape_utils (1.3.0) et-orbi (1.4.0) tzinfo + event_stream_parser (1.0.0) eventmachine (1.2.7) eventmachine_httpserver (0.2.1) excon (1.5.0) @@ -586,8 +587,12 @@ GEM logger faraday-follow_redirects (0.5.0) faraday (>= 1, < 3) + faraday-multipart (1.2.0) + multipart-post (~> 2.0) faraday-net_http (3.4.4) net-http (~> 0.5) + faraday-retry (2.4.0) + faraday (~> 2.0) ferrum (0.17.2) addressable (~> 2.5) base64 (~> 0.2) @@ -848,6 +853,7 @@ GEM prism (~> 1.5) msgpack (1.8.4) multi_json (1.21.1) + multipart-post (2.4.1) mustermann (4.0.0) net-http (0.9.1) uri (>= 0.11.1) @@ -1390,6 +1396,17 @@ GEM ruby-vips (2.3.0) ffi (~> 1.12) logger + ruby_llm (1.16.0) + base64 + event_stream_parser (~> 1) + faraday (>= 1.10.0) + faraday-multipart (>= 1) + faraday-net_http (>= 1) + faraday-retry (>= 1) + marcel (~> 1) + ruby_llm-schema (~> 0) + zeitwerk (~> 2) + ruby_llm-schema (0.4.0) rubytree (2.2.1) json (~> 2.0, >= 2.19.9) rubyzip (2.4.1) @@ -1746,6 +1763,7 @@ DEPENDENCIES ruby-duration (~> 3.2.0) ruby-prof ruby-progressbar (~> 1.13.0) + ruby_llm (~> 1.16) rubytree (~> 2.2.1) sanitize (~> 7.0.0) scimitar (~> 2.13) @@ -1842,6 +1860,7 @@ CHECKSUMS browser (6.2.0) sha256=281d5295788825c9396427c292c2d2be0a5c91875c93c390fde6e5d61a5ace2d budgets (1.0.0) builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f + bundler (4.0.16) sha256=d6ca5dd440c24f9abce9844cf44cc8e18c6a553de65a47efb4544137af92c47d byebug (13.0.0) sha256=d2263efe751941ca520fa29744b71972d39cbc41839496706f5d9b22e92ae05d capybara (3.40.0) sha256=42dba720578ea1ca65fd7a41d163dd368502c191804558f6e0f71b391054aeef capybara-screenshot (1.0.27) sha256=afa1896cc23df77be1774e8d3b3ce3953bf060aeaa04ff87607b5daf689174f2 @@ -1909,6 +1928,7 @@ CHECKSUMS erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9 escape_utils (1.3.0) sha256=dffb7010922880ace6ceed642156c64e2a64620f27e0849f43bc4f68fd3c2c09 et-orbi (1.4.0) sha256=6c7e3c90779821f9e3b324c5e96fda9767f72995d6ae435b96678a4f3e2de8bc + event_stream_parser (1.0.0) sha256=a2683bab70126286f8184dc88f7968ffc4028f813161fb073ec90d171f7de3c8 eventmachine (1.2.7) sha256=994016e42aa041477ba9cff45cbe50de2047f25dd418eba003e84f0d16560972 eventmachine_httpserver (0.2.1) sha256=5db5e8a23754204d43592e5fcc2160457c57c870babe6307c4e61fc95019b809 excon (1.5.0) sha256=c503ad1d0123bc8ab2a062ff3789dc891ec368cb9e13765ab88a9c58c8bb6d50 @@ -1916,7 +1936,9 @@ CHECKSUMS factory_bot_rails (6.5.1) sha256=d3cc4851eae4dea8a665ec4a4516895045e710554d2b5ac9e68b94d351bc6d68 faraday (2.14.3) sha256=1882247e6766615c8220b4392bf1d27f6ebb63d8e28267587cef1fb0bf37f278 faraday-follow_redirects (0.5.0) sha256=5cde93c894b30943a5d2b93c2fe9284216a6b756f7af406a1e55f211d97d10ad + faraday-multipart (1.2.0) sha256=7d89a949693714176f612323ca13746a2ded204031a6ba528adee788694ef757 faraday-net_http (3.4.4) sha256=0e78af151747ed1b00f33e25973b4bc220d7f16c00c39676817c8b12331eb588 + faraday-retry (2.4.0) sha256=7b79c48fb7e56526faf247b12d94a680071ff40c9fda7cf1ec1549439ad11ebe ferrum (0.17.2) sha256=2c2540a850b211a46f4d81de21bfd62048f507e4c327d1807225c3823c17e6ee ffi (1.17.4-aarch64-linux-gnu) sha256=b208f06f91ffd8f5e1193da3cae3d2ccfc27fc36fba577baf698d26d91c080df ffi (1.17.4-aarch64-linux-musl) sha256=9286b7a615f2676245283aef0a0a3b475ae3aae2bb5448baace630bb77b91f39 @@ -2016,6 +2038,7 @@ CHECKSUMS minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1 msgpack (1.8.4) sha256=4411c22d350dd1c20250f7eada3cca2695438c2f769cf0782f0cd065d90a3e7b multi_json (1.21.1) sha256=e6126a31808e3b4d19f483c775ceac34df190dffa62adfb63a165ee14ba68080 + multipart-post (2.4.1) sha256=9872d03a8e552020ca096adadbf5e3cb1cd1cdd6acd3c161136b8a5737cdb4a8 mustermann (4.0.0) sha256=91f67411bb208d1d93c41e6128cb3b0f8ddd9ec7c45966f1007e1c43c08040d7 my_page (1.0.0) net-http (0.9.1) sha256=25ba0b67c63e89df626ed8fac771d0ad24ad151a858af2cc8e6a716ca4336996 @@ -2238,6 +2261,8 @@ CHECKSUMS ruby-rc4 (0.1.5) sha256=00cc40a39d20b53f5459e7ea006a92cf584e9bc275e2a6f7aa1515510e896c03 ruby-saml (1.18.1) sha256=1b0e7a44aef150b4197955f5e015d593672e242cfdc5d06aa7554ec2350b9107 ruby-vips (2.3.0) sha256=e685ec02c13969912debbd98019e50492e12989282da5f37d05f5471442f5374 + ruby_llm (1.16.0) sha256=26bd5310cf2ce55f74a60f8aae0b0d0327b586ff4532c84828103c3b2b905a18 + ruby_llm-schema (0.4.0) sha256=e930f5a5316f9301bff3fb7fe572e44727d05bb8e50621001bbb49a47d63b8da rubytree (2.2.1) sha256=237f930795479fad48d8860cc8ec057534864b7da92dbee6a3c81470ad00c5c8 rubyzip (2.4.1) sha256=8577c88edc1fde8935eb91064c5cb1aef9ad5494b940cf19c775ee833e075615 safety_net_attestation (0.5.0) sha256=c8cd01dd550dbe8553862918af6355a04672db11d218ec96104ce3955293f2aa diff --git a/app/components/health_reports/report_component.html.erb b/app/components/health_reports/report_component.html.erb index 0afbae54edd1..cb0922c4f74d 100644 --- a/app/components/health_reports/report_component.html.erb +++ b/app/components/health_reports/report_component.html.erb @@ -67,7 +67,7 @@ See COPYRIGHT and LICENSE files for more details. result_group.results.each do |value| box.with_row do - render(HealthReports::ResultComponent.new(group: result_group.key, result: value, i18n_scope:)) + render(HealthReports::ResultComponent.new(group: result_group.key, result: value, i18n_scope:, docs_href:)) end end end diff --git a/app/components/health_reports/report_component.rb b/app/components/health_reports/report_component.rb index 9778ff26ff45..08a988cd7d79 100644 --- a/app/components/health_reports/report_component.rb +++ b/app/components/health_reports/report_component.rb @@ -37,14 +37,18 @@ class ReportComponent < ApplicationComponent # The i18n_scope parameter defines the I18n scope that should be used to resolve # names of groups, checks and error messages indicated by the results. - def initialize(*, i18n_scope:, **) + # + # docs_href overrides where each result's "More information" link points; + # without it, results link to the file storages troubleshooting page. + def initialize(*, i18n_scope:, docs_href: nil, **) super(*, **) @i18n_scope = i18n_scope + @docs_href = docs_href end private - attr_reader :i18n_scope + attr_reader :i18n_scope, :docs_href def summary_scheme(check_tally) case check_tally diff --git a/app/components/health_reports/result_component.html.erb b/app/components/health_reports/result_component.html.erb index 67b17ee95405..8335ceba3e65 100644 --- a/app/components/health_reports/result_component.html.erb +++ b/app/components/health_reports/result_component.html.erb @@ -38,18 +38,20 @@ See COPYRIGHT and LICENSE files for more details. line.with_column(mr: 2) do render(Primer::Beta::Text.new(font_size: :small, color: status_color)) { status_text } end - if error_code.present? + if error_code.present? && docs_href.present? line.with_column do render(Primer::Beta::Label.new(scheme: status_color)) { error_code } end end end - if error_code.present? + if error_code.present? && docs_href.present? row.with_column do - helpers.static_link_to(href: docs_href, - label: I18n.t(:label_more_information), - underline: true) + helpers.static_link_to( + href: docs_href, + label: I18n.t(:label_more_information), + underline: true + ) end end end diff --git a/app/components/health_reports/result_component.rb b/app/components/health_reports/result_component.rb index 5a57739f09f7..24cccb0d8d6d 100644 --- a/app/components/health_reports/result_component.rb +++ b/app/components/health_reports/result_component.rb @@ -32,10 +32,18 @@ module HealthReports class ResultComponent < ApplicationComponent include OpPrimer::ComponentHelpers - def initialize(group:, result:, i18n_scope:) + # Where "More information" points. Defaults to the file storages + # documentation because that was this component's only consumer for a long + # time; a subject with its own troubleshooting page passes its own, and one + # with no page yet passes false to suppress the link rather than send an + # administrator somewhere unrelated. + DEFAULT_DOCS_HREF = -> { ::OpenProject::Static::Links.url_for(:storage_docs, :health_status) } + + def initialize(group:, result:, i18n_scope:, docs_href: nil) super(result) @group = group @i18n_scope = i18n_scope + @docs_href = docs_href end private @@ -49,7 +57,11 @@ def error_text I18n.t("errors.#{model.code}", scope: @i18n_scope, **model.context&.symbolize_keys) end - def docs_href = ::OpenProject::Static::Links.url_for(:storage_docs, :health_status) + def docs_href + return @docs_href if @docs_href == false + + @docs_href || DEFAULT_DOCS_HREF.call + end def error_code if model.failure? diff --git a/app/components/llm_connections/delete_api_key_dialog_component.html.erb b/app/components/llm_connections/delete_api_key_dialog_component.html.erb new file mode 100644 index 000000000000..0c317a9534ec --- /dev/null +++ b/app/components/llm_connections/delete_api_key_dialog_component.html.erb @@ -0,0 +1,22 @@ +<%= + render( + Primer::OpenProject::DangerDialog.new( + title: t("admin.llm_connections.delete_api_key.title"), + form_arguments:, + # Defaults to "Delete", which is wrong here: neither action deletes anything. + confirm_button_text: t("admin.llm_connections.delete_api_key.title"), + test_selector: TEST_SELECTOR + ) + ) do |dialog| + dialog.with_confirmation_message do |message| + message.with_heading(tag: :h2) { t("admin.llm_connections.delete_api_key.heading") } + message.with_description_content( + if loses_admin_verdicts? + t("admin.llm_connections.delete_api_key.description_verdicts") + else + t("admin.llm_connections.delete_api_key.description") + end + ) + end + end +%> diff --git a/app/components/llm_connections/delete_api_key_dialog_component.rb b/app/components/llm_connections/delete_api_key_dialog_component.rb new file mode 100644 index 000000000000..6c0aa7280fee --- /dev/null +++ b/app/components/llm_connections/delete_api_key_dialog_component.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Confirms removing the stored API key. + # + # No confirmation checkbox: the key itself can simply be pasted again. The + # dialog exists for what is *not* recoverable -- see #loses_admin_verdicts?. + class DeleteApiKeyDialogComponent < ApplicationComponent + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + TEST_SELECTOR = "llm-connection--delete-api-key-dialog" + + alias_method :connection, :model + + def form_arguments + { action: url_helpers.api_key_llm_connection_path, method: :delete } + end + + # The catalogue sync fingerprints base_url and api_key together, so the next + # refresh after the key changes treats the endpoint as a different deployment + # and discards every capability verdict -- including the ones an + # administrator asserted by hand, which nothing else in the system throws + # away. Worth saying out loud before the key goes. + def loses_admin_verdicts? + connection.capability_verdicts.exists?(source: "admin") + end + end +end diff --git a/app/components/llm_connections/delete_model_dialog_component.html.erb b/app/components/llm_connections/delete_model_dialog_component.html.erb new file mode 100644 index 000000000000..5fe5f2d5816c --- /dev/null +++ b/app/components/llm_connections/delete_model_dialog_component.html.erb @@ -0,0 +1,20 @@ +<%= + render( + Primer::OpenProject::DangerDialog.new( + title: t("admin.llm_models.destroy.title"), + form_arguments:, + test_selector: TEST_SELECTOR + ) + ) do |dialog| + dialog.with_confirmation_message do |message| + message.with_heading(tag: :h2) { t("admin.llm_models.destroy.heading", model: llm_model.external_id) } + message.with_description_content( + if bound_features.any? + t("admin.llm_models.destroy.description_bound", features: bound_features.to_sentence) + else + t("admin.llm_models.destroy.description") + end + ) + end + end +%> diff --git a/app/components/llm_connections/delete_model_dialog_component.rb b/app/components/llm_connections/delete_model_dialog_component.rb new file mode 100644 index 000000000000..d166425e2e3c --- /dev/null +++ b/app/components/llm_connections/delete_model_dialog_component.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class DeleteModelDialogComponent < ApplicationComponent + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + TEST_SELECTOR = "llm-model--delete-dialog" + + alias_method :llm_model, :model + + def form_arguments + { action: url_helpers.llm_model_path(llm_model), method: :delete } + end + + # Named so the message says what is actually at stake: features bound to this + # model stop resolving, rather than silently falling back to another one. + def bound_features + llm_model.llm_connection + .feature_bindings + .where(model_id: llm_model.external_id) + .filter_map { |binding| binding.feature&.label } + end + end +end diff --git a/app/components/llm_connections/disconnect_dialog_component.html.erb b/app/components/llm_connections/disconnect_dialog_component.html.erb new file mode 100644 index 000000000000..b5affa99625d --- /dev/null +++ b/app/components/llm_connections/disconnect_dialog_component.html.erb @@ -0,0 +1,37 @@ +<%= + render( + Primer::OpenProject::DangerDialog.new( + title: t("admin.llm_connections.disconnect.title"), + form_arguments:, + # Defaults to "Delete", which is wrong here: neither action deletes anything. + confirm_button_text: t("admin.llm_connections.disconnect.title"), + test_selector: TEST_SELECTOR + ) + ) do |dialog| + dialog.with_confirmation_message do |message| + message.with_heading(tag: :h2) { t("admin.llm_connections.disconnect.heading") } + message.with_description_content(t("admin.llm_connections.disconnect.description")) + end + + dialog.with_additional_details do + render(Primer::OpenProject::FlexLayout.new) do |flex| + flex.with_row do + content_tag(:ul) do + safe_join( + [ + content_tag(:li, t("admin.llm_connections.disconnect.keeps_settings")), + content_tag(:li, t("admin.llm_connections.disconnect.keeps_models")), + if bound_features.any? + content_tag( + :li, + t("admin.llm_connections.disconnect.keeps_bindings", features: bound_features.to_sentence) + ) + end + ].compact + ) + end + end + end + end + end +%> diff --git a/app/components/llm_connections/disconnect_dialog_component.rb b/app/components/llm_connections/disconnect_dialog_component.rb new file mode 100644 index 000000000000..4df992cf05da --- /dev/null +++ b/app/components/llm_connections/disconnect_dialog_component.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Confirms disconnecting from the LLM server. + # + # Disconnecting clears the credential and switches the connection off. It does + # not delete anything: the endpoint, the model catalogue, the capability + # verdicts and every feature binding are kept, so reconnecting is a matter of + # entering the key again. + # + # That is also why there is no confirmation checkbox -- nothing here is + # irreversible. A destroying variant would have been, and would have taken the + # locked embedding bindings with it: those are the only record that a vector + # index exists and which model and dimension it was written under, and + # dependent: :delete_all bypasses the guard that protects them. + class DisconnectDialogComponent < ApplicationComponent + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + TEST_SELECTOR = "llm-connection--disconnect-dialog" + + alias_method :connection, :model + + def form_arguments + { action: url_helpers.disconnect_llm_connection_path, method: :post } + end + + def bound_features + connection.feature_bindings.filter_map { |binding| binding.feature&.label if binding.model_id.present? } + end + end +end diff --git a/app/components/llm_connections/feature_binding_component.html.erb b/app/components/llm_connections/feature_binding_component.html.erb new file mode 100644 index 000000000000..762e4a7b5705 --- /dev/null +++ b/app/components/llm_connections/feature_binding_component.html.erb @@ -0,0 +1,53 @@ +<%= render(Primer::Box.new(border: true, border_radius: 2, p: 3, mb: 3)) do %> + <%= render(Primer::Beta::Text.new(tag: :h3, font_size: 4, font_weight: :bold, mb: 1)) { feature.label } %> + + <% if feature.caption.present? %> + <%= render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 2)) { feature.caption } %> + <% end %> + + <% if dangling? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :alert)) do %> + <%= t("admin.llm_feature_bindings.dangling", model: binding.resolved_model_id) %> + <% end %> + <% end %> + + <% if deactivated? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :alert)) do %> + <%= t("admin.llm_feature_bindings.deactivated", model: binding.resolved_model_id) %> + <% end %> + <% end %> + + <% if locked? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :lock)) do %> + <%= t("admin.llm_feature_bindings.locked", model: binding.model_id) %> + <% end %> + <% end %> + + <%= primer_form_with(model: form_model, url: form_url, method: :patch, scope: :llm_feature_binding) do |f| %> + <%= render( + LlmConnections::FeatureBindingForm.new( + f, + options: model_options, + inherit_label:, + feature_key: feature.key, + locked: locked?, + embedding: feature.embedding?, + selected_model_id: binding&.model_id, + dimensions_hint: probed_dimensions + ) + ) %> + <% end %> + + <% if locked? && feature.embedding? %> + <%# Rendered as text rather than disabled inputs: a disabled input submits + nothing, so the values would arrive blank and wipe the columns. %> + <%= render(Primer::Beta::Text.new(tag: :p, font_weight: :bold, mt: 2, mb: 1)) do %> + <%= t("admin.llm_feature_bindings.locked_values_heading") %> + <% end %> + <% locked_values.each do |label, value| %> + <%= render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 0)) do %> + <%= "#{label}: #{value}" %> + <% end %> + <% end %> + <% end %> +<% end %> diff --git a/app/components/llm_connections/feature_binding_component.rb b/app/components/llm_connections/feature_binding_component.rb new file mode 100644 index 000000000000..9d5d16e5a555 --- /dev/null +++ b/app/components/llm_connections/feature_binding_component.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # One feature's row on the model assignment page. + class FeatureBindingComponent < ApplicationComponent + include ApplicationHelper + include OpPrimer::ComponentHelpers + + def initialize(feature:, connection:, binding: nil) + super(feature) + @feature = feature + @connection = connection + @binding = binding + end + + # The record the select binds to. A feature without a stored binding still + # needs one so the form has a model_id to read. + def form_model + binding || connection.feature_bindings.new(feature_key: feature.key.to_s) + end + + # Not named +options+: ApplicationComponent already owns that name and + # initialises it to an empty hash, which silently swallowed the memoisation. + def model_options + @model_options ||= SelectableModelsQuery.new(connection, feature).call + end + + def inherit_label + if default_model_id.present? + I18n.t("admin.llm_feature_bindings.inherit_with_default", model: default_model_id) + else + I18n.t("admin.llm_feature_bindings.inherit_without_default") + end + end + + def locked? = binding&.locked? + + def dangling? = binding&.dangling? + + # What the embeddings probe last saw, offered as information. Never filled + # into the field: the server decides the vector size at index time. + def probed_dimensions + return unless feature.embedding? + + model_id = binding&.resolved_model_id + return if model_id.blank? + + connection.capability_verdicts.for_model(model_id).for_capability(:embeddings).first&.dimensions + end + + # Quoted so a trailing space -- load-bearing for the E5 and BGE families -- + # is visible rather than invisible. + def locked_values + [ + [LlmFeatureBinding.human_attribute_name(:model_id), binding.model_id], + [LlmFeatureBinding.human_attribute_name(:dimensions), binding.dimensions || "—"], + [LlmFeatureBinding.human_attribute_name(:input_prefix), binding.input_prefix.to_s.inspect], + [LlmFeatureBinding.human_attribute_name(:query_prefix), binding.query_prefix.to_s.inspect] + ] + end + + # Still resolvable, so not dangling -- but an administrator has hidden it + # from the pickers, so say so rather than let the choice look unremarkable. + def deactivated? + model_id = binding&.resolved_model_id + return false if model_id.blank? + + connection.models.deactivated.exists?(external_id: model_id) + end + + private + + attr_reader :feature, :connection, :binding + + def default_model_id + feature.embedding? ? connection.default_embedding_model_id : connection.default_chat_model_id + end + + def form_url + url_helpers.llm_feature_binding_path(feature.key) + end + end +end diff --git a/app/components/llm_connections/form_component.html.erb b/app/components/llm_connections/form_component.html.erb new file mode 100644 index 000000000000..401606ee473a --- /dev/null +++ b/app/components/llm_connections/form_component.html.erb @@ -0,0 +1,34 @@ +<%= + component_wrapper(tag: "turbo-frame", **wrapper_options) do + if connection.configured_from_env? + concat( + render(Primer::Alpha::Banner.new(mb: 3, icon: :info)) do + t("admin.banners.environment_configured_readonly") + end + ) + end + + concat( + settings_primer_form_with(**form_options) do |f| + render(LlmConnections::ConnectionForm.new(f)) + end + ) + + concat( + render( + Primer::Beta::Text.new( + tag: :div, + mt: 3, + hidden: true, + display: :flex, + align_items: :center, + color: :muted, + data: { "admin--llm-connection-form-target": "progressBanner" } + ) + ) do + concat(render(Primer::Beta::Spinner.new(size: :small, mr: 2))) + concat(t("admin.llm_connections.form.label_connecting")) + end + ) + end +%> diff --git a/app/components/llm_connections/form_component.rb b/app/components/llm_connections/form_component.rb new file mode 100644 index 000000000000..18313947a162 --- /dev/null +++ b/app/components/llm_connections/form_component.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class FormComponent < ApplicationComponent + include ApplicationHelper + include OpPrimer::ComponentHelpers + include OpTurbo::Streamable + + def self.wrapper_key = :llm_connection_form + + alias_method :connection, :model + + private + + def wrapper_options + { + data: { + controller: "admin--llm-connection-form", + test_selector: "llm-connection--form" + } + } + end + + def form_options + { + model: connection, + url: llm_connection_path, + method: :patch + } + end + end +end diff --git a/app/components/llm_connections/models/index_component.html.erb b/app/components/llm_connections/models/index_component.html.erb new file mode 100644 index 000000000000..84d1c5941804 --- /dev/null +++ b/app/components/llm_connections/models/index_component.html.erb @@ -0,0 +1,3 @@ +<%= component_wrapper do %> + <%= render(LlmConnections::ModelsTableComponent.new(rows:, connection:)) %> +<% end %> diff --git a/app/components/llm_connections/models/index_component.rb b/app/components/llm_connections/models/index_component.rb new file mode 100644 index 000000000000..844b94fe199a --- /dev/null +++ b/app/components/llm_connections/models/index_component.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + module Models + # A turbo-frame around the model table, so filtering can replace just the + # table rather than reloading the settings page. + class IndexComponent < ApplicationComponent + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + alias_method :rows, :model + + def initialize(rows, connection:) + super(rows) + @connection = connection + end + + attr_reader :connection + end + end +end diff --git a/app/components/llm_connections/models/sub_header_component.html.erb b/app/components/llm_connections/models/sub_header_component.html.erb new file mode 100644 index 000000000000..6ecd35edc0f9 --- /dev/null +++ b/app/components/llm_connections/models/sub_header_component.html.erb @@ -0,0 +1,36 @@ +<%= render(Primer::OpenProject::SubHeader.new(collapsed_search: false, data: sub_header_data_attributes)) do |subheader| %> + <% subheader.with_filter_input( + name: "name", + label: t("admin.llm_connections.models.filter_label"), + visually_hide_label: true, + value: filter_input_value, + placeholder: t("admin.llm_connections.models.filter_label"), + leading_visual: { icon: :search, size: :small }, + show_clear_button: true, + clear_button_id:, + data: filter_input_data_attributes + ) %> + + <% subheader.with_action_button( + scheme: :primary, + label: t("admin.llm_connections.show.add_model_submit"), + mobile_label: t("admin.llm_connections.show.add_model_submit"), + mobile_icon: :plus, + leading_icon: :plus, + tag: :a, + href: helpers.new_llm_model_path, + test_selector: "llm-model--add-button" + ) { t("admin.llm_connections.show.add_model_submit") } %> + + <% subheader.with_action_button( + scheme: :default, + label: t("admin.llm_connections.show.refresh_models"), + mobile_label: t("admin.llm_connections.show.refresh_models"), + mobile_icon: :sync, + leading_icon: :sync, + tag: :a, + href: helpers.refresh_models_llm_connection_path, + data: { turbo_method: :post, controller: "disable-when-clicked" }, + test_selector: "llm-model--refresh-button" + ) { t("admin.llm_connections.show.refresh_models") } %> +<% end %> diff --git a/app/components/llm_connections/models/sub_header_component.rb b/app/components/llm_connections/models/sub_header_component.rb new file mode 100644 index 000000000000..57cb69e49ec6 --- /dev/null +++ b/app/components/llm_connections/models/sub_header_component.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + module Models + # The filter bar above the model list. + class SubHeaderComponent < ApplicationComponent + include OpPrimer::ComponentHelpers + + alias_method :query, :model + + def filter_input_value + query.find_active_filter(:name)&.values&.first + end + + def clear_button_id = "llm-models-filter-clear" + + def sub_header_data_attributes + { + controller: "filter--filters-form", + "filter--filters-form-perform-turbo-requests-value": true, + "filter--filters-form-output-format-value": "json", + "filter--filters-form-url-path-name-value": helpers.search_models_llm_connection_path, + "filter--filters-form-clear-button-id-value": clear_button_id + } + end + + def filter_input_data_attributes + { + "filter-name": "name", + "filter-type": "string", + "filter-operator": "~", + "filter--filters-form-target": "simpleFilter filterValueContainer simpleValue" + } + end + end + end +end diff --git a/app/components/llm_connections/models_row_component.rb b/app/components/llm_connections/models_row_component.rb new file mode 100644 index 000000000000..1c0ba2abc2ed --- /dev/null +++ b/app/components/llm_connections/models_row_component.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # A row of the model list. Note +model+ is the row's record (an LlmModel), + # not the connection -- aliased to avoid confusion with either. + class ModelsRowComponent < OpPrimer::BorderBoxRowComponent + alias_method :llm_model, :model + + def identifier + render(Primer::Beta::Text.new(font_weight: :bold)) { llm_model.name } + end + + # vLLM and SGLang report the operator's real --max-model-len here, which is + # more trustworthy than any published figure for the model. Servers that do + # not report it simply show nothing rather than a guess. + def context_window + window = llm_model.context_window + return render(Primer::Beta::Text.new(color: :muted)) { "—" } if window.blank? + + number_with_delimiter(window) + end + + # Derived from the embeddings verdict rather than stored separately: a model + # that produces vectors is an embedding model, and that is the same fact. + def kind + case table.embeddings_states[llm_model.external_id] + when "supported" + render(Primer::Beta::Label.new(scheme: :success)) { I18n.t("llm.model_kinds.embedding") } + when "unsupported" + render(Primer::Beta::Label.new(scheme: :secondary)) { I18n.t("llm.model_kinds.chat") } + else + render(Primer::Beta::Label.new(scheme: :secondary, inline: true)) { I18n.t("llm.model_kinds.unknown") } + end + end + + def source + scheme, key = source_label + + render(Primer::Beta::Label.new(scheme:)) { I18n.t("admin.llm_connections.models.#{key}") } + end + + def source_label + return %i[attention source_deactivated] if llm_model.deactivated? + return %i[accent source_manual] if llm_model.manual? + return %i[attention source_withdrawn] if llm_model.withdrawn? + + %i[secondary source_discovered] + end + + # Whether a feature may choose this model. A withdrawn model has nothing to + # switch on -- the server stopped offering it -- so its toggle is inert + # rather than absent, which keeps the column aligned and says why. + def status + render(Primer::Alpha::ToggleSwitch.new(**toggle_options)) + end + + def toggle_options + options = { + checked: llm_model.selectable?, + enabled: togglable?, + size: :small, + # A bare ToggleSwitch has no accessible name, and axe fails without one. + aria: { label: I18n.t("admin.llm_connections.models.toggle_aria_label", model: llm_model.name) }, + test_selector: "llm-model--toggle-#{llm_model.id}" + } + + togglable? ? options.merge(mutation_options) : options + end + + def mutation_options + { + src: url_helpers.toggle_llm_model_path(llm_model), + csrf_token: helpers.form_authenticity_token, + data: { "turbo-method": :post, "turbo-stream": true }, + classes: "op-primer-adjustments__toggle-switch--hidden-loading-indicator" + } + end + + def togglable? = llm_model.active? + + def button_links + llm_model.manual? ? [edit_link, delete_link] : [edit_link] + end + + def edit_link + link_to(helpers.op_icon("icon-edit"), + url_helpers.edit_llm_model_path(llm_model), + data: { test_selector: "llm-model--edit-#{llm_model.id}" }, + title: I18n.t(:button_edit)) + end + + # Opens a DangerDialog rather than a browser confirm, so the message can say + # which features are bound to the model. + def delete_link + link_to(helpers.op_icon("icon-delete"), + url_helpers.delete_dialog_llm_model_path(llm_model), + data: { controller: "async-dialog", test_selector: "llm-model--delete-#{llm_model.id}" }, + title: I18n.t(:button_delete)) + end + end +end diff --git a/app/components/llm_connections/models_table_component.rb b/app/components/llm_connections/models_table_component.rb new file mode 100644 index 000000000000..e57ca958cada --- /dev/null +++ b/app/components/llm_connections/models_table_component.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Lists the models the remote server reported, read from the cached catalogue. + # + # Rendering never issues an HTTP request: the catalogue is refreshed explicitly + # through the "Refresh models" action. + class ModelsTableComponent < OpPrimer::BorderBoxTableComponent + columns :identifier, :kind, :context_window, :source, :status + + mobile_columns :identifier + + # The connection is passed in rather than derived from the first row: with a + # paginated, filtered list a page can legitimately be empty, and deriving it + # would silently degrade the kind column to "Unknown". + def initialize(connection:, **) + super(**) + @connection = connection + end + + attr_reader :connection + + def initial_sort = %i[identifier asc] + + def has_footer? = false + + # Without this the row's button_links are never rendered. + def has_actions? = true + + def mobile_title = I18n.t("admin.llm_connections.show.models_heading") + + # The row class is otherwise derived by convention as LlmConnections::RowComponent. + def row_class = ModelsRowComponent + + def headers + [ + [:identifier, { caption: I18n.t("admin.llm_connections.models.identifier") }], + [:kind, { caption: I18n.t("admin.llm_connections.models.kind") }], + [:context_window, { caption: I18n.t("admin.llm_connections.models.context_window") }], + [:source, { caption: I18n.t("admin.llm_connections.models.source") }], + [:status, { caption: I18n.t("admin.llm_connections.models.status") }] + ] + end + + # Built once for the whole table so each row does not query for its own + # verdict. Rows reach this through their +table+ accessor. + def embeddings_states + @embeddings_states ||= load_embeddings_states + end + + def load_embeddings_states + connection.capability_verdicts.for_capability(:embeddings).pluck(:model_id, :state).to_h + end + + def blank_title = I18n.t("admin.llm_connections.models.blank_title") + + def blank_description = I18n.t("admin.llm_connections.models.blank_description") + + def blank_icon = :sparkle + end +end diff --git a/app/components/llm_connections/side_panel/health_status_component.html.erb b/app/components/llm_connections/side_panel/health_status_component.html.erb new file mode 100644 index 000000000000..88ca7f6e6924 --- /dev/null +++ b/app/components/llm_connections/side_panel/health_status_component.html.erb @@ -0,0 +1,58 @@ +<%= + component_wrapper(tag: :turbo_frame, refresh: :morph) do + render(Primer::OpenProject::SidePanel::Section.new) do |section| + section.with_title(tag: :h3) { t(".title") } + + flex_layout do |container| + if report.present? + container.with_row do + header = summary_header + concat(render(Primer::Beta::Octicon.new(icon: header[:icon], color: header[:icon_color], mr: 2))) + concat(render(Primer::Beta::Text.new(font_weight: :bold)) { header[:text] }) + end + + container.with_row(mt: 2) do + render( + Primer::Beta::Text.new(color: :muted, test_selector: "llm-connection--health-summary") + ) { summary_description } + end + + container.with_row(mt: 2) do + render( + Primer::Beta::Button.new( + scheme: :link, color: :default, font_weight: :bold, + tag: :a, href: llm_connection_health_status_report_path, + test_selector: "llm-connection--open-health-report" + ) + ) do |button| + button.with_leading_visual_icon(icon: :meter) + t(".open_report") + end + end + else + container.with_row do + render(Primer::Beta::Text.new(color: :muted)) { t(".never_checked") } + end + + container.with_row(mt: 2) do + primer_form_with( + url: create_health_status_report_llm_connection_health_status_report_path, + method: :post, + data: { turbo: true } + ) do + render( + Primer::Beta::Button.new( + scheme: :link, color: :default, font_weight: :bold, type: :submit, + test_selector: "llm-connection--run-health-checks" + ) + ) do |button| + button.with_leading_visual_icon(icon: :meter) + t(".run_checks") + end + end + end + end + end + end + end +%> diff --git a/app/components/llm_connections/side_panel/health_status_component.rb b/app/components/llm_connections/side_panel/health_status_component.rb new file mode 100644 index 000000000000..e7145a678387 --- /dev/null +++ b/app/components/llm_connections/side_panel/health_status_component.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + module SidePanel + # "Has this connection ever actually worked?", answered on the settings page. + class HealthStatusComponent < ApplicationComponent + include ApplicationHelper + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + alias_method :connection, :model + + private + + def report + connection.latest_health_report + end + + def summary_header + tally = report.tally + + case tally + in { failure: 1.. } + { icon: :alert, icon_color: :danger, + text: I18n.t("health_reports.common.checks.failures", count: tally[:failure]) } + in { warning: 1.. } + { icon: :alert, icon_color: :attention, + text: I18n.t("health_reports.common.checks.warnings", count: tally[:warning]) } + else + { icon: :"check-circle", icon_color: :success, text: I18n.t("health_reports.common.checks.success") } + end + end + + def summary_description + text = if report.healthy? + I18n.t("health_reports.common.summary.success") + elsif report.unhealthy? + I18n.t("health_reports.common.summary.failure") + else + I18n.t("health_reports.common.summary.warning") + end + + "#{text} #{t('.last_check', datetime: helpers.format_time(report.created_at))}" + end + end + end +end diff --git a/app/contracts/llm_connections/base_contract.rb b/app/contracts/llm_connections/base_contract.rb new file mode 100644 index 000000000000..c1881e680a10 --- /dev/null +++ b/app/contracts/llm_connections/base_contract.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Validations that hold for every write, including provisioning from the + # environment. Deliberately makes no network request -- see UpdateContract. + class BaseContract < ModelContract + attribute :enabled + attribute :api_format + attribute :base_url + attribute :api_key + attribute :default_chat_model_id + attribute :default_embedding_model_id + + validates :base_url, presence: true + validates :api_format, inclusion: { in: Llm::Adapters::FORMATS } + # Bedrock and Vertex AI can be discovered against, but not called: they need + # a secret key and region, or a project and location, and there is one + # api_key column. Offering them would only fail at request time. + validates :api_format, exclusion: { in: Llm::Session::UNSUPPORTED_FORMATS, message: :not_supported }, + unless: -> { api_format.blank? } + # Resolves to the validate_url gem, which defaults to http and https. Plain + # http is deliberately allowed: an on-premise LLM server on an internal + # network commonly terminates TLS elsewhere, or not at all. + validates :base_url, url: { message: :invalid_url }, unless: -> { base_url.blank? } + + validate :enabled_requires_connection + validate :default_models_offered_by_server + validate :default_embedding_model_can_embed + validate :not_configured_from_env + + def not_configured_from_env + return unless model.configured_from_env? + + errors.add :base, :configured_via_env + end + + private + + # A model the server has positively told us cannot embed is not a candidate + # for the embedding default, however it got submitted. An unknown verdict + # does not block: that is the normal state for a server that publishes + # nothing about its models. + def default_embedding_model_can_embed + model_id = model.default_embedding_model_id + return if model_id.blank? + return unless model.changed_attributes.include?("default_embedding_model_id") + + unsupported = model.capability_verdicts + .for_capability(:embeddings) + .exists?(model_id:, state: "unsupported") + + errors.add(:default_embedding_model_id, :cannot_embed) if unsupported + end + + def enabled_requires_connection + return unless model.enabled? + return if model.base_url.present? + + errors.add :enabled, :requires_connection + end + + # A designated default must be a model the server actually reported. Validated + # only when it changes, so a catalogue that shrinks underneath a stored + # selection does not block every unrelated save; the dangling state is + # surfaced in the UI instead. + def default_models_offered_by_server + %i[default_chat_model_id default_embedding_model_id].each do |attribute| + value = model.public_send(attribute) + next if value.blank? + next unless model.changed_attributes.include?(attribute.to_s) + next if model.available_model_ids.include?(value) + + errors.add attribute, :not_available + end + end + end +end diff --git a/app/contracts/llm_connections/environment_update_contract.rb b/app/contracts/llm_connections/environment_update_contract.rb new file mode 100644 index 000000000000..72529132bef7 --- /dev/null +++ b/app/contracts/llm_connections/environment_update_contract.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Used when the connection is provisioned from the environment. + # + # Inherits from BaseContract, not UpdateContract: seeding must never reach out + # to the LLM server, because the container it runs in may well start before the + # server does. It also lifts the "configured from environment is read-only" + # guard, since this is the code path that legitimately writes those values. + class EnvironmentUpdateContract < BaseContract + def not_configured_from_env = nil + + # On a fresh installation the seed runs before any model synchronisation, so + # there is no catalogue to validate a default model against. A wrong id is + # surfaced afterwards, the same way as a model that vanished: the binding + # shows as no longer offered. + def default_models_offered_by_server = nil + end +end diff --git a/app/contracts/llm_connections/update_contract.rb b/app/contracts/llm_connections/update_contract.rb new file mode 100644 index 000000000000..9c4ec676e034 --- /dev/null +++ b/app/contracts/llm_connections/update_contract.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # The contract used when an administrator saves the form. + # + # This is the only contract that talks to the remote server, so that a failed + # probe persists nothing. It must NOT move to BaseContract: provisioning from + # the environment reuses that base, and a validates line cannot be un-declared + # by a subclass -- an outbound request during seeding would fail the boot of a + # container whose LLM server has not started yet. + # + # The probe itself is guarded on changed attributes, so toggling +enabled+ or + # picking a default model contacts nothing. + class UpdateContract < BaseContract + validates :base_url, llm_server: true, unless: -> { errors.include?(:base_url) } + end +end diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb new file mode 100644 index 000000000000..341150136e8d --- /dev/null +++ b/app/controllers/admin/llm_connections_controller.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + class LlmConnectionsController < ApplicationController + include OpTurbo::ComponentStream + include PaginationHelper + + layout "admin" + menu_item :llm_connection + + before_action :require_admin + before_action :set_connection + + def show + @query = ParamsToQueryService + .new(LlmModel, current_user, query_class: Queries::LlmModels::LlmModelQuery) + .call(params) + @models = @query.results.paginate(page: page_param, per_page: per_page_param) + end + + # Answers the sub-header's filter input, replacing just the table. + def search_models + show + + replace_via_turbo_stream( + component: LlmConnections::Models::IndexComponent.new(@models, connection: @connection) + ) + turbo_streams << turbo_stream.push_state(llm_connection_path(params.permit(:filters))) + + respond_with_turbo_streams + end + + def update + result = ::LlmConnections::UpdateService + .new(user: current_user, model: @connection) + .call(**llm_connection_params) + + result.on_success { redirect_after_save } + result.on_failure { render_form_with_errors } + end + + def refresh_models + result = ::LlmConnections::SyncModelsService.new(@connection).call + + if result.success? + redirect_with_notice(t(".success")) + else + redirect_with_error(t(".failure")) + end + end + + def disconnect_dialog + respond_with_dialog LlmConnections::DisconnectDialogComponent.new(@connection) + end + + # Clears the credential and switches the connection off, keeping the endpoint, + # the catalogue and every feature binding. Deliberately not a destroy: the + # cascade would take the locked embedding bindings with it, and those are the + # only record that a vector index exists and what it was written with. + def disconnect + return redirect_with_error(t(".configured_from_env")) if @connection.configured_from_env? + + @connection.update!(api_key: nil, enabled: false) + Llm::HealthCheckJob.toggle_cron_job + + redirect_with_notice(t(".success")) + end + + def delete_api_key_dialog + respond_with_dialog LlmConnections::DeleteApiKeyDialogComponent.new(@connection) + end + + # Deliberately not routed through UpdateService. Its contract probes the + # server whenever credentials change, and a server that requires + # authentication would reject the now-keyless probe -- so going through the + # service would let a server refuse an administrator permission to remove a + # credential. Removing one must always be possible. + # + # The one thing the contract did give us is the environment guard, which is + # therefore checked explicitly here: without it a hand-crafted request could + # wipe a key the UI correctly refuses to touch. + def delete_api_key + return redirect_with_error(t(".configured_from_env")) if @connection.configured_from_env? + + @connection.update!(api_key: nil) + + redirect_with_notice(t(".success")) + end + + private + + def set_connection + @connection = LlmConnection.instance + end + + # A connection can be perfectly usable without offering a model list, so the + # save succeeds either way; the administrator is told what to do next rather + # than being left with an empty table and no explanation. + def redirect_after_save + if @connection.reload.models.none? + flash[:warning] = t(".no_models") + else + flash[:notice] = t(".success") + end + + redirect_to llm_connection_path, status: :see_other + end + + def render_form_with_errors + update_via_turbo_stream(component: ::LlmConnections::FormComponent.new(@connection)) + # The HTML fallback re-renders the whole page, which needs everything the + # show action assigns -- not just the form that failed. + respond_with_turbo_streams do |format| + format.html do + show + render :show + end + end + end + + def redirect_with_notice(message) + flash[:notice] = message + redirect_to llm_connection_path, status: :see_other + end + + def redirect_with_error(message) + flash[:error] = message + redirect_to llm_connection_path, status: :see_other + end + + # A blank API key means "keep the stored one": the form never renders the + # saved value, so submitting it unchanged posts an empty string. + def llm_connection_params + permitted = params.expect( + llm_connection: %i[enabled api_format base_url api_key default_chat_model_id default_embedding_model_id] + ) + permitted.delete(:api_key) if permitted[:api_key].blank? + permitted.to_h.symbolize_keys + end + end +end diff --git a/app/controllers/admin/llm_feature_bindings_controller.rb b/app/controllers/admin/llm_feature_bindings_controller.rb new file mode 100644 index 000000000000..a4c65390a064 --- /dev/null +++ b/app/controllers/admin/llm_feature_bindings_controller.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + # Assigns a model to each registered AI feature. + class LlmFeatureBindingsController < ApplicationController + layout "admin" + menu_item :llm_feature_bindings + + before_action :require_admin + before_action :set_connection + + def index + @features = OpenProject::Llm::Features.available + @bindings = bindings_by_feature_key + end + + def update + feature = OpenProject::Llm::Features[params[:id]] + assign(feature) + + redirect_to llm_feature_bindings_path, status: :see_other + rescue OpenProject::Llm::UnknownFeature + render_404 + end + + private + + def set_connection + @connection = LlmConnection.instance + end + + def bindings_by_feature_key + @connection.feature_bindings.index_by(&:feature_key) + end + + def binding_for(feature) + @connection.feature_bindings.find_or_initialize_by(feature_key: feature.key.to_s) + end + + def assign(feature) + binding = build_binding(feature) + + if binding.save + probe_capabilities(feature, binding) + flash[:notice] = t("admin.llm_feature_bindings.update.success", feature: feature.label) + else + flash[:error] = binding.errors.full_messages.join(", ") + end + end + + def build_binding(feature) + binding = binding_for(feature) + binding.model_id = params.dig(:llm_feature_binding, :model_id).presence + + # Only ever accepted for the kind of feature they describe; the model + # rejects them elsewhere, and they are not read at all for a chat feature. + assign_embedding_settings(binding) if feature.embedding? + + binding + end + + # The prefixes are stored exactly as typed. The trailing space in "passage: " + # is load-bearing for the E5 and BGE families, so stripping would silently + # degrade retrieval. + def assign_embedding_settings(binding) + settings = params.fetch(:llm_feature_binding, {}) + + binding.dimensions = settings[:dimensions].presence + binding.input_prefix = settings[:input_prefix] + binding.query_prefix = settings[:query_prefix] + end + + # The verdict that actually matters is the one for the model an administrator + # just chose, so it is fetched now rather than left unknown until first use. + def probe_capabilities(feature, binding) + return if feature.requires.empty? || binding.model_id.blank? + + LlmConnections::DetectCapabilitiesService.new(@connection).detect(binding.model_id) + end + end +end diff --git a/app/controllers/admin/llm_health_status_controller.rb b/app/controllers/admin/llm_health_status_controller.rb new file mode 100644 index 000000000000..3ddb70f926d6 --- /dev/null +++ b/app/controllers/admin/llm_health_status_controller.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + class LlmHealthStatusController < ApplicationController + include OpTurbo::ComponentStream + + layout :admin_or_frame_layout + + before_action :require_admin + before_action :find_connection + + menu_item :llm_connection + + def show + @report = @connection.latest_health_report + + respond_to do |format| + format.html + format.text do + return head :not_found if @report.nil? + + timestamp = @report.created_at.iso8601 + send_data text_report(timestamp), + filename: "llm_connection_health_report_#{timestamp}.txt", + type: "text/plain", + disposition: :attachment + end + end + end + + # A full run, including the billed completion: an administrator clicking + # "Run checks" is asking whether the connection actually works. + def create + run_checks + redirect_to llm_connection_health_status_report_path, status: :see_other + end + + def create_health_status_report + run_checks + update_via_turbo_stream(component: LlmConnections::SidePanel::HealthStatusComponent.new(@connection)) + respond_with_turbo_streams + end + + private + + def admin_or_frame_layout + turbo_frame_request? ? "turbo_rails/frame" : "admin" + end + + def run_checks + @connection.deep_health_check = true + report = Llm::Validators::ConnectionValidator.new(@connection).call + report.save! + report + end + + # Downloaded and pasted into support tickets, so it must not contain the API + # key or the custom headers -- a gateway header routinely carries a second + # credential. + def text_report(timestamp) + { + connection: @connection.name, + configuration: @connection.non_confidential_configuration, + ran_at: timestamp, + results: @report ? @report.results.map(&:to_h) : [] + }.to_yaml(stringify_names: true) + end + + # HealthReports::Validator builds the report through the association, which + # would insert an unpersisted singleton along with it. + def find_connection + @connection = LlmConnection.instance + + redirect_to llm_connection_path unless @connection.persisted? + end + end +end diff --git a/app/controllers/admin/llm_models_controller.rb b/app/controllers/admin/llm_models_controller.rb new file mode 100644 index 000000000000..374425ca040e --- /dev/null +++ b/app/controllers/admin/llm_models_controller.rb @@ -0,0 +1,181 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + # Models an administrator enters by hand. + # + # Necessary because not every OpenAI-compatible server exposes a model list. + # A gateway may route /v1/chat/completions and nothing else, in which case the + # operator knows the model name and OpenProject cannot discover it. + class LlmModelsController < ApplicationController + include OpTurbo::ComponentStream + + layout "admin" + menu_item :llm_connection + + before_action :require_admin + before_action :set_connection + + def new + @llm_model = @connection.models.new + end + + def edit + @llm_model = @connection.models.find(params.expect(:id)) + @verdicts = @connection.capability_verdicts.for_model(@llm_model.external_id).index_by(&:capability) + end + + def create + llm_model = @connection.models.new(llm_model_params.except(*capability_param_names).merge(manual: true)) + + if save_with_capabilities(llm_model) + flash[:notice] = t(".success", model: llm_model.external_id) + redirect_to llm_connection_path, status: :see_other + else + # Re-rendered rather than redirected so the Primer form shows the error + # inline against the field that caused it. + @llm_model = llm_model + render :new, status: :unprocessable_entity + end + end + + def update + @llm_model = @connection.models.find(params.expect(:id)) + + if update_with_capabilities(@llm_model) + flash[:notice] = t(".success", model: @llm_model.external_id) + redirect_to llm_connection_path, status: :see_other + else + # Re-rendered rather than redirected so the Primer form shows the error + # inline against the field that caused it, e.g. a rename that collides + # with an existing model id. + @verdicts = @connection.capability_verdicts.for_model(@llm_model.external_id_was).index_by(&:capability) + render :edit, status: :unprocessable_entity + end + end + + def delete_dialog + llm_model = @connection.models.manual.find(params.expect(:id)) + + respond_with_dialog LlmConnections::DeleteModelDialogComponent.new(llm_model) + end + + def destroy + llm_model = @connection.models.manual.find(params.expect(:id)) + llm_model.destroy! + + flash[:notice] = t(".success", model: llm_model.external_id) + redirect_to llm_connection_path, status: :see_other + end + + # Hides a model from the pickers, or puts it back. Deliberately does not + # touch +active+, which the catalogue sync owns and would overwrite. + def toggle + llm_model = @connection.models.find(params.expect(:id)) + + # A withdrawn model has nothing to switch on; its toggle is rendered + # disabled, and this refuses a request that got here anyway. + return render(json: {}, status: :unprocessable_entity) unless llm_model.active? + + llm_model.update!(deactivated_at: llm_model.deactivated? ? nil : Time.current) + + render json: {}, status: :ok + end + + private + + def set_connection + @connection = LlmConnection.instance + end + + def update_with_capabilities(llm_model) + attributes = llm_model_params.except(*capability_param_names) + # A discovered model is named by the server; only a hand-entered one may be + # renamed here, and everything referencing the old name follows it. + attributes = attributes.except(:external_id) unless llm_model.manual? + + saved = false + ActiveRecord::Base.transaction do + previous_external_id = llm_model.external_id + llm_model.assign_attributes(attributes) + raise ActiveRecord::Rollback unless llm_model.save + + llm_model.cascade_rename!(previous_external_id) + apply_capabilities(llm_model) + saved = true + end + saved + end + + # external_id is accepted on create, and on update for manually added models. + def llm_model_params + params.expect(llm_model: [:external_id, :display_name, :admin_context_window, *capability_param_names]) + end + + def capability_param_names + Llm::Capabilities::ALL.map { |capability| :"capability_#{capability}" } + end + + def save_with_capabilities(llm_model) + ActiveRecord::Base.transaction do + llm_model.save! + apply_capabilities(llm_model) + end + + true + rescue ActiveRecord::RecordInvalid + false + end + + # Stored as admin-sourced verdicts, which survive re-detection: an + # administrator knows things about their deployment that neither a published + # registry nor a probe can determine. + def apply_capabilities(llm_model) + submitted = llm_model_params + + Llm::Capabilities::ALL.each do |capability| + assert(llm_model.external_id, capability, submitted[:"capability_#{capability}"].presence) + end + end + + def assert(model_id, capability, state) + verdict = @connection.capability_verdicts + .find_or_initialize_by(model_id:, capability: capability.to_s) + + if state.blank? + # "Not specified" clears an assertion rather than recording ignorance as + # fact; detection may fill it in later. + verdict.destroy! if verdict.persisted? && verdict.source_admin? + else + verdict.update!(state:, source: "admin", checked_at: Time.current) + end + end + end +end diff --git a/app/forms/llm_connections/connection_form.rb b/app/forms/llm_connections/connection_form.rb new file mode 100644 index 000000000000..e00b936c7985 --- /dev/null +++ b/app/forms/llm_connections/connection_form.rb @@ -0,0 +1,230 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class ConnectionForm < ApplicationForm + form do |f| + f.check_box( + name: :enabled, + label: LlmConnection.human_attribute_name(:enabled), + caption: I18n.t("admin.llm_connections.form.enabled_caption"), + disabled: read_only? + ) + + f.select_list( + name: :api_format, + label: LlmConnection.human_attribute_name(:api_format), + caption: I18n.t("admin.llm_connections.form.api_format_caption"), + include_blank: false, + input_width: :medium, + disabled: read_only? + ) do |select| + # Only formats a request can actually be sent in. The contract rejects + # the rest as a backstop, but they should not be offered in the first place. + Llm::Adapters::FORMATS.select { |format| Llm::Session.supports?(format) }.each do |format| + select.option(value: format, label: I18n.t("llm.api_formats.#{format}")) + end + end + + f.text_field( + name: :base_url, + label: LlmConnection.human_attribute_name(:base_url), + caption: I18n.t("admin.llm_connections.form.base_url_caption"), + placeholder: "https://example.com/v1", + required: true, + type: :url, + input_width: :large, + disabled: read_only? + ) + + f.text_field( + name: :api_key, + label: LlmConnection.human_attribute_name(:api_key), + caption: api_key_caption, + # The stored key is never sent to the browser. A blank submission means + # "keep the current key", handled in the controller. + value: nil, + placeholder: api_key_placeholder, + type: :password, + autocomplete: "off", + input_width: :large, + disabled: read_only?, + data: { "admin--llm-connection-form-target": "secretInput" } + ) + + if models_available? + # An autocompleter rather than a select: a gateway reports hundreds of + # models, and every one of them would otherwise be inlined as an option + # in the page body. decorated: true serialises the list into the element, + # so this needs no endpoint of its own. + f.autocompleter( + name: :default_chat_model_id, + label: LlmConnection.human_attribute_name(:default_chat_model_id), + caption: I18n.t("admin.llm_connections.form.default_chat_model_caption"), + disabled: read_only?, + autocomplete_options: { + decorated: true, + inputValue: model.default_chat_model_id, + placeholder: I18n.t("label_none_parentheses") + } + ) do |list| + list.option(label: I18n.t("label_none_parentheses"), value: "", + selected: model.default_chat_model_id.blank?) + + default_chat_model_options.each do |model_id| + list.option(label: model_id, value: model_id, selected: model.default_chat_model_id == model_id) + end + end + + # Only worth asking for once something embeds. The column, the contract + # attribute, its validation and every translation for this field already + # existed; the input was simply never rendered, so the value could not be + # set through the UI at all. + if embedding_features? + f.autocompleter( + name: :default_embedding_model_id, + label: LlmConnection.human_attribute_name(:default_embedding_model_id), + caption: default_embedding_model_caption, + disabled: read_only?, + autocomplete_options: { + decorated: true, + inputValue: model.default_embedding_model_id, + placeholder: I18n.t("label_none_parentheses") + } + ) do |list| + list.option(label: I18n.t("label_none_parentheses"), value: "", + selected: model.default_embedding_model_id.blank?) + + default_embedding_model_options.each do |model_id| + list.option(label: embedding_option_label(model_id, embeddings_state(model_id)), + value: model_id, + selected: model.default_embedding_model_id == model_id) + end + end + end + end + + unless read_only? + f.submit( + name: :submit, + label: submit_label, + scheme: :primary, + data: { "admin--llm-connection-form-target": "submitButton" } + ) + end + end + + private + + def read_only? + model.configured_from_env? + end + + def models_available? + model.available_model_ids.any? + end + + # Deactivated models are hidden, except the one already chosen -- dropping + # that would silently blank the field on the next save. + def default_chat_model_options + (model.selectable_model_ids + [model.default_chat_model_id]).compact_blank.uniq + end + + # Only models actually known to embed. + # + # An unconfirmed capability is not a capability: offering a model here on the + # grounds that nothing has ruled it out invites an administrator to pick one + # that cannot embed, and the failure would surface much later, at index time. + # A catalogue from a registry-backed provider makes that vivid -- 132 models, + # 3 of which embed. + # + # This leaves nothing to choose when no model is known to embed, and that is + # the honest state rather than a dead end: an administrator who knows better + # than the registry says so on the model itself, by setting its embeddings + # capability, which is what default_embedding_model_hint points at. + # + # The model already chosen is kept regardless, so a save cannot silently + # blank a working configuration. + def default_embedding_model_options + capable = model.selectable_model_ids.select { |id| embeddings_state(id) == :supported } + + (capable + [model.default_embedding_model_id]).compact_blank.uniq + end + + # Says how to make a model eligible when none is, rather than leaving an + # empty select with no explanation. + def default_embedding_model_caption + return I18n.t("admin.llm_connections.form.default_embedding_model_caption") if + default_embedding_model_options.any? + + I18n.t("admin.llm_connections.form.default_embedding_model_none") + end + + def embedding_features? + OpenProject::Llm::Features.for_kind(:embedding).any? + end + + # No verdict at all is the same as an inconclusive one: we do not know. + def embeddings_state(model_id) + embeddings_verdicts[model_id]&.to_sym || :unknown + end + + def embeddings_verdicts + @embeddings_verdicts ||= model.capability_verdicts + .for_capability(:embeddings) + .pluck(:model_id, :state) + .to_h + end + + def embedding_option_label(model_id, state) + case state + when :unsupported + I18n.t("admin.llm_connections.form.embedding_option_unsupported", model: model_id) + when :unknown + I18n.t("admin.llm_connections.form.embedding_option_unknown", model: model_id) + else + model_id + end + end + + def submit_label + model.persisted? ? I18n.t(:button_save) : I18n.t("admin.llm_connections.form.button_connect") + end + + def api_key_caption + key = model.persisted? && model.api_key.present? + I18n.t("admin.llm_connections.form.api_key_caption#{'_stored' if key}") + end + + def api_key_placeholder + "••••••••••••••••" if model.api_key.present? + end + end +end diff --git a/app/forms/llm_connections/feature_binding_form.rb b/app/forms/llm_connections/feature_binding_form.rb new file mode 100644 index 000000000000..cf7a50fbd5f7 --- /dev/null +++ b/app/forms/llm_connections/feature_binding_form.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # The model select for one registered feature. + class FeatureBindingForm < ApplicationForm + # Primer::Forms::Base.new assigns the builder itself and calls this with the + # remaining keywords, so the builder must not appear in the signature. + def initialize(options:, inherit_label:, feature_key:, locked: false, embedding: false, dimensions_hint: nil, + selected_model_id: nil) + super() + @selected_model_id = selected_model_id + @model_options = options + @inherit_label = inherit_label + @feature_key = feature_key + @locked = locked + @embedding = embedding + @dimensions_hint = dimensions_hint + end + + form do |f| + # An autocompleter rather than a select, so a model can be found by typing + # among the hundreds a gateway reports. decorated: true serialises the list + # into the element, so no endpoint is needed. + f.autocompleter( + name: :model_id, + label: LlmFeatureBinding.human_attribute_name(:model_id), + disabled: locked, + autocomplete_options: { + decorated: true, + inputValue: selected_model_id, + placeholder: inherit_label + }, + data: { test_selector: "llm-feature-binding--model-#{feature_key}" } + ) do |list| + list.option(label: inherit_label, value: "", selected: selected_model_id.blank?) + + model_options.each do |option| + # Listed but not choosable when a required capability is known to be + # missing: hiding it would leave the reason invisible too. + list.option(label: option_label(option), + value: option.model_id, + selected: selected_model_id == option.model_id, + disabled: !option.selectable?) + end + end + + # Only for an embedding feature, and only while unlocked. A locked binding + # renders these as text instead: a disabled input submits nothing, so the + # values would arrive blank and wipe the columns. + if embedding && !locked + f.text_field( + name: :dimensions, + type: :number, + min: 1, + label: LlmFeatureBinding.human_attribute_name(:dimensions), + caption: dimensions_caption, + input_width: :small, + data: { test_selector: "llm-feature-binding--dimensions-#{feature_key}" } + ) + + f.text_field( + name: :input_prefix, + label: LlmFeatureBinding.human_attribute_name(:input_prefix), + caption: I18n.t("admin.llm_feature_bindings.form.input_prefix_caption"), + input_width: :medium, + data: { test_selector: "llm-feature-binding--input-prefix-#{feature_key}" } + ) + + f.text_field( + name: :query_prefix, + label: LlmFeatureBinding.human_attribute_name(:query_prefix), + caption: I18n.t("admin.llm_feature_bindings.form.query_prefix_caption"), + input_width: :medium, + data: { test_selector: "llm-feature-binding--query-prefix-#{feature_key}" } + ) + end + + unless locked + f.submit( + name: :submit, + label: I18n.t(:button_save), + scheme: :secondary, + data: { test_selector: "llm-feature-binding--submit-#{feature_key}" } + ) + end + end + + private + + attr_reader :model_options, :inherit_label, :feature_key, :locked, :embedding, :dimensions_hint, + :selected_model_id + + # Blank is the right default: the server decides the vector size, and baking + # in a number it may contradict helps nobody. Where the probe has already + # seen a vector, its size is offered as information rather than filled in. + def dimensions_caption + return I18n.t("admin.llm_feature_bindings.form.dimensions_caption") if dimensions_hint.blank? + + I18n.t("admin.llm_feature_bindings.form.dimensions_caption_probed", dimensions: dimensions_hint) + end + + def option_label(option) + case option.state + when :unsupported + I18n.t("admin.llm_feature_bindings.option_unsupported", + model: option.model_id, + capability: option.reasons.map { |reason| Llm::Capabilities.label(reason) }.join(", ")) + when :unknown + I18n.t("admin.llm_feature_bindings.option_unknown", model: option.model_id) + else + option.model_id + end + end + end +end diff --git a/app/forms/llm_models/form.rb b/app/forms/llm_models/form.rb new file mode 100644 index 000000000000..444767ca85bf --- /dev/null +++ b/app/forms/llm_models/form.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmModels + class Form < ApplicationForm + form do |f| + # A discovered model is named by the server, so its identifier is not ours + # to change -- the next refresh would only put it back. One entered by hand + # is editable, because a typo in it is otherwise unfixable except by + # deleting the model and losing everything asserted about it. Renaming + # cascades; see LlmModel#cascade_rename!. + if new_record? || model.manual? + f.text_field( + name: :external_id, + label: LlmModel.human_attribute_name(:external_id), + caption: I18n.t("admin.llm_models.form.external_id_caption"), + placeholder: "qwen3.6-35b-a3b", + required: true, + autocomplete: "off", + input_width: :large, + data: { test_selector: "llm-model--external-id" } + ) + end + + f.text_field( + name: :display_name, + label: LlmModel.human_attribute_name(:display_name), + caption: I18n.t("admin.llm_models.form.display_name_caption"), + autocomplete: "off", + input_width: :large, + data: { test_selector: "llm-model--display-name" } + ) + + f.text_field( + name: :admin_context_window, + label: I18n.t("admin.llm_models.form.context_window"), + caption: context_window_caption, + type: :number, + min: 1, + autocomplete: "off", + input_width: :medium, + data: { test_selector: "llm-model--context-window" } + ) + + Llm::Capabilities::ALL.each do |capability| + f.select_list( + name: :"capability_#{capability}", + label: Llm::Capabilities.label(capability), + caption: capability_caption(capability), + include_blank: false, + input_width: :medium, + data: { test_selector: "llm-model--capability-#{capability}" } + ) do |select| + select.option(value: "", label: inherited_state_label(capability)) + select.option(value: "supported", label: I18n.t("admin.llm_models.form.state_supported")) + select.option(value: "unsupported", label: I18n.t("admin.llm_models.form.state_unsupported")) + end + end + + f.submit( + name: :submit, + label: new_record? ? I18n.t("admin.llm_models.form.create_submit") : I18n.t(:button_save), + scheme: :primary, + data: { test_selector: "llm-model--submit" } + ) + end + + private + + def new_record? = model.new_record? + + def context_window_caption + source = model.context_window_source + + if source.present? && source != :admin + I18n.t("admin.llm_models.form.context_window_known", + value: model.context_window, + source: I18n.t("llm.context_window_sources.#{source}")) + else + I18n.t("admin.llm_models.form.context_window_caption") + end + end + + # A verdict established by a probe or a registry is never loaded into the + # field -- saving must not turn someone else's finding into the + # administrator's assertion -- so the blank option names what applies while + # nothing is asserted here. Labelling it "Not specified" next to a caption + # reading "Currently Supported" stated two contradictory things at once. + def inherited_state_label(capability) + verdict = model.verdict_for(capability) + return I18n.t("admin.llm_models.form.state_unspecified") if verdict.nil? || verdict.source_admin? + + I18n.t("admin.llm_models.form.state_inherited", + state: I18n.t("llm.verdict_states.#{verdict.state}"), + source: I18n.t("llm.verdict_sources.#{verdict.source}")) + end + + # The state is now carried by the option itself; the caption only has to say + # what choosing something else here means. + def capability_caption(capability) + verdict = model.verdict_for(capability) + return if verdict.nil? || verdict.source_admin? + + I18n.t("admin.llm_models.form.capability_override_caption") + end + end +end diff --git a/app/models/llm_capability_verdict.rb b/app/models/llm_capability_verdict.rb new file mode 100644 index 000000000000..b6fe7d1e73df --- /dev/null +++ b/app/models/llm_capability_verdict.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# What we know about one capability of one model on one server. +# +# Three states rather than a boolean, because the honest answer is very often +# "we cannot tell": the OpenAI model list carries no capability information at +# all, and only some servers offer a non-standard endpoint that does. +# +# The rule the rest of the system depends on: only :unsupported blocks. An +# :unknown verdict warns and lets the administrator proceed, because refusing +# on "we could not tell" would make most self-hosted servers unusable. +class LlmCapabilityVerdict < ApplicationRecord + belongs_to :llm_connection + + enum :state, { supported: "supported", unsupported: "unsupported", unknown: "unknown" }, validate: true + + # Where the verdict came from. Orthogonal to the state, so that an + # administrator's assertion can be shown as such without adding a fourth state + # that every caller would have to handle. + enum :source, + { metadata: "metadata", probe: "probe", admin: "admin", observed: "observed" }, + prefix: true, + validate: true + + validates :model_id, presence: true + validates :capability, presence: true, uniqueness: { scope: %i[llm_connection_id model_id] } + + scope :for_model, ->(model_id) { where(model_id:) } + scope :for_capability, ->(capability) { where(capability: capability.to_s) } + + # An administrator's assertion survives re-detection: they know something about + # their deployment that we could not determine. + scope :sticky, -> { where(source: "admin") } + + def blocking? + unsupported? + end + + def dimensions + detail["dimensions"] + end +end diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb new file mode 100644 index 000000000000..2ec8cda20877 --- /dev/null +++ b/app/models/llm_connection.rb @@ -0,0 +1,160 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# The connection to an OpenAI-API-compatible LLM server. +# +# Only a single connection is supported today. That is enforced by a validation +# rather than by the schema, so lifting the restriction later is a one-line change: +# every association is already scoped by +llm_connection_id+ and the STI +type+ +# column is in place. +class LlmConnection < ApplicationRecord + include Redmine::Ciphering + + SINGLETON_NAME = "default" + + has_many :health_reports, as: :subject, dependent: :delete_all + has_many :models, class_name: "LlmModel", dependent: :delete_all + has_many :capability_verdicts, class_name: "LlmCapabilityVerdict", dependent: :delete_all + has_many :feature_bindings, class_name: "LlmFeatureBinding", dependent: :delete_all + + validates :base_url, presence: true + validate :only_one_connection, on: :create + + class << self + # The connection record, whether or not it has been persisted yet. + # + # Identifying attributes are left unset here and filled in by + # LlmConnections::SetAttributesService as system changes, so that they do not + # register as user-made changes to non-writable attributes. + def instance + first || new + end + + # Cheap enough to call from a menu visibility lambda. + def enabled? + exists?(enabled: true) + end + + # Whether LLM-backed features may run right now. This is the predicate + # sibling features gate on; see #77783. + def available? + OpenProject::FeatureDecisions.llm_connection_active? && + enabled? && + instance.configured? + end + end + + def api_key + read_ciphered_attribute(:api_key) + end + + def api_key=(value) + write_ciphered_attribute(:api_key, value) + end + + # Deliberately does not consider +last_connected_at+: a connection provisioned + # from the environment is never probed, and must still count as configured. + def configured? + base_url.present? + end + + def configured_from_env? + Setting.llm_connection.present? + end + + # Every model that can be addressed today: discovered and still offered, plus + # anything an administrator entered by hand. + # + # Deliberately includes models an administrator has deactivated. This is what + # Llm::Runtime resolves against, and hiding a model from the pickers must not + # break a feature that is already bound to it. + def available_model_ids + models.active.by_identifier.pluck(:external_id) + end + + # What a picker should offer: the above, minus what an administrator has + # switched off. + def selectable_model_ids + models.selectable.by_identifier.pluck(:external_id) + end + + def server_flavour + options["server_flavour"].presence&.to_sym + end + + # Whether a health check run may spend a real completion. + # + # Not a column: it describes the run, not the connection. An administrator + # asking to test the connection wants it; the scheduled re-check leaves it off + # so that a connection to a paid provider is not billed four times a day. + attr_accessor :deep_health_check + + def deep_health_check? = ActiveModel::Type::Boolean.new.cast(deep_health_check).present? + + def latest_health_report + health_reports.order(created_at: :asc).last + end + + # Derived rather than stored, for the same reason LlmFeatureBinding#dangling? + # is: a status column would be a cache with no invalidation trigger, and would + # be stale exactly when it matters. + def health_state + report = latest_health_report + + return :unknown if report.nil? + return :unhealthy if report.unhealthy? + return :warning if report.warning? + + :healthy + end + + # Feeds the downloadable health report. Deliberately excludes api_key *and* + # custom_headers: a gateway header routinely carries a second credential. + def non_confidential_configuration + { + base_url:, + api_format:, + enabled:, + server_flavour:, + catalogue_fetched_at:, + last_connected_at:, + model_count: models.count, + manual_model_count: models.where(manual: true).count + } + end + + private + + def only_one_connection + return unless self.class.where.not(id:).exists? + + errors.add(:base, :singleton) + end +end diff --git a/app/models/llm_feature_binding.rb b/app/models/llm_feature_binding.rb new file mode 100644 index 000000000000..8595c24a3f1b --- /dev/null +++ b/app/models/llm_feature_binding.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Which model a registered feature uses. +# +# One row per feature, not per binding: rows are reconciled against the registry +# and are never destroyed when a feature deregisters, so flipping a feature flag +# does not lose the administrator's choice. +class LlmFeatureBinding < ApplicationRecord + belongs_to :llm_connection + + # Settings that describe how vectors are written, and so only mean anything + # for an embedding feature. + EMBEDDING_SETTINGS = %i[dimensions input_prefix query_prefix].freeze + + # Everything a stored index depends on. Changing any of it invalidates the + # vectors already written, not just the model. + LOCKED_SETTINGS = ([:model_id] + EMBEDDING_SETTINGS).freeze + + validates :feature_key, presence: true, uniqueness: { scope: :llm_connection_id } + validates :dimensions, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true + validate :feature_registered + validate :embedding_settings_only_for_embedding_features + validate :locked_settings_unchanged + + def feature + OpenProject::Llm::Features[feature_key] + rescue OpenProject::Llm::UnknownFeature + nil + end + + # NULL means "use the connection default for this kind of model". + def resolved_model_id + model_id.presence || default_model_id + end + + def inherits_default? = model_id.blank? + + # Derived, never stored. A status column would be a cache with no invalidation + # trigger, and would be stale exactly when it matters -- right after the remote + # catalogue changed. + def dangling? + resolved = resolved_model_id + resolved.present? && llm_connection.available_model_ids.exclude?(resolved) + end + + def locked? = locked_at.present? + + private + + def default_model_id + return if feature.nil? + + feature.embedding? ? llm_connection.default_embedding_model_id : llm_connection.default_chat_model_id + end + + def feature_registered + return if feature.present? + + errors.add(:feature_key, :not_registered) + end + + def embedding_settings_only_for_embedding_features + return if feature.nil? || feature.embedding? + + EMBEDDING_SETTINGS.each do |attribute| + next if public_send(attribute).blank? + + errors.add(attribute, :not_for_chat_feature) + end + end + + # Vectors written under one embedding model are meaningless under another, and + # the dimension count is baked into the index, so a locked binding can only be + # changed by an explicit re-index. + # + # The prefixes are locked for the same reason and matter just as much: an index + # built with "passage: " but queried under a different prefix does not error, + # it quietly returns worse results, which is the hardest kind of failure to + # notice. + # + # TODO(#69620): re-indexing is what clears locked_at. Until that job exists a + # locked binding can only be changed in the database. + def locked_settings_unchanged + # Only constrains later edits. On the save that records the lock -- and on + # create -- every attribute reads as changed from nil, and there is nothing + # indexed yet for them to contradict. + return unless locked? && locked_at_was.present? + + LOCKED_SETTINGS.each do |attribute| + next unless public_send(:"#{attribute}_changed?") + + errors.add(attribute, :locked) + end + end +end diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb new file mode 100644 index 000000000000..f7fbd34630c1 --- /dev/null +++ b/app/models/llm_model.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# A model this connection can address. +# +# Rows come from two places: discovered from the server's model list, or entered +# by an administrator. Both are addressed by +external_id+, which is whatever the +# deployment calls the model -- provider-specific and not comparable across +# vendors, which is why it is never used as a lookup key into a public catalogue. +class LlmModel < ApplicationRecord + belongs_to :llm_connection + + validates :external_id, presence: true, uniqueness: { scope: :llm_connection_id } + + scope :active, -> { where(active: true) } + scope :discovered, -> { where(manual: false) } + scope :manual, -> { where(manual: true) } + scope :by_identifier, -> { order(:external_id) } + + # What an administrator is willing to have chosen. Distinct from +active+, + # which the catalogue sync owns and rewrites on every refresh. + scope :deactivated, -> { where.not(deactivated_at: nil) } + scope :selectable, -> { active.where(deactivated_at: nil) } + + def deactivated? = deactivated_at.present? + + # Everything that points at a model does so by its identifier string, so a + # rename has to carry them along or it silently orphans them. + # + # Renaming is a correction of the name, not a change of model, which is why + # this writes directly: a locked binding must not refuse to follow the model + # it is locked to, and the connection defaults are pointing at this very row. + def cascade_rename!(previous_external_id) + return if previous_external_id.blank? || previous_external_id == external_id + + llm_connection.capability_verdicts.where(model_id: previous_external_id).update_all(model_id: external_id) + llm_connection.feature_bindings.where(model_id: previous_external_id).update_all(model_id: external_id) + + rename_connection_defaults(previous_external_id) + end + + # Offerable in a picker. Note that this is *not* what decides whether a model + # still resolves: a feature already bound to a deactivated model keeps working, + # and is surfaced as a warning instead. Switching a row off must never silently + # break a running feature. + def selectable? = active? && !deactivated? + + def name = display_name.presence || external_id + + def rename_connection_defaults(previous_external_id) + defaults = %i[default_chat_model_id default_embedding_model_id] + .select { |attribute| llm_connection.public_send(attribute) == previous_external_id } + .index_with { external_id } + + llm_connection.update_columns(defaults) if defaults.any? + end + + # Precedence: what an administrator set, then what the server reported (vLLM + # and SGLang publish the operator's actual --max-model-len), then what a + # registry believes about the model in general. + def context_window + raw_metadata["admin_context_window"] || + raw_metadata["max_model_len"] || + raw_metadata["context_window"] + end + + def admin_context_window = raw_metadata["admin_context_window"] + + def context_window_source + return :admin if raw_metadata["admin_context_window"].present? + return :server if raw_metadata["max_model_len"].present? + return :registry if raw_metadata["context_window"].present? + + nil + end + + def admin_context_window=(value) + self.raw_metadata = if value.blank? + raw_metadata.except("admin_context_window") + else + raw_metadata.merge("admin_context_window" => value.to_i) + end + end + + # Capability assertions are stored as verdicts, not columns. These virtual + # attributes let the edit form treat them as ordinary fields, so the whole + # screen can be a single Primer form rather than hand-written inputs. + Llm::Capabilities::ALL.each do |capability| + define_method(:"capability_#{capability}") do + capability_overrides.fetch(capability.to_s) { admin_capability_state(capability) } + end + + define_method(:"capability_#{capability}=") do |value| + capability_overrides[capability.to_s] = value.presence + end + end + + def capability_overrides = @capability_overrides ||= {} + + # Only an administrator's own assertion is shown as the field's value. A + # verdict from a probe or a registry is displayed alongside instead, so that + # saving the form does not silently adopt it as the administrator's. + def admin_capability_state(capability) + verdict_for(capability)&.then { |verdict| verdict.source_admin? ? verdict.state : nil } + end + + def verdict_for(capability) + llm_connection.capability_verdicts + .for_model(external_id) + .for_capability(capability) + .first + end + + # Discovered models that the server stopped offering are deactivated rather + # than deleted, so a binding or verdict pointing at one still has something to + # name. Manual entries are never deactivated by a refresh: nothing confirms + # them, so nothing can un-confirm them either. + def withdrawn? = !active? && !manual? +end diff --git a/app/models/queries/llm_models.rb b/app/models/queries/llm_models.rb new file mode 100644 index 000000000000..50492ff3b215 --- /dev/null +++ b/app/models/queries/llm_models.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Queries::LlmModels + ::Queries::Register.register(LlmModelQuery) do + filter Filters::NameFilter + end +end diff --git a/app/models/queries/llm_models/filters/llm_model_filter.rb b/app/models/queries/llm_models/filters/llm_model_filter.rb new file mode 100644 index 000000000000..bf0be12e74b5 --- /dev/null +++ b/app/models/queries/llm_models/filters/llm_model_filter.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class Queries::LlmModels::Filters::LlmModelFilter < Queries::Filters::Base + self.model = LlmModel +end diff --git a/app/models/queries/llm_models/filters/name_filter.rb b/app/models/queries/llm_models/filters/name_filter.rb new file mode 100644 index 000000000000..4fe27abbaa63 --- /dev/null +++ b/app/models/queries/llm_models/filters/name_filter.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class Queries::LlmModels::Filters::NameFilter < Queries::LlmModels::Filters::LlmModelFilter + def self.key + :name + end + + def type + :string + end + + def human_name + I18n.t("admin.llm_connections.models.filter_label") + end + + # Matches the identifier the server uses and the friendly name an + # administrator may have given it, since either is what someone types. + def where + escaped = ActiveRecord::Base.sanitize_sql_like(values.first) + + case operator + when "~", "**" + ["llm_models.external_id ILIKE :q OR llm_models.display_name ILIKE :q", { q: "%#{escaped}%" }] + when "!~" + ["llm_models.external_id NOT ILIKE :q AND (llm_models.display_name IS NULL OR llm_models.display_name NOT ILIKE :q)", + { q: "%#{escaped}%" }] + else + raise "Unsupported operator #{operator}" + end + end +end diff --git a/app/models/queries/llm_models/llm_model_query.rb b/app/models/queries/llm_models/llm_model_query.rb new file mode 100644 index 000000000000..e6d096a22749 --- /dev/null +++ b/app/models/queries/llm_models/llm_model_query.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class Queries::LlmModels::LlmModelQuery + include Queries::BaseQuery + include Queries::UnpersistedQuery + + def self.model + LlmModel + end + + # There is exactly one LLM connection, so every model belongs to it. + def default_scope + LlmModel.by_identifier + end +end diff --git a/app/seeders/env_data/llm_connection_seeder.rb b/app/seeders/env_data/llm_connection_seeder.rb new file mode 100644 index 000000000000..c62948cf72b7 --- /dev/null +++ b/app/seeders/env_data/llm_connection_seeder.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module EnvData + # Provisions the LLM connection from OPENPROJECT_LLM__CONNECTION_* variables so + # a container comes up connected without anyone opening the administration UI. + # + # Never contacts the LLM server: the catalogue refresh is enqueued, so seeding + # succeeds even when the server starts after OpenProject does. + class LlmConnectionSeeder < Seeder + KNOWN_KEYS = %w[base_url api_key default_chat_model default_embedding_model enabled].freeze + + def seed_data! + print_status " ↳ Creating LLM connection" do + validate_options!(config) + + result = LlmConnections::EnvSyncService.new(config).call + raise result.errors.full_messages.join(", ") if result.failure? + + Llm::SyncModelsJob.perform_later + end + end + + def applicable? + config.present? + end + + def not_applicable_message + "No LLM connection configured through environment variables." + end + + private + + def config + Setting.llm_connection + end + + def validate_options!(options) + check_unknown_keys!(options, KNOWN_KEYS) + return if options["base_url"].present? + + raise "LLM connection: #{env_form('base_url')} is required." + end + + def check_unknown_keys!(options, known_keys) + unknown = options.keys - known_keys + return if unknown.empty? + + raise <<~MSG.strip + LLM connection: unknown configuration key(s): #{unknown.map { |k| env_form(k) }.join(', ')}. + Accepted keys: #{known_keys.map { |k| env_form(k) }.join(', ')}. + Note: in environment variable names, single underscores split path segments and double underscores encode a literal underscore (e.g. BASE__URL, not BASE_URL). + MSG + end + + def env_form(key) + key.gsub("_", "__").upcase + end + end +end diff --git a/app/seeders/env_data_seeder.rb b/app/seeders/env_data_seeder.rb index 48a5959cced8..91300692b014 100644 --- a/app/seeders/env_data_seeder.rb +++ b/app/seeders/env_data_seeder.rb @@ -31,6 +31,7 @@ def data_seeder_classes [ EnvData::CustomDesignSeeder, EnvData::LdapSeeder, + EnvData::LlmConnectionSeeder, EnvData::ScimClientSeeder, EnvData::TokenSeeder ] diff --git a/app/services/llm/adapters.rb b/app/services/llm/adapters.rb new file mode 100644 index 000000000000..f319abcfc1c6 --- /dev/null +++ b/app/services/llm/adapters.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Resolves a connection's api_format to the adapter that speaks it. + # + # There is no universal model-discovery standard, so each dialect needs its own + # translation: OpenAI-compatible servers answer GET /models with data[].id, + # Gemini uses /v1beta/models with richer metadata, Bedrock needs AWS signing + # rather than a bearer token, and Azure indirects through deployment names. + # + # Only the OpenAI adapter is implemented. The seam exists so that adding one is + # a new class rather than a migration. + module Adapters + class UnsupportedFormat < StandardError; end + + # Formats an administrator can choose. "openai" covers OpenAI itself and the + # great majority of gateways and self-hosted inference servers; the rest are + # RubyLLM providers whose model lists come from its registry. + OPENAI_COMPATIBLE = "openai" + + FORMATS = %w[ + openai + anthropic + gemini + mistral + deepseek + openrouter + perplexity + xai + ollama + gpustack + azure + bedrock + vertexai + ].freeze + + def self.for(connection) + format = connection.api_format.to_s + raise UnsupportedFormat, format unless FORMATS.include?(format) + + if format == OPENAI_COMPATIBLE + # Queried live, so the list is what this endpoint actually serves. + Openai.new(connection) + else + RegistryBacked.new(connection) + end + end + end +end diff --git a/app/services/llm/adapters/openai.rb b/app/services/llm/adapters/openai.rb new file mode 100644 index 000000000000..68da5bf06e2f --- /dev/null +++ b/app/services/llm/adapters/openai.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Adapters + # Servers speaking the OpenAI API: OpenAI itself, and the great majority of + # gateways and self-hosted inference servers. + class Openai + def initialize(connection) + @connection = connection + end + + # Normalised model cards. + # + # The raw card is kept alongside, because the fields worth having are the + # non-standard ones: the OpenAI schema itself carries only id, object, + # created and owned_by, while vLLM adds max_model_len -- the operator's + # actual --max-model-len, and the only trustworthy context window for this + # deployment. + # + # @return [Array] cards with :id and :raw + def models + @models ||= Array(client.models["data"]).filter_map do |card| + id = card["id"] + next if id.blank? + + { id:, raw: card } + end + end + + def embeddings(model_id:, input:) + client.embeddings(model: model_id, input:) + end + + # Which server we are talking to decides which non-standard metadata is + # worth reading later. +owned_by+ is the documented hint; the structural + # fallback catches an operator who overrode it. + def server_flavour + cards = models + owner = cards.first&.dig(:raw, "owned_by").to_s.downcase + + return owner if %w[vllm sglang llamacpp openai].include?(owner) + + cards.any? { |card| card[:raw].key?("max_model_len") || card[:raw].key?("root") } ? "vllm" : "unknown" + end + + private + + attr_reader :connection + + def client + @client ||= Llm::Client.new(base_url: connection.base_url, + api_key: connection.api_key, + headers: connection.custom_headers) + end + end + end +end diff --git a/app/services/llm/adapters/registry_backed.rb b/app/services/llm/adapters/registry_backed.rb new file mode 100644 index 000000000000..f551f878cef7 --- /dev/null +++ b/app/services/llm/adapters/registry_backed.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Adapters + # Discovery for providers that do not speak the OpenAI model-list API. + # + # Anthropic, Gemini, Bedrock and the rest each list models differently -- and + # some, like Bedrock, need request signing rather than a bearer token. Rather + # than implement each, the model list comes from RubyLLM's registry for that + # provider, which is a published catalogue rather than a live query. + # + # The consequence is worth stating: this list is what the provider offers in + # general, not what these particular credentials can reach. An administrator + # can add anything missing by hand. + class RegistryBacked + def initialize(connection) + @connection = connection + end + + def models + RubyLLM.models.by_provider(connection.api_format.to_sym).map do |info| + { + id: info.id, + display_name: info.name, + raw: { "context_window" => info.context_window, "owned_by" => connection.api_format } + } + end + rescue StandardError => e + raise Llm::Client::ApiError.new("Model registry lookup failed: #{e.class} #{e.message}", status: nil) + end + + # Nothing is queried, so there is no server to characterise. + def server_flavour = connection.api_format + + private + + attr_reader :connection + end + end +end diff --git a/app/services/llm/capabilities.rb b/app/services/llm/capabilities.rb new file mode 100644 index 000000000000..cf610a3067c3 --- /dev/null +++ b/app/services/llm/capabilities.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # The capabilities a model may have, and how to read the published ones. + module Capabilities + ALL = %i[embeddings function_calling structured_output vision reasoning].freeze + + # Capabilities a chat model can be asked about. +embeddings+ is deliberately + # absent: it is a different kind of model, not a feature of a chat one. + CHAT = (ALL - %i[embeddings]).freeze + EMBEDDING = %i[embeddings].freeze + + module_function + + # What a public registry says about this model id. + # + # Advisory only. The registry describes a model as some vendor deploys it, + # which is not the same as this deployment: the same weights are catalogued + # with contradictory capability flags and context windows an order of + # magnitude apart across providers. Verdicts derived from it are therefore + # recorded with source "metadata", and an administrator can overrule them. + # + # @return [Hash{Symbol => Symbol}, nil] capability => :supported / :unsupported, + # or nil when the registry does not know the model -- the normal case for a + # self-hosted server. + def published_for(model_id) + info = RubyLLM.models.find(model_id) + + { states: states_from(info), context_window: info.context_window, display_name: info.name } + rescue RubyLLM::ModelNotFoundError + nil + rescue StandardError => e + # Registry lookup is an enrichment; it must never break a model sync. + Rails.logger.info { "LLM capability lookup for #{model_id} failed: #{e.class} #{e.message}" } + nil + end + + def states_from(info) + embedding = info.type.to_s == "embedding" + published = Array(info.capabilities).map(&:to_sym) + relevant = embedding ? EMBEDDING : CHAT + + states = relevant.index_with { |capability| published.include?(capability) ? :supported : :unsupported } + states.merge(embeddings: embedding ? :supported : :unsupported) + end + + def label(capability) + I18n.t("llm.capabilities.#{capability}.label") + end + end +end diff --git a/app/services/llm/client.rb b/app/services/llm/client.rb new file mode 100644 index 000000000000..00641c991d48 --- /dev/null +++ b/app/services/llm/client.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # A thin model-discovery client for an OpenAI-API-compatible server. + # + # Inference goes through Llm::Session and RubyLLM. Discovery deliberately + # stays here, for two reasons. RubyLLM's model parsing discards max_model_len + # and root, which are the only trustworthy statement of a self-hosted + # deployment's real context window, and it substitutes OpenAI's capability + # heuristics for arbitrary model ids. And this path runs on OpenProject.httpx, + # so it is covered by the SSRF filter that RubyLLM's own Faraday stack bypasses. + # + # The configured base URL is expected to already contain the API version segment + # (for example +https://example.com/v1+), matching what every provider documents + # and what every OpenAI client library expects. This client only appends the + # endpoint path. + # + # Errors come from Llm::Errors, which is shared with the RubyLLM-backed + # inference path. They are aliased here because every existing caller rescues + # them by their Llm::Client:: name, and both names refer to the same classes. + class Client + Error = Llm::Errors::Error + ConnectionError = Llm::Errors::ConnectionError + SsrfError = Llm::Errors::SsrfError + TimeoutError = Llm::Errors::TimeoutError + AuthenticationError = Llm::Errors::AuthenticationError + ApiError = Llm::Errors::ApiError + ParseError = Llm::Errors::ParseError + + # The global httpx defaults (connect 3s / read 3s / request 10s, all + # writable: false) are tuned for storage and webhook calls and are far too + # tight for an inference endpoint. Every call site must override them. + PROBE_TIMEOUT = { + timeout: { connect_timeout: 5, read_timeout: 15, request_timeout: 20 } + }.freeze + + def initialize(base_url:, api_key: nil, timeout: PROBE_TIMEOUT, headers: {}) + @base_url = base_url.to_s.chomp("/") + @api_key = api_key + @timeout = timeout + @headers = (headers || {}).compact_blank + end + + # The model catalogue as the server reports it, verbatim. + # + # Kept verbatim on purpose: vLLM adds +max_model_len+ and +root+ to each card, + # which is the only trustworthy source for a deployment's real context window. + # + # @return [Hash] the parsed +GET /models+ body + def models + body = get("/models") + + raise ParseError, "Response does not contain a model list" unless body.is_a?(Hash) && body["data"].is_a?(Array) + + body + end + + private + + attr_reader :base_url, :api_key, :timeout, :headers + + def get(path) + response = session.get(uri_for(path)) + # A connection-level failure yields an HTTPX::ErrorResponse. A real response + # carrying a 4xx/5xx is an ordinary HTTPX::Response — note that its #error + # is also populated (it delegates to #raise_for_status), so the response + # class, not #error, is what distinguishes the two. + handle_transport_error(response) if response.is_a?(HTTPX::ErrorResponse) + handle_status(response) + parse(response) + rescue OpenProject::ServerSideRequestForgeryError + # The SSRF plugin either raises directly or surfaces the error as + # response.error, which #handle_transport_error covers. + raise SsrfError, "Host resolves to a blocked address" + end + + def session + request = OpenProject.httpx.with(timeout) + request = request.with(headers:) if headers.any? + api_key.present? ? request.plugin(:auth).bearer_auth(api_key) : request + end + + def uri_for(path) + URI.parse("#{base_url}#{path}") + rescue URI::InvalidURIError + raise ConnectionError, "Invalid URL" + end + + def handle_transport_error(response) + error = response.error + + case error + when OpenProject::ServerSideRequestForgeryError + raise SsrfError, "Host resolves to a blocked address" + when HTTPX::TimeoutError + raise TimeoutError, "Request timed out" + else + raise ConnectionError, error.class.name + end + end + + def handle_status(response) + status = response.status + return if status.in?(200..299) + + raise AuthenticationError, "Server rejected the API key (#{status})" if status.in?([401, 403]) + + raise ApiError.new("Server responded with #{status}", status:) + end + + # Parses the body directly rather than through HTTPX's +#json+, which insists on a + # JSON content type. Self-hosted servers behind a proxy do not reliably set one, + # and a content-type mismatch is not a reason to call a working server broken. + def parse(response) + JSON.parse(response.body.to_s) + rescue JSON::ParserError + raise ParseError, "Response is not valid JSON" + end + end +end diff --git a/app/services/llm/errors.rb b/app/services/llm/errors.rb new file mode 100644 index 000000000000..14d4c585bd6c --- /dev/null +++ b/app/services/llm/errors.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # The error taxonomy shared by every path that talks to an LLM server. + # + # Callers map these onto per-attribute contract errors or health check codes, + # so the set is deliberately small and describes *what the administrator has to + # fix*, not what the transport did. + # + # Response bodies never appear in a message. An OpenAI-compatible gateway + # routinely echoes the submitted Authorization header, upstream provider URLs + # and internal hostnames in its error payloads, and RubyLLM puts exactly that + # payload into its exception messages (+RubyLLM::Error#initialize+ falls back + # to +response.body+, and the middleware's own defaults come from parsing the + # body). #translate therefore discards the incoming message and logs it instead. + module Errors + class Error < StandardError; end + + # The server could not be reached at all. + class ConnectionError < Error; end + # The host resolved to an address blocked by the SSRF policy. + class SsrfError < ConnectionError; end + # The server took too long to answer. + class TimeoutError < ConnectionError; end + # The server answered, but rejected our credentials. + class AuthenticationError < Error; end + + # The server answered with an unexpected status. + class ApiError < Error + attr_reader :status + + def initialize(message, status: nil) + super(message) + @status = status + end + end + + # The prompt exceeded the model's context window. + class ContextLengthError < ApiError; end + # The server is throttling us. + class RateLimitedError < ApiError; end + + # The server answered successfully with something we cannot read. + class ParseError < Error; end + + # The connection cannot be expressed at all -- an api_format we cannot supply + # credentials for, or a model the provider refuses to accept. + class ConfigurationError < Error; end + + # A feature asked for a client before its model resolved. Carries the + # Llm::Runtime::Resolution status so a caller can tell "no server configured" + # apart from "this model cannot do that". + class NotReady < Error + attr_reader :status + + def initialize(status) + super("LLM is not ready to serve this feature (#{status})") + @status = status + end + end + + module_function + + # Maps anything RubyLLM or Faraday raised onto this taxonomy. + # + # @param error [StandardError] + # @return [Llm::Errors::Error] + def translate(error) + log(error) + + case error + when Llm::Errors::Error + error + when RubyLLM::Error, RubyLLM::ConfigurationError, RubyLLM::ModelNotFoundError + from_ruby_llm(error) + else + from_transport(error) + end + end + + def from_ruby_llm(error) + status = status_of(error) + + case error + when RubyLLM::UnauthorizedError, RubyLLM::ForbiddenError + AuthenticationError.new("Server rejected the API key (#{status})") + when RubyLLM::ContextLengthExceededError + ContextLengthError.new("Prompt exceeds the model's context window", status:) + when RubyLLM::RateLimitError + RateLimitedError.new("Server is rate limiting requests", status:) + when RubyLLM::ConfigurationError, RubyLLM::ModelNotFoundError + ConfigurationError.new("The connection is not usable as configured") + else + # Covers BadRequestError, PaymentRequiredError, ServerError, + # ServiceUnavailableError, OverloadedError and the middleware's catch-all + # -- the last of which is how 404/405/501 arrive, i.e. a server that does + # not implement the endpoint we asked for. + ApiError.new("Server responded with #{status || 'an error'}", status:) + end + end + + def from_transport(error) + case error + when Faraday::TimeoutError, Timeout::Error, Errno::ETIMEDOUT + TimeoutError.new("Request timed out") + when Faraday::ConnectionFailed, Faraday::SSLError, SocketError, Errno::ECONNREFUSED + ConnectionError.new(error.class.name) + when JSON::ParserError + ParseError.new("Response is not valid JSON") + else + Error.new(error.class.name) + end + end + + # Runs the block, re-raising any RubyLLM or Faraday failure as an Llm::Errors. + def wrap + yield + rescue StandardError => e + raise translate(e) + end + + def status_of(error) + error.respond_to?(:response) ? error.response&.status : nil + end + + # The original message is the only place the upstream detail exists, and an + # administrator debugging a gateway needs it -- it just must not travel back + # up to a view or a contract error. + def log(error) + Rails.logger.info { "LLM request failed: #{error.class}: #{error.message}" } + end + end +end diff --git a/app/services/llm/probes/embeddings_probe.rb b/app/services/llm/probes/embeddings_probe.rb new file mode 100644 index 000000000000..4bf2fc225606 --- /dev/null +++ b/app/services/llm/probes/embeddings_probe.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Probes + # Determines whether a model can produce embeddings, by asking it to. + # + # This is the only behavioural probe in the system, and it qualifies because + # it cannot lie: a server either returns a vector or it does not. Probes for + # tool calling deliberately do not exist -- several current vLLM releases + # return HTTP 200 with the tool call as plain text on a fully capable server, + # deterministically, so retries would launder a wrong answer into a confident + # one rather than correcting it. + # + # The 200 case is checked by shape rather than by status, because unknown + # parameters are silently dropped by vLLM, llama.cpp and Ollama alike: a 200 + # on its own proves nothing. + class EmbeddingsProbe + PROBE_INPUT = "openproject" + + # The server understood the request and refused it for this model. Anything + # else -- 5xx, throttling -- says something about the server, not the model. + REFUSED_STATUSES = [400, 404, 405, 501].freeze + + Result = Data.define(:state, :detail) do + def supported? = state == :supported + end + + def initialize(connection) + @connection = connection + end + + def call(model_id) + classify(session.embed(PROBE_INPUT, model: model_id)) + rescue Llm::Errors::ApiError => e + return unsupported(e.status) if e.status.in?(REFUSED_STATUSES) + + unknown("http_#{e.status}") + rescue Llm::Errors::AuthenticationError + unknown("unauthorized") + rescue Llm::Errors::Error => e + unknown(e.class.name.demodulize.underscore) + end + + private + + attr_reader :connection + + # Never retried: a refusal is the answer we are looking for, and repeating + # it would only slow the probe down. + def session + @session ||= Llm::Session.for(connection, timeout: Llm::Session::PROBE_TIMEOUT, max_retries: 0) + end + + def classify(embedding) + vector = embedding.vectors + vector = vector.first if vector.is_a?(Array) && vector.first.is_a?(Array) + + if vector.is_a?(Array) && vector.any? && vector.all?(Numeric) + Result.new(state: :supported, detail: { "dimensions" => vector.length }) + else + # A 200 whose body is not an embedding response: the server accepted the + # request but answered with something else entirely. + unknown("unexpected_body") + end + end + + def unsupported(status) + Result.new(state: :unsupported, detail: { "http_status" => status }) + end + + def unknown(reason) + Result.new(state: :unknown, detail: { "reason" => reason }) + end + end + end +end diff --git a/app/services/llm/runtime.rb b/app/services/llm/runtime.rb new file mode 100644 index 000000000000..8acfcb0aa763 --- /dev/null +++ b/app/services/llm/runtime.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Answers "which model should this feature use, and can it run right now?". + # + # The single place model resolution happens, so that every feature agrees on + # what an unset value means: + # + # per-item override -> feature binding -> connection default -> unbound + # + # A blank value at any level means "inherit from the level below". + class Runtime + # :ready - go ahead + # :feature_disabled - the feature's own toggle is off + # :no_connection - no LLM server configured, or AI switched off globally + # :unbound - nothing has chosen a model for this feature yet + # :model_missing - the chosen model is not in the server's catalogue + # :incapable - the chosen model is known not to support what is needed + Resolution = Data.define(:feature, :connection, :model_id, :status, :missing_capabilities) do + def ready? = status == :ready + + # A chat builder for the resolved model. + # + # Note that RubyLLM enforces none of the feature's declared requirements: + # Chat#with_schema performs no capability check, and a model absent from + # RubyLLM's registry is described by Model::Info.default, which claims + # structured output, vision and function calling for everything. The + # capability verdicts consulted in #call above are the only real gate. + # + # @return [RubyLLM::Chat] + def chat(**) + ensure_usable!(:chat) + session(**).chat(model_id) + end + + # @return [RubyLLM::Embedding] + def embed(input, dimensions: nil, **) + ensure_usable!(:embedding) + session(**).embed(input, model: model_id, dimensions:) + end + + # @return [Llm::Session] + def session(**) + Llm::Session.for(connection, **) + end + + private + + # Features are resolved by kind, so asking a chat feature to embed means a + # caller has confused two features -- a bug, not a configuration problem. + def ensure_usable!(kind) + raise Llm::Errors::NotReady, status unless ready? + return if feature.public_send(:"#{kind}?") + + raise Llm::Errors::NotReady, :wrong_kind + end + end + + class << self + # @param feature_key [Symbol] a key registered with OpenProject::Llm::Features + # @param override [String, nil] a per-item model choice, e.g. one stored on + # a description assistant action. Blank means inherit. + def for(feature_key, override: nil) + new(OpenProject::Llm::Features[feature_key], override:).call + end + end + + def initialize(feature, override: nil) + @feature = feature + @override = override + end + + def call + return resolution(:feature_disabled) unless feature.available? + return resolution(:no_connection) unless LlmConnection.available? + + model_id = resolved_model_id + return resolution(:unbound) if model_id.blank? + return resolution(:model_missing, model_id:) unless connection.available_model_ids.include?(model_id) + + missing = unsupported_capabilities(model_id) + return resolution(:incapable, model_id:, missing_capabilities: missing) if missing.any? + + resolution(:ready, model_id:) + end + + private + + attr_reader :feature, :override + + def connection + @connection ||= LlmConnection.instance + end + + def resolved_model_id + override.presence || binding_model_id || connection_default + end + + def binding_model_id + connection.feature_bindings.find_by(feature_key: feature.key.to_s)&.model_id.presence + end + + def connection_default + feature.embedding? ? connection.default_embedding_model_id : connection.default_chat_model_id + end + + # Only a definite :unsupported blocks. An :unknown verdict -- which is the + # normal state for a server that reports nothing about its models -- is + # surfaced in the UI as a warning but never prevents a call. + def unsupported_capabilities(model_id) + return [] if feature.requires.empty? + + blocking = connection.capability_verdicts + .for_model(model_id) + .where(capability: feature.requires.map(&:to_s), state: "unsupported") + + blocking.pluck(:capability).map(&:to_sym) + end + + def resolution(status, model_id: nil, missing_capabilities: []) + Resolution.new(feature:, connection: status == :feature_disabled ? nil : connection, + model_id:, status:, missing_capabilities:) + end + end +end diff --git a/app/services/llm/session.rb b/app/services/llm/session.rb new file mode 100644 index 000000000000..2a672852f962 --- /dev/null +++ b/app/services/llm/session.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Turns an LlmConnection into something that can issue requests. + # + # Every inference request in OpenProject goes through here, so that the + # administrator's stored settings -- endpoint, credential, custom headers -- + # are applied in exactly one place. + # + # Note that RubyLLM builds its own Faraday connection over net_http and so + # does *not* pass through OpenProject.httpx and its SSRF filter. Inference + # traffic to an administrator-supplied base URL is therefore not SSRF + # filtered. This is a deliberate, recorded decision. Model discovery + # deliberately stays on Llm::Client, which is filtered. + class Session + # RubyLLM exposes a single Faraday timeout rather than Llm::Client's + # connect/read/request triple, so those three values collapse to one. + # RubyLLM's own default is 300s, which is far too long to hold a Rails + # worker for a check. + PROBE_TIMEOUT = 20 + INFERENCE_TIMEOUT = 180 + + # Formats whose credentials an LlmConnection cannot express: Bedrock needs a + # secret key and a region as well as a key, Vertex AI a project and a + # location. The model has a single api_key column. + UNSUPPORTED_FORMATS = %w[bedrock vertexai].freeze + + # A self-hosted OpenAI-compatible server frequently needs no credential at + # all, but RubyLLM's ensure_configured! raises when a provider declares + # _api_key as required and none is set. A placeholder gets us past + # that check; a server that wants no credential ignores it. + PLACEHOLDER_API_KEY = "-" + + class << self + def for(connection, timeout: INFERENCE_TIMEOUT, max_retries: 1) + new(connection, timeout:, max_retries:) + end + + def supports?(api_format) + UNSUPPORTED_FORMATS.exclude?(api_format.to_s) + end + end + + def initialize(connection, timeout: INFERENCE_TIMEOUT, max_retries: 1) + @connection = connection + @timeout = timeout + @max_retries = max_retries + + unless self.class.supports?(connection.api_format) + raise Llm::Errors::ConfigurationError, + "#{connection.api_format} needs credentials an LLM connection cannot store" + end + end + + def provider + @provider ||= connection.api_format.to_sym + end + + # Translates failures raised by a chat's own request methods. + # + # RubyLLM::Chat is a builder: the request happens later, when the caller + # invokes #ask, long after this class has returned. Extending the instance + # is what makes the error taxonomy hold for that call too, rather than + # relying on every caller to remember to wrap it. The with_* builder methods + # return self, so the extension survives them. + module TranslatesErrors + def complete(...) + Llm::Errors.wrap { super } + end + end + + # @param model_id [String] + # @return [RubyLLM::Chat] + def chat(model_id) + Llm::Errors.wrap do + context.chat(model: model_id, provider:, assume_model_exists: true) + .extend(TranslatesErrors) + end + end + + # @param input [String, Array] + # @return [RubyLLM::Embedding] + def embed(input, model:, dimensions: nil) + Llm::Errors.wrap do + context.embed(input, model:, provider:, assume_model_exists: true, dimensions:) + end + end + + private + + attr_reader :connection, :timeout, :max_retries + + # A per-call context rather than RubyLLM.configure: the global configuration + # is process-wide, and writing an administrator's endpoint and credential + # into it would leak them across requests and across tenants. + def context + @context ||= RubyLLM.context do |config| + config.public_send(:"#{provider}_api_base=", connection.base_url) + apply_credentials(config) + + config.openproject_custom_headers = connection.custom_headers + config.request_timeout = timeout + # RubyLLM retries POSTs three times by default, so one completion can be + # billed four times. Callers state what they are willing to pay for. + config.max_retries = max_retries + config.logger = Rails.logger + end + end + + def apply_credentials(config) + key = connection.api_key.presence || (PLACEHOLDER_API_KEY if api_key_required?) + + config.public_send(:"#{provider}_api_key=", key) if key + end + + # Asked of the gem rather than hard-coded: which providers insist on a key, + # as opposed to accepting a bare base URL, is RubyLLM's business and changes + # between releases. + def api_key_required? + RubyLLM::Provider.resolve(provider) + .configuration_requirements + .include?(:"#{provider}_api_key") + end + end +end diff --git a/app/services/llm/structured_output.rb b/app/services/llm/structured_output.rb new file mode 100644 index 000000000000..5aa0af7f0dd9 --- /dev/null +++ b/app/services/llm/structured_output.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Turns a schema-constrained answer into a Hash, or fails loudly. + # + # RubyLLM's Chat#with_schema fails soft: when the answer does not parse as + # JSON it rescues and leaves the content as a String, so a caller expecting a + # Hash gets a String and only notices further downstream. + # + # That matters most on a self-hosted server, which is exactly where the + # structured_output capability verdict is :unknown rather than :supported, and + # so exactly where the model is most likely to answer in prose. + module StructuredOutput + module_function + + # @param message [RubyLLM::Message] the answer from Chat#ask + # @raise [Llm::Errors::ParseError] when the model did not honour the schema + # @return [Hash] + def parse!(message) + content = message.respond_to?(:content) ? message.content : message + + return content.deep_symbolize_keys if content.is_a?(Hash) + + raise Llm::Errors::ParseError, "Model did not answer with the requested structure" + end + end +end diff --git a/app/services/llm/validators/configuration_validator.rb b/app/services/llm/validators/configuration_validator.rb new file mode 100644 index 000000000000..0170d38bb62c --- /dev/null +++ b/app/services/llm/validators/configuration_validator.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # What can be said about the connection without asking the server anything. + class ConfigurationValidator < HealthReports::ValidatorGroup + def self.key = :configuration + + private + + def validate + register_checks(:feature_flag, :base_url_present, :api_format_supported, :credentials_present, :enabled) + + feature_flag + base_url_present + api_format_supported + credentials_present + connection_enabled + end + + def feature_flag + if OpenProject::FeatureDecisions.llm_connection_active? + pass_check(:feature_flag) + else + # Not a failure: an administrator may reasonably configure and test the + # connection before switching the feature on. + warn_check(:feature_flag, :feature_flag_off) + end + end + + def base_url_present + return pass_check(:base_url_present) if subject.base_url.present? + + fail_check(:base_url_present, :not_configured) + end + + def api_format_supported + return pass_check(:api_format_supported) if Llm::Session.supports?(subject.api_format) + + fail_check(:api_format_supported, :unsupported_api_format, context: { api_format: subject.api_format.to_s }) + end + + # Only ever a warning. A self-hosted server on a trusted network legitimately + # needs no credential, and a key that is genuinely required but wrong is + # reported precisely by the server group rather than guessed at here. + def credentials_present + return pass_check(:credentials_present) if subject.api_key.present? + + warn_check(:credentials_present, :api_key_missing) + end + + def connection_enabled + return pass_check(:enabled) if subject.enabled? + + warn_check(:enabled, :connection_disabled) + end + end + end +end diff --git a/app/services/llm/validators/connection_validator.rb b/app/services/llm/validators/connection_validator.rb new file mode 100644 index 000000000000..77ee9858b7d7 --- /dev/null +++ b/app/services/llm/validators/connection_validator.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # The health report for an LLM connection. + # + # Nothing here reads User.current: the whole report has to be reproducible + # from a background job, which is what lets the same code answer both the + # administrator's "Run checks" and the scheduled re-check. + class ConnectionValidator < HealthReports::Validator + register_group ConfigurationValidator + + register_group ServerValidator, + precondition: ->(_, report) { report.group(:configuration).non_failure? } + + # Costs a real completion, so it is not part of the scheduled run. + register_group InferenceValidator, + precondition: lambda { |connection, report| + connection.deep_health_check? && report.group(:configuration).non_failure? + } + + register_group ModelValidator + register_group FeatureValidator + end + end +end diff --git a/app/services/llm/validators/feature_validator.rb b/app/services/llm/validators/feature_validator.rb new file mode 100644 index 000000000000..0d587a7e0443 --- /dev/null +++ b/app/services/llm/validators/feature_validator.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # Whether each registered feature can actually run. + # + # Driven by Llm::Runtime, so the health report and the runtime can never + # disagree about what is usable. That makes this the most informative group + # in the report: it answers "will the AI features work" rather than "is the + # server up". + # + # The check keys are fixed and the feature names travel in the result + # context. Per-feature keys would need a translation per feature and would + # break HealthReports::ResultComponent#text, which resolves a check's label + # from its key. + class FeatureValidator < HealthReports::ValidatorGroup + def self.key = :features + + private + + def validate + register_checks(:bindings_resolvable, :locked_bindings_intact) + + bindings_resolvable + locked_bindings_intact + end + + def bindings_resolvable + by_status = resolutions.group_by(&:status) + + if (incapable = by_status[:incapable]).present? + fail_check(:bindings_resolvable, :features_incapable, + context: { features: labels(incapable), + capabilities: capability_labels(incapable) }) + end + + if (missing = by_status[:model_missing]).present? + fail_check(:bindings_resolvable, :features_model_missing, context: { features: labels(missing) }) + end + + if (unbound = by_status[:unbound]).present? + warn_check(:bindings_resolvable, :features_unbound, context: { features: labels(unbound) }) + end + + pass_check(:bindings_resolvable) + end + + # A locked binding is the record that a vector index exists and which model + # and dimension it was written under. If that model has left the catalogue + # the index can no longer be extended or queried consistently, which is a + # data problem rather than a configuration one. + def locked_bindings_intact + broken = subject.feature_bindings.select { |binding| binding.locked? && binding.dangling? } + + return pass_check(:locked_bindings_intact) if broken.empty? + + fail_check(:locked_bindings_intact, :locked_model_missing, + context: { features: broken.filter_map { |b| b.feature&.label }.join(", ") }) + end + + def resolutions + @resolutions ||= OpenProject::Llm::Features.available.map do |feature| + Llm::Runtime.for(feature.key) + end + end + + def labels(resolutions) = resolutions.map { |resolution| resolution.feature.label }.join(", ") + + def capability_labels(resolutions) + resolutions.flat_map(&:missing_capabilities) + .uniq + .map { |capability| Llm::Capabilities.label(capability) } + .join(", ") + end + end + end +end diff --git a/app/services/llm/validators/inference_validator.rb b/app/services/llm/validators/inference_validator.rb new file mode 100644 index 000000000000..fe7d2c39aa1e --- /dev/null +++ b/app/services/llm/validators/inference_validator.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # Whether the server will actually answer a request. + # + # This is the only check that proves a connection works, and the only one + # available at all for a deployment that publishes no model list. It is also + # the only one that costs money, so it runs on demand rather than on the + # schedule -- see LlmConnection#deep_health_check. + # + # Deliberately absent: probes for tool calling and structured output. Several + # current vLLM releases answer a tool call with plain text on a fully capable + # server, deterministically, so a probe would turn a wrong answer into a + # confident one rather than discovering anything. + class InferenceValidator < HealthReports::ValidatorGroup + PROMPT = "ping" + + def self.key = :inference + + private + + def validate + register_checks(:chat_round_trip) + + model_id = chat_model_id + return warn_check(:chat_round_trip, :no_model_to_test) if model_id.blank? + + round_trip(model_id) + end + + def round_trip(model_id) + answer = session.chat(model_id).with_temperature(0).ask(PROMPT) + + if answer.content.to_s.strip.empty? + warn_check(:chat_round_trip, :empty_completion, context: { model: model_id }) + else + pass_check(:chat_round_trip) + end + rescue Llm::Errors::AuthenticationError + fail_check(:chat_round_trip, :invalid_api_key) + rescue Llm::Errors::TimeoutError + fail_check(:chat_round_trip, :request_timed_out) + rescue Llm::Errors::ApiError => e + fail_check(:chat_round_trip, :chat_failed, context: { model: model_id, status: e.status.to_s }) + rescue Llm::Errors::Error + fail_check(:chat_round_trip, :connection_error) + end + + # The configured default first, so the check exercises what features will + # actually use rather than an arbitrary entry in the catalogue. + def chat_model_id + subject.default_chat_model_id.presence || subject.available_model_ids.first + end + + # Never retried: a failing check should report the failure, not pay for it + # three more times. + def session + Llm::Session.for(subject, timeout: Llm::Session::PROBE_TIMEOUT, max_retries: 0) + end + end + end +end diff --git a/app/services/llm/validators/model_validator.rb b/app/services/llm/validators/model_validator.rb new file mode 100644 index 000000000000..e02f0f6ad809 --- /dev/null +++ b/app/services/llm/validators/model_validator.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # What the stored catalogue says, without asking the server anything. + class ModelValidator < HealthReports::ValidatorGroup + STALE_AFTER = 30.days + + def self.key = :models + + private + + def validate + register_checks(:catalogue_present, :catalogue_fresh, :default_chat_model, :default_embedding_model) + + catalogue_present + catalogue_fresh + default_chat_model + default_embedding_model + end + + # Never a failure. A gateway may expose chat and no catalogue at all, which + # is why models can be entered by hand in the first place. + def catalogue_present + return pass_check(:catalogue_present) if subject.available_model_ids.any? + + warn_check(:catalogue_present, :no_models) + end + + def catalogue_fresh + fetched_at = subject.catalogue_fetched_at + + if fetched_at.blank? + warn_check(:catalogue_fresh, :catalogue_never_fetched) + elsif fetched_at < STALE_AFTER.ago + # Context is serialised to jsonb, so a Time has to be formatted here. + warn_check(:catalogue_fresh, :catalogue_stale, context: { fetched_at: I18n.l(fetched_at.to_date) }) + else + pass_check(:catalogue_fresh) + end + end + + def default_chat_model + check_default(:default_chat_model, subject.default_chat_model_id) + end + + # Only meaningful once something wants embeddings; otherwise an unset + # default is not a defect. + def default_embedding_model + return pass_check(:default_embedding_model) if OpenProject::Llm::Features.for_kind(:embedding).empty? + + check_default(:default_embedding_model, subject.default_embedding_model_id) + end + + def check_default(key, model_id) + if model_id.blank? + warn_check(key, :default_model_unset) + elsif subject.available_model_ids.exclude?(model_id) + warn_check(key, :default_model_missing, context: { model: model_id }) + else + pass_check(key) + end + end + end + end +end diff --git a/app/services/llm/validators/server_validator.rb b/app/services/llm/validators/server_validator.rb new file mode 100644 index 000000000000..13d15dd763c0 --- /dev/null +++ b/app/services/llm/validators/server_validator.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # Whether the server answers, and whether it accepts our credentials. + # + # Uses the model list, which costs nothing on any provider. That only exists + # for OpenAI-compatible endpoints -- for the others the catalogue comes from + # RubyLLM's registry, so there is nothing free to ask, and reachability can + # only be established by the inference group's billed request. + class ServerValidator < HealthReports::ValidatorGroup + # A gateway may implement chat and nothing else. That is a supported + # deployment, not a broken one, so it must not read as unreachable. + MODELS_ENDPOINT_ABSENT = LlmServerValidator::MODELS_ENDPOINT_ABSENT + + def self.key = :server + + private + + def validate + register_checks(:reachable, :credentials_accepted) + + return unless queries_the_server? + + list_models + end + + def queries_the_server? + subject.api_format == Llm::Adapters::OPENAI_COMPATIBLE + end + + def list_models + client.models + pass_check(:reachable) + pass_check(:credentials_accepted) + rescue Llm::Errors::AuthenticationError + pass_check(:reachable) + fail_check(:credentials_accepted, :invalid_api_key) + rescue Llm::Errors::TimeoutError + fail_check(:reachable, :request_timed_out) + rescue Llm::Errors::SsrfError + fail_check(:reachable, :ssrf_filtered) + rescue Llm::Errors::ApiError => e + answered_with_error(e) + rescue Llm::Errors::ParseError + pass_check(:reachable) + fail_check(:credentials_accepted, :not_openai_compatible) + rescue Llm::Errors::Error + fail_check(:reachable, :connection_error) + end + + # The server answered, so it is reachable and did not reject us. + def answered_with_error(error) + pass_check(:reachable) + + if error.status.in?(MODELS_ENDPOINT_ABSENT) + # It simply does not publish a catalogue. A supported deployment. + warn_check(:credentials_accepted, :no_models_endpoint) + else + fail_check(:credentials_accepted, :server_error, context: { status: error.status.to_s }) + end + end + + def client + Llm::Client.new(base_url: subject.base_url, + api_key: subject.api_key, + headers: subject.custom_headers) + end + end + end +end diff --git a/app/services/llm_connections/detect_capabilities_service.rb b/app/services/llm_connections/detect_capabilities_service.rb new file mode 100644 index 000000000000..c9e9efe78d4f --- /dev/null +++ b/app/services/llm_connections/detect_capabilities_service.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Records what a model on this server can do. + # + # Probing every listed model would be wrong: a gateway can list hundreds, each + # probe is a request, and some providers bill per request. So the models worth + # asking about are either the one an administrator is about to bind, or a small + # number whose names suggest they are embedding models. + class DetectCapabilitiesService + # Naming is a hint for which models are worth spending a probe on, never a + # verdict in itself. + EMBEDDING_NAME_HINT = /embed|bge|e5|gte|nomic|minilm/i + BACKGROUND_LIMIT = 10 + + def initialize(connection) + @connection = connection + end + + # Probes a specific model, synchronously. Used when an administrator binds a + # model to a feature that requires embeddings -- the verdict that matters. + def detect(model_id) + return existing_admin_verdict(model_id) if admin_asserted?(model_id) + + result = probe.call(model_id) + record(model_id, result) + end + + # Pre-colours the model list after a connect, without spending a request per + # model. Everything not probed stays unknown, which never blocks. + def detect_likely_embedding_models + candidates.each { |model_id| detect(model_id) } + end + + private + + attr_reader :connection + + def probe + @probe ||= Llm::Probes::EmbeddingsProbe.new(connection) + end + + def candidates + connection.available_model_ids + .grep(EMBEDDING_NAME_HINT) + .reject { |model_id| admin_asserted?(model_id) } + .first(BACKGROUND_LIMIT) + end + + # An administrator knows things about their deployment that a probe cannot + # determine, so their assertion is never overwritten by re-detection. + def admin_asserted?(model_id) + verdicts.for_model(model_id).for_capability(:embeddings).sticky.exists? + end + + def existing_admin_verdict(model_id) + verdicts.for_model(model_id).for_capability(:embeddings).first + end + + def record(model_id, result) + verdict = verdicts.find_or_initialize_by(model_id:, capability: "embeddings") + verdict.update!(state: result.state.to_s, + source: "probe", + detail: result.detail, + checked_at: Time.current) + verdict + end + + def verdicts + connection.capability_verdicts + end + end +end diff --git a/app/services/llm_connections/enrich_capabilities_service.rb b/app/services/llm_connections/enrich_capabilities_service.rb new file mode 100644 index 000000000000..e5d53976d4b0 --- /dev/null +++ b/app/services/llm_connections/enrich_capabilities_service.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Fills in what a public registry publishes about the models a connection offers. + # + # This is the "the provider tells us" path. A hosted provider's model ids are + # catalogued, so capabilities arrive without asking the server anything. A + # self-hosted deployment naming its model "default" or "my-finetune-v3" is not + # catalogued, nothing is filled in, and the administrator enters capabilities + # by hand instead. + # + # An administrator's assertion is never overwritten: they know things about + # their deployment that no registry can. + class EnrichCapabilitiesService + def initialize(connection) + @connection = connection + end + + def call + connection.models.active.find_each { |llm_model| enrich(llm_model) } + + ServiceResult.success(result: connection) + end + + private + + attr_reader :connection + + def enrich(llm_model) + published = Llm::Capabilities.published_for(llm_model.external_id) + return if published.nil? + + apply_metadata(llm_model, published) + published[:states].each { |capability, state| record(llm_model.external_id, capability, state) } + end + + def apply_metadata(llm_model, published) + attributes = {} + attributes[:display_name] = published[:display_name] if llm_model.display_name.blank? + + if published[:context_window].present? && llm_model.raw_metadata["max_model_len"].blank? + attributes[:raw_metadata] = llm_model.raw_metadata.merge("context_window" => published[:context_window]) + end + + llm_model.update!(attributes) if attributes.any? + end + + def record(model_id, capability, state) + verdict = connection.capability_verdicts.find_or_initialize_by(model_id:, capability: capability.to_s) + # Anything an administrator or a probe established beats a published claim: + # both looked at this deployment, the registry did not. + return if verdict.persisted? && verdict.source.in?(%w[admin probe]) + + verdict.update!(state: state.to_s, source: "metadata", checked_at: Time.current) + end + end +end diff --git a/app/services/llm_connections/env_sync_service.rb b/app/services/llm_connections/env_sync_service.rb new file mode 100644 index 000000000000..9b7c0184bfa8 --- /dev/null +++ b/app/services/llm_connections/env_sync_service.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Applies an environment-provided configuration to the connection record. + # + # Uses EnvironmentUpdateContract, which lifts the "configured from environment + # is read-only" guard and, crucially, does not probe the LLM server: the + # container running the seed may well start before the server does. + class EnvSyncService + def initialize(env_config) + @config = env_config.deep_symbolize_keys + end + + def call + UpdateService + .new(user: User.system, + model: LlmConnection.instance, + contract_class: EnvironmentUpdateContract, + sync_models: false) + .call(**attributes) + end + + private + + attr_reader :config + + # Absent keys are written as nil on purpose: the environment is the source + # of truth here, and the form is read-only while it is. Keeping a stored + # value that was removed from the environment would leave, for example, an + # obsolete API key in use with no supported way to clear it. + def attributes + { + base_url: config.fetch(:base_url), + api_key: config[:api_key], + default_chat_model_id: config[:default_chat_model], + default_embedding_model_id: config[:default_embedding_model], + enabled: ActiveRecord::Type::Boolean.new.deserialize(config.fetch(:enabled, true)) + } + end + end +end diff --git a/app/services/llm_connections/selectable_models_query.rb b/app/services/llm_connections/selectable_models_query.rb new file mode 100644 index 000000000000..1ca7d8a4a974 --- /dev/null +++ b/app/services/llm_connections/selectable_models_query.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # The models offerable to a feature, each with why it is or is not usable. + # + # Models are never hidden. Hiding one produces the single support question + # nobody can answer -- "why can I not pick the model I know works" -- and it is + # exactly wrong when most verdicts are unknown. Instead each option carries a + # state the UI renders: selectable, selectable with a warning, or disabled with + # a reason. + class SelectableModelsQuery + Option = Data.define(:model_id, :state, :reasons) do + def selectable? = state != :unsupported + + def warning? = state == :unknown + end + + def initialize(connection, feature) + @connection = connection + @feature = feature + end + + def call + offerable_model_ids.map { |model_id| option_for(model_id) } + end + + private + + attr_reader :connection, :feature + + # Models an administrator has switched off are not offered, but the one this + # feature is already bound to stays listed -- otherwise the select silently + # shows nothing where a working binding exists. + def offerable_model_ids + (connection.selectable_model_ids + [bound_model_id]).compact_blank.uniq + end + + def bound_model_id + connection.feature_bindings.find_by(feature_key: feature.key.to_s)&.model_id + end + + def option_for(model_id) + states = feature.requires.index_with { |capability| verdict_state(model_id, capability) } + + if states.value?(:unsupported) + Option.new(model_id:, state: :unsupported, + reasons: states.select { |_, s| s == :unsupported }.keys) + elsif states.value?(:unknown) + Option.new(model_id:, state: :unknown, + reasons: states.select { |_, s| s == :unknown }.keys) + else + Option.new(model_id:, state: :supported, reasons: []) + end + end + + # No verdict at all is the same as an inconclusive one: we do not know. + def verdict_state(model_id, capability) + verdicts.dig(model_id, capability.to_s)&.to_sym || :unknown + end + + def verdicts + @verdicts ||= connection.capability_verdicts + .pluck(:model_id, :capability, :state) + .group_by(&:first) + .transform_values { |rows| rows.to_h { |(_, capability, state)| [capability, state] } } + end + end +end diff --git a/app/services/llm_connections/set_attributes_service.rb b/app/services/llm_connections/set_attributes_service.rb new file mode 100644 index 000000000000..4a24ffe2bb37 --- /dev/null +++ b/app/services/llm_connections/set_attributes_service.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class SetAttributesService < BaseServices::SetAttributes + private + + def set_attributes(params) + super + + model.base_url = normalized_base_url if model.base_url.present? + set_singleton_defaults + end + + # +name+ and +type+ identify the record but are never user-editable, so they + # are set as system changes: the contract's readonly check only looks at + # attributes the user changed. + def set_singleton_defaults + model.change_by_system do + model.name ||= LlmConnection::SINGLETON_NAME + model.type ||= LlmConnection.name + end + end + + # Only trailing whitespace and slashes are removed. The /v1 segment is + # deliberately not added or stripped: silently rewriting an administrator's + # URL makes the eventual failure harder to diagnose, so a wrong shape is + # reported by the probe instead. + def normalized_base_url + model.base_url.strip.chomp("/") + end + end +end diff --git a/app/services/llm_connections/sync_models_service.rb b/app/services/llm_connections/sync_models_service.rb new file mode 100644 index 000000000000..0453b49a6c3c --- /dev/null +++ b/app/services/llm_connections/sync_models_service.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # Refreshes the model list from the remote server. + # + # Kept separate from the contract probe so the same path serves the "Refresh + # models" button, the update service and the environment seeder. + class SyncModelsService + def initialize(connection) + @connection = connection + end + + def call + store(adapter.models) + + ServiceResult.success(result: connection) + rescue Llm::Client::Error => e + Rails.logger.info { "LLM model sync for #{connection.base_url} failed: #{e.class} #{e.message}" } + ServiceResult.failure(errors: e.message) + end + + private + + attr_reader :connection + + def adapter + @adapter ||= Llm::Adapters.for(connection) + end + + def store(cards) + ActiveRecord::Base.transaction do + discard_verdicts_for_a_different_deployment(fingerprint) + connection.update!(connection_attributes) + upsert(cards) + withdraw_models_absent_from(cards) + discard_verdicts_for_vanished_models + end + + EnrichCapabilitiesService.new(connection).call + end + + def connection_attributes + now = Time.current + + { + catalogue_fetched_at: now, + last_connected_at: now, + connection_fingerprint: fingerprint, + options: connection.options.merge("server_flavour" => adapter.server_flavour) + } + end + + def fingerprint + @fingerprint ||= Digest::SHA256.hexdigest("#{connection.api_format}\0#{connection.base_url}\0#{connection.api_key}") + end + + def upsert(cards) + now = Time.current + + cards.each do |card| + model = connection.models.find_or_initialize_by(external_id: card.fetch(:id)) + model.update!(display_name: card[:display_name], + raw_metadata: card.fetch(:raw, {}), + last_seen_at: now, + active: true) + end + end + + # Deactivated rather than deleted, so a binding or verdict pointing at one + # still has something to name. Manual entries are left alone: the server was + # never the thing that confirmed them. + def withdraw_models_absent_from(cards) + connection.models + .discovered + .where.not(external_id: cards.map { |card| card.fetch(:id) }) + .update_all(active: false) + end + + # A changed base URL or key means a different deployment, so everything we + # learned about the old one is void -- including administrator assertions, + # which were about that deployment. + def discard_verdicts_for_a_different_deployment(new_fingerprint) + return if connection.connection_fingerprint.blank? + return if connection.connection_fingerprint == new_fingerprint + + connection.capability_verdicts.delete_all + end + + # Same deployment, but a model is gone. Its verdict is meaningless now, + # except an administrator's assertion: an operator restarting a server must + # not silently lose one. + def discard_verdicts_for_vanished_models + known = connection.available_model_ids + return if known.empty? + + connection.capability_verdicts.where.not(model_id: known).where.not(source: "admin").delete_all + end + end +end diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb new file mode 100644 index 000000000000..790f53b1bd04 --- /dev/null +++ b/app/services/llm_connections/update_service.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + class UpdateService < BaseServices::Update + # @param sync_models [Boolean] whether to refresh the model catalogue inline + # after a successful save. Provisioning from the environment passes false: + # seeding must not block on an LLM server that has not started yet, and + # enqueues Llm::SyncModelsJob instead. + def initialize(*, sync_models: true, **) + super(*, **) + @sync_models = sync_models + end + + private + + # The contract has already proven the server reachable when the credentials + # changed, so refreshing the catalogue here cannot be the thing that fails + # the save. A sync failure is therefore logged, not surfaced. + def after_perform(service_call) + super.tap do + next unless service_call.success? + + # Enabling or disabling the connection decides whether the scheduled + # health check has anything to do. + Llm::HealthCheckJob.toggle_cron_job + + next unless @sync_models + next unless connection_changed?(service_call.result) + + SyncModelsService.new(service_call.result).call + Llm::DetectCapabilitiesJob.perform_later + end + end + + def connection_changed?(connection) + connection.saved_changes.keys.intersect?(LlmServerValidator::CONNECTION_ATTRIBUTES) + end + end +end diff --git a/app/validators/llm_server_validator.rb b/app/validators/llm_server_validator.rb new file mode 100644 index 000000000000..2bd1924547fc --- /dev/null +++ b/app/validators/llm_server_validator.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Verifies that the configured URL really is an OpenAI-API-compatible server that +# accepts the configured credentials, by fetching its model list. +# +# Runs inside the contract so that a server which cannot be reached never gets +# persisted. It is guarded on the attributes that identify the connection: +# without that guard every unrelated save -- and every form render that builds a +# model through SetAttributesService -- would fire an outbound HTTP request. +class LlmServerValidator < ActiveModel::EachValidator + # Changing any of these means talking to a different server, or to the same + # server in a different dialect, so the connection must be proven again and + # its models resynchronised. + CONNECTION_ATTRIBUTES = %w[base_url api_key api_format].freeze + + # Statuses that mean "this server has no model list here", as opposed to "this + # server is broken". A gateway may route chat completions and nothing else. + MODELS_ENDPOINT_ABSENT = [404, 405, 501].freeze + + def validate_each(contract, attribute, value) + return if value.blank? + return unless queries_the_server?(contract) + return unless connection_changed?(contract) + return unless host_allowed?(contract, attribute, value) + + probe(contract, attribute, value) + end + + private + + # Only OpenAI-compatible connections discover models from the server. For the + # other formats the list comes from the model registry, so there is nothing at + # this URL to probe. + def queries_the_server?(contract) + contract.model.api_format == Llm::Adapters::OPENAI_COMPATIBLE + end + + def connection_changed?(contract) + contract.model.changed_attributes.keys.intersect?(CONNECTION_ATTRIBUTES) + end + + # A pre-flight check purely so the administrator gets an actionable message + # naming the environment variable, rather than a bare connection failure from + # the transport-level SSRF filter. + def host_allowed?(contract, attribute, value) + host = URI.parse(value).host + return false if host.blank? + return true if OpenProject::SsrfProtection.safe_ip?(host) + + contract.errors.add(attribute, :ssrf_filtered, env_name: ssrf_allowlist_env_name) + false + rescue URI::InvalidURIError + false + end + + def ssrf_allowlist_env_name + Settings::Definition[:ssrf_protection_ip_allowlist].env_name + end + + def probe(contract, attribute, value) + client(contract, value).models + rescue Llm::Client::Error => e + log(value, e) + add_error(contract, attribute, e) + end + + # Order matters: SsrfError and TimeoutError are both ConnectionError subclasses. + def add_error(contract, attribute, error) + case error + when Llm::Client::SsrfError + contract.errors.add(attribute, :ssrf_filtered, env_name: ssrf_allowlist_env_name) + when Llm::Client::AuthenticationError + contract.errors.add(:api_key, :invalid_api_key) + when Llm::Client::TimeoutError + contract.errors.add(attribute, :request_timed_out) + when Llm::Client::ConnectionError + contract.errors.add(attribute, :cannot_be_connected_to) + when Llm::Client::ApiError + add_api_error(contract, attribute, error) + else + contract.errors.add(attribute, :not_openai_compatible) + end + end + + # A missing model list does not block the save. + # + # The server answered, so it is reachable and the credentials were accepted; it + # simply does not offer a list here. OpenProject's own hosted gateway is exactly + # this case, and it will not be the only one. Blocking would leave an + # administrator unable to save a working connection -- and unable to reach the + # manual model entry that exists for precisely this situation. + def add_api_error(contract, attribute, error) + return if error.status.in?(MODELS_ENDPOINT_ABSENT) + + contract.errors.add(attribute, :not_openai_compatible) + end + + def client(contract, base_url) + Llm::Client.new(base_url:, api_key: contract.model.api_key) + end + + # The upstream message is logged but never surfaced: an OpenAI-compatible + # gateway routinely echoes credentials and internal hostnames in error bodies. + def log(base_url, error) + Rails.logger.info { "LLM connection probe to #{base_url} failed: #{error.class} #{error.message}" } + end +end diff --git a/app/views/admin/llm_connections/show.html.erb b/app/views/admin/llm_connections/show.html.erb new file mode 100644 index 000000000000..38a8e2702ee5 --- /dev/null +++ b/app/views/admin/llm_connections/show.html.erb @@ -0,0 +1,110 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection") %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t("menus.admin.llm_connection") } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + t("menus.admin.llm_connection")] + ) + + # Nothing here applies to a connection provisioned from the environment: + # those settings belong to the deployment, not to the administrator. + if @connection.persisted? && !@connection.configured_from_env? + header.with_action_menu( + menu_arguments: { anchor_align: :end }, + button_arguments: { + icon: "kebab-horizontal", + "aria-label": t(:label_actions), + test_selector: "llm-connection--actions" + } + ) do |menu| + if @connection.api_key.present? + menu.with_item( + label: t("admin.llm_connections.delete_api_key.menu_label"), + scheme: :danger, + tag: :a, + href: delete_api_key_dialog_llm_connection_path, + content_arguments: { + data: { controller: "async-dialog" }, + test_selector: "llm-connection--delete-api-key" + } + ) do |item| + item.with_leading_visual_icon(icon: :key) + end + end + + menu.with_item( + label: t("admin.llm_connections.disconnect.menu_label"), + scheme: :danger, + tag: :a, + href: disconnect_dialog_llm_connection_path, + content_arguments: { + data: { controller: "async-dialog" }, + test_selector: "llm-connection--disconnect" + } + ) do |item| + item.with_leading_visual_icon(icon: :"circle-slash") + end + end + end + end +%> + +<% if @connection.persisted? %> + <%= render(Primer::Alpha::Layout.new(stacking_breakpoint: :lg)) do |layout| %> + <% layout.with_main do %> + <%= render(LlmConnections::FormComponent.new(@connection)) %> + + <%= + render(Primer::Beta::Subhead.new(mt: 4)) do |component| + component.with_heading(tag: :h3) { t(".models_heading") } + component.with_description { t(".models_description", fetched_at: format_time(@connection.catalogue_fetched_at)) } + end + %> + + <%= render(LlmConnections::Models::SubHeaderComponent.new(@query)) %> + <%= render(LlmConnections::Models::IndexComponent.new(@models, connection: @connection)) %> + <% end %> + + <% layout.with_sidebar(col_placement: :end, row_placement: :end) do %> + <%= render(Primer::OpenProject::SidePanel.new) do |panel| %> + <% panel.with_section(LlmConnections::SidePanel::HealthStatusComponent.new(@connection)) %> + <% end %> + <% end %> + <% end %> +<% else %> + <%# Nothing to show alongside the form until a connection exists. %> + <%= render(LlmConnections::FormComponent.new(@connection)) %> +<% end %> diff --git a/app/views/admin/llm_feature_bindings/index.html.erb b/app/views/admin/llm_feature_bindings/index.html.erb new file mode 100644 index 000000000000..7c026aac983d --- /dev/null +++ b/app/views/admin/llm_feature_bindings/index.html.erb @@ -0,0 +1,63 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_feature_bindings") %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t("menus.admin.llm_feature_bindings") } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + t("menus.admin.llm_feature_bindings")] + ) + end +%> + +<% if !@connection.configured? %> + <%= + render(Primer::Beta::Blankslate.new(border: true)) do |component| + component.with_visual_icon(icon: :sparkle) + component.with_heading(tag: :h2) { t(".blank_title") } + component.with_description { t(".blank_description") } + component.with_primary_action(href: llm_connection_path) { t("menus.admin.llm_connection") } + end + %> +<% else %> + <% @features.each do |feature| %> + <%= render( + LlmConnections::FeatureBindingComponent.new( + feature:, + connection: @connection, + binding: @bindings[feature.key.to_s] + ) + ) %> + <% end %> +<% end %> diff --git a/app/views/admin/llm_health_status/show.html.erb b/app/views/admin/llm_health_status/show.html.erb new file mode 100644 index 000000000000..5af9df1f4d86 --- /dev/null +++ b/app/views/admin/llm_health_status/show.html.erb @@ -0,0 +1,101 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection"), t(".title") %> + +<%= turbo_frame_tag "llm_health_status_report", refresh: :morph do %> + <%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t(".title") } + if @report.present? + header.with_description { t(".last_check", datetime: format_time(@report.created_at)) } + end + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + { href: llm_connection_path, text: t("menus.admin.llm_connection") }, + t(".title")] + ) + end + %> + + <% if @report.present? %> + <%= + render(Primer::OpenProject::SubHeader.new) do |subheader| + subheader.with_action_button( + scheme: :secondary, + label: t(".download"), + mobile_label: t(".download"), + mobile_icon: :download, + leading_icon: :download, + tag: :a, + href: llm_connection_health_status_report_path(format: :txt), + data: { turbo: false } + ) { t(".download") } + + subheader.with_action_button( + scheme: :primary, + label: t(".run_checks"), + mobile_label: t(".run_checks"), + mobile_icon: "op-reload", + leading_icon: "op-reload", + tag: :a, + href: llm_connection_health_status_report_path, + data: { turbo_method: :post, turbo: true }, + test_selector: "llm-connection--rerun-health-checks" + ) { t(".run_checks") } + end + %> + + <%= + render( + HealthReports::ReportComponent.new( + @report, + i18n_scope: "llm.health_checks", + # No admin guide page exists for this yet. Every check explains itself + # in its own message, so a "More information" link would only lead + # somewhere unrelated -- suppressed until that page is written. + docs_href: false + ) + ) + %> + <% else %> + <%= + render(Primer::Beta::Blankslate.new(test_selector: "llm-connection--health-blankslate")) do |blankslate| + blankslate.with_visual_icon(icon: :meter) + blankslate.with_heading(tag: :h2) { t(".blankslate_heading") } + blankslate.with_description { t(".blankslate_description") } + blankslate.with_primary_action( + href: llm_connection_health_status_report_path, + data: { turbo_method: :post, turbo: true } + ) { t(".run_checks") } + end + %> + <% end %> +<% end %> diff --git a/app/views/admin/llm_models/edit.html.erb b/app/views/admin/llm_models/edit.html.erb new file mode 100644 index 000000000000..c6a5cfcbb6cd --- /dev/null +++ b/app/views/admin/llm_models/edit.html.erb @@ -0,0 +1,52 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection"), @llm_model.external_id %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { @llm_model.external_id } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + { href: llm_connection_path, text: t("menus.admin.llm_connection") }, + @llm_model.external_id] + ) + end +%> + +<%= settings_primer_form_with( + model: @llm_model, + url: llm_model_path(@llm_model), + method: :patch, + data: { test_selector: "llm-model--edit-form" } + ) do |f| %> + <%= render(LlmModels::Form.new(f)) %> +<% end %> diff --git a/app/views/admin/llm_models/new.html.erb b/app/views/admin/llm_models/new.html.erb new file mode 100644 index 000000000000..ebba7e58487c --- /dev/null +++ b/app/views/admin/llm_models/new.html.erb @@ -0,0 +1,54 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection"), t(".title") %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t(".title") } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + { href: llm_connection_path, text: t("menus.admin.llm_connection") }, + t(".title")] + ) + end +%> + +<%= + settings_primer_form_with( + model: @llm_model, + url: llm_models_path, + method: :post, + data: { test_selector: "llm-model--add-form" } + ) do |f| + render(LlmModels::Form.new(f)) + end +%> diff --git a/app/workers/llm/detect_capabilities_job.rb b/app/workers/llm/detect_capabilities_job.rb new file mode 100644 index 000000000000..5c37305b2ac7 --- /dev/null +++ b/app/workers/llm/detect_capabilities_job.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Pre-colours the model list after a connect, out of band so that saving the + # connection does not wait on one request per candidate model. + class DetectCapabilitiesJob < ApplicationJob + def perform + connection = LlmConnection.first + return if connection.nil? || !connection.configured? + + LlmConnections::DetectCapabilitiesService.new(connection).detect_likely_embedding_models + end + end +end diff --git a/app/workers/llm/health_check_job.rb b/app/workers/llm/health_check_job.rb new file mode 100644 index 000000000000..059b68ac5bf8 --- /dev/null +++ b/app/workers/llm/health_check_job.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Re-checks the configured connection, so a server that dies after setup does + # not stay green until somebody happens to look. + # + # Deliberately does *not* set deep_health_check: the inference group spends a + # real completion, which is billed on a hosted provider. An unattended job must + # not run up a bill, so the schedule covers everything that is free -- an + # expired key, a withdrawn model, a binding that stopped resolving -- and the + # billed round trip stays behind the administrator's "Run checks". + class HealthCheckJob < ApplicationJob + CRON_JOB_KEY = :"Llm::HealthCheckJob" + + queue_with_priority :low + + class << self + # Keeps the cron idle while there is nothing to check, rather than waking + # every six hours to find no connection. + def toggle_cron_job + if runnable? + GoodJob::Setting.cron_key_enable(CRON_JOB_KEY) unless GoodJob::Setting.cron_key_enabled?(CRON_JOB_KEY) + elsif GoodJob::Setting.cron_key_enabled?(CRON_JOB_KEY) + GoodJob::Setting.cron_key_disable(CRON_JOB_KEY) + end + end + + def runnable? + return false unless OpenProject::FeatureDecisions.llm_connection_active? + + connection = LlmConnection.first + connection.present? && connection.configured? && connection.enabled? + end + end + + def perform + return unless self.class.runnable? + + # .first, not .instance: the latter builds an unsaved record, and the + # validator writes through the health_reports association. + connection = LlmConnection.first + report = Llm::Validators::ConnectionValidator.new(connection).call + report.save! + report + end + end +end diff --git a/app/workers/llm/prune_health_reports_job.rb b/app/workers/llm/prune_health_reports_job.rb new file mode 100644 index 000000000000..aae37cc65ae3 --- /dev/null +++ b/app/workers/llm/prune_health_reports_job.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Keeps the health report history bounded. + # + # Until the scheduled check existed, health_reports only grew when somebody + # clicked "Run checks", so nothing anywhere pruned the table. A four-times-daily + # writer changes that, and this is the first pruner it has had. + # + # Only ever prunes reports belonging to an LLM connection: storages and wikis + # share the table and still write a row only on demand. + class PruneHealthReportsJob < ApplicationJob + # Enough history to see a pattern in when a flaky server fails, without + # keeping a year of identical green reports. + KEEP = 50 + MAX_AGE = 90.days + + queue_with_priority :low + + def perform + LlmConnection.find_each do |connection| + recent = connection.health_reports.order(created_at: :desc).limit(KEEP).pluck(:id) + + connection.health_reports + .where.not(id: recent) + .where(created_at: ...MAX_AGE.ago) + .delete_all + end + end + end +end diff --git a/app/workers/llm/sync_models_job.rb b/app/workers/llm/sync_models_job.rb new file mode 100644 index 000000000000..eb72d9a6075b --- /dev/null +++ b/app/workers/llm/sync_models_job.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Refreshes the cached model catalogue out of band. + # + # Used by the environment seeder, which must not block on -- or fail because of + # -- an LLM server that has not finished starting. + class SyncModelsJob < ApplicationJob + def perform + connection = LlmConnection.first + return if connection.nil? || !connection.configured? + + LlmConnections::SyncModelsService.new(connection).call + end + end +end diff --git a/config/constants/settings/definition.rb b/config/constants/settings/definition.rb index 54a5896eca29..f12ade4d1c2f 100644 --- a/config/constants/settings/definition.rb +++ b/config/constants/settings/definition.rb @@ -731,6 +731,13 @@ class Definition format: :boolean, default: false }, + llm_connection: { + description: "Configure the connection to an OpenAI-API-compatible LLM server through environment variables", + writable: false, + default: {}, + format: :hash, + string_values: true + }, log_level: { description: "Set the OpenProject logger level", default: Rails.env.development? ? "debug" : "info", diff --git a/config/initializers/cronjobs.rb b/config/initializers/cronjobs.rb index 1b9a8fc628af..586336079024 100644 --- a/config/initializers/cronjobs.rb +++ b/config/initializers/cronjobs.rb @@ -72,6 +72,14 @@ cron: "30 23 * * *", class: Ldap::SynchronizationJob.name }, + "Llm::HealthCheckJob": { + cron: "7 */6 * * *", # every six hours at xx:07 + class: Llm::HealthCheckJob.name + }, + "Llm::PruneHealthReportsJob": { + cron: "25 3 * * *", # runs at 3:25 nightly + class: Llm::PruneHealthReportsJob.name + }, "RecurringMeetings::InitNextOccurrenceWatchdogJob": { cron: "11 05 * * *", class: RecurringMeetings::InitNextOccurrenceWatchdogJob.name diff --git a/config/initializers/feature_decisions.rb b/config/initializers/feature_decisions.rb index 2630a2d46c45..4a0d5fc2a6c3 100644 --- a/config/initializers/feature_decisions.rb +++ b/config/initializers/feature_decisions.rb @@ -56,6 +56,14 @@ OpenProject::FeatureDecisions.add :type_variants, description: "Enables work package type variants." +OpenProject::FeatureDecisions.add :llm_connection, + description: "Enables the administration page connecting OpenProject to an " \ + "OpenAI-API-compatible LLM server, and the AI features built on it." + +OpenProject::FeatureDecisions.add :work_package_multiple_versions, + description: "Enables assigning multiple (target) versions to a work package. " \ + "Experimental; the user-facing setting and admin switch follow later." + OpenProject::FeatureDecisions.add :sprint_reports, description: "Enables sprint reporting within the backlogs module. " \ "It shows a dashboard with various widgets regarding the sprint progress." diff --git a/config/initializers/llm_features.rb b/config/initializers/llm_features.rb new file mode 100644 index 000000000000..568d977063fa --- /dev/null +++ b/config/initializers/llm_features.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require_relative "../../lib_static/open_project/llm/features" + +# Features that send requests to the configured LLM server. +# +# Add a feature here (or from a module engine initializer) so that +# administrators can assign it a model on the "AI models" page. + +# The description assistant rewrites work package text on explicit user action. +# Plain chat completions only: no tools, no JSON mode, no streaming. Individual +# actions may override the model, which is why it is overridable. +OpenProject::Llm::Features.register :description_assistant, + kind: :chat, + prefers: %i[structured_output], + overridable: true + +# Semantic search embeds work packages into a pgvector index. Pinned because the +# stored vectors are meaningless under a different model: changing it is a +# destructive re-index rather than a swap. +OpenProject::Llm::Features.register :semantic_search, + kind: :embedding, + requires: %i[embeddings], + pinned: true diff --git a/config/initializers/menus.rb b/config/initializers/menus.rb index 8897b8bd86f0..2872ac21a09a 100644 --- a/config/initializers/menus.rb +++ b/config/initializers/menus.rb @@ -504,6 +504,18 @@ caption: I18n.t("menus.admin.ai"), icon: :sparkle + menu.push :llm_connection, + { controller: "/admin/llm_connections", action: :show }, + if: ->(_) { User.current.admin? && OpenProject::FeatureDecisions.llm_connection_active? }, + caption: I18n.t("menus.admin.llm_connection"), + parent: :ai + + menu.push :llm_feature_bindings, + { controller: "/admin/llm_feature_bindings", action: :index }, + if: ->(_) { User.current.admin? && OpenProject::FeatureDecisions.llm_connection_active? }, + caption: I18n.t("menus.admin.llm_feature_bindings"), + parent: :ai + menu.push :mcp_configurations, { controller: "/admin/mcp_configurations", action: :index }, if: ->(_) { User.current.admin? }, diff --git a/config/locales/en.yml b/config/locales/en.yml index 1bad3c7b69fc..a1cfd7cdaab7 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -46,7 +46,6 @@ en: version: "A dynamic link to a version placed using a macro." view: "A dynamic link to a view placed using a macro." work_package: "A dynamic link to a work package placed using a macro." - account: auth_source_login_html: Please login as %{login} to activate your account. delete: "Delete account" @@ -77,14 +76,11 @@ en: omniauth_login: Please login to activate your account. signup_title: "Create an account in %{app_title}" signup_with_external_account: "Sign up with an external account" - actionview_instancetag_blank_option: "Please select" - activemodel: attributes: projects/copy_options: dependencies: "Dependencies" - activerecord: attributes: announcements: @@ -201,6 +197,31 @@ en: onthefly: "Automatic user creation" port: "Port" tls_certificate_string: "LDAP server SSL certificate" + llm_capability_verdict: + capability: "Capability" + model_id: "Model" + state: "State" + llm_connection: + api_format: "API format" + api_key: "API key" + base_url: "Host URL" + # ActiveRecord::Base.human_attribute_name strips the _id suffix, so these + # keys deliberately do not carry it (see lib/open_project/patches/active_record_i18n.rb). + default_chat_model: "Default chat model" + default_embedding_model: "Default embedding model" + enabled: "Enable LLMs for this instance" + last_connected_at: "Last connected" + llm_feature_binding: + dimensions: "Dimensions" + feature_key: "Feature" + input_prefix: "Document prefix" + # ActiveRecord::Base.human_attribute_name strips the _id suffix. + model: "Model" + query_prefix: "Query prefix" + llm_model: + display_name: "Display name" + # human_attribute_name strips the _id suffix, so the key omits it. + external: "Model name" mcp_configuration: description: Description enabled: Enabled @@ -671,6 +692,42 @@ en: tls_certificate_string: format: "%{message}" invalid_certificate: "The provided SSL certificate is invalid: %{additional_message}" + llm_connection: + attributes: + api_format: + not_supported: "needs credentials that OpenProject cannot store, such as a separate secret key or region." + api_key: + invalid_api_key: "was rejected by the LLM server." + unknown_error: "could not be validated with the LLM server. Please verify that the connection is functioning properly." + base: + singleton: "Only one LLM connection can be configured." + base_url: + cannot_be_connected_to: "could not be reached. Please ensure the LLM server is running and reachable from OpenProject." + not_openai_compatible: "did not return a valid model list. Please ensure the URL points at an OpenAI-API-compatible endpoint, including the API version segment (for example https://example.com/v1)." + request_timed_out: "did not respond in time. Please ensure the LLM server is reachable and not overloaded." + ssrf_filtered: "resolves to a blocked address. If the LLM server runs on an internal network, allow its IP via the %{env_name} environment variable." + default_chat_model_id: + not_available: "is not offered by the configured LLM server." + default_embedding_model_id: + cannot_embed: "cannot create embeddings, according to the LLM server." + not_available: "is not offered by the configured LLM server." + enabled: + requires_connection: "cannot be turned on before a connection has been configured." + llm_feature_binding: + attributes: + dimensions: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." + feature_key: + not_registered: "does not belong to a known AI feature." + input_prefix: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." + model_id: + locked: "cannot be changed while data indexed with it still exists. Re-index to switch models." + query_prefix: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." meeting: error_conflict: "Unable to save because the meeting was updated by someone else in the meantime. Please reload the page." member: @@ -1317,7 +1374,6 @@ en: no_results_description_text: 'Choose "Show everything" to show all activity and comments' no_results_title_text: No activity to display unsaved_changes_confirmation_message: "Are you sure you want to dismiss your comment? The content that you have written will be lost." - activity: filter: changeset: "Changesets" @@ -1363,7 +1419,6 @@ en: phase_and_both_gates: "%{phase_message}. %{start_gate_message}, and %{finish_gate_message}" phase_and_one_gate: "%{phase_message}. %{gate_message}" removed_date: "date deleted %{date}" - admin: banners: environment_configured_readonly: "These values are configured via environment variables and cannot be edited here." @@ -1430,7 +1485,6 @@ en:
We have prepared [upgrade guides for all installation methods](upgrade_guide). You can perform the upgrade ahead of the next release at any time by following the guides. - jemalloc_allocator: Jemalloc memory allocator jira: actions: @@ -1673,7 +1727,124 @@ en: journal_aggregation: caption_with_maximum: > User actions on a work package (changing description, status, values, or writing comments) are grouped if performed within this period. It also controls notification and [webhook](webhook_link) delays. The maximum is %{max} minutes. - + llm_connections: + delete_api_key: + configured_from_env: "This connection is configured through the environment and cannot be changed here." + description: "OpenProject will stop sending an API key with its requests. Features will keep working only if the server requires no authentication. You can enter a new key at any time." + description_verdicts: "OpenProject will stop sending an API key with its requests, and the next model refresh will discard what is known about each model's capabilities, including the assertions you made yourself. You can enter a new key at any time." + heading: "Remove the stored API key?" + menu_label: "Remove API key" + success: "The API key has been removed." + title: "Remove API key" + disconnect: + configured_from_env: "This connection is configured through the environment and cannot be changed here." + description: "OpenProject will stop using the LLM server. AI features will be unavailable until you connect again." + heading: "Disconnect from the LLM server?" + keeps_bindings: "The model chosen for each feature is kept: %{features}." + keeps_models: "The model list, including any models you added manually, is kept." + keeps_settings: "The endpoint and API format are kept. Only the stored API key is removed." + menu_label: "Disconnect" + success: "OpenProject has disconnected from the LLM server." + title: "Disconnect" + form: + api_format_caption: "Which API the server speaks. Choose OpenAI-compatible for most gateways and self-hosted servers." + api_key_caption: "The key OpenProject authenticates with. Leave blank if the server requires no authentication." + api_key_caption_stored: "A key is stored. Leave blank to keep it, or enter a new one to replace it." + base_url_caption: > + The full base URL of the server, including the API version segment, exactly as your provider documents it (for example https://example.com/v1). OpenProject appends only the endpoint path. + button_connect: "Connect" + default_chat_model_caption: "Used by AI features that do not select a model themselves." + default_embedding_model_caption: "Used by AI features that index text for search and do not select a model themselves. Only models known to create embeddings are offered." + default_embedding_model_none: "No model is known to create embeddings. If you know one does, open it under Available models and set its embeddings capability." + embedding_option_unknown: "%{model} (not verified as an embedding model)" + embedding_option_unsupported: "%{model} (cannot create embeddings)" + enabled_caption: "When turned off, all LLM-backed AI features stop working. The connection settings are kept." + label_connecting: "Contacting the LLM server…" + models: + blank_description: "The server reported no models. If it does not offer a model list, add the model names you want to use below." + blank_title: "No models available" + context_window: "Context window" + display_name_caption: "An optional friendly name shown instead of the model id." + filter_label: "Filter models" + identifier: "Model" + kind: "Type" + source: "Source" + source_deactivated: "Hidden by administrator" + source_discovered: "Reported by server" + source_manual: "Added manually" + source_withdrawn: "No longer reported" + status: "Available for use" + toggle_aria_label: "Make %{model} available to AI features" + refresh_models: + failure: "The model list could not be refreshed. Please check that the LLM server is still reachable." + success: "The model list has been refreshed." + show: + add_model_description: "If the server does not expose a model list, enter the model name exactly as the server expects it. Manually added models are kept when the list is refreshed." + add_model_submit: "Add model" + description: "Connect OpenProject to a server that speaks the OpenAI API, so AI features can use it." + models_description: "Reported by the server on %{fetched_at}." + models_heading: "Available models" + refresh_models: "Refresh models" + update: + no_models: "Saved, but the server did not return a model list. Either it does not offer one, or the endpoint is wrong — a URL missing its API version segment (for example /v1) looks exactly the same from here. Add the models you want to use below, then run the health checks to confirm the server answers." + success: "Successfully connected to the LLM server." + llm_feature_bindings: + dangling: "%{model} is no longer offered by the LLM server. This feature will not run until another model is selected." + deactivated: "%{model} has been hidden by an administrator. This feature keeps using it, but it can no longer be chosen elsewhere." + form: + dimensions_caption: "How many numbers each vector has. Leave blank to use whatever the server returns." + dimensions_caption_probed: "The server returned %{dimensions}-dimension vectors for this model. Leave blank to use whatever it returns at index time." + input_prefix_caption: "Prepended to each document before it is indexed. Some models expect one, for example \"passage: \" including the trailing space." + query_prefix_caption: "Prepended to each search query. Some models expect one, for example \"query: \" including the trailing space." + index: + blank_description: "Connect OpenProject to an LLM server first. Models can be assigned once the server reports which ones it offers." + blank_title: "No LLM server configured" + description: "Choose which model each AI feature uses. Features without a choice use the instance default." + inherit_with_default: "Use the default (%{model})" + inherit_without_default: "Use the default (none set)" + locked: "%{model} is in use by indexed data and cannot be changed here. Re-index to switch models." + locked_values_heading: "Values fixed by the existing index" + option_unknown: "%{model} — not verified" + option_unsupported: "%{model} — no %{capability} support" + update: + success: "The model for %{feature} has been saved." + llm_health_status: + show: + blankslate_description: "Run the checks to find out whether OpenProject can reach the LLM server, whether it accepts the API key, and whether every AI feature has a usable model." + blankslate_heading: "This connection has not been checked yet" + download: "Download report" + last_check: "Last checked %{datetime}." + run_checks: "Run checks" + title: "Health status" + llm_models: + create: + success: "%{model} has been added." + destroy: + description: "The model will no longer be offered to AI features." + description_bound: "This model is currently used by %{features}. Those features will stop working until another model is selected." + heading: "Remove %{model}?" + success: "%{model} has been removed." + title: "Remove model" + edit: + description: "Edit how OpenProject uses this model." + form: + capabilities_description: "Leave a capability unspecified to let OpenProject determine it. Marking Embeddings as supported makes this an embedding model, offered only to features that need one." + capability_override_caption: "Choose a value to override this." + context_window: "Context window" + context_window_caption: "How many tokens this model accepts. Leave blank if you do not know." + context_window_known: "Leave blank to use %{value}, %{source}." + create_submit: "Add model" + display_name_caption: "An optional friendly name shown instead of the model id." + external_id_caption: "The model name exactly as the server expects it." + state_inherited: "%{state} (%{source})" + state_supported: "Supported" + state_unspecified: "Not specified" + state_unsupported: "Not supported" + new: + description: "Name a model this server can use but does not advertise, and describe what it can do." + title: "Add a model" + update: + success: "%{model} has been updated." mcp_configurations: index: description: "The model context protocol allows AI agents to provide its users with tools and resources exposed by this OpenProject instance. This feature is still in beta." @@ -1925,7 +2096,6 @@ en: is_active: currently displayed is_inactive: currently not displayed show_until: Show until - antivirus_scan: deleted_by_admin: "The quarantined file '%{filename}' has been deleted by an administrator." deleted_message: "A virus was detected in file '%{filename}'. The file has been deleted." @@ -2012,7 +2182,6 @@ en: parent: Undisclosed - The parent is invisible because of lacking permissions. project: Undisclosed - The project is invisible because of lacking permissions. workPackage: Undisclosed - The work package is invisible because of lacking permissions. - attribute_help_texts: add_new: "Add help text" caption: "This short version will be displayed as caption of the attribute." @@ -2129,12 +2298,10 @@ en: work_package_id: "Work package" authentication: login_and_registration: "Login and registration" - background_jobs: status: cancelled_due_to: "Job was cancelled due to error: %{message}" error_requeue: "Job experienced an error but is retrying. The error was: %{message}" - backup: error: backup_pending: There is already a backup pending. @@ -2170,7 +2337,6 @@ en: When you create a new token you will only be allowed to request a backup after 24 hours. This is a safety measure. After that you can request a backup any time using that token. text_token_deleted: Backup token deleted. Backups are now disabled. - button_actions: "Actions" button_add: "Add" button_add_comment: "Add comment" @@ -2280,7 +2446,6 @@ en: mathematical: "The mathematical formula leads to an error. Please review the project calculation attribute and try again." missing_value: The attribute "%{custom_field_name}" is required by this Calculated value, but is empty. unknown: "An unknown error occurred. Please review the formula for this Calculated value." - colors: edit: label_edit_color: "Edit Color" @@ -2299,10 +2464,8 @@ en: new: label_new_color: "New Color" - concatenation: single: "or" - consent: checkbox_label: I have noted and do consent to the above. contact_mail_instructions: Define the mail address that users can reach a data controller to perform data change or removal requests. @@ -2315,7 +2478,6 @@ en: title: User Consent update_consent_last_time: "Last update of consent: %{update_time}" user_has_consented: The user gave their consent to your [configured consent information text](consent_settings). - copy_project: copy_options: dependencies_label: "Copy from project" @@ -2339,13 +2501,11 @@ en: description: Start from scratch. Manually add portfolio attributes, members and modules. label: "Blank portfolio" template_heading: "Select a portfolio template to work with the most common project management methods, or create a portfolio from scratch." - create_program: blank_template: description: Start from scratch. Manually add program attributes, members and modules. label: "Blank program" template_heading: "Select a program template to work with the most common project management methods, or create a program from scratch." - create_project: attributes_heading: "Fill in this mandatory information to work on your projects." blank_description: No description provided. @@ -2356,7 +2516,6 @@ en: dependencies_label: "Copy from template" template_heading: "Select a project template to work with the most common project management methods, or create a project from scratch." template_label: "Use template" - create_wiki_page: "Create new wiki page" create_wiki_page_button: "Wiki page" custom_actions: @@ -2370,7 +2529,6 @@ en: execute: "Execute %{name}" new: "New custom action" plural: "Custom actions" - custom_fields: admin: custom_field_projects: @@ -2476,11 +2634,9 @@ en: text_add_new_custom_field: > To add new custom fields to a project you first need to create them before you can add them to this project. - danger_dialog: confirmation_live_message_checked: "The button to proceed is now active." confirmation_live_message_unchecked: "The button to proceed is now inactive. You need to tick the checkbox to continue." - date: abbr_day_names: ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"] abbr_month_names: @@ -2539,7 +2695,6 @@ en: - :year - :month - :day - datetime: distance_in_words: about_x_hours: @@ -2604,7 +2759,6 @@ en: minute_abbreviated: one: "min" other: "mins" - departments: add_department: "Add department" add_department_form: @@ -2641,7 +2795,6 @@ en: managed_heading: "This user cannot be moved" managed_title: "User managed by LDAP" title: "User already in a department" - description_active: "Active?" description_attachment_toggle: "Show/Hide attachments" description_autocomplete: > @@ -2681,7 +2834,6 @@ en: # Text direction: Left-to-Right (ltr) or Right-to-Left (rtl) direction: ltr - doorkeeper: access_token_url: "Access token URL" auth_url: "Auth URL" @@ -2717,7 +2869,6 @@ en: unsupported_response_type: "The authorization server does not support this response type." pre_authorization: status: "Pre-authorization" - ee: features: baseline_comparison: Baseline Comparisons @@ -2867,7 +3018,6 @@ en: work_package_subject_generation: description: "Create automatically generated subjects using referenced attributes and text." - email_calendar_updates: button: disabled: "Enable" @@ -2877,11 +3027,9 @@ en: enabled: "Enabled." enterprise_plans: legacy_enterprise: "Enterprise Plan" - enterprise_trials: dialog_component: title: Enterprise Trial - enumeration_activities: "Time tracking activities" enumeration_caption_order_changed: "Order successfully changed." enumeration_could_not_be_moved: "Enumeration could not be moved." @@ -2889,7 +3037,6 @@ en: enumeration_reported_project_statuses: "Reported status" enumeration_work_package_priorities: "Work package priorities" error_attribute_not_highlightable: "Attribute(s) not highlightable: %{attributes}" - error_auth_source_sso_failed: "Single Sign-On (SSO) for user '%{value}' failed" error_can_not_archive_project: "This project cannot be archived: %{errors}" error_can_not_deactivate_type_invisible_projects: "There are also work packages in projects you cannot see." @@ -2946,12 +3093,10 @@ en: error_wiki_root_menu_item_conflict: > Can't rename "%{old_name}" to "%{new_name}" due to a conflict in the resulting menu item with the existing menu item "%{existing_caption}" (%{existing_identifier}). - error_work_package_id_not_found: "The work package was not found." error_work_package_not_found_in_project: "The work package was not found or does not belong to this project" error_workflow_copy_source: "Please select a source type or role" error_workflow_copy_target: "Please select target type(s) and role(s)" - errors: field_erroneous_label: "This field is invalid: %{full_errors}\nPlease enter a valid value." header_additional_invalid_fields: @@ -2966,7 +3111,6 @@ en: must_be_template: "must be template" storage_error: "There was an error with the storage connection." unsupported_storage_type: "is not a supported storage type." - events: changeset: "Changeset edited" message: Message edited @@ -2983,7 +3127,6 @@ en: work_package_closed: "Work Package closed" work_package_edit: "Work Package edited" work_package_note: "Work Package note added" - export: demo: button_text: Generate Demo PDF @@ -3084,7 +3227,6 @@ en: continue_message: "Are you sure you want to proceed to the following external link?" title: "Leaving OpenProject" warning_message: "You are about to leave OpenProject and visit an external website. Please be aware that external websites are not under our control and may have different privacy and security policies." - extraction: available: catdoc: "Catdoc available (optional)" @@ -3093,7 +3235,6 @@ en: tesseract: "Tesseract available (optional)" unrtf: "Unrtf available (optional)" xls2csv: "Xls2csv available (optional)" - filterable_tree_view: filter_mode: all: "All" @@ -3105,7 +3246,6 @@ en: show: no_results_title_text: There are currently no posts for the forum. general_csv_decimal_separator: "." - general_csv_encoding: "UTF-8" general_csv_separator: "," general_first_day_of_week: "7" @@ -3126,7 +3266,6 @@ en: all_projects: 'Search for "%{search_term}" in all projects' current_project: 'Search for "%{search_term}" in %{project_name}' project_and_subprojects: 'Search for "%{search_term}" in %{project_name} and all subprojects' - groups: edit: synchronized_groups: "Synchronized groups" @@ -3146,10 +3285,8 @@ en: user_count: User count users: no_results_title_text: There are currently no users part of this group. - gui_validation_error: "1 error" gui_validation_error_plural: "%{count} errors" - header: project_select_component: all_projects: "All projects" @@ -3157,7 +3294,6 @@ en: leave_project: "Leave project" no_favorite_projects: "You have no favorite projects." title: "Projects" - health_reports: common: checks: @@ -3186,7 +3322,6 @@ en: You can only select projects here where the %{plural} module is active. After creating a %{singular} you can add work packages from other projects to it. public: "Publish this view, allowing other users to access your view. Users with the 'Manage public views' permission can modify or remove public query. This does not affect the visibility of work package results in that view and depending on their permissions, users may see different results." - homescreen: additional: favorite_projects: @@ -3233,19 +3368,15 @@ en: missing_authorization: "The server side request failed due to missing authorization information." response: unexpected: "Unexpected response received." - image_conversion: imagemagick: "Imagemagick" - incoming_mails: ignore_filenames: > Specify a list of names to ignore when processing attachments for incoming mails (e.g., signatures or icons). Enter one filename per line. - instructions_after_error_link: "You can try to sign in again by clicking [here](signin_url). If the error persists, ask your admin for help." instructions_after_logout_link: "You can sign in again by clicking [here](signin_url)." instructions_after_registration_link: "You can sign in as soon as your account has been activated by clicking [here](signin_url)." - journals: cause_descriptions: budget_deleted: Budget has been deleted @@ -3335,7 +3466,6 @@ en: version: "a non-visible version" work_package: "a non-visible work package" label_accessibility: "Accessibility" - label_account: "Account" label_actions: "Actions" label_activate_user: "Activate user" @@ -3642,7 +3772,6 @@ en: label_internal: "Internal" label_introduction_video: "Getting started video" label_invitation: Invitation - label_invite_user: "Invite user" label_item: "Item" label_item_plural: "Items" @@ -4104,7 +4233,6 @@ en: other: "Time off: %{count} working days" label_yesterday: "yesterday" label_zen_mode: "Zen mode" - ldap_auth_sources: attribute_texts: admin_map_html: "Optional: The attribute key in LDAP that if present marks the OpenProject user an admin. Leave empty when in doubt." @@ -4187,6 +4315,113 @@ en: update: failure: "The modified list cannot be saved: %{errors}" success: "The modified list has been saved" + llm: + api_formats: + anthropic: "Anthropic" + azure: "Azure OpenAI" + bedrock: "AWS Bedrock" + deepseek: "DeepSeek" + gemini: "Google Gemini" + gpustack: "GPUStack" + mistral: "Mistral" + ollama: "Ollama" + openai: "OpenAI-compatible" + openrouter: "OpenRouter" + perplexity: "Perplexity" + vertexai: "Google Vertex AI" + xai: "xAI" + capabilities: + embeddings: + label: "Embeddings" + function_calling: + label: "Tool calling" + reasoning: + label: "Reasoning" + structured_output: + label: "Structured output" + vision: + label: "Vision" + context_window_sources: + registry: "the figure published for this model" + server: "reported by the server" + features: + description_assistant: + caption: "Rewrites and restructures work package text on request." + label: "Description assistant" + semantic_search: + caption: "Indexes work packages so they can be found by meaning rather than by keyword." + label: "Semantic search" + health_checks: + configuration: + api_format_supported: "Endpoint format is usable" + base_url_present: "Endpoint is configured" + credentials_present: "API key is stored" + enabled: "Connection is switched on" + feature_flag: "AI features are switched on" + header: "Configuration" + errors: + api_key_missing: "No API key is stored. This is only correct if the LLM server requires no authentication." + catalogue_never_fetched: "The model list has never been retrieved from the server." + catalogue_stale: "The model list was last retrieved on %{fetched_at} and may no longer match what the server offers." + chat_failed: "The server refused a request for %{model} with status %{status}." + connection_disabled: "The connection is configured but switched off, so no feature will use it." + connection_error: "The server could not be reached. Please ensure it is running and reachable from OpenProject." + default_model_missing: "%{model} is no longer offered by the server." + default_model_unset: "No default model has been chosen, so features fall back to their own selection." + empty_completion: "%{model} answered with an empty message." + feature_flag_off: "The LLM connection feature is switched off, so the connection is not used yet." + features_incapable: "%{features} cannot run, because the chosen model does not provide: %{capabilities}." + features_model_missing: "%{features} point at a model the server no longer offers." + features_unbound: "%{features} have no model chosen yet." + invalid_api_key: "The server rejected the stored API key." + locked_model_missing: "%{features} indexed data with a model the server no longer offers. The index cannot be extended until it is rebuilt." + no_model_to_test: "No model is available to test with. Add a model or choose a default first." + no_models: "The server does not publish a model list. Either it does not offer one, or the endpoint is missing its API version segment. Models can be added manually; the inference check is what proves the endpoint is right." + no_models_endpoint: "The server does not offer a model list, so the API key could not be verified. Run the inference check to verify it." + not_configured: "No endpoint has been configured." + not_openai_compatible: "The endpoint did not return a valid model list. Please ensure it points at an OpenAI-API-compatible endpoint, including the API version segment." + request_timed_out: "The server did not respond in time." + server_error: "The server responded with status %{status}." + ssrf_filtered: "The endpoint resolves to a blocked address." + unsupported_api_format: "%{api_format} needs credentials that OpenProject cannot store." + features: + bindings_resolvable: "Every feature has a usable model" + header: "AI features" + locked_bindings_intact: "Indexed data still has its model" + inference: + chat_round_trip: "The server answers a request" + header: "Inference" + models: + catalogue_fresh: "Model list is up to date" + catalogue_present: "Models are available" + default_chat_model: "Default chat model" + default_embedding_model: "Default embedding model" + header: "Models" + server: + credentials_accepted: "The server accepts the API key" + header: "Server" + reachable: "The server can be reached" + model_kinds: + chat: "Chat" + embedding: "Embedding" + unknown: "Unknown" + verdict_sources: + admin: "set by an administrator" + metadata: "from the model registry" + observed: "observed in use" + probe: "verified against the server" + verdict_states: + supported: "Supported" + unknown: "Not verified" + unsupported: "Not supported" + llm_connections: + side_panel: + health_status_component: + last_check: "Last checked %{datetime}." + never_checked: "This connection has not been checked yet." + open_report: "Open full health report" + run_checks: "Run checks now" + title: "Health status" macro_execution_error: "Error executing the macro %{macro_name}" macro_unavailable: "Macro %{macro_name} cannot be displayed." macro_unknown: "Unknown or unsupported macro." @@ -4322,7 +4557,6 @@ en: without_message: "%{user} added you as a member to the project '%{project}'." roles: "You have the following roles:" subject: "%{project} - You have been added as a member" - mail_member_updated_global: body: roles: "You now have the following roles:" @@ -4330,7 +4564,6 @@ en: with_message: "%{user} updated the roles you have globally writing:" without_message: "%{user} updated the roles you have globally." subject: "Your global permissions have been updated" - mail_member_updated_project: body: roles: "You now have the following roles:" @@ -4350,7 +4583,6 @@ en: mail_subject_register: "Your %{value} account activation" mail_subject_wiki_content_added: "'%{id}' wiki page has been added" mail_subject_wiki_content_updated: "'%{id}' wiki page has been updated" - mail_user_activation_limit_reached: message_html: | A new user (%{email}) tried to create an account on an OpenProject environment that you manage (%{host}). @@ -4364,7 +4596,6 @@ en: server_url_component: caption: "The URL at which the OpenProject MCP server will be reachable. Required for setting up MCP clients." label: "Server URL" - members: columns: shared: "Shared" @@ -4426,20 +4657,19 @@ en: send_invite_to: "Send invite to" menu_item: "Menu item" menu_item_setting: "Visibility" - menus: admin: aggregation: "Aggregation" ai: "Artificial Intelligence (AI)" api_and_webhooks: "API and webhooks" + llm_connection: "LLM settings" + llm_feature_bindings: "AI models" mail_notification: "Email notifications" mails_and_notifications: "Emails and notifications" mcp_configurations: "Model Context Protocol (MCP)" quick_add: label: "Add…" - more_actions: "More functions" - my: access_token: created_dialog: @@ -4467,7 +4697,6 @@ en: password_confirmation_dialog: confirmation_required: "You need to enter your account password to confirm this change." title: "Confirm your password to continue" - my_account: access_tokens: access_tokens: "Access tokens" @@ -4607,21 +4836,17 @@ en: email_reminders: "Email reminders" notifications: "Notification settings" title: "Notification and email" - news: index: no_results_content_text: Add a news item no_results_title_text: There is currently no news to report. no_results_title_text: There is currently nothing to display. - noscript_description: "You need to activate JavaScript in order to use OpenProject!" noscript_heading: "JavaScript disabled" noscript_learn_more: "Learn more" note: Note - note_password_login_disabled_link: "Password login has been disabled through a [configuration setting](configuration_url)." nothing_to_preview: "Nothing to preview" - notice_accessibility_mode: The accessibility mode can be enabled in your [account settings](url). notice_account_activated: "Your account has been activated. You can now log in." notice_account_already_activated: The account has already been activated. @@ -4644,7 +4869,6 @@ en: notice_attachment_migration_wiki_page: > This page was generated automatically during the update of OpenProject. It contains all attachments previously associated with the %{container_type} "%{container_name}". - notice_auth_stage_error: "Authentication stage '%{stage}' failed." notice_auth_stage_verification_error: "Could not verify stage '%{stage}'." notice_auth_stage_wrong_stage: "Expected to finish authentication stage '%{expected}', but '%{actual}' returned." @@ -4656,7 +4880,6 @@ en: notice_email_sent: "An email was sent to %{value}" notice_failed_to_save_members: "Failed to save member(s): %{errors}." notice_failed_to_save_work_packages: "Failed to save %{count} work package(s) on %{total} selected: %{ids}." - notice_file_not_found: "The page you were trying to access doesn't exist or has been removed." notice_forced_logout: "You have been automatically logged out after %{ttl_time} minutes of inactivity." notice_internal_server_error: "An error occurred on the page you were trying to access. If you continue to experience problems please contact your %{app_title} administrator for assistance." @@ -4699,7 +4922,6 @@ en: notice_user_invitation_resent: An invitation has been sent to %{email}. notice_user_missing_authentication_method: User has yet to choose a password or another way to sign in. notice_wont_delete_auth_source: The LDAP connection cannot be deleted as long as there are still users using it. - notifications: facets: all: "All" @@ -4814,7 +5036,6 @@ en: api_v3_text: "Application will receive full read & write access to the OpenProject API v3 to perform actions on your behalf." mcp: "Access to MCP" mcp_text: "Application will receive access to the OpenProject MCP endpoints. They are a limited subset of APIv3, that's tailored for usage with AI agents." - oauth_client: errors: oauth_authorization_code_grant_had_errors: "OAuth2 Authorization grant unsuccessful" @@ -4862,13 +5083,11 @@ en: error: "Error" failed_authorization: "Authorization failed" not_connected: "Not connected" - onboarding: heading_getting_started: "Get an overview" select_language: "Please select your language" text_getting_started_description: "Get a quick overview of project management and team collaboration with OpenProject. You can restart this video from the help menu." welcome: "Welcome to %{app_title}" - op_dry_validation: errors: array?: "must be an array." @@ -4919,7 +5138,6 @@ en: short: "Short name" weight: "Weight" open_link_in_a_new_tab: "Open link in a new tab" - open_project: common: work_package_card_component: @@ -4931,7 +5149,6 @@ en: undisclosed: "Undisclosed" page: text: "Text" - pagination: label: "Pagination" next: "Next" @@ -4985,7 +5202,6 @@ en: permission_add_subprojects: "Create subprojects" permission_add_work_package_attachments: "Add attachments" permission_add_work_package_attachments_explanation: "Allows adding attachments without Edit work packages permission" - permission_add_work_package_comments: "Add comments" permission_add_work_package_watchers: "Add watchers" permission_add_work_packages: "Add work packages" @@ -5112,18 +5328,15 @@ en: right_to_manage_members_missing: > You are not allowed to delete the placeholder user. You do not have the right to manage members for all projects that the placeholder user is a member of. - placeholders: default: "-" templated_hint: Automatically generated through type %{type} - plugin_openproject_auth_plugins: description: "Integration of OmniAuth strategy providers for authentication in OpenProject." name: "OpenProject Auth Plugins" plugin_openproject_auth_saml: description: "Adds the OmniAuth SAML provider to OpenProject" name: "OmniAuth SAML / Single-Sign On" - portfolio: count: one: "1 portfolio" @@ -5150,7 +5363,6 @@ en: priority_color_text: | Click to assign or change the color of this priority. It can be used for highlighting work packages in the table. - program: count: one: "1 program" @@ -5158,7 +5370,6 @@ en: zero: "0 programs" filters: name: "Part of Program" - project: archive: archived: "Archived" @@ -5196,14 +5407,12 @@ en: make_template: "Set as template" remove_from_templates: "Remove from templates" use_template: "Use template" - project_module_activity: "Activity" project_module_forums: "Forums" project_module_news: "News" project_module_repository: "Repository" project_module_wiki: "Wiki" project_module_work_package_tracking: "Work packages" - projects: # Contains custom strings for options when copying a project that cannot be found elsewhere. copy: @@ -5413,7 +5622,6 @@ en: automatic: description: "Order the %{plural} by one or more sorting criteria. You will lose the previous sorting." heading: "Automatic" - query: attribute_and_direction: "%{attribute} (%{direction})" @@ -5438,12 +5646,10 @@ en: and_user: "and %{user}" react_with: "React with %{reaction}" reaction_by: "%{reaction} by" - reportings: index: no_results_content_text: Add a status reporting no_results_title_text: There are currently no status reportings. - repositories: at_identifier: "at %{identifier}" atom_revision_feed: "Atom revision feed" @@ -5551,7 +5757,6 @@ en: url: "URL to repository" warnings: cannot_annotate: "This file cannot be annotated." - roles: edit: default_for_new_projects_warning: >- @@ -5568,11 +5773,9 @@ en: matrix_check_uncheck_all_in_row_label_html: "Toggle %{permission} permission for all roles" matrix_checkbox_label: "Assign %{permission} permission to %{role} role" matrix_uncheck_all_label: "Unassign all %{module} permissions from all roles" - scheduling: automatic: "set to Automatic" manual: "set to Manual" - search_input_placeholder: "Search ..." setting_accessibility_mode_for_anonymous: "Enable accessibility mode for anonymous users" setting_activity_days_default: "Days displayed on project activity" @@ -5586,7 +5789,6 @@ en: Set a default path to redirect users after login, if no back link was provided. Redirects to home page if not set.
Example: %{example_code} - setting_allowed_link_protocols: "Allowed link protocols" setting_allowed_link_protocols_text_html: >- Allow these protocols to be rendered as links in work package descriptions, long text fields and comments. @@ -5835,7 +6037,6 @@ en: Maximum number of work packages or projects that can be included in a single export. Larger exports are truncated to this limit. setting_working_days: "Working days" - settings: antivirus: actions: @@ -6149,14 +6350,12 @@ en: Adding additional users will exceed the current limit. Please [upgrade your plan](upgrade_url) to be able to ensure external users are able to access this %{entity}. status_active: "active" - status_archived: "archived" status_blocked: "blocked" status_deleted: deleted status_invited: invited status_locked: locked status_registered: registered - statuses: edit: status_color_text: | @@ -6191,7 +6390,6 @@ en: sentence_connector: "and" skip_last_comma: "false" text_access_token_hint: "Access tokens allow you to grant external applications access to resources in OpenProject." - text_accessibility_hint: "The accessibility mode is designed for users who are blind, motorically handicaped or have a bad eyesight. For the latter focused elements are specially highlighted. Please notice, that the Backlogs module is not available in this mode." text_analyze: "Further analyze: %{subject}" text_are_you_sure: "Are you sure?" @@ -6353,7 +6551,6 @@ en: text_wrote: "wrote" text_zoom_in: "Zoom in" text_zoom_out: "Zoom out" - themes: dark: "Dark" light: "Light" @@ -6366,19 +6563,15 @@ en: short: "%d %b %H:%M" time: "%I:%M %p" pm: "pm" - timeframe: end: "to" show: "Show timeframe" start: "from" title_enterprise_upgrade: "Upgrade to unlock more users." - title_remove_and_delete_user: Remove the invited user from the project and delete him/her. - toggle_switch: label_off: "Off" label_on: "On" - token: hashed_token: display_value_placeholder: "***" @@ -6387,23 +6580,18 @@ en: If enabled a user will be unable to chose a login during registration. Instead their given email address will serve as the login. An administrator may still change the login separately. - tooltip_resend_invitation: > Sends another invitation email with a fresh token in case the old one expired or the user did not get the original email. Can also be used for active users to choose a new authentication method. When used with active users their status will be changed to 'invited'. - tooltip_user_default_timezone: > The default time zone for new users. Can be changed in a user's settings. - top_menu: additional_resources: "Additional resources" getting_started: "Getting started" help_and_support: "Help and support" - total_progress: "Total progress" - types: creation_wizard: add_variant: "Add variant to %{name}" @@ -6642,13 +6830,11 @@ en: one: "1 variant" other: "%{count} variants" milestone_indicator: "Milestone" - unsupported_browser: close_warning: "Ignore this warning." message: "You may run into errors and degraded experience on this page." title: "Your browser is outdated and unsupported." update_message: "Please update your browser." - user: activate: "Activate" activate_and_reset_failed_logins: "Activate and reset failed logins" @@ -6689,7 +6875,6 @@ en: user_preferences: disable_keyboard_shortcuts_caption: > You can choose to disable default [keyboard shortcuts](docs_url) if you use a screen reader or want to avoid accidentally triggering an action with a shortcut. - users: autologins: prompt: "Stay logged in for %{num_days}" @@ -6830,11 +7015,9 @@ en: one: "1 working day" other: "%{count} working days" work_hours: "Work hours" - version_status_closed: "closed" version_status_locked: "locked" version_status_open: "open" - versions: overview: no_results_title_text: There are currently no work packages assigned to this version. @@ -6862,7 +7045,6 @@ en: Please [upgrade your plan](upgrade_url) or block existing users in order to allow invited and registered users to join. warning_protocol_mismatch_html: > - warning_registration_token_expired: | The activation email has expired. We sent you a new one to %{email}. Please click the link inside of it to activate your account. @@ -6875,7 +7057,6 @@ en: warning_user_limit_reached_instructions: > You reached your user limit (%{current}/%{max} active users). Please contact sales@openproject.com to upgrade your Enterprise edition plan and add additional users. - wiki: index: no_results_content_text: Add a new wiki page @@ -6884,7 +7065,6 @@ en: page_not_editable_index: The requested page does not (yet) exist. You have been redirected to the index of all wiki pages. print_hint: This will print the content of this wiki page without any navigation bars. wiki_menu_item_delete_not_permitted: The wiki menu item of the only wiki page cannot be deleted. - wiki_menu_item_for: 'Menu item for wikipage "%{title}"' wiki_menu_item_new_main_item_explanation: > You are deleting the only main wiki menu item. You now have to choose a wiki page for which a new main item will be generated. @@ -7016,7 +7196,6 @@ en: relates_to_description: "Creates a visible link between the two work packages with no additional effect" required_description: "Marks this work package as being a requirement to the related one" requires_description: "Marks the related work package as a requirement to this one" - work_packages: bulk: copy_failed: "The work packages could not be copied." @@ -7184,7 +7363,6 @@ en: x_descendants: one: "One descendant work package" other: "%{count} work package descendants" - workflows: copies: form: @@ -7257,5 +7435,4 @@ en: warning: > Changing which days of the week are considered working days or non-working days can affect the start and finish days of all work packages and life cycles in all projects in this instance. - you: you diff --git a/config/routes.rb b/config/routes.rb index 0bf4e07cc385..d113c9c7d130 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -742,6 +742,31 @@ end end + resource :llm_connection, only: %i[show update], controller: "admin/llm_connections" do + post :refresh_models + delete :api_key, action: :delete_api_key + get :delete_api_key_dialog + get :disconnect_dialog + post :disconnect + get :search_models, defaults: { format: :turbo_stream } + + resource :health_status_report, only: %i[show create], controller: "admin/llm_health_status" do + post :create_health_status_report + end + end + + # Manual entries only; discovered models are managed by the sync. + resources :llm_models, only: %i[new create edit update destroy], controller: "admin/llm_models" do + member do + get :delete_dialog + post :toggle + end + end + + # Keyed by feature key rather than by record id: the binding is an attribute + # of a registered feature, and a feature may not have a row yet. + resources :llm_feature_bindings, only: %i[index update], controller: "admin/llm_feature_bindings" + resources :mcp_configurations, only: %i[index update], controller: "admin/mcp_configurations" do collection do post :multi_update diff --git a/db/migrate/20260811090000_create_llm_connections.rb b/db/migrate/20260811090000_create_llm_connections.rb new file mode 100644 index 000000000000..4028aef13393 --- /dev/null +++ b/db/migrate/20260811090000_create_llm_connections.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmConnections < ActiveRecord::Migration[8.1] + def change + create_table :llm_connections do |t| + t.string :name, null: false, index: { unique: true } + t.string :type, null: false, index: true + t.boolean :enabled, null: false, default: false + t.string :base_url, null: false + # Ciphered through Redmine::Ciphering when database_cipher_key is configured. + # Nullable: an unauthenticated self-hosted server needs no key. + t.string :api_key + t.jsonb :options, null: false, default: {} + # Raw /v1/models payload plus any server-specific metadata, stored verbatim. + t.jsonb :catalogue, null: false, default: {} + t.datetime :catalogue_fetched_at + t.string :connection_fingerprint + # Model references are strings, never foreign keys: a selection must survive + # the model disappearing from the remote catalogue. + t.string :default_chat_model_id + t.string :default_embedding_model_id + t.datetime :last_connected_at + + t.timestamps null: false + end + end +end diff --git a/db/migrate/20260811140000_create_llm_capability_verdicts.rb b/db/migrate/20260811140000_create_llm_capability_verdicts.rb new file mode 100644 index 000000000000..1e94a71fa5e0 --- /dev/null +++ b/db/migrate/20260811140000_create_llm_capability_verdicts.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmCapabilityVerdicts < ActiveRecord::Migration[8.1] + def change + create_table :llm_capability_verdicts do |t| + t.references :llm_connection, null: false, foreign_key: true + # A plain string, not a foreign key: the catalogue is a cache of a remote + # list, and a model may vanish from it without invalidating what we learned. + t.string :model_id, null: false + t.string :capability, null: false + t.string :state, null: false + t.string :source, null: false + t.jsonb :detail, null: false, default: {} + t.datetime :checked_at, null: false + + t.timestamps null: false + end + + add_index :llm_capability_verdicts, + %i[llm_connection_id model_id capability], + unique: true, + name: "index_llm_capability_verdicts_on_connection_model_capability" + end +end diff --git a/db/migrate/20260811140100_create_llm_feature_bindings.rb b/db/migrate/20260811140100_create_llm_feature_bindings.rb new file mode 100644 index 000000000000..b9bbc61683e1 --- /dev/null +++ b/db/migrate/20260811140100_create_llm_feature_bindings.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmFeatureBindings < ActiveRecord::Migration[8.1] + def change + create_table :llm_feature_bindings do |t| + t.references :llm_connection, null: false, foreign_key: true + t.string :feature_key, null: false + # NULL means "use the connection default for this kind of model". + t.string :model_id + # Embedding features only. Frozen together with model_id once vectors exist. + t.integer :dimensions + t.string :input_prefix + t.string :query_prefix + # Set once the binding has data depending on it, after which the model + # cannot be swapped without a destructive re-index. + t.datetime :locked_at + t.datetime :last_seen_at + + t.timestamps null: false + end + + add_index :llm_feature_bindings, %i[llm_connection_id feature_key], unique: true + end +end diff --git a/db/migrate/20260812090000_create_llm_models.rb b/db/migrate/20260812090000_create_llm_models.rb new file mode 100644 index 000000000000..9e8d7298c944 --- /dev/null +++ b/db/migrate/20260812090000_create_llm_models.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmModels < ActiveRecord::Migration[8.1] + def change + create_table :llm_models do |t| + t.references :llm_connection, null: false, foreign_key: true + # Whatever this deployment calls the model. Provider-specific: Scaleway + # serves "qwen3.6-35b-a3b" for weights another catalogue lists as + # "Qwen/Qwen3.6-35B-A3B". + t.string :external_id, null: false + t.string :display_name + t.boolean :active, null: false, default: true + # Entered by an administrator rather than discovered. Survives a refresh + # that cannot see it, which is what makes a server offering + # /v1/chat/completions but no /v1/models usable. + t.boolean :manual, null: false, default: false + t.datetime :last_seen_at + t.jsonb :raw_metadata, null: false, default: {} + + t.timestamps null: false + end + + add_index :llm_models, %i[llm_connection_id external_id], unique: true + + # Superseded by the table above. + remove_column :llm_connections, :catalogue, :jsonb, null: false, default: {} + end +end diff --git a/db/migrate/20260812090100_add_api_format_to_llm_connections.rb b/db/migrate/20260812090100_add_api_format_to_llm_connections.rb new file mode 100644 index 000000000000..97aeedb6978b --- /dev/null +++ b/db/migrate/20260812090100_add_api_format_to_llm_connections.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Named with a capital API: config/initializers/inflections.rb registers "API" +# as an acronym, so Rails resolves this file to AddAPIFormatToLlmConnections and +# silently skips a class spelled any other way. +class AddAPIFormatToLlmConnections < ActiveRecord::Migration[8.1] + def change + # Which dialect the server speaks. Only "openai" is implemented; the column + # exists so that adding Azure -- the one non-OpenAI format with a real + # constituency under #62215's bring-your-own-infrastructure goal -- is a new + # adapter rather than a migration. + change_table :llm_connections, bulk: true do |t| + t.string :api_format, null: false, default: "openai" + # Provider-specific headers sent with every request, e.g. Azure's + # api-version or a gateway's own key header. + t.jsonb :custom_headers, null: false, default: {} + end + end +end diff --git a/db/migrate/20260813090000_add_created_at_index_to_health_reports.rb b/db/migrate/20260813090000_add_created_at_index_to_health_reports.rb new file mode 100644 index 000000000000..dfcba13aa170 --- /dev/null +++ b/db/migrate/20260813090000_add_created_at_index_to_health_reports.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Every consumer of a health report reads the newest one for a subject +# (`health_reports.order(created_at: :asc).last`), and the table has only a +# [subject_type, subject_id] index, so that read sorts. Until now the table grew +# a row per manual "Run checks" click; the LLM connection adds a scheduled check, +# which makes it grow unattended and adds a pruning delete that filters on age. +# +# Storages and wikis benefit from this too. +class AddCreatedAtIndexToHealthReports < ActiveRecord::Migration[8.1] + disable_ddl_transaction! + + def change + add_index :health_reports, + %i[subject_type subject_id created_at], + algorithm: :concurrently, + if_not_exists: true + end +end diff --git a/db/migrate/20260813100000_add_deactivated_at_to_llm_models.rb b/db/migrate/20260813100000_add_deactivated_at_to_llm_models.rb new file mode 100644 index 000000000000..8f59a81367c9 --- /dev/null +++ b/db/migrate/20260813100000_add_deactivated_at_to_llm_models.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Separates "the server stopped offering this" from "an administrator does not +# want this used". +# +# +active+ is owned by the catalogue sync, which sets it on every refresh and +# clears it for models the server no longer reports. An administrator's choice +# cannot live there: the next "Refresh models" would silently undo it, and the +# model would be labelled "no longer reported" when it is merely hidden. +class AddDeactivatedAtToLlmModels < ActiveRecord::Migration[8.1] + def change + add_column :llm_models, :deactivated_at, :datetime + end +end diff --git a/docs/installation-and-operations/configuration/environment/README.md b/docs/installation-and-operations/configuration/environment/README.md index 60405bd8994a..693a98f380d4 100644 --- a/docs/installation-and-operations/configuration/environment/README.md +++ b/docs/installation-and-operations/configuration/environment/README.md @@ -234,11 +234,13 @@ OPENPROJECT_ENTERPRISE__PLAN (default="enterprise-on-premises---basic---euro---1 OPENPROJECT_ENTERPRISE__TRIAL__CREATION__HOST (default="https://start.openproject.com") Host for EE trial service OPENPROJECT_FEATURE__BUILT__IN__OAUTH__APPLICATIONS__ACTIVE (default=true) Allows the display and use of built-in OAuth applications. OPENPROJECT_FEATURE__DEPLOY__TARGETS__ACTIVE (default=false) +OPENPROJECT_FEATURE__LLM__CONNECTION__ACTIVE (default=false) Enables the administration page connecting OpenProject to an OpenAI-API-compatible LLM server, and the AI features built on it. OPENPROJECT_FEATURE__MINUTES__STYLING__MEETING__PDF__ACTIVE (default=false) Allow exporting a meeting with FITKO styling. See #65124 for details. OPENPROJECT_FEATURE__SPRINT__REPORTS__ACTIVE (default=false) Enables sprint reporting within the backlogs module. It shows a dashboard with various widgets regarding the sprint progress. OPENPROJECT_FEATURE__STORAGE__FILE__PICKING__SELECT__ALL__ACTIVE (default=false) OPENPROJECT_FEATURE__TYPE__VARIANTS__ACTIVE (default=false) Enables work package type variants. OPENPROJECT_FEATURE__WIKI__ENHANCEMENTS__ACTIVE (default=true) Enables Wiki enhancements, such as the Wikis tab and XWiki integration. +OPENPROJECT_FEATURE__WORK__PACKAGE__MULTIPLE__VERSIONS__ACTIVE (default=false) Enables assigning multiple (target) versions to a work package. Experimental; the user-facing setting and admin switch follow later. OPENPROJECT_FEEDS__ENABLED (default=true) Enable Feeds OPENPROJECT_FEEDS__LIMIT (default=15) Feed content limit OPENPROJECT_FILE__MAX__SIZE__DISPLAYED (default=512) Max size of text files displayed inline @@ -285,6 +287,7 @@ OPENPROJECT_LDAP__FORCE__NO__PAGE (default=nil) Force LDAP to respond as a singl OPENPROJECT_LDAP__GROUPS__DISABLE__SYNC__JOB (default=false) Deactivate regular synchronization job for groups in case scheduled as a separate cronjob OPENPROJECT_LDAP__USERS__DISABLE__SYNC__JOB (default=false) Deactivate user attributes synchronization from LDAP OPENPROJECT_LDAP__USERS__SYNC__STATUS (default=false) Enable user status (locked/unlocked) synchronization from LDAP +OPENPROJECT_LLM__CONNECTION (default={}) Configure the connection to an OpenAI-API-compatible LLM server through environment variables OPENPROJECT_LOG__LEVEL (default="info") Set the OpenProject logger level OPENPROJECT_LOG__REQUESTING__USER (default=false) Log user login, name, and mail address for all requests OPENPROJECT_LOGIN__REQUIRED (default=true) Authentication required diff --git a/frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts b/frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts new file mode 100644 index 000000000000..1ee8cc63ced1 --- /dev/null +++ b/frontend/src/stimulus/controllers/dynamic/admin/llm-connection-form.controller.ts @@ -0,0 +1,69 @@ +//-- copyright +// OpenProject is an open source project management software. +// Copyright (C) the OpenProject GmbH +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License version 3. +// +// OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +// Copyright (C) 2006-2013 Jean-Philippe Lang +// Copyright (C) 2010-2013 the ChiliProject Team +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +// +// See COPYRIGHT and LICENSE files for more details. +//++ + +import { Controller } from '@hotwired/stimulus'; + +// Saving the connection contacts the LLM server, which can take seconds. This +// gives the administrator feedback while that happens, and keeps the typed API +// key out of Turbo's page cache. +export default class extends Controller { + static targets = ['submitButton', 'progressBanner', 'secretInput']; + + declare readonly submitButtonTargets:HTMLButtonElement[]; + declare readonly progressBannerTarget:HTMLElement; + declare readonly secretInputTargets:HTMLInputElement[]; + + connect():void { + this.element.addEventListener('submit', this.showProgress); + document.addEventListener('turbo:before-cache', this.clearSecrets); + } + + disconnect():void { + this.element.removeEventListener('submit', this.showProgress); + document.removeEventListener('turbo:before-cache', this.clearSecrets); + } + + // Turbo caches the rendered page for the back button. Without this the typed + // API key would be restored into the DOM on navigating back. + private clearSecrets = ():void => { + this.secretInputTargets.forEach((input:HTMLInputElement) => { + input.value = ''; + }); + }; + + private showProgress = ():void => { + this.progressBannerTarget.hidden = false; + + // aria-disabled rather than disabled: disabling a button while it holds + // focus moves focus to , silently dropping keyboard users to the top + // of the page. It also keeps the button in the submitted form data. + this.submitButtonTargets.forEach((button:HTMLButtonElement) => { + button.setAttribute('aria-disabled', 'true'); + }); + }; +} diff --git a/lib/open_project/patches/ruby_llm_provider_headers.rb b/lib/open_project/patches/ruby_llm_provider_headers.rb new file mode 100644 index 000000000000..a3ed85160ab0 --- /dev/null +++ b/lib/open_project/patches/ruby_llm_provider_headers.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Lets an LlmConnection's custom headers reach every request RubyLLM makes. +# +# An OpenAI-compatible deployment is routinely fronted by a gateway that +# authenticates on its own terms -- APISIX's key-auth wants an +apikey+ header, +# not +Authorization+ -- so an administrator must be able to add, and override, +# whatever the provider would send. +# +# RubyLLM's public API cannot express that: +# +# * Chat#with_headers reaches chat only. Provider#embed takes no headers +# argument at all, so an embedding request could never carry them. +# * Even for chat it merges the wrong way round -- +additional_headers.merge( +# req.headers)+ -- so the provider's own headers win every collision and an +# administrator can never replace Authorization. +# +# Provider#headers is the one hook that reaches chat, embeddings and model +# listing alike, and merging our values last makes an override actually override. +# +# The module is prepended onto each registered provider class rather than onto +# RubyLLM::Provider, because all thirteen providers define #headers themselves +# without calling super -- a patch on the base class would never be reached. +# Azure inherits OpenAI, so it receives the module twice; the inner copy is +# simply never consulted. +module OpenProject + module Patches + module RubyLLMProviderHeaders + def headers + custom = config.openproject_custom_headers + + return super if custom.blank? + + super.merge(custom.stringify_keys) + end + end + end +end + +OpenProject::Patches.patch_gem_version("ruby_llm", "1.16.0") do + RubyLLM::Configuration.register_provider_options(%i[openproject_custom_headers]) + + RubyLLM::Provider.providers.each_value do |provider_class| + provider_class.prepend(OpenProject::Patches::RubyLLMProviderHeaders) + end +end diff --git a/lib_static/open_project/llm/features.rb b/lib_static/open_project/llm/features.rb new file mode 100644 index 000000000000..5165d24ca2e8 --- /dev/null +++ b/lib_static/open_project/llm/features.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module OpenProject + module Llm + class UnknownFeature < StandardError; end + + # A feature that sends requests to the configured LLM server. + # + # Features declare the capabilities they need so the administration UI can + # tell an administrator which models are usable for which job, and so a + # feature never silently runs against a model that cannot serve it. + Feature = Data.define(:key, :kind, :requires, :prefers, :overridable, :pinned, :available, :i18n_scope) do + def available? = available.call + + def chat? = kind == :chat + + def embedding? = kind == :embedding + + def label = I18n.t("label", scope: i18n_scope) + + def caption = I18n.t("caption", scope: i18n_scope, default: nil) + end + + # The registry of LLM-consuming features. + # + # Lives in lib_static because it is populated from initializers, which run + # before eager loading; constants defined under app/ would be unloaded on a + # development reload and lose their registrations. This is the same reason + # OpenProject::FeatureDecisions lives here. + # + # Register from config/initializers/llm_features.rb for core features, or + # from a module's engine: + # + # initializer "openproject_foo.llm_features" do + # OpenProject::Llm::Features.register :foo, kind: :chat + # end + module Features + module_function + + KINDS = %i[chat embedding].freeze + + # Mirrors Llm::Capabilities, which owns the vocabulary and knows how to + # read published values from the model registry. Duplicated as literals + # here because lib_static is autoloaded once, before app/ is available. + CAPABILITIES = { + chat: %i[function_calling structured_output vision reasoning].freeze, + embedding: %i[embeddings].freeze + }.freeze + + def register(key, + kind:, + requires: [], + prefers: [], + overridable: false, + pinned: false, + available: -> { true }, + i18n_scope: nil) + key = key.to_sym + validate!(key, kind, requires + prefers) + + all[key] = Feature.new(key:, kind:, requires: requires.map(&:to_sym).freeze, + prefers: prefers.map(&:to_sym).freeze, + overridable:, pinned:, available:, + i18n_scope: i18n_scope || "llm.features.#{key}") + end + + def all = @all ||= {} + + def [](key) + all.fetch(key.to_sym) { raise UnknownFeature, key.to_s } + end + + def registered?(key) = all.key?(key.to_sym) + + # Features whose own toggle is on. A feature that is switched off keeps its + # stored binding: flipping a flag must not lose an administrator's choice. + def available = all.values.select(&:available?) + + def for_kind(kind) = available.select { |feature| feature.kind == kind } + + def validate!(key, kind, capabilities) + raise ArgumentError, "unknown kind #{kind.inspect}" unless KINDS.include?(kind) + raise ArgumentError, "LLM feature #{key} is already registered" if all.key?(key) + + unknown = capabilities.map(&:to_sym) - CAPABILITIES.fetch(kind) + return if unknown.empty? + + raise ArgumentError, "#{unknown.join(', ')} not valid for a #{kind} feature" + end + end + end +end diff --git a/spec/contracts/llm_connections/update_contract_spec.rb b/spec/contracts/llm_connections/update_contract_spec.rb new file mode 100644 index 000000000000..96ed9c75bdfb --- /dev/null +++ b/spec/contracts/llm_connections/update_contract_spec.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" +require_relative "../shared/model_contract_shared_context" + +RSpec.describe LlmConnections::UpdateContract, :check_errors_i18n, :llm_server_helpers, :webmock do + include_context "ModelContract shared context" + + let(:current_user) { build_stubbed(:admin) } + let(:base_url) { "https://example.com/v1" } + # Persisted, so that only the attributes an example changes are dirty. A + # freshly built record has every factory attribute in changed_attributes, + # which the contract would rightly reject as writes to non-writable columns. + let(:connection) { create(:llm_connection, base_url: "https://previous.example/v1") } + let(:contract) { described_class.new(connection, current_user) } + + let!(:models_request) { mock_llm_models_response(base_url) } + + before { connection.base_url = base_url } + + context "when the server answers with a model list" do + include_examples "contract is valid" + + it "probes the server exactly once" do + contract.validate + + expect(models_request).to have_been_made.once + end + end + + context "when the server rejects the credentials" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: 401) } + + include_examples "contract is invalid", api_key: :invalid_api_key + end + + context "when the server cannot be reached" do + let!(:models_request) { mock_llm_models_response(base_url, timeout: true) } + + include_examples "contract is invalid", base_url: :request_timed_out + end + + # A server can speak the OpenAI API for chat and still not expose a model list: + # OpenProject's own hosted gateway does exactly that. Blocking the save would + # leave the administrator unable to configure a working connection at all. + [404, 405, 501].each do |status| + context "when the server answers #{status} for the model list" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: status) } + + include_examples "contract is valid" + end + end + + # Formats whose model list comes from the registry have nothing to probe here. + context "with a format that does not discover models from the server" do + let(:connection) { create(:llm_connection, api_format: "anthropic", base_url: "https://previous.example") } + + it "does not contact the base URL" do + contract.validate + + expect(models_request).not_to have_been_made + end + end + + context "when the endpoint is not OpenAI-compatible" do + let!(:models_request) { mock_llm_models_response(base_url, body: "login") } + + include_examples "contract is invalid", base_url: :not_openai_compatible + end + + context "when the host is blocked by the SSRF policy" do + before { allow_llm_host("something.else") } + + include_examples "contract is invalid", base_url: :ssrf_filtered + + it "does not contact the server" do + contract.validate + + expect(models_request).not_to have_been_made + end + end + + context "when the base URL is not a URL at all" do + let(:base_url) { "not a url" } + + # The validate_url gem always records :url in errors.details; the + # message: :invalid_url option controls the rendered text, not the symbol. + include_examples "contract is invalid", base_url: :url + + it "does not contact the server" do + contract.validate + + expect(models_request).not_to have_been_made + end + end + + # Without the changed-attributes guard every unrelated save -- and every form + # render that builds a model through SetAttributesService -- would fire an + # outbound request at the LLM server. + describe "the changed-attributes guard" do + it "probes only when the credentials changed" do + contract.validate + expect(models_request).to have_been_made.once + + WebMock.reset_executed_requests! + connection.save! + contract.validate + + expect(models_request).not_to have_been_made + end + + it "probes again when only the API key changed" do + connection.save! + WebMock.reset_executed_requests! + + connection.api_key = "sk-rotated" + contract.validate + + expect(models_request).to have_been_made.once + end + + # Switching the dialect is switching the server, even at the same URL: a + # connection previously talking to a registry-backed provider must be proven + # again when it becomes OpenAI-compatible. + it "probes when only the API format changed to an OpenAI-compatible one" do + connection.update_columns(api_format: "anthropic", base_url: base_url) + connection.reload.api_format = "openai" + + expect(contract.validate).to be(true) + expect(models_request).to have_been_made.once + end + + it "does not probe when only the enabled flag changed" do + connection.save! + WebMock.reset_executed_requests! + + connection.enabled = true + + expect(contract.validate).to be(true) + expect(models_request).not_to have_been_made + end + end + + describe "default model selection" do + let(:connection) { create(:llm_connection, :with_models, base_url:) } + + context "with a model the server offers" do + before { connection.default_chat_model_id = "bge-m3" } + + include_examples "contract is valid" + end + + context "with a model the server does not offer" do + before { connection.default_chat_model_id = "not-there" } + + include_examples "contract is invalid", default_chat_model_id: :not_available + end + end +end diff --git a/spec/factories/llm_connection_factory.rb b/spec/factories/llm_connection_factory.rb new file mode 100644 index 000000000000..7fd84f91a744 --- /dev/null +++ b/spec/factories/llm_connection_factory.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +FactoryBot.define do + factory :llm_connection do + name { LlmConnection::SINGLETON_NAME } + type { "LlmConnection" } + base_url { "https://example.com/v1" } + api_key { "sk-test-key" } + enabled { false } + + trait :enabled do + enabled { true } + end + + trait :with_models do + catalogue_fetched_at { Time.current } + last_connected_at { Time.current } + + after(:create) do |connection| + create(:llm_model, llm_connection: connection, external_id: "qwen3.6-27b", + raw_metadata: { "owned_by" => "vllm", "max_model_len" => 262_144 }) + create(:llm_model, llm_connection: connection, external_id: "bge-m3", + raw_metadata: { "owned_by" => "vllm", "max_model_len" => 8_192 }) + end + end + end +end diff --git a/spec/factories/llm_model_factory.rb b/spec/factories/llm_model_factory.rb new file mode 100644 index 000000000000..59527ef65278 --- /dev/null +++ b/spec/factories/llm_model_factory.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +FactoryBot.define do + factory :llm_model do + llm_connection + sequence(:external_id) { |n| "model-#{n}" } + active { true } + manual { false } + last_seen_at { Time.current } + + trait :manual do + manual { true } + last_seen_at { nil } + end + + trait :withdrawn do + active { false } + end + + # Still offered by the server; hidden by an administrator. + trait :deactivated do + deactivated_at { Time.current } + end + end +end diff --git a/spec/features/admin/llm_connection_spec.rb b/spec/features/admin/llm_connection_spec.rb new file mode 100644 index 000000000000..79dc4100d1d1 --- /dev/null +++ b/spec/features/admin/llm_connection_spec.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +# The markup on these pages is hand-written rather than generated, so this spec +# exists mainly to put it through axe. The row toggle in particular has no +# accessible name of its own and depends on an explicit aria-label. +# :selenium is required, not incidental: axe-core-api drives the browser through +# Selenium's #manage API, so be_axe_clean does not work under cuprite. Every other +# axe spec in this repository is tagged the same way for the same reason. +RSpec.describe "LLM connection administration", + :js, :llm_server_helpers, :selenium, :webmock, + driver: :firefox_de, + with_flag: { llm_connection: true } do + shared_let(:admin) { create(:admin) } + + let(:base_url) { "https://example.com/v1" } + + current_user { admin } + + # The kebab is a Primer ActionMenu: clicking it before its behaviour is + # attached silently does nothing, so wait for the page to settle first and + # for the item itself to become visible. + def choose_action(item) + expect(page).to have_test_selector("llm-connection--actions") + find_test_selector("llm-connection--actions").click + expect(page).to have_test_selector(item) + find_test_selector(item).click + end + + context "when nothing is configured yet" do + it "renders an accessible, empty settings page" do + visit llm_connection_path + + expect(page).to have_field("Host URL") + expect(page).to be_axe_clean.within("#content") + end + end + + context "with a configured connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before { mock_llm_models_response(base_url) } + + it "renders the model list accessibly" do + visit llm_connection_path + + expect(page).to have_test_selector("llm-model--toggle-#{connection.models.first.id}") + expect(page).to be_axe_clean.within("#content") + end + + it "hides a model from the feature pickers when it is switched off" do + llm_model = connection.models.find_by(external_id: "qwen3.6-27b") + + visit llm_connection_path + find_test_selector("llm-model--toggle-#{llm_model.id}").click + + wait_for { llm_model.reload.deactivated_at }.not_to be_nil + + # The toggle acknowledges with JSON rather than re-rendering the row, so + # the source label only catches up on the next load. + visit llm_connection_path + expect(page).to have_text("Hidden by administrator") + end + + it "removes the stored API key" do + visit llm_connection_path + expect(page).to have_test_selector("llm-model--refresh-button") + + choose_action("llm-connection--delete-api-key") + + within_test_selector("llm-connection--delete-api-key-dialog") do + expect(page).to be_axe_clean + click_on "Remove API key" + end + + wait_for { connection.reload.api_key }.to be_blank + expect(connection.base_url).to eq(base_url) + end + + it "disconnects without losing the configuration" do + visit llm_connection_path + expect(page).to have_test_selector("llm-model--refresh-button") + + choose_action("llm-connection--disconnect") + + within_test_selector("llm-connection--disconnect-dialog") do + expect(page).to be_axe_clean + click_on "Disconnect" + end + + wait_for { connection.reload.enabled? }.to be(false) + expect(connection.api_key).to be_blank + # The point of disconnecting rather than deleting. + expect(connection.models.count).to eq(2) + end + + describe "the health report" do + before { mock_llm_chat_response(base_url) } + + it "runs the checks and renders the report accessibly" do + visit llm_connection_path + + find_test_selector("llm-connection--run-health-checks").click + + wait_for { connection.health_reports.count }.to eq(1) + + find_test_selector("llm-connection--open-health-report").click + + expect(page).to have_current_path(llm_connection_health_status_report_path) + expect(page).to have_text("Configuration") + expect(page).to be_axe_clean.within("#content") + end + end + end + + describe "the AI models page" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before { mock_llm_embeddings_response(base_url) } + + it "offers the vector settings only for features that embed" do + visit llm_feature_bindings_path + + expect(page).to have_test_selector("llm-feature-binding--dimensions-semantic_search") + expect(page).to have_no_test_selector("llm-feature-binding--dimensions-description_assistant") + expect(page).to be_axe_clean.within("#content") + end + end +end diff --git a/spec/lib/open_project/patches/ruby_llm_provider_headers_spec.rb b/spec/lib/open_project/patches/ruby_llm_provider_headers_spec.rb new file mode 100644 index 000000000000..9373d5fdc729 --- /dev/null +++ b/spec/lib/open_project/patches/ruby_llm_provider_headers_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe OpenProject::Patches::RubyLLMProviderHeaders do + def provider_for(slug, custom_headers) + context = RubyLLM.context do |config| + config.public_send(:"#{slug}_api_key=", "test-key") + config.openproject_custom_headers = custom_headers + config.logger = Rails.logger + end + + RubyLLM::Provider.resolve(slug).new(context.config) + end + + it "adds the connection's custom headers to a provider's own" do + headers = provider_for(:openai, { "apikey" => "gateway-secret" }).headers + + expect(headers).to include("apikey" => "gateway-secret") + expect(headers).to include("Authorization" => "Bearer test-key") + end + + # The point of the patch: a gateway may authenticate on its own terms, so an + # administrator has to be able to replace what the provider would send. + # RubyLLM's own Chat#with_headers merges the other way round and cannot. + it "lets a custom header override the provider's" do + headers = provider_for(:openai, { "Authorization" => "Bearer override" }).headers + + expect(headers).to include("Authorization" => "Bearer override") + end + + it "accepts symbol keys" do + headers = provider_for(:openai, { apikey: "gateway-secret" }).headers + + expect(headers).to include("apikey" => "gateway-secret") + end + + it "leaves the provider's headers untouched when none are configured" do + expect(provider_for(:openai, {}).headers).to eq(provider_for(:openai, nil).headers) + expect(provider_for(:openai, nil).headers).to include("Authorization" => "Bearer test-key") + end + + # All thirteen providers define #headers without calling super, so the patch + # has to be prepended onto each one rather than onto the base class. + it "applies to every registered provider" do + RubyLLM::Provider.providers.each_value do |provider_class| + expect(provider_class.ancestors).to include(described_class) + end + end + + it "reaches a provider that is not openai" do + headers = provider_for(:anthropic, { "apikey" => "gateway-secret" }).headers + + expect(headers).to include("apikey" => "gateway-secret") + end +end diff --git a/spec/models/llm_model_deactivation_spec.rb b/spec/models/llm_model_deactivation_spec.rb new file mode 100644 index 000000000000..be415f977b5c --- /dev/null +++ b/spec/models/llm_model_deactivation_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe LlmModel, "deactivation", :llm_server_helpers, :webmock, + with_flag: { llm_connection: true } do + let(:base_url) { "https://example.com/v1" } + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + let(:model) { connection.models.find_by(external_id: "qwen3.6-27b") } + + before { model.update!(deactivated_at: Time.current) } + + it "hides the model from the pickers" do + expect(connection.selectable_model_ids).not_to include("qwen3.6-27b") + expect(connection.selectable_model_ids).to include("bge-m3") + end + + # The decision that makes the toggle safe: curation, not enforcement. A row an + # administrator switches off must never silently break a running feature. + it "keeps a feature already bound to it working" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + expect(connection.available_model_ids).to include("qwen3.6-27b") + expect(Llm::Runtime.for(:description_assistant)).to be_ready + end + + it "still offers it to the feature that is bound to it" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + options = LlmConnections::SelectableModelsQuery + .new(connection, OpenProject::Llm::Features[:description_assistant]) + .call + + expect(options.map(&:model_id)).to include("qwen3.6-27b") + end + + # The reason deactivated_at exists rather than reusing active: the sync writes + # active on every refresh, so an administrator's choice stored there would be + # undone by the next "Refresh models". + it "survives a catalogue sync that still reports the model" do + mock_llm_models_response(base_url) + + LlmConnections::SyncModelsService.new(connection).call + + expect(model.reload).to be_deactivated + expect(model).to be_active + expect(model).not_to be_selectable + end + + it "is distinct from a model the server withdrew" do + withdrawn = create(:llm_model, :withdrawn, llm_connection: connection, external_id: "gone") + + expect(withdrawn).to be_withdrawn + expect(model).not_to be_withdrawn + end +end diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb new file mode 100644 index 000000000000..6f3c5df2c874 --- /dev/null +++ b/spec/requests/admin/llm_connections_spec.rb @@ -0,0 +1,449 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe "Admin LLM connection", :llm_server_helpers, :skip_csrf, :webmock, type: :rails_request do + let(:admin) { create(:admin) } + let(:non_admin) { create(:user) } + let(:base_url) { "https://example.com/v1" } + + describe "GET /admin/llm_connection" do + it "is not reachable for non-admins" do + login_as non_admin + get llm_connection_path + + expect(response).not_to have_http_status(:ok) + end + + context "when logged in as admin" do + before { login_as admin } + + it "renders without a connection configured" do + get llm_connection_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Host URL") + end + + it "lists the cached models without contacting the server" do + create(:llm_connection, :with_models, base_url:) + + get llm_connection_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("qwen3.6-27b") + expect(a_request(:get, "#{base_url}/models")).not_to have_been_made + end + end + end + + describe "PATCH /admin/llm_connection" do + before { login_as admin } + + context "with a reachable server" do + let!(:models_request) { mock_llm_models_response(base_url) } + + it "stores the connection and caches the catalogue" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-test" } } + + expect(response).to have_http_status(:see_other) + connection = LlmConnection.first + expect(connection.base_url).to eq(base_url) + expect(connection.api_key).to eq("sk-test") + expect(connection.available_model_ids).to contain_exactly("qwen3.6-27b", "bge-m3") + end + end + + # The case that matters for OpenProject's own gateway: chat completions are + # routed, the model list is not. + context "with a server that exposes no model list" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: 404) } + + it "still saves the connection and says models must be added by hand" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-test" } } + + expect(response).to have_http_status(:see_other) + expect(LlmConnection.first.base_url).to eq(base_url) + expect(flash[:warning]).to be_present + end + end + + context "with an unreachable server" do + let!(:models_request) { mock_llm_models_response(base_url, timeout: true) } + + it "persists nothing" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-test" } } + + expect(LlmConnection.count).to eq(0) + end + end + + # Reachability and credentials still gate the save; only the model list is + # treated as optional. + context "with rejected credentials" do + let!(:models_request) { mock_llm_models_response(base_url, response_code: 401) } + + it "persists nothing" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-wrong" } } + + expect(LlmConnection.count).to eq(0) + end + end + + context "when an API key is already stored" do + let!(:connection) { create(:llm_connection, base_url:, api_key: "sk-original") } + let!(:models_request) { mock_llm_models_response(base_url) } + + it "keeps the stored key when the field is submitted blank" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "" } } + + expect(connection.reload.api_key).to eq("sk-original") + end + + it "replaces the stored key when a new one is submitted" do + patch llm_connection_path, params: { llm_connection: { base_url:, api_key: "sk-rotated" } } + + expect(connection.reload.api_key).to eq("sk-rotated") + end + end + end + + describe "DELETE /admin/llm_connection/api_key" do + before { login_as admin } + + it "removes the key but keeps the connection" do + connection = create(:llm_connection, base_url:, api_key: "sk-original") + + delete api_key_llm_connection_path + + expect(response).to have_http_status(:see_other) + expect(connection.reload.api_key).to be_nil + expect(connection.base_url).to eq(base_url) + end + end + + describe "POST /admin/llm_connection/refresh_models" do + before { login_as admin } + + it "refetches the catalogue" do + create(:llm_connection, base_url:) + request = mock_llm_models_response(base_url) + + post refresh_models_llm_connection_path + + expect(request).to have_been_made.once + expect(LlmConnection.first.available_model_ids).to include("bge-m3") + end + end + + describe "GET /admin/llm_connection/delete_api_key_dialog" do + let!(:connection) { create(:llm_connection, base_url: "https://example.com/v1", api_key: "sk-test") } + + before { login_as admin } + + it "offers the confirmation" do + # Requested by the async-dialog Stimulus controller, which asks for a + # turbo stream rather than HTML. + get delete_api_key_dialog_llm_connection_path, + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Remove the stored API key?") + end + + # The catalogue sync fingerprints base_url and api_key together, so changing + # the key discards every verdict -- including hand-made ones, which nothing + # else throws away. + it "warns when hand-made capability assertions would be lost" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "supported", source: "admin", + checked_at: Time.current) + + get delete_api_key_dialog_llm_connection_path, + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response.body).to include("assertions you made yourself") + end + end + + describe "DELETE /admin/llm_connection/api_key when provisioned" do + before { login_as admin } + + it "clears the stored key and keeps everything else" do + connection = create(:llm_connection, base_url: "https://example.com/v1", api_key: "sk-test") + + delete api_key_llm_connection_path + + expect(connection.reload.api_key).to be_blank + expect(connection.base_url).to eq("https://example.com/v1") + end + + # Previously update! bypassed the contract, so this could be wiped by a + # hand-crafted request even though the UI never offers it. + it "refuses when the connection comes from the environment" do + connection = create(:llm_connection, base_url: "https://example.com/v1", api_key: "sk-test") + allow(Setting).to receive(:llm_connection).and_return({ "base_url" => "https://example.com/v1" }) + + delete api_key_llm_connection_path + + expect(connection.reload.api_key).to eq("sk-test") + end + end + + describe "disconnecting" do + let!(:connection) do + create(:llm_connection, :with_models, :enabled, + base_url: "https://example.com/v1", api_key: "sk-test") + end + + before { login_as admin } + + it "offers the confirmation, naming what is kept" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + get disconnect_dialog_llm_connection_path, + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Disconnect from the LLM server?") + expect(response.body).to include("Description assistant") + end + + # Disconnecting is reversible on purpose: destroying the connection would + # cascade to the models, the verdicts and every binding. + it "clears the credential and switches the connection off, keeping everything else" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + post disconnect_llm_connection_path + + connection.reload + expect(connection.api_key).to be_blank + expect(connection).not_to be_enabled + expect(connection.base_url).to eq("https://example.com/v1") + expect(connection.models.count).to eq(2) + expect(connection.feature_bindings.first.model_id).to eq("qwen3.6-27b") + end + + it "refuses when the connection comes from the environment" do + allow(Setting).to receive(:llm_connection).and_return({ "base_url" => "https://example.com/v1" }) + + post disconnect_llm_connection_path + + expect(connection.reload.api_key).to eq("sk-test") + expect(connection).to be_enabled + end + + it "is refused to a non-admin" do + login_as create(:user) + + post disconnect_llm_connection_path + + expect(connection.reload.api_key).to eq("sk-test") + end + end + + describe "the default embedding model field" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url: "https://example.com/v1") } + + before { login_as admin } + + # Column, contract attribute, validation, error key, locale key and permitted + # param all existed; the input was never rendered, so the value could not be + # set through the UI at all. + it "is rendered once something embeds" do + get llm_connection_path + + expect(response.body).to include("llm_connection[default_embedding_model_id]") + end + + it "is saved" do + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "bge-m3" } } + + expect(connection.reload.default_embedding_model_id).to eq("bge-m3") + end + end + + describe "paginating the model list" do + let!(:connection) { create(:llm_connection, :enabled, base_url: "https://example.com/v1") } + + before do + login_as admin + # A gateway can report hundreds; OpenRouter returns 341. + 25.times { |n| create(:llm_model, llm_connection: connection, external_id: format("model-%03d", n)) } + end + + # Counted by row, not by model id: the default-model select still lists + # every model, so the ids appear in the body regardless of the table. + def rendered_rows(body) = body.scan("llm-model--toggle-").size + + it "shows one page of rows at a time rather than every model" do + get llm_connection_path, params: { per_page: 20 } + + expect(rendered_rows(response.body)).to eq(20) + expect(response.body).to include("op-pagination") + end + + it "serves the remainder on the next page" do + get llm_connection_path, params: { per_page: 20, page: 2 } + + expect(rendered_rows(response.body)).to eq(5) + end + end + + describe "filtering the model list" do + let!(:connection) { create(:llm_connection, :enabled, base_url: "https://example.com/v1") } + let(:filters) { [{ name: { operator: "~", values: ["bge"] } }].to_json } + + before do + login_as admin + create(:llm_model, llm_connection: connection, external_id: "qwen3.6-27b") + create(:llm_model, llm_connection: connection, external_id: "bge-m3") + create(:llm_model, llm_connection: connection, external_id: "e5-large", display_name: "BGE compatible") + end + + def rendered_rows(body) = body.scan("llm-model--toggle-").size + + it "narrows the table to matching models" do + get search_models_llm_connection_path, params: { filters: } + + expect(response).to have_http_status(:ok) + # The identifier and the friendly name both match, since either is what + # somebody would type. + expect(rendered_rows(response.body)).to eq(2) + expect(response.body).to include("bge-m3") + end + + it "answers with a turbo stream so only the table is replaced" do + get search_models_llm_connection_path, params: { filters: } + + expect(response.media_type).to eq("text/vnd.turbo-stream.html") + end + + it "applies the filter to the full page too, so a shared link works" do + get llm_connection_path, params: { filters: } + + expect(rendered_rows(response.body)).to eq(2) + end + + it "shows everything without a filter" do + get llm_connection_path + + expect(rendered_rows(response.body)).to eq(3) + end + end + + describe "choosing a default embedding model" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url: "https://example.com/v1") } + + before { login_as admin } + + def verdict(model_id, state) + connection.capability_verdicts.create!(model_id:, capability: "embeddings", state:, + source: "probe", checked_at: Time.current) + end + + it "does not offer a model the server says cannot embed" do + verdict("qwen3.6-27b", "unsupported") + verdict("bge-m3", "supported") + + get llm_connection_path + + expect(response.body).to include("bge-m3") + expect(response.body).not_to include("cannot create embeddings") + end + + # An unconfirmed capability is not a capability: offering such a model + # invites a choice that fails much later, at index time. + it "does not offer a model whose capability is merely unconfirmed" do + get llm_connection_path + + expect(response.body).not_to include("not verified as an embedding model") + end + + it "says how to make a model eligible when none is" do + get llm_connection_path + + expect(response.body).to include("set its embeddings capability") + end + + it "offers one an administrator has asserted can embed" do + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "supported", source: "admin", checked_at: Time.current) + + get llm_connection_path + + expect(response.body).to include("bge-m3") + expect(response.body).not_to include("set its embeddings capability") + end + + # Otherwise a save would silently blank a working configuration. + it "keeps the chosen model listed even once it is ruled out" do + connection.update_column(:default_embedding_model_id, "qwen3.6-27b") + verdict("qwen3.6-27b", "unsupported") + + get llm_connection_path + + expect(response.body).to include("qwen3.6-27b") + end + + it "refuses a model the server says cannot embed" do + verdict("qwen3.6-27b", "unsupported") + + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "qwen3.6-27b" } } + + expect(connection.reload.default_embedding_model_id).to be_nil + end + + it "accepts one that can" do + verdict("bge-m3", "supported") + + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "bge-m3" } } + + expect(connection.reload.default_embedding_model_id).to eq("bge-m3") + end + + # Unknown is the normal state for a server that publishes nothing, so it + # must not block the choice. + it "allows one with no verdict at all" do + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "bge-m3" } } + + expect(connection.reload.default_embedding_model_id).to eq("bge-m3") + end + end +end diff --git a/spec/requests/admin/llm_feature_bindings_spec.rb b/spec/requests/admin/llm_feature_bindings_spec.rb new file mode 100644 index 000000000000..4b58a11eda8e --- /dev/null +++ b/spec/requests/admin/llm_feature_bindings_spec.rb @@ -0,0 +1,185 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe "Admin AI model assignment", :llm_server_helpers, :skip_csrf, :webmock, type: :rails_request do + let(:admin) { create(:admin) } + let(:base_url) { "https://example.com/v1" } + + describe "GET /admin/llm_feature_bindings" do + before { login_as admin } + + it "prompts to configure a connection when there is none" do + get llm_feature_bindings_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("No LLM server configured") + end + + context "with a configured connection" do + let!(:connection) { create(:llm_connection, :with_models, base_url:) } + + it "lists every registered feature" do + get llm_feature_bindings_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Description assistant") + expect(response.body).to include("Semantic search") + end + + # Hiding an unusable model is the one thing that produces an unanswerable + # support question, so it stays listed and says why it cannot be chosen. + it "offers a model with no verdict, marked as unverified" do + get llm_feature_bindings_path + + expect(response.body).to include("qwen3.6-27b — not verified") + end + + it "disables a model known not to support a required capability" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: Time.current) + + get llm_feature_bindings_path + + expect(response.body).to include("qwen3.6-27b — no Embeddings support") + end + end + end + + describe "PATCH /admin/llm_feature_bindings/:id" do + let!(:connection) { create(:llm_connection, :with_models, base_url:) } + + before { login_as admin } + + it "stores the chosen model" do + patch llm_feature_binding_path("description_assistant"), + params: { llm_feature_binding: { model_id: "qwen3.6-27b" } } + + expect(response).to have_http_status(:see_other) + expect(connection.feature_bindings.find_by(feature_key: "description_assistant").model_id) + .to eq("qwen3.6-27b") + end + + it "treats a blank choice as inheriting the default" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + patch llm_feature_binding_path("description_assistant"), params: { llm_feature_binding: { model_id: "" } } + + expect(connection.feature_bindings.find_by(feature_key: "description_assistant").model_id).to be_nil + end + + # The verdict that matters is the one for the model just chosen, so it is + # fetched now rather than left unknown until the feature first runs. + it "probes the model when the feature requires a capability" do + request = stub_request(:post, "#{base_url}/embeddings") + .to_return(status: 200, + headers: { "Content-Type" => "application/json" }, + body: { data: [{ embedding: [0.1, 0.2] }] }.to_json) + + patch llm_feature_binding_path("semantic_search"), params: { llm_feature_binding: { model_id: "bge-m3" } } + + expect(request).to have_been_made.once + verdict = connection.capability_verdicts.find_by(model_id: "bge-m3", capability: "embeddings") + expect(verdict.state).to eq("supported") + expect(verdict.dimensions).to eq(2) + end + + it "does not probe for a feature that requires nothing" do + patch llm_feature_binding_path("description_assistant"), + params: { llm_feature_binding: { model_id: "qwen3.6-27b" } } + + expect(a_request(:post, "#{base_url}/embeddings")).not_to have_been_made + end + + it "404s for a feature that is not registered" do + patch llm_feature_binding_path("no_such_feature"), params: { llm_feature_binding: { model_id: "x" } } + + expect(response).to have_http_status(:not_found) + end + end + + describe "embedding settings" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before do + login_as admin + # Binding an embedding feature probes the model for a vector. + mock_llm_embeddings_response(base_url) + end + + it "stores the vector settings, keeping the prefixes exactly as typed" do + patch llm_feature_binding_path(:semantic_search), + params: { llm_feature_binding: { model_id: "bge-m3", + dimensions: "1024", + input_prefix: "passage: ", + query_prefix: "query: " } } + + binding = connection.feature_bindings.find_by(feature_key: "semantic_search") + + expect(binding.dimensions).to eq(1024) + # The trailing space is load-bearing for the E5 and BGE families. + expect(binding.input_prefix).to eq("passage: ") + expect(binding.query_prefix).to eq("query: ") + end + + it "rejects a dimension count that is not a positive integer" do + patch llm_feature_binding_path(:semantic_search), + params: { llm_feature_binding: { model_id: "bge-m3", dimensions: "0" } } + + expect(connection.feature_bindings.find_by(feature_key: "semantic_search")&.dimensions).to be_nil + end + + it "ignores vector settings sent to a chat feature" do + patch llm_feature_binding_path(:description_assistant), + params: { llm_feature_binding: { model_id: "qwen3.6-27b", dimensions: "1024" } } + + binding = connection.feature_bindings.find_by(feature_key: "description_assistant") + + expect(binding.model_id).to eq("qwen3.6-27b") + expect(binding.dimensions).to be_nil + end + + # A locked binding is the record that a vector index exists. Everything the + # index depends on is frozen, not just the model. + it "refuses to change anything a locked index depends on" do + binding = connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "bge-m3", + dimensions: 1024, input_prefix: "passage: ", + locked_at: Time.current) + + patch llm_feature_binding_path(:semantic_search), + params: { llm_feature_binding: { model_id: "bge-m3", dimensions: "512", input_prefix: "other: " } } + + binding.reload + expect(binding.dimensions).to eq(1024) + expect(binding.input_prefix).to eq("passage: ") + end + end +end diff --git a/spec/requests/admin/llm_health_status_spec.rb b/spec/requests/admin/llm_health_status_spec.rb new file mode 100644 index 000000000000..ad04b17ffa8f --- /dev/null +++ b/spec/requests/admin/llm_health_status_spec.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe "LLM connection health status", :llm_server_helpers, :skip_csrf, :webmock, + type: :rails_request, with_flag: { llm_connection: true } do + shared_let(:admin) { create(:admin) } + + let(:base_url) { "https://example.com/v1" } + + before { login_as(admin) } + + describe "GET /admin/llm_connection/health_status_report" do + it "redirects to the settings page when nothing is configured yet" do + get llm_connection_health_status_report_path + + expect(response).to redirect_to(llm_connection_path) + end + + context "with a connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + it "offers to run the checks when none have run" do + get llm_connection_health_status_report_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("has not been checked yet") + end + + it "renders the report once it exists" do + mock_llm_models_response(base_url) + mock_llm_chat_response(base_url) + post llm_connection_health_status_report_path + + get llm_connection_health_status_report_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Configuration") + end + end + end + + describe "POST /admin/llm_connection/health_status_report" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:, default_chat_model_id: "qwen3.6-27b") } + + before do + mock_llm_models_response(base_url) + mock_llm_chat_response(base_url) + end + + it "stores a report and sends the completion an administrator asked for" do + expect { post llm_connection_health_status_report_path } + .to change(connection.health_reports, :count).by(1) + + expect(response).to redirect_to(llm_connection_health_status_report_path) + expect(WebMock).to have_requested(:post, "#{base_url}/chat/completions").once + end + end + + describe "GET /admin/llm_connection/health_status_report.txt" do + let!(:connection) do + create(:llm_connection, :with_models, :enabled, base_url:, + api_key: "sk-super-secret", + custom_headers: { "apikey" => "gateway-secret" }) + end + + before do + mock_llm_models_response(base_url) + mock_llm_chat_response(base_url) + post llm_connection_health_status_report_path + end + + it "downloads the report" do + get llm_connection_health_status_report_path(format: :txt) + + expect(response).to have_http_status(:ok) + expect(response.body).to include("base_url") + end + + # The report is pasted into support tickets. A gateway header routinely + # carries a second credential, so neither it nor the API key may appear. + it "contains no credentials" do + get llm_connection_health_status_report_path(format: :txt) + + expect(response.body).not_to include("sk-super-secret") + expect(response.body).not_to include("gateway-secret") + end + end + + describe "authorisation" do + let!(:connection) { create(:llm_connection, :enabled, base_url:) } + + it "is refused to a non-admin" do + login_as(create(:user)) + + get llm_connection_health_status_report_path + + expect(response).not_to have_http_status(:ok) + end + end +end diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb new file mode 100644 index 000000000000..51f1dd3a41a5 --- /dev/null +++ b/spec/requests/admin/llm_models_spec.rb @@ -0,0 +1,402 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +# Manual entry exists for servers that route /v1/chat/completions but expose no +# model list -- OpenProject's own hosted gateway does exactly that today. +RSpec.describe "Admin manual LLM models", :llm_server_helpers, :skip_csrf, :webmock, type: :rails_request do + let(:admin) { create(:admin) } + let(:base_url) { "https://example.com/v1" } + let!(:connection) { create(:llm_connection, base_url:) } + + before { login_as admin } + + describe "POST /admin/llm_models" do + it "accepts everything the edit screen accepts" do + post llm_models_path, params: { llm_model: { external_id: "bge-m3", + display_name: "BGE M3", + admin_context_window: "8192", + capability_embeddings: "supported", + capability_vision: "unsupported" } } + + llm_model = connection.models.find_by(external_id: "bge-m3") + expect(llm_model.display_name).to eq("BGE M3") + expect(llm_model.context_window).to eq(8192) + + verdicts = connection.capability_verdicts.for_model("bge-m3").pluck(:capability, :state, :source) + expect(verdicts).to include(["embeddings", "supported", "admin"], ["vision", "unsupported", "admin"]) + end + + it "adds a model an administrator names" do + post llm_models_path, params: { llm_model: { external_id: "qwen3.6-35b-a3b" } } + + expect(response).to have_http_status(:see_other) + llm_model = connection.models.find_by(external_id: "qwen3.6-35b-a3b") + expect(llm_model).to be_manual + expect(connection.available_model_ids).to include("qwen3.6-35b-a3b") + end + + it "rejects a duplicate" do + create(:llm_model, llm_connection: connection, external_id: "already-there") + + post llm_models_path, params: { llm_model: { external_id: "already-there" } } + + expect(connection.models.where(external_id: "already-there").count).to eq(1) + end + + it "makes the model bindable straight away" do + post llm_models_path, params: { llm_model: { external_id: "qwen3.6-35b-a3b" } } + + patch llm_feature_binding_path("description_assistant"), + params: { llm_feature_binding: { model_id: "qwen3.6-35b-a3b" } } + + expect(connection.feature_bindings.find_by(feature_key: "description_assistant").model_id) + .to eq("qwen3.6-35b-a3b") + end + end + + describe "a refresh that cannot see the manual model" do + it "keeps it, and withdraws discovered models instead" do + create(:llm_model, llm_connection: connection, external_id: "was-discovered") + post llm_models_path, params: { llm_model: { external_id: "hand-typed" } } + mock_llm_models_response(base_url) + + post refresh_models_llm_connection_path + + expect(connection.models.find_by(external_id: "hand-typed")).to be_active + expect(connection.models.find_by(external_id: "was-discovered")).not_to be_active + expect(connection.available_model_ids).to include("hand-typed", "qwen3.6-27b") + end + end + + describe "PATCH /admin/llm_models/:id" do + let!(:llm_model) { create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") } + + it "stores capabilities an administrator asserts" do + patch llm_model_path(llm_model), + params: { llm_model: { display_name: "Hand typed", + capability_embeddings: "supported", + capability_vision: "unsupported" } } + + expect(response).to have_http_status(:see_other) + expect(llm_model.reload.display_name).to eq("Hand typed") + + verdicts = connection.capability_verdicts.for_model("hand-typed").pluck(:capability, :state, :source) + expect(verdicts).to include(["embeddings", "supported", "admin"], ["vision", "unsupported", "admin"]) + end + + it "stores a context window an administrator supplies" do + patch llm_model_path(llm_model), params: { llm_model: { admin_context_window: "32768" } } + + expect(llm_model.reload.context_window).to eq(32_768) + expect(llm_model.context_window_source).to eq(:admin) + end + + # The administrator's figure wins over whatever the server or a registry said. + it "prefers the administrator's context window over a reported one" do + llm_model.update!(raw_metadata: { "max_model_len" => 8192 }) + + patch llm_model_path(llm_model), params: { llm_model: { admin_context_window: "32768" } } + + expect(llm_model.reload.context_window).to eq(32_768) + end + + it "falls back to the reported figure when cleared" do + llm_model.update!(raw_metadata: { "max_model_len" => 8192, "admin_context_window" => 32_768 }) + + patch llm_model_path(llm_model), params: { llm_model: { admin_context_window: "" } } + + expect(llm_model.reload.context_window).to eq(8192) + expect(llm_model.context_window_source).to eq(:server) + end + + it "makes an asserted capability satisfy a feature that requires it" do + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } + + patch llm_feature_binding_path("semantic_search"), + params: { llm_feature_binding: { model_id: "hand-typed" } } + + expect(connection.feature_bindings.find_by(feature_key: "semantic_search").model_id).to eq("hand-typed") + end + + # Clearing an assertion records nothing rather than recording ignorance as + # fact, so detection can still fill it in later. + it "clears an assertion when set back to unspecified" do + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "" } } + + expect(connection.capability_verdicts.for_model("hand-typed").for_capability(:embeddings)).to be_empty + end + + # An administrator looked at this deployment; a published registry did not. + it "is not overwritten by registry enrichment" do + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } + + LlmConnections::EnrichCapabilitiesService.new(connection).call + + verdict = connection.capability_verdicts.find_by(model_id: "hand-typed", capability: "embeddings") + expect(verdict.source).to eq("admin") + expect(verdict.state).to eq("supported") + end + end + + describe "the model type shown in the list" do + it "reads as an embedding model once embeddings are supported" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "bge-m3") + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } + + get llm_connection_path + + expect(response.body).to include("Embedding") + end + + it "reads as a chat model when embeddings are not supported" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "qwen") + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "unsupported" } } + + get llm_connection_path + + expect(response.body).to include("Chat") + end + end + + describe "GET /admin/llm_models/new" do + it "renders the add-model form" do + get new_llm_model_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Model name") + end + + it "re-renders with the error inline when the name is taken" do + create(:llm_model, llm_connection: connection, external_id: "already-there") + + post llm_models_path, params: { llm_model: { external_id: "already-there" } } + + expect(response).to have_http_status(:unprocessable_entity) + expect(connection.models.where(external_id: "already-there").count).to eq(1) + end + end + + describe "GET /admin/llm_models/:id/delete_dialog" do + it "offers a confirmation naming the features that would break" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "hand-typed") + + # Requested by the async-dialog Stimulus controller, which asks for a + # turbo stream rather than HTML. + get delete_dialog_llm_model_path(llm_model), + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Description assistant") + end + end + + describe "renaming to a taken id" do + it "re-renders the form with the error instead of failing" do + create(:llm_model, llm_connection: connection, external_id: "taken") + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "mine") + + patch llm_model_path(llm_model), params: { llm_model: { external_id: "taken" } } + + expect(response).to have_http_status(:unprocessable_entity) + expect(llm_model.reload.external_id).to eq("mine") + end + end + + describe "DELETE /admin/llm_models/:id" do + it "removes a manual model" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") + + delete llm_model_path(llm_model) + + expect(response).to have_http_status(:see_other) + expect(LlmModel.where(id: llm_model.id)).to be_empty + end + + # Discovered models are the server's to add and remove, not the administrator's. + it "refuses to remove a discovered model" do + llm_model = create(:llm_model, llm_connection: connection, external_id: "from-server") + + delete llm_model_path(llm_model) + + expect(response).to have_http_status(:not_found) + expect(LlmModel.where(id: llm_model.id)).to exist + end + end + + describe "POST /admin/llm_models/:id/toggle" do + let!(:llm_model) { create(:llm_model, llm_connection: connection, external_id: "qwen3.6-27b") } + + it "hides the model from the pickers and puts it back" do + post toggle_llm_model_path(llm_model) + + expect(response).to have_http_status(:ok) + expect(llm_model.reload).to be_deactivated + expect(connection.selectable_model_ids).not_to include("qwen3.6-27b") + + post toggle_llm_model_path(llm_model) + + expect(llm_model.reload).not_to be_deactivated + expect(connection.selectable_model_ids).to include("qwen3.6-27b") + end + + # Curation, not enforcement: a feature already pointing at the model keeps + # resolving, so switching a row off cannot silently break anything. + it "leaves an existing binding working" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + post toggle_llm_model_path(llm_model) + + expect(connection.available_model_ids).to include("qwen3.6-27b") + end + + it "refuses a model the server has withdrawn" do + withdrawn = create(:llm_model, :withdrawn, llm_connection: connection, external_id: "gone") + + post toggle_llm_model_path(withdrawn) + + expect(response).to have_http_status(:unprocessable_entity) + expect(withdrawn.reload).not_to be_deactivated + end + + it "is refused to a non-admin" do + login_as create(:user) + + post toggle_llm_model_path(llm_model) + + expect(llm_model.reload).not_to be_deactivated + end + end + + describe "renaming a manually added model" do + let!(:llm_model) do + create(:llm_model, :manual, llm_connection: connection, external_id: "qwen/qwen3.6-35b-a3b") + end + + before do + connection.update!(default_chat_model_id: "qwen/qwen3.6-35b-a3b") + connection.feature_bindings.create!(feature_key: "description_assistant", + model_id: "qwen/qwen3.6-35b-a3b") + connection.capability_verdicts.create!(model_id: "qwen/qwen3.6-35b-a3b", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: Time.current) + end + + it "offers the identifier field on the edit page" do + get edit_llm_model_path(llm_model) + + expect(response).to have_http_status(:ok) + expect(response.body).to include("llm_model[external_id]") + end + + it "does not offer it for a discovered model" do + discovered = create(:llm_model, llm_connection: connection, external_id: "server-named") + + get edit_llm_model_path(discovered) + + expect(response).to have_http_status(:ok) + expect(response.body).not_to include("llm_model[external_id]") + end + + # A typo in a hand-typed identifier was previously only fixable by deleting + # the model, which threw away everything asserted about it. + it "renames it and carries every reference along" do + patch llm_model_path(llm_model), params: { llm_model: { external_id: "qwen/qwen3.6-35b-a3b:bf16" } } + + expect(llm_model.reload.external_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + expect(connection.reload.default_chat_model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + expect(connection.feature_bindings.first.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + expect(connection.capability_verdicts.first.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + end + + it "keeps the feature resolving afterwards", with_flag: { llm_connection: true } do + connection.update!(enabled: true) + + patch llm_model_path(llm_model), params: { llm_model: { external_id: "qwen/qwen3.6-35b-a3b:bf16" } } + + expect(Llm::Runtime.for(:description_assistant).model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + end + + it "follows a model a locked binding depends on" do + binding = connection.feature_bindings.first + binding.update!(locked_at: Time.current) + + patch llm_model_path(llm_model), params: { llm_model: { external_id: "qwen/qwen3.6-35b-a3b:bf16" } } + + expect(binding.reload.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + end + + # The server names its own models; renaming one here would only be undone by + # the next refresh. + it "refuses to rename a discovered model" do + discovered = create(:llm_model, llm_connection: connection, external_id: "server-named") + + patch llm_model_path(discovered), params: { llm_model: { external_id: "renamed" } } + + expect(discovered.reload.external_id).to eq("server-named") + end + end + + describe "how an inherited capability verdict is shown" do + let!(:llm_model) { create(:llm_model, :manual, llm_connection: connection, external_id: "qwen3.6-27b") } + + # The blank option used to read "Not specified" while the caption underneath + # read "Currently Supported, from the model registry" -- two contradictory + # statements about the same field. + it "names the inherited value in the option itself" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "function_calling", + state: "supported", source: "metadata", checked_at: Time.current) + + get edit_llm_model_path(llm_model) + + expect(response.body).to include("Supported (from the model registry)") + expect(response.body).not_to include("Currently Supported") + end + + it "falls back to Not specified when nothing is known" do + get edit_llm_model_path(llm_model) + + expect(response.body).to include("Not specified") + end + + # An administrator's own assertion is loaded into the field, so the blank + # option must not claim it as inherited. + it "does not present an administrator's own assertion as inherited" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "function_calling", + state: "supported", source: "admin", checked_at: Time.current) + + get edit_llm_model_path(llm_model) + + expect(response.body).not_to include("Supported (set by an administrator)") + end + end +end diff --git a/spec/seeders/env_data/llm_connection_seeder_spec.rb b/spec/seeders/env_data/llm_connection_seeder_spec.rb new file mode 100644 index 000000000000..f1bc7a834e12 --- /dev/null +++ b/spec/seeders/env_data/llm_connection_seeder_spec.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe EnvData::LlmConnectionSeeder do + subject(:seed) { described_class.new(seed_data).seed! } + + let(:seed_data) { Source::SeedData.new({}) } + + it "does not seed a connection without configuration" do + expect { seed }.not_to change(LlmConnection, :count) + end + + # On a fresh installation the seed runs before any model synchronisation, so + # a configured default model cannot be validated against a catalogue yet. + # Provisioning must still complete; a wrong id surfaces later as dangling. + context "with a default model configured on a fresh installation", with_settings: { + llm_connection: { + "base_url" => "https://example.com/v1", + "api_key" => "sk-from-env", + "default_chat_model" => "qwen3.6-35b-a3b" + } + } do + it "seeds the connection without contacting the server" do + expect { seed }.to change(LlmConnection, :count).from(0).to(1) + + connection = LlmConnection.first + expect(connection.default_chat_model_id).to eq("qwen3.6-35b-a3b") + expect(connection.api_key).to eq("sk-from-env") + end + end + + # The environment is the source of truth while the form is read-only under it, + # so a value removed from the environment must not linger in the database. + context "when a previously set key is removed from the environment", with_settings: { + llm_connection: { "base_url" => "https://example.com/v1" } + } do + before do + create(:llm_connection, base_url: "https://example.com/v1", + api_key: "sk-stale", + default_chat_model_id: "old-default") + end + + it "clears the values the environment no longer provides" do + expect { seed }.not_to change(LlmConnection, :count) + + connection = LlmConnection.first + expect(connection.api_key).to be_nil + expect(connection.default_chat_model_id).to be_nil + expect(connection.base_url).to eq("https://example.com/v1") + end + end +end diff --git a/spec/services/llm/errors_spec.rb b/spec/services/llm/errors_spec.rb new file mode 100644 index 000000000000..45c9dc4abd41 --- /dev/null +++ b/spec/services/llm/errors_spec.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Errors do + def rubyllm_response(status) + instance_double(Faraday::Env, status:) + end + + describe ".translate" do + it "maps an unauthorized response onto an authentication error" do + translated = described_class.translate(RubyLLM::UnauthorizedError.new(rubyllm_response(401), "nope")) + + expect(translated).to be_a(Llm::Errors::AuthenticationError) + end + + it "maps a forbidden response onto an authentication error" do + translated = described_class.translate(RubyLLM::ForbiddenError.new(rubyllm_response(403), "nope")) + + expect(translated).to be_a(Llm::Errors::AuthenticationError) + end + + it "maps an exceeded context window onto a context length error" do + translated = described_class.translate(RubyLLM::ContextLengthExceededError.new(rubyllm_response(400), "too long")) + + expect(translated).to be_a(Llm::Errors::ContextLengthError) + expect(translated.status).to eq(400) + end + + it "maps throttling onto a rate limited error" do + translated = described_class.translate(RubyLLM::RateLimitError.new(rubyllm_response(429), "slow down")) + + expect(translated).to be_a(Llm::Errors::RateLimitedError) + end + + it "maps a bad request onto an api error carrying the status" do + translated = described_class.translate(RubyLLM::BadRequestError.new(rubyllm_response(400), "bad")) + + expect(translated).to be_a(Llm::Errors::ApiError) + expect(translated.status).to eq(400) + end + + # A server that does not implement the endpoint answers 404/405/501, which + # RubyLLM's middleware raises as a bare Error. This is the path that tells a + # probe "this model cannot do that" rather than "the server is broken". + it "maps an unimplemented endpoint onto an api error carrying the status" do + translated = described_class.translate(RubyLLM::Error.new(rubyllm_response(404), "not found")) + + expect(translated).to be_a(Llm::Errors::ApiError) + expect(translated.status).to eq(404) + end + + it "maps a misconfiguration onto a configuration error" do + expect(described_class.translate(RubyLLM::ConfigurationError.new("missing key"))) + .to be_a(Llm::Errors::ConfigurationError) + expect(described_class.translate(RubyLLM::ModelNotFoundError.new("no such model"))) + .to be_a(Llm::Errors::ConfigurationError) + end + + it "maps transport failures onto connection errors" do + expect(described_class.translate(Faraday::TimeoutError.new("timeout"))) + .to be_a(Llm::Errors::TimeoutError) + expect(described_class.translate(Faraday::ConnectionFailed.new("refused"))) + .to be_a(Llm::Errors::ConnectionError) + expect(described_class.translate(JSON::ParserError.new("unexpected token"))) + .to be_a(Llm::Errors::ParseError) + end + + it "passes an already translated error through untouched" do + original = Llm::Errors::SsrfError.new("Host resolves to a blocked address") + + expect(described_class.translate(original)).to be(original) + end + + # An OpenAI-compatible gateway echoes the submitted Authorization header, + # upstream provider URLs and internal hostnames in its error payloads, and + # RubyLLM puts that payload straight into the exception message. + it "never carries the upstream message into the translated error" do + secret = "Bearer sk-super-secret upstream=http://10.0.0.5:8000" + + http_errors = [RubyLLM::Error, RubyLLM::BadRequestError, RubyLLM::ForbiddenError, + RubyLLM::ContextLengthExceededError, RubyLLM::OverloadedError, + RubyLLM::PaymentRequiredError, RubyLLM::RateLimitError, RubyLLM::ServerError, + RubyLLM::ServiceUnavailableError, RubyLLM::UnauthorizedError] + plain_errors = [RubyLLM::ConfigurationError, RubyLLM::ModelNotFoundError] + + errors = http_errors.map { |klass| klass.new(rubyllm_response(400), secret) } + + plain_errors.map { |klass| klass.new(secret) } + + errors.each do |error| + message = described_class.translate(error).message + + expect(message).not_to include("sk-super-secret") + expect(message).not_to include("10.0.0.5") + end + end + end + + describe ".wrap" do + it "returns the block's value when nothing is raised" do + expect(described_class.wrap { :fine }).to be(:fine) + end + + it "re-raises a RubyLLM failure as an Llm::Errors" do + expect { described_class.wrap { raise RubyLLM::UnauthorizedError.new(rubyllm_response(401), "nope") } } + .to raise_error(Llm::Errors::AuthenticationError) + end + end + + describe "the Llm::Client aliases" do + it "resolve to the same classes, so existing rescues keep working" do + expect(Llm::Client::Error).to be(Llm::Errors::Error) + expect(Llm::Client::ApiError).to be(Llm::Errors::ApiError) + expect(Llm::Client::SsrfError).to be(Llm::Errors::SsrfError) + end + end +end diff --git a/spec/services/llm/probes/embeddings_probe_spec.rb b/spec/services/llm/probes/embeddings_probe_spec.rb new file mode 100644 index 000000000000..9fdfc0439b40 --- /dev/null +++ b/spec/services/llm/probes/embeddings_probe_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Probes::EmbeddingsProbe, :webmock do + subject(:result) { described_class.new(connection).call("some-model") } + + let(:connection) { build(:llm_connection, base_url: "https://example.com/v1") } + + def stub_embeddings(status:, body:) + stub_request(:post, "https://example.com/v1/embeddings") + .to_return(status:, headers: { "Content-Type" => "application/json" }, body: body.to_json) + end + + context "when the server returns a vector" do + before { stub_embeddings(status: 200, body: { data: [{ embedding: [0.1, 0.2, 0.3] }] }) } + + it "is supported and captures the dimension count" do + expect(result.state).to eq(:supported) + expect(result.detail["dimensions"]).to eq(3) + end + end + + # vLLM, llama.cpp and Ollama all silently drop parameters they do not + # understand, so a 200 on its own proves nothing about the model. + context "when the server returns 200 with something that is not an embedding" do + before { stub_embeddings(status: 200, body: { data: [{ message: "hello" }] }) } + + it "is unknown rather than supported" do + expect(result.state).to eq(:unknown) + expect(result.detail["reason"]).to eq("unexpected_body") + end + end + + context "when the server returns an empty data array" do + before { stub_embeddings(status: 200, body: { data: [] }) } + + it { expect(result.state).to eq(:unknown) } + end + + [400, 404, 501].each do |status| + context "when the server refuses the request with #{status}" do + before { stub_embeddings(status:, body: { error: "nope" }) } + + it "is unsupported" do + expect(result.state).to eq(:unsupported) + expect(result.detail["http_status"]).to eq(status) + end + end + end + + # A 5xx says something about the server, not about the model. + context "when the server errors" do + before { stub_embeddings(status: 500, body: { error: "boom" }) } + + it { expect(result.state).to eq(:unknown) } + end + + context "when the credentials are rejected" do + before { stub_embeddings(status: 401, body: { error: "no" }) } + + it "is unknown, since this says nothing about the model" do + expect(result.state).to eq(:unknown) + expect(result.detail["reason"]).to eq("unauthorized") + end + end + + context "when the server cannot be reached" do + before { stub_request(:post, "https://example.com/v1/embeddings").to_timeout } + + it { expect(result.state).to eq(:unknown) } + end +end diff --git a/spec/services/llm/runtime_spec.rb b/spec/services/llm/runtime_spec.rb new file mode 100644 index 000000000000..b578b75bc93f --- /dev/null +++ b/spec/services/llm/runtime_spec.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Runtime, with_flag: { llm_connection: true } do + subject(:resolution) { described_class.for(feature_key, override:) } + + let(:feature_key) { :description_assistant } + let(:override) { nil } + + context "without a connection" do + it { expect(resolution.status).to eq(:no_connection) } + end + + context "with a connection that is not enabled" do + before { create(:llm_connection, :with_models, enabled: false) } + + it { expect(resolution.status).to eq(:no_connection) } + end + + context "with an enabled connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled) } + + it "is unbound until a model is chosen" do + expect(resolution.status).to eq(:unbound) + expect(resolution.model_id).to be_nil + end + + it "falls back to the connection default" do + connection.update!(default_chat_model_id: "qwen3.6-27b") + + expect(resolution).to be_ready + expect(resolution.model_id).to eq("qwen3.6-27b") + end + + it "prefers the feature binding over the connection default" do + connection.update!(default_chat_model_id: "qwen3.6-27b") + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "bge-m3") + + expect(resolution.model_id).to eq("bge-m3") + end + + context "with a per-item override" do + let(:override) { "qwen3.6-27b" } + + it "wins over the binding" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "bge-m3") + + expect(resolution.model_id).to eq("qwen3.6-27b") + end + end + + # Substituting the default here would silently change the output of a + # transform an administrator configured deliberately. + context "when the chosen model is gone from the catalogue" do + let(:override) { "vanished-model" } + + before { connection.update!(default_chat_model_id: "qwen3.6-27b") } + + it "fails closed rather than falling back" do + expect(resolution.status).to eq(:model_missing) + expect(resolution.model_id).to eq("vanished-model") + end + end + end + + describe "capability gating" do + let(:feature_key) { :semantic_search } + let!(:connection) { create(:llm_connection, :with_models, :enabled) } + + before { connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "qwen3.6-27b") } + + it "blocks on a definite unsupported verdict" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: Time.current) + + expect(resolution.status).to eq(:incapable) + expect(resolution.missing_capabilities).to eq([:embeddings]) + end + + # Refusing on "we could not tell" would make most self-hosted servers + # unusable, since the model list carries no capability information at all. + it "does not block when the verdict is unknown" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "unknown", source: "probe", checked_at: Time.current) + + expect(resolution).to be_ready + end + + it "does not block when there is no verdict at all" do + expect(resolution).to be_ready + end + end + + describe "running a request", :llm_server_helpers, :webmock do + let!(:connection) { create(:llm_connection, :with_models, :enabled, default_chat_model_id: "qwen3.6-27b") } + + it "sends a completion for the resolved model" do + mock_llm_chat_response("https://example.com/v1", content: "pong") + + expect(resolution.chat(max_retries: 0).ask("ping").content).to eq("pong") + expect(WebMock).to have_requested(:post, "https://example.com/v1/chat/completions") + .with(body: hash_including("model" => "qwen3.6-27b")) + end + + it "refuses when the feature is not ready" do + connection.update!(enabled: false) + + expect { resolution.chat }.to raise_error(Llm::Errors::NotReady) { |e| expect(e.status).to eq(:no_connection) } + end + + # Features are resolved by kind, so asking a chat feature to embed means a + # caller has confused two features. + it "refuses to embed through a chat feature" do + expect { resolution.embed("hello") } + .to raise_error(Llm::Errors::NotReady) { |e| expect(e.status).to eq(:wrong_kind) } + end + + context "with an embedding feature" do + let(:feature_key) { :semantic_search } + + before { connection.update!(default_embedding_model_id: "bge-m3") } + + it "requests a vector for the resolved model" do + mock_llm_embeddings_response("https://example.com/v1", dimensions: 8) + + expect(resolution.embed("hello", max_retries: 0).vectors.length).to eq(8) + end + + it "refuses to chat through an embedding feature" do + expect { resolution.chat } + .to raise_error(Llm::Errors::NotReady) { |e| expect(e.status).to eq(:wrong_kind) } + end + end + end +end diff --git a/spec/services/llm/session_spec.rb b/spec/services/llm/session_spec.rb new file mode 100644 index 000000000000..3871da3264a3 --- /dev/null +++ b/spec/services/llm/session_spec.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Session, :llm_server_helpers, :webmock do + let(:base_url) { "https://example.com/v1" } + let(:connection) { create(:llm_connection, base_url:, api_key: "sk-test-key") } + + describe ".supports?" do + it "rejects the formats a connection cannot supply credentials for" do + expect(described_class.supports?("bedrock")).to be(false) + expect(described_class.supports?("vertexai")).to be(false) + end + + it "accepts the rest" do + expect(described_class.supports?("openai")).to be(true) + expect(described_class.supports?("anthropic")).to be(true) + end + end + + describe "#initialize" do + it "refuses a format whose credentials cannot be stored" do + connection.update_column(:api_format, "bedrock") + + expect { described_class.for(connection) } + .to raise_error(Llm::Errors::ConfigurationError, /bedrock/) + end + end + + describe "#chat" do + it "sends the completion to the configured base URL with the stored key" do + stub = mock_llm_chat_response(base_url, content: "pong") + + answer = described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") + + expect(answer.content).to eq("pong") + expect(stub).to have_been_requested.once + expect(WebMock).to have_requested(:post, "#{base_url}/chat/completions") + .with(headers: { "Authorization" => "Bearer sk-test-key" }) + end + + it "sends the connection's custom headers" do + connection.update!(custom_headers: { "apikey" => "gateway-secret" }) + mock_llm_chat_response(base_url) + + described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") + + expect(WebMock).to have_requested(:post, "#{base_url}/chat/completions") + .with(headers: { "apikey" => "gateway-secret" }) + end + + it "translates a rejected key into an authentication error" do + mock_llm_chat_response(base_url, response_code: 401) + + expect { described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") } + .to raise_error(Llm::Errors::AuthenticationError) + end + + # RubyLLM retries POSTs three times by default, so an unbounded retry would + # bill four completions for one call. + it "does not retry when told not to" do + stub = mock_llm_chat_response(base_url, response_code: 500) + + expect { described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") } + .to raise_error(Llm::Errors::ApiError) + expect(stub).to have_been_requested.once + end + + # A self-hosted OpenAI-compatible server commonly needs no credential, but + # RubyLLM's ensure_configured! raises unless one is set. + it "reaches a server that needs no API key" do + connection.update!(api_key: nil) + mock_llm_chat_response(base_url) + + expect { described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") } + .not_to raise_error + end + end + + describe "#embed" do + it "requests a vector and carries the custom headers" do + connection.update!(custom_headers: { "apikey" => "gateway-secret" }) + mock_llm_embeddings_response(base_url, dimensions: 8) + + embedding = described_class.for(connection, max_retries: 0).embed("hello", model: "bge-m3") + + expect(embedding.vectors.length).to eq(8) + expect(WebMock).to have_requested(:post, "#{base_url}/embeddings") + .with(headers: { "apikey" => "gateway-secret" }) + end + end + + describe "global configuration" do + it "never writes the connection's settings into RubyLLM's global config" do + mock_llm_chat_response(base_url) + + described_class.for(connection, max_retries: 0).chat("qwen3.6-27b").ask("ping") + + expect(RubyLLM.config.openai_api_base).to be_nil + expect(RubyLLM.config.openai_api_key).to be_nil + expect(RubyLLM.config.openproject_custom_headers).to be_nil + end + end +end diff --git a/spec/services/llm/structured_output_spec.rb b/spec/services/llm/structured_output_spec.rb new file mode 100644 index 000000000000..b7ee9a7d4357 --- /dev/null +++ b/spec/services/llm/structured_output_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::StructuredOutput do + def message(content) + instance_double(RubyLLM::Message, content:) + end + + it "returns the structure with symbol keys" do + expect(described_class.parse!(message({ "summary" => "hi", "tags" => %w[a b] }))) + .to eq(summary: "hi", tags: %w[a b]) + end + + it "symbolizes nested keys" do + expect(described_class.parse!(message({ "a" => { "b" => 1 } }))).to eq(a: { b: 1 }) + end + + # RubyLLM rescues a JSON parse failure and leaves the content a String, so + # without this the caller gets a String where it expected a Hash and only + # notices much further downstream. + it "raises when the model answered in prose instead" do + expect { described_class.parse!(message("Sure! Here is the summary you asked for.")) } + .to raise_error(Llm::Errors::ParseError) + end + + it "accepts a bare structure as well as a message" do + expect(described_class.parse!({ "a" => 1 })).to eq(a: 1) + end +end diff --git a/spec/services/llm/validators/connection_validator_spec.rb b/spec/services/llm/validators/connection_validator_spec.rb new file mode 100644 index 000000000000..43ba395a2d8e --- /dev/null +++ b/spec/services/llm/validators/connection_validator_spec.rb @@ -0,0 +1,179 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Validators::ConnectionValidator, :llm_server_helpers, :webmock, + with_flag: { llm_connection: true } do + subject(:report) { described_class.new(connection).call } + + let(:base_url) { "https://example.com/v1" } + let(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + def result_for(group, key) + report.group(group)&.result_for(key) + end + + context "with a healthy connection" do + before do + mock_llm_models_response(base_url) + connection.update!(default_chat_model_id: "qwen3.6-27b") + end + + it "passes the configuration and server groups" do + expect(result_for(:configuration, :base_url_present).state).to eq(:success) + expect(result_for(:server, :reachable).state).to eq(:success) + expect(result_for(:server, :credentials_accepted).state).to eq(:success) + end + + # The billed request only runs when an administrator asks for it. + it "skips inference unless a deep check was asked for" do + expect(report.group(:inference)).to be_nil + expect(WebMock).not_to have_requested(:post, "#{base_url}/chat/completions") + end + + context "when a deep check is asked for" do + before do + connection.deep_health_check = true + mock_llm_chat_response(base_url, content: "pong") + end + + it "sends a completion and reports the round trip" do + expect(result_for(:inference, :chat_round_trip).state).to eq(:success) + expect(WebMock).to have_requested(:post, "#{base_url}/chat/completions").once + end + end + end + + context "when the server publishes no model list" do + before { mock_llm_models_response(base_url, response_code: 404) } + + # The case that motivated manual model entry: a gateway exposing only chat. + it "is still reachable, and says the key could not be verified" do + expect(result_for(:server, :reachable).state).to eq(:success) + expect(result_for(:server, :credentials_accepted).state).to eq(:warning) + expect(result_for(:server, :credentials_accepted).code).to eq(:no_models_endpoint) + end + + it "can still prove the connection works through inference" do + connection.deep_health_check = true + connection.update!(default_chat_model_id: "qwen3.6-27b") + mock_llm_chat_response(base_url) + + expect(result_for(:inference, :chat_round_trip).state).to eq(:success) + end + end + + context "when the key is rejected" do + before { mock_llm_models_response(base_url, response_code: 401) } + + it "separates reachability from authentication" do + expect(result_for(:server, :reachable).state).to eq(:success) + expect(result_for(:server, :credentials_accepted).state).to eq(:failure) + expect(report).to be_unhealthy + end + end + + context "when the server cannot be reached" do + before { mock_llm_models_response(base_url, timeout: true) } + + it "fails reachability and skips the credential check" do + expect(result_for(:server, :reachable).state).to eq(:failure) + expect(result_for(:server, :credentials_accepted).state).to eq(:skipped) + end + end + + context "without a base URL" do + let(:connection) { create(:llm_connection, :enabled).tap { |c| c.update_column(:base_url, "") } } + + it "fails and does not ask the server anything" do + expect(result_for(:configuration, :base_url_present).state).to eq(:failure) + expect(report.group(:server)).to be_nil + expect(WebMock).not_to have_requested(:get, "#{base_url}/models") + end + end + + describe "the features group" do + before { mock_llm_models_response(base_url) } + + it "warns about features with no model chosen" do + result = result_for(:features, :bindings_resolvable) + + expect(result.state).to eq(:warning) + expect(result.code).to eq(:features_unbound) + end + + it "fails when a binding points at a model the server no longer offers" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "vanished") + + result = result_for(:features, :bindings_resolvable) + + expect(result.state).to eq(:failure) + expect(result.code).to eq(:features_model_missing) + end + end + + # HealthReports::ResultComponent resolves a check's label from its key and its + # explanation from its code, so a key with no translation renders as + # "translation missing" rather than failing anywhere a spec would notice. + describe "translations" do + let(:scenarios) do + [ + -> { mock_llm_models_response(base_url) }, + -> { mock_llm_models_response(base_url, response_code: 401) }, + -> { mock_llm_models_response(base_url, response_code: 404) }, + -> { mock_llm_models_response(base_url, timeout: true) } + ] + end + + it "exist for every check and every code the validator can emit" do + # A dangling binding, so the features group emits its failure codes too. + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "vanished") + + scenarios.each do |setup| + WebMock.reset! + setup.call + + described_class.new(connection).call.results.each do |group| + expect(I18n.t("#{group.key}.header", scope: "llm.health_checks", raise: true)).to be_present + + group.results.each do |result| + expect(I18n.t("#{group.key}.#{result.key}", scope: "llm.health_checks", raise: true)).to be_present + next if result.code.nil? + + context_vars = result.context&.symbolize_keys || {} + expect(I18n.t("errors.#{result.code}", scope: "llm.health_checks", raise: true, **context_vars)) + .to be_present + end + end + end + end + end +end diff --git a/spec/support/llm_server_helpers.rb b/spec/support/llm_server_helpers.rb new file mode 100644 index 000000000000..52fc9689e519 --- /dev/null +++ b/spec/support/llm_server_helpers.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmServerHelpers + DEFAULT_MODELS = [ + { id: "qwen3.6-27b", object: "model", owned_by: "vllm", max_model_len: 262_144 }, + { id: "bge-m3", object: "model", owned_by: "vllm", max_model_len: 8_192 } + ].freeze + + # Stubs GET /models. Returns the stub so specs can assert on how + # often it was called -- which is how the changed-attributes guard is pinned. + def mock_llm_models_response(base_url, + models: DEFAULT_MODELS, + response_code: 200, + body: nil, + timeout: false) + stub = stub_request(:get, "#{base_url.chomp('/')}/models") + + return stub.to_timeout if timeout + + stub.to_return( + status: response_code, + headers: { "Content-Type" => "application/json" }, + body: body || { object: "list", data: models }.to_json + ) + end + + DEFAULT_CHAT_BODY = { + id: "chatcmpl-test", + object: "chat.completion", + model: "qwen3.6-27b", + choices: [{ index: 0, message: { role: "assistant", content: "pong" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } + }.freeze + + # Stubs POST /chat/completions. + # + # Note that RubyLLM retries POSTs up to max_retries times on 429 and 5xx, so a + # spec stubbing one of those against a session built with the default + # max_retries sees more than one request. Build the session with an explicit + # max_retries when the request count matters. + def mock_llm_chat_response(base_url, + content: "pong", + response_code: 200, + body: nil, + timeout: false) + stub = stub_request(:post, "#{base_url.chomp('/')}/chat/completions") + + return stub.to_timeout if timeout + + payload = body || DEFAULT_CHAT_BODY.merge( + choices: [{ index: 0, message: { role: "assistant", content: }, finish_reason: "stop" }] + ).to_json + + stub.to_return( + status: response_code, + headers: { "Content-Type" => "application/json" }, + body: payload.is_a?(String) ? payload : payload.to_json + ) + end + + # Stubs POST /embeddings, returning a vector of the requested size. + def mock_llm_embeddings_response(base_url, + dimensions: 4, + response_code: 200, + body: nil, + timeout: false) + stub = stub_request(:post, "#{base_url.chomp('/')}/embeddings") + + return stub.to_timeout if timeout + + payload = body || { + object: "list", + model: "bge-m3", + data: [{ object: "embedding", index: 0, embedding: Array.new(dimensions) { 0.1 } }], + usage: { prompt_tokens: 1, total_tokens: 1 } + } + + stub.to_return( + status: response_code, + headers: { "Content-Type" => "application/json" }, + body: payload.is_a?(String) ? payload : payload.to_json + ) + end + + # example.com resolves publicly, but a spec that needs a literal or private + # host has to say so explicitly rather than opening the allowlist to 0.0.0.0/0. + def allow_llm_host(*hosts) + allow(OpenProject::SsrfProtection).to receive(:safe_ip?) do |host| + hosts.include?(host) ? IPAddr.new("93.184.216.34") : nil + end + end +end + +RSpec.configure do |config| + config.include LlmServerHelpers, :llm_server_helpers +end diff --git a/spec/workers/llm/health_check_job_spec.rb b/spec/workers/llm/health_check_job_spec.rb new file mode 100644 index 000000000000..92603af70ff2 --- /dev/null +++ b/spec/workers/llm/health_check_job_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::HealthCheckJob, :llm_server_helpers, :webmock, with_flag: { llm_connection: true } do + let(:base_url) { "https://example.com/v1" } + + describe "#perform" do + context "with an enabled connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before { mock_llm_models_response(base_url) } + + it "stores a report" do + expect { described_class.perform_now }.to change(connection.health_reports, :count).by(1) + end + + # The whole reason the inference group is gated: an unattended job must not + # run up a bill on a hosted provider four times a day. + # Group keys come back from jsonb as strings, so a stored report is + # queried by string where an in-memory one is queried by symbol. + it "does not spend a completion" do + described_class.perform_now + + expect(WebMock).not_to have_requested(:post, "#{base_url}/chat/completions") + expect(connection.latest_health_report.results.map(&:key)).not_to include("inference") + end + + it "still reports what the free checks found" do + described_class.perform_now + + expect(connection.latest_health_report.group("server").result_for("reachable").state).to eq(:success) + end + end + + it "does nothing without a connection" do + expect { described_class.perform_now }.not_to change(HealthReport, :count) + end + + it "does nothing while the connection is switched off" do + create(:llm_connection, :with_models, enabled: false, base_url:) + + expect { described_class.perform_now }.not_to change(HealthReport, :count) + end + + context "with the feature flag off", with_flag: { llm_connection: false } do + it "does nothing" do + create(:llm_connection, :with_models, :enabled, base_url:) + + expect { described_class.perform_now }.not_to change(HealthReport, :count) + end + end + end + + describe ".toggle_cron_job" do + it "enables the cron once a connection is usable" do + create(:llm_connection, :with_models, :enabled, base_url:) + + described_class.toggle_cron_job + + expect(GoodJob::Setting.cron_key_enabled?(described_class::CRON_JOB_KEY)).to be(true) + end + + it "disables the cron while there is nothing to check" do + GoodJob::Setting.cron_key_enable(described_class::CRON_JOB_KEY) + + described_class.toggle_cron_job + + expect(GoodJob::Setting.cron_key_enabled?(described_class::CRON_JOB_KEY)).to be(false) + end + end +end diff --git a/spec/workers/llm/prune_health_reports_job_spec.rb b/spec/workers/llm/prune_health_reports_job_spec.rb new file mode 100644 index 000000000000..7d3af3097978 --- /dev/null +++ b/spec/workers/llm/prune_health_reports_job_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::PruneHealthReportsJob do + shared_let(:connection) { create(:llm_connection) } + + def report(created_at:, subject: connection) + subject.health_reports.create!( + results: [HealthReport::ResultGroup.new(key: :configuration, results: [])], + created_at: + ) + end + + it "deletes reports that are both old and beyond the retention count" do + stub_const("#{described_class}::KEEP", 2) + oldest = report(created_at: 300.days.ago) + report(created_at: 200.days.ago) + report(created_at: 100.days.ago) + + expect { described_class.perform_now }.to change { HealthReport.exists?(oldest.id) }.from(true).to(false) + end + + it "keeps recent reports however many there are" do + recent = report(created_at: 1.day.ago) + + described_class.perform_now + + expect(HealthReport.exists?(recent.id)).to be(true) + end + + # Age alone is not enough: a connection checked once a year ago should still + # show that check rather than nothing at all. + it "keeps the newest reports even when they are old" do + oldest = report(created_at: 200.days.ago) + + described_class.perform_now + + expect(HealthReport.exists?(oldest.id)).to be(true) + end + + it "leaves other subjects' reports alone" do + storage = create(:nextcloud_storage) + foreign = report(created_at: 200.days.ago, subject: storage) + + described_class.perform_now + + expect(HealthReport.exists?(foreign.id)).to be(true) + end +end