From 761b2d408f911153037e3ff842d9cff66f40c485 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Sat, 22 Aug 2026 18:12:59 +0200 Subject: [PATCH] [AI-3] Report the health of the LLM connection Adds the health side panel and report page, reusing the HealthReports framework and preparing it for a third consumer. Six validator groups check the configuration, the server, the model catalogue, the capability verdicts, the feature bindings and, only when an administrator explicitly asks, a real inference call, so a scheduled check never spends a billed completion. The scheduled Llm::HealthCheckJob runs only while a connection is enabled, toggled from the update service and the disconnect action. Reports are kept as an audit trail with their own lifecycle, distinct from verdicts, and pruned by a cron job because nothing else prunes health_reports. Part 11 of the AI-3 stack. https://community.openproject.org/work_packages/66020 --- .../health_reports/report_component.html.erb | 2 +- .../health_reports/report_component.rb | 8 +- .../health_reports/result_component.html.erb | 12 +- .../health_reports/result_component.rb | 16 +- .../health_status_component.html.erb | 58 ++++++ .../side_panel/health_status_component.rb | 75 ++++++++ .../admin/llm_connections_controller.rb | 1 + .../admin/llm_health_status_controller.rb | 112 +++++++++++ app/models/llm_connection.rb | 43 +++++ .../llm/validators/configuration_validator.rb | 87 +++++++++ .../llm/validators/connection_validator.rb | 54 ++++++ .../llm/validators/feature_validator.rb | 105 ++++++++++ .../llm/validators/inference_validator.rb | 91 +++++++++ .../llm/validators/model_validator.rb | 94 +++++++++ .../llm/validators/server_validator.rb | 102 ++++++++++ .../llm_connections/update_service.rb | 4 + app/views/admin/llm_connections/show.html.erb | 38 ++-- .../admin/llm_health_status/show.html.erb | 101 ++++++++++ app/workers/llm/health_check_job.rb | 75 ++++++++ app/workers/llm/prune_health_reports_job.rb | 59 ++++++ config/initializers/cronjobs.rb | 8 + config/locales/en.yml | 65 +++++++ config/routes.rb | 4 + ..._add_created_at_index_to_health_reports.rb | 47 +++++ spec/features/admin/llm_connection_spec.rb | 18 ++ spec/requests/admin/llm_health_status_spec.rb | 129 +++++++++++++ .../validators/connection_validator_spec.rb | 179 ++++++++++++++++++ spec/workers/llm/health_check_job_spec.rb | 100 ++++++++++ .../llm/prune_health_reports_job_spec.rb | 78 ++++++++ 29 files changed, 1741 insertions(+), 24 deletions(-) create mode 100644 app/components/llm_connections/side_panel/health_status_component.html.erb create mode 100644 app/components/llm_connections/side_panel/health_status_component.rb create mode 100644 app/controllers/admin/llm_health_status_controller.rb create mode 100644 app/services/llm/validators/configuration_validator.rb create mode 100644 app/services/llm/validators/connection_validator.rb create mode 100644 app/services/llm/validators/feature_validator.rb create mode 100644 app/services/llm/validators/inference_validator.rb create mode 100644 app/services/llm/validators/model_validator.rb create mode 100644 app/services/llm/validators/server_validator.rb create mode 100644 app/views/admin/llm_health_status/show.html.erb create mode 100644 app/workers/llm/health_check_job.rb create mode 100644 app/workers/llm/prune_health_reports_job.rb create mode 100644 db/migrate/20260813090000_add_created_at_index_to_health_reports.rb create mode 100644 spec/requests/admin/llm_health_status_spec.rb create mode 100644 spec/services/llm/validators/connection_validator_spec.rb create mode 100644 spec/workers/llm/health_check_job_spec.rb create mode 100644 spec/workers/llm/prune_health_reports_job_spec.rb diff --git a/app/components/health_reports/report_component.html.erb b/app/components/health_reports/report_component.html.erb index 0afbae54edd1..cb0922c4f74d 100644 --- a/app/components/health_reports/report_component.html.erb +++ b/app/components/health_reports/report_component.html.erb @@ -67,7 +67,7 @@ See COPYRIGHT and LICENSE files for more details. result_group.results.each do |value| box.with_row do - render(HealthReports::ResultComponent.new(group: result_group.key, result: value, i18n_scope:)) + render(HealthReports::ResultComponent.new(group: result_group.key, result: value, i18n_scope:, docs_href:)) end end end diff --git a/app/components/health_reports/report_component.rb b/app/components/health_reports/report_component.rb index 9778ff26ff45..08a988cd7d79 100644 --- a/app/components/health_reports/report_component.rb +++ b/app/components/health_reports/report_component.rb @@ -37,14 +37,18 @@ class ReportComponent < ApplicationComponent # The i18n_scope parameter defines the I18n scope that should be used to resolve # names of groups, checks and error messages indicated by the results. - def initialize(*, i18n_scope:, **) + # + # docs_href overrides where each result's "More information" link points; + # without it, results link to the file storages troubleshooting page. + def initialize(*, i18n_scope:, docs_href: nil, **) super(*, **) @i18n_scope = i18n_scope + @docs_href = docs_href end private - attr_reader :i18n_scope + attr_reader :i18n_scope, :docs_href def summary_scheme(check_tally) case check_tally diff --git a/app/components/health_reports/result_component.html.erb b/app/components/health_reports/result_component.html.erb index 67b17ee95405..8335ceba3e65 100644 --- a/app/components/health_reports/result_component.html.erb +++ b/app/components/health_reports/result_component.html.erb @@ -38,18 +38,20 @@ See COPYRIGHT and LICENSE files for more details. line.with_column(mr: 2) do render(Primer::Beta::Text.new(font_size: :small, color: status_color)) { status_text } end - if error_code.present? + if error_code.present? && docs_href.present? line.with_column do render(Primer::Beta::Label.new(scheme: status_color)) { error_code } end end end - if error_code.present? + if error_code.present? && docs_href.present? row.with_column do - helpers.static_link_to(href: docs_href, - label: I18n.t(:label_more_information), - underline: true) + helpers.static_link_to( + href: docs_href, + label: I18n.t(:label_more_information), + underline: true + ) end end end diff --git a/app/components/health_reports/result_component.rb b/app/components/health_reports/result_component.rb index 5a57739f09f7..24cccb0d8d6d 100644 --- a/app/components/health_reports/result_component.rb +++ b/app/components/health_reports/result_component.rb @@ -32,10 +32,18 @@ module HealthReports class ResultComponent < ApplicationComponent include OpPrimer::ComponentHelpers - def initialize(group:, result:, i18n_scope:) + # Where "More information" points. Defaults to the file storages + # documentation because that was this component's only consumer for a long + # time; a subject with its own troubleshooting page passes its own, and one + # with no page yet passes false to suppress the link rather than send an + # administrator somewhere unrelated. + DEFAULT_DOCS_HREF = -> { ::OpenProject::Static::Links.url_for(:storage_docs, :health_status) } + + def initialize(group:, result:, i18n_scope:, docs_href: nil) super(result) @group = group @i18n_scope = i18n_scope + @docs_href = docs_href end private @@ -49,7 +57,11 @@ def error_text I18n.t("errors.#{model.code}", scope: @i18n_scope, **model.context&.symbolize_keys) end - def docs_href = ::OpenProject::Static::Links.url_for(:storage_docs, :health_status) + def docs_href + return @docs_href if @docs_href == false + + @docs_href || DEFAULT_DOCS_HREF.call + end def error_code if model.failure? diff --git a/app/components/llm_connections/side_panel/health_status_component.html.erb b/app/components/llm_connections/side_panel/health_status_component.html.erb new file mode 100644 index 000000000000..88ca7f6e6924 --- /dev/null +++ b/app/components/llm_connections/side_panel/health_status_component.html.erb @@ -0,0 +1,58 @@ +<%= + component_wrapper(tag: :turbo_frame, refresh: :morph) do + render(Primer::OpenProject::SidePanel::Section.new) do |section| + section.with_title(tag: :h3) { t(".title") } + + flex_layout do |container| + if report.present? + container.with_row do + header = summary_header + concat(render(Primer::Beta::Octicon.new(icon: header[:icon], color: header[:icon_color], mr: 2))) + concat(render(Primer::Beta::Text.new(font_weight: :bold)) { header[:text] }) + end + + container.with_row(mt: 2) do + render( + Primer::Beta::Text.new(color: :muted, test_selector: "llm-connection--health-summary") + ) { summary_description } + end + + container.with_row(mt: 2) do + render( + Primer::Beta::Button.new( + scheme: :link, color: :default, font_weight: :bold, + tag: :a, href: llm_connection_health_status_report_path, + test_selector: "llm-connection--open-health-report" + ) + ) do |button| + button.with_leading_visual_icon(icon: :meter) + t(".open_report") + end + end + else + container.with_row do + render(Primer::Beta::Text.new(color: :muted)) { t(".never_checked") } + end + + container.with_row(mt: 2) do + primer_form_with( + url: create_health_status_report_llm_connection_health_status_report_path, + method: :post, + data: { turbo: true } + ) do + render( + Primer::Beta::Button.new( + scheme: :link, color: :default, font_weight: :bold, type: :submit, + test_selector: "llm-connection--run-health-checks" + ) + ) do |button| + button.with_leading_visual_icon(icon: :meter) + t(".run_checks") + end + end + end + end + end + end + end +%> diff --git a/app/components/llm_connections/side_panel/health_status_component.rb b/app/components/llm_connections/side_panel/health_status_component.rb new file mode 100644 index 000000000000..e7145a678387 --- /dev/null +++ b/app/components/llm_connections/side_panel/health_status_component.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + module SidePanel + # "Has this connection ever actually worked?", answered on the settings page. + class HealthStatusComponent < ApplicationComponent + include ApplicationHelper + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + alias_method :connection, :model + + private + + def report + connection.latest_health_report + end + + def summary_header + tally = report.tally + + case tally + in { failure: 1.. } + { icon: :alert, icon_color: :danger, + text: I18n.t("health_reports.common.checks.failures", count: tally[:failure]) } + in { warning: 1.. } + { icon: :alert, icon_color: :attention, + text: I18n.t("health_reports.common.checks.warnings", count: tally[:warning]) } + else + { icon: :"check-circle", icon_color: :success, text: I18n.t("health_reports.common.checks.success") } + end + end + + def summary_description + text = if report.healthy? + I18n.t("health_reports.common.summary.success") + elsif report.unhealthy? + I18n.t("health_reports.common.summary.failure") + else + I18n.t("health_reports.common.summary.warning") + end + + "#{text} #{t('.last_check', datetime: helpers.format_time(report.created_at))}" + end + end + end +end diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb index dfba30ac110b..663cdf8bd532 100644 --- a/app/controllers/admin/llm_connections_controller.rb +++ b/app/controllers/admin/llm_connections_controller.rb @@ -90,6 +90,7 @@ def disconnect return redirect_with_error(t(".configured_from_env")) if @connection.configured_from_env? @connection.update!(api_key: nil, enabled: false) + Llm::HealthCheckJob.toggle_cron_job redirect_with_notice(t(".success")) end diff --git a/app/controllers/admin/llm_health_status_controller.rb b/app/controllers/admin/llm_health_status_controller.rb new file mode 100644 index 000000000000..5b090ac4b6b1 --- /dev/null +++ b/app/controllers/admin/llm_health_status_controller.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + class LlmHealthStatusController < ApplicationController + include OpTurbo::ComponentStream + + layout :admin_or_frame_layout + + before_action :require_feature + before_action :require_admin + before_action :find_connection + + menu_item :llm_connection + + def show + @report = @connection.latest_health_report + + respond_to do |format| + format.html + format.text do + return head :not_found if @report.nil? + + timestamp = @report.created_at.iso8601 + send_data text_report(timestamp), + filename: "llm_connection_health_report_#{timestamp}.txt", + type: "text/plain", + disposition: :attachment + end + end + end + + # A full run, including the billed completion: an administrator clicking + # "Run checks" is asking whether the connection actually works. + def create + run_checks + redirect_to llm_connection_health_status_report_path, status: :see_other + end + + def create_health_status_report + run_checks + update_via_turbo_stream(component: LlmConnections::SidePanel::HealthStatusComponent.new(@connection)) + respond_with_turbo_streams + end + + private + + def admin_or_frame_layout + turbo_frame_request? ? "turbo_rails/frame" : "admin" + end + + def run_checks + @connection.deep_health_check = true + report = Llm::Validators::ConnectionValidator.new(@connection).call + report.save! + report + end + + # Downloaded and pasted into support tickets, so it must not contain the API + # key or the custom headers -- a gateway header routinely carries a second + # credential. + def text_report(timestamp) + { + connection: @connection.name, + configuration: @connection.non_confidential_configuration, + ran_at: timestamp, + results: @report ? @report.results.map(&:to_h) : [] + }.to_yaml(stringify_names: true) + end + + # HealthReports::Validator builds the report through the association, which + # would insert an unpersisted singleton along with it. + def find_connection + @connection = LlmConnection.instance + + redirect_to llm_connection_path unless @connection.persisted? + end + + # The flag gates the endpoints, not only the menu entry: an unfinished page + # must not accept writes just because somebody knows the URL. + def require_feature + render_404 unless OpenProject::FeatureDecisions.llm_connection_active? + end + end +end diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb index e146d378d3ab..2ec8cda20877 100644 --- a/app/models/llm_connection.rb +++ b/app/models/llm_connection.rb @@ -38,10 +38,12 @@ class LlmConnection < ApplicationRecord include Redmine::Ciphering SINGLETON_NAME = "default" + has_many :health_reports, as: :subject, dependent: :delete_all has_many :models, class_name: "LlmModel", dependent: :delete_all has_many :capability_verdicts, class_name: "LlmCapabilityVerdict", dependent: :delete_all has_many :feature_bindings, class_name: "LlmFeatureBinding", dependent: :delete_all + validates :base_url, presence: true validate :only_one_connection, on: :create @@ -107,6 +109,47 @@ def server_flavour options["server_flavour"].presence&.to_sym end + # Whether a health check run may spend a real completion. + # + # Not a column: it describes the run, not the connection. An administrator + # asking to test the connection wants it; the scheduled re-check leaves it off + # so that a connection to a paid provider is not billed four times a day. + attr_accessor :deep_health_check + + def deep_health_check? = ActiveModel::Type::Boolean.new.cast(deep_health_check).present? + + def latest_health_report + health_reports.order(created_at: :asc).last + end + + # Derived rather than stored, for the same reason LlmFeatureBinding#dangling? + # is: a status column would be a cache with no invalidation trigger, and would + # be stale exactly when it matters. + def health_state + report = latest_health_report + + return :unknown if report.nil? + return :unhealthy if report.unhealthy? + return :warning if report.warning? + + :healthy + end + + # Feeds the downloadable health report. Deliberately excludes api_key *and* + # custom_headers: a gateway header routinely carries a second credential. + def non_confidential_configuration + { + base_url:, + api_format:, + enabled:, + server_flavour:, + catalogue_fetched_at:, + last_connected_at:, + model_count: models.count, + manual_model_count: models.where(manual: true).count + } + end + private def only_one_connection diff --git a/app/services/llm/validators/configuration_validator.rb b/app/services/llm/validators/configuration_validator.rb new file mode 100644 index 000000000000..0170d38bb62c --- /dev/null +++ b/app/services/llm/validators/configuration_validator.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # What can be said about the connection without asking the server anything. + class ConfigurationValidator < HealthReports::ValidatorGroup + def self.key = :configuration + + private + + def validate + register_checks(:feature_flag, :base_url_present, :api_format_supported, :credentials_present, :enabled) + + feature_flag + base_url_present + api_format_supported + credentials_present + connection_enabled + end + + def feature_flag + if OpenProject::FeatureDecisions.llm_connection_active? + pass_check(:feature_flag) + else + # Not a failure: an administrator may reasonably configure and test the + # connection before switching the feature on. + warn_check(:feature_flag, :feature_flag_off) + end + end + + def base_url_present + return pass_check(:base_url_present) if subject.base_url.present? + + fail_check(:base_url_present, :not_configured) + end + + def api_format_supported + return pass_check(:api_format_supported) if Llm::Session.supports?(subject.api_format) + + fail_check(:api_format_supported, :unsupported_api_format, context: { api_format: subject.api_format.to_s }) + end + + # Only ever a warning. A self-hosted server on a trusted network legitimately + # needs no credential, and a key that is genuinely required but wrong is + # reported precisely by the server group rather than guessed at here. + def credentials_present + return pass_check(:credentials_present) if subject.api_key.present? + + warn_check(:credentials_present, :api_key_missing) + end + + def connection_enabled + return pass_check(:enabled) if subject.enabled? + + warn_check(:enabled, :connection_disabled) + end + end + end +end diff --git a/app/services/llm/validators/connection_validator.rb b/app/services/llm/validators/connection_validator.rb new file mode 100644 index 000000000000..77ee9858b7d7 --- /dev/null +++ b/app/services/llm/validators/connection_validator.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # The health report for an LLM connection. + # + # Nothing here reads User.current: the whole report has to be reproducible + # from a background job, which is what lets the same code answer both the + # administrator's "Run checks" and the scheduled re-check. + class ConnectionValidator < HealthReports::Validator + register_group ConfigurationValidator + + register_group ServerValidator, + precondition: ->(_, report) { report.group(:configuration).non_failure? } + + # Costs a real completion, so it is not part of the scheduled run. + register_group InferenceValidator, + precondition: lambda { |connection, report| + connection.deep_health_check? && report.group(:configuration).non_failure? + } + + register_group ModelValidator + register_group FeatureValidator + end + end +end diff --git a/app/services/llm/validators/feature_validator.rb b/app/services/llm/validators/feature_validator.rb new file mode 100644 index 000000000000..a503c34f4ce8 --- /dev/null +++ b/app/services/llm/validators/feature_validator.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # Whether each registered feature can actually run. + # + # Driven by Llm::Runtime, so the health report and the runtime can never + # disagree about what is usable. That makes this the most informative group + # in the report: it answers "will the AI features work" rather than "is the + # server up". + # + # The check keys are fixed and the feature names travel in the result + # context. Per-feature keys would need a translation per feature and would + # break HealthReports::ResultComponent#text, which resolves a check's label + # from its key. + class FeatureValidator < HealthReports::ValidatorGroup + def self.key = :features + + private + + def validate + register_checks(:bindings_resolvable, :locked_bindings_intact) + + bindings_resolvable + locked_bindings_intact + end + + def bindings_resolvable + by_status = resolutions.group_by(&:status) + + if (incapable = by_status[:incapable]).present? + fail_check(:bindings_resolvable, :features_incapable, + context: { features: labels(incapable), + capabilities: capability_labels(incapable) }) + end + + if (missing = by_status[:model_missing]).present? + fail_check(:bindings_resolvable, :features_model_missing, context: { features: labels(missing) }) + end + + if (unbound = by_status[:unbound]).present? + return warn_check(:bindings_resolvable, :features_unbound, context: { features: labels(unbound) }) + end + + pass_check(:bindings_resolvable) + end + + # A locked binding is the record that a vector index exists and which model + # and dimension it was written under. If that model has left the catalogue + # the index can no longer be extended or queried consistently, which is a + # data problem rather than a configuration one. + def locked_bindings_intact + broken = subject.feature_bindings.select { |binding| binding.locked? && binding.dangling? } + + return pass_check(:locked_bindings_intact) if broken.empty? + + fail_check(:locked_bindings_intact, :locked_model_missing, + context: { features: broken.filter_map { |b| b.feature&.label }.join(", ") }) + end + + def resolutions + @resolutions ||= OpenProject::Llm::Features.available.map do |feature| + Llm::Runtime.for(feature.key) + end + end + + def labels(resolutions) = resolutions.map { |resolution| resolution.feature.label }.join(", ") + + def capability_labels(resolutions) + resolutions.flat_map(&:missing_capabilities) + .uniq + .map { |capability| Llm::Capabilities.label(capability) } + .join(", ") + end + end + end +end diff --git a/app/services/llm/validators/inference_validator.rb b/app/services/llm/validators/inference_validator.rb new file mode 100644 index 000000000000..fe7d2c39aa1e --- /dev/null +++ b/app/services/llm/validators/inference_validator.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # Whether the server will actually answer a request. + # + # This is the only check that proves a connection works, and the only one + # available at all for a deployment that publishes no model list. It is also + # the only one that costs money, so it runs on demand rather than on the + # schedule -- see LlmConnection#deep_health_check. + # + # Deliberately absent: probes for tool calling and structured output. Several + # current vLLM releases answer a tool call with plain text on a fully capable + # server, deterministically, so a probe would turn a wrong answer into a + # confident one rather than discovering anything. + class InferenceValidator < HealthReports::ValidatorGroup + PROMPT = "ping" + + def self.key = :inference + + private + + def validate + register_checks(:chat_round_trip) + + model_id = chat_model_id + return warn_check(:chat_round_trip, :no_model_to_test) if model_id.blank? + + round_trip(model_id) + end + + def round_trip(model_id) + answer = session.chat(model_id).with_temperature(0).ask(PROMPT) + + if answer.content.to_s.strip.empty? + warn_check(:chat_round_trip, :empty_completion, context: { model: model_id }) + else + pass_check(:chat_round_trip) + end + rescue Llm::Errors::AuthenticationError + fail_check(:chat_round_trip, :invalid_api_key) + rescue Llm::Errors::TimeoutError + fail_check(:chat_round_trip, :request_timed_out) + rescue Llm::Errors::ApiError => e + fail_check(:chat_round_trip, :chat_failed, context: { model: model_id, status: e.status.to_s }) + rescue Llm::Errors::Error + fail_check(:chat_round_trip, :connection_error) + end + + # The configured default first, so the check exercises what features will + # actually use rather than an arbitrary entry in the catalogue. + def chat_model_id + subject.default_chat_model_id.presence || subject.available_model_ids.first + end + + # Never retried: a failing check should report the failure, not pay for it + # three more times. + def session + Llm::Session.for(subject, timeout: Llm::Session::PROBE_TIMEOUT, max_retries: 0) + end + end + end +end diff --git a/app/services/llm/validators/model_validator.rb b/app/services/llm/validators/model_validator.rb new file mode 100644 index 000000000000..e02f0f6ad809 --- /dev/null +++ b/app/services/llm/validators/model_validator.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # What the stored catalogue says, without asking the server anything. + class ModelValidator < HealthReports::ValidatorGroup + STALE_AFTER = 30.days + + def self.key = :models + + private + + def validate + register_checks(:catalogue_present, :catalogue_fresh, :default_chat_model, :default_embedding_model) + + catalogue_present + catalogue_fresh + default_chat_model + default_embedding_model + end + + # Never a failure. A gateway may expose chat and no catalogue at all, which + # is why models can be entered by hand in the first place. + def catalogue_present + return pass_check(:catalogue_present) if subject.available_model_ids.any? + + warn_check(:catalogue_present, :no_models) + end + + def catalogue_fresh + fetched_at = subject.catalogue_fetched_at + + if fetched_at.blank? + warn_check(:catalogue_fresh, :catalogue_never_fetched) + elsif fetched_at < STALE_AFTER.ago + # Context is serialised to jsonb, so a Time has to be formatted here. + warn_check(:catalogue_fresh, :catalogue_stale, context: { fetched_at: I18n.l(fetched_at.to_date) }) + else + pass_check(:catalogue_fresh) + end + end + + def default_chat_model + check_default(:default_chat_model, subject.default_chat_model_id) + end + + # Only meaningful once something wants embeddings; otherwise an unset + # default is not a defect. + def default_embedding_model + return pass_check(:default_embedding_model) if OpenProject::Llm::Features.for_kind(:embedding).empty? + + check_default(:default_embedding_model, subject.default_embedding_model_id) + end + + def check_default(key, model_id) + if model_id.blank? + warn_check(key, :default_model_unset) + elsif subject.available_model_ids.exclude?(model_id) + warn_check(key, :default_model_missing, context: { model: model_id }) + else + pass_check(key) + end + end + end + end +end diff --git a/app/services/llm/validators/server_validator.rb b/app/services/llm/validators/server_validator.rb new file mode 100644 index 000000000000..ea3cf9de1488 --- /dev/null +++ b/app/services/llm/validators/server_validator.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + module Validators + # Whether the server answers, and whether it accepts our credentials. + # + # Uses the model list, which costs nothing on any provider. That only exists + # for OpenAI-compatible endpoints -- for the others the catalogue comes from + # RubyLLM's registry, so there is nothing free to ask, and reachability can + # only be established by the inference group's billed request. + class ServerValidator < HealthReports::ValidatorGroup + # A gateway may implement chat and nothing else. That is a supported + # deployment, not a broken one, so it must not read as unreachable. + MODELS_ENDPOINT_ABSENT = LlmServerValidator::MODELS_ENDPOINT_ABSENT + + def self.key = :server + + private + + def validate + # For a registry-backed format there is nothing free to ask, so the + # group is omitted entirely rather than rendered as two skipped checks + # that read as neither healthy nor warning. + return unless queries_the_server? + + register_checks(:reachable, :credentials_accepted) + + list_models + end + + def queries_the_server? + subject.api_format == Llm::Adapters::OPENAI_COMPATIBLE + end + + def list_models + client.models + pass_check(:reachable) + pass_check(:credentials_accepted) + rescue Llm::Errors::AuthenticationError + pass_check(:reachable) + fail_check(:credentials_accepted, :invalid_api_key) + rescue Llm::Errors::TimeoutError + fail_check(:reachable, :request_timed_out) + rescue Llm::Errors::SsrfError + fail_check(:reachable, :ssrf_filtered) + rescue Llm::Errors::ApiError => e + answered_with_error(e) + rescue Llm::Errors::ParseError + pass_check(:reachable) + fail_check(:credentials_accepted, :not_openai_compatible) + rescue Llm::Errors::Error + fail_check(:reachable, :connection_error) + end + + # The server answered, so it is reachable and did not reject us. + def answered_with_error(error) + pass_check(:reachable) + + if error.status.in?(MODELS_ENDPOINT_ABSENT) + # It simply does not publish a catalogue. A supported deployment. + warn_check(:credentials_accepted, :no_models_endpoint) + else + fail_check(:credentials_accepted, :server_error, context: { status: error.status.to_s }) + end + end + + def client + Llm::Client.new(base_url: subject.base_url, + api_key: subject.api_key, + headers: subject.custom_headers) + end + end + end +end diff --git a/app/services/llm_connections/update_service.rb b/app/services/llm_connections/update_service.rb index 4b35926861d3..790f53b1bd04 100644 --- a/app/services/llm_connections/update_service.rb +++ b/app/services/llm_connections/update_service.rb @@ -48,6 +48,10 @@ def after_perform(service_call) super.tap do next unless service_call.success? + # Enabling or disabling the connection decides whether the scheduled + # health check has anything to do. + Llm::HealthCheckJob.toggle_cron_job + next unless @sync_models next unless connection_changed?(service_call.result) diff --git a/app/views/admin/llm_connections/show.html.erb b/app/views/admin/llm_connections/show.html.erb index e5b974c85093..65c940915a5c 100644 --- a/app/views/admin/llm_connections/show.html.erb +++ b/app/views/admin/llm_connections/show.html.erb @@ -83,23 +83,33 @@ See COPYRIGHT and LICENSE files for more details. %> <% if @connection.persisted? %> - <%= render(LlmConnections::FormComponent.new(@connection)) %> + <%= render(Primer::Alpha::Layout.new(stacking_breakpoint: :lg)) do |layout| %> + <% layout.with_main do %> + <%= render(LlmConnections::FormComponent.new(@connection)) %> - <%= - render(Primer::Beta::Subhead.new(mt: 4)) do |component| - component.with_heading(tag: :h3) { t(".models_heading") } - component.with_description do - if @connection.catalogue_fetched_at - t(".models_description", fetched_at: format_time(@connection.catalogue_fetched_at)) - else - t(".models_description_unfetched") + <%= + render(Primer::Beta::Subhead.new(mt: 4)) do |component| + component.with_heading(tag: :h3) { t(".models_heading") } + component.with_description do + if @connection.catalogue_fetched_at + t(".models_description", fetched_at: format_time(@connection.catalogue_fetched_at)) + else + t(".models_description_unfetched") + end + end end - end - end - %> + %> + + <%= render(LlmConnections::Models::SubHeaderComponent.new(@query)) %> + <%= render(LlmConnections::Models::IndexComponent.new(@models, connection: @connection)) %> + <% end %> - <%= render(LlmConnections::Models::SubHeaderComponent.new(@query)) %> - <%= render(LlmConnections::Models::IndexComponent.new(@models, connection: @connection)) %> + <% layout.with_sidebar(col_placement: :end, row_placement: :end) do %> + <%= render(Primer::OpenProject::SidePanel.new) do |panel| %> + <% panel.with_section(LlmConnections::SidePanel::HealthStatusComponent.new(@connection)) %> + <% end %> + <% end %> + <% end %> <% else %> <%# Nothing to show alongside the form until a connection exists. %> <%= render(LlmConnections::FormComponent.new(@connection)) %> diff --git a/app/views/admin/llm_health_status/show.html.erb b/app/views/admin/llm_health_status/show.html.erb new file mode 100644 index 000000000000..5af9df1f4d86 --- /dev/null +++ b/app/views/admin/llm_health_status/show.html.erb @@ -0,0 +1,101 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_connection"), t(".title") %> + +<%= turbo_frame_tag "llm_health_status_report", refresh: :morph do %> + <%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t(".title") } + if @report.present? + header.with_description { t(".last_check", datetime: format_time(@report.created_at)) } + end + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + { href: llm_connection_path, text: t("menus.admin.llm_connection") }, + t(".title")] + ) + end + %> + + <% if @report.present? %> + <%= + render(Primer::OpenProject::SubHeader.new) do |subheader| + subheader.with_action_button( + scheme: :secondary, + label: t(".download"), + mobile_label: t(".download"), + mobile_icon: :download, + leading_icon: :download, + tag: :a, + href: llm_connection_health_status_report_path(format: :txt), + data: { turbo: false } + ) { t(".download") } + + subheader.with_action_button( + scheme: :primary, + label: t(".run_checks"), + mobile_label: t(".run_checks"), + mobile_icon: "op-reload", + leading_icon: "op-reload", + tag: :a, + href: llm_connection_health_status_report_path, + data: { turbo_method: :post, turbo: true }, + test_selector: "llm-connection--rerun-health-checks" + ) { t(".run_checks") } + end + %> + + <%= + render( + HealthReports::ReportComponent.new( + @report, + i18n_scope: "llm.health_checks", + # No admin guide page exists for this yet. Every check explains itself + # in its own message, so a "More information" link would only lead + # somewhere unrelated -- suppressed until that page is written. + docs_href: false + ) + ) + %> + <% else %> + <%= + render(Primer::Beta::Blankslate.new(test_selector: "llm-connection--health-blankslate")) do |blankslate| + blankslate.with_visual_icon(icon: :meter) + blankslate.with_heading(tag: :h2) { t(".blankslate_heading") } + blankslate.with_description { t(".blankslate_description") } + blankslate.with_primary_action( + href: llm_connection_health_status_report_path, + data: { turbo_method: :post, turbo: true } + ) { t(".run_checks") } + end + %> + <% end %> +<% end %> diff --git a/app/workers/llm/health_check_job.rb b/app/workers/llm/health_check_job.rb new file mode 100644 index 000000000000..059b68ac5bf8 --- /dev/null +++ b/app/workers/llm/health_check_job.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Re-checks the configured connection, so a server that dies after setup does + # not stay green until somebody happens to look. + # + # Deliberately does *not* set deep_health_check: the inference group spends a + # real completion, which is billed on a hosted provider. An unattended job must + # not run up a bill, so the schedule covers everything that is free -- an + # expired key, a withdrawn model, a binding that stopped resolving -- and the + # billed round trip stays behind the administrator's "Run checks". + class HealthCheckJob < ApplicationJob + CRON_JOB_KEY = :"Llm::HealthCheckJob" + + queue_with_priority :low + + class << self + # Keeps the cron idle while there is nothing to check, rather than waking + # every six hours to find no connection. + def toggle_cron_job + if runnable? + GoodJob::Setting.cron_key_enable(CRON_JOB_KEY) unless GoodJob::Setting.cron_key_enabled?(CRON_JOB_KEY) + elsif GoodJob::Setting.cron_key_enabled?(CRON_JOB_KEY) + GoodJob::Setting.cron_key_disable(CRON_JOB_KEY) + end + end + + def runnable? + return false unless OpenProject::FeatureDecisions.llm_connection_active? + + connection = LlmConnection.first + connection.present? && connection.configured? && connection.enabled? + end + end + + def perform + return unless self.class.runnable? + + # .first, not .instance: the latter builds an unsaved record, and the + # validator writes through the health_reports association. + connection = LlmConnection.first + report = Llm::Validators::ConnectionValidator.new(connection).call + report.save! + report + end + end +end diff --git a/app/workers/llm/prune_health_reports_job.rb b/app/workers/llm/prune_health_reports_job.rb new file mode 100644 index 000000000000..aae37cc65ae3 --- /dev/null +++ b/app/workers/llm/prune_health_reports_job.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Keeps the health report history bounded. + # + # Until the scheduled check existed, health_reports only grew when somebody + # clicked "Run checks", so nothing anywhere pruned the table. A four-times-daily + # writer changes that, and this is the first pruner it has had. + # + # Only ever prunes reports belonging to an LLM connection: storages and wikis + # share the table and still write a row only on demand. + class PruneHealthReportsJob < ApplicationJob + # Enough history to see a pattern in when a flaky server fails, without + # keeping a year of identical green reports. + KEEP = 50 + MAX_AGE = 90.days + + queue_with_priority :low + + def perform + LlmConnection.find_each do |connection| + recent = connection.health_reports.order(created_at: :desc).limit(KEEP).pluck(:id) + + connection.health_reports + .where.not(id: recent) + .where(created_at: ...MAX_AGE.ago) + .delete_all + end + end + end +end diff --git a/config/initializers/cronjobs.rb b/config/initializers/cronjobs.rb index 1b9a8fc628af..586336079024 100644 --- a/config/initializers/cronjobs.rb +++ b/config/initializers/cronjobs.rb @@ -72,6 +72,14 @@ cron: "30 23 * * *", class: Ldap::SynchronizationJob.name }, + "Llm::HealthCheckJob": { + cron: "7 */6 * * *", # every six hours at xx:07 + class: Llm::HealthCheckJob.name + }, + "Llm::PruneHealthReportsJob": { + cron: "25 3 * * *", # runs at 3:25 nightly + class: Llm::PruneHealthReportsJob.name + }, "RecurringMeetings::InitNextOccurrenceWatchdogJob": { cron: "11 05 * * *", class: RecurringMeetings::InitNextOccurrenceWatchdogJob.name diff --git a/config/locales/en.yml b/config/locales/en.yml index 827106843325..210be9d73c05 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1811,6 +1811,14 @@ en: update: model_incapable: "The model for %{feature} has been saved, but the server just reported that it does not support %{capability}. Pick a different model, or assert the capability on the model if you know better." success: "The model for %{feature} has been saved." + llm_health_status: + show: + blankslate_description: "Run the checks to find out whether OpenProject can reach the LLM server, whether it accepts the API key, and whether every AI feature has a usable model." + blankslate_heading: "This connection has not been checked yet" + download: "Download report" + last_check: "Last checked %{datetime}." + run_checks: "Run checks" + title: "Health status" llm_models: create: success: "%{model} has been added." @@ -4346,6 +4354,56 @@ en: semantic_search: caption: "Indexes work packages so they can be found by meaning rather than by keyword." label: "Semantic search" + health_checks: + configuration: + api_format_supported: "Endpoint format is usable" + base_url_present: "Endpoint is configured" + credentials_present: "API key is stored" + enabled: "Connection is switched on" + feature_flag: "AI features are switched on" + header: "Configuration" + errors: + api_key_missing: "No API key is stored. This is only correct if the LLM server requires no authentication." + catalogue_never_fetched: "The model list has never been retrieved from the server." + catalogue_stale: "The model list was last retrieved on %{fetched_at} and may no longer match what the server offers." + chat_failed: "The server refused a request for %{model} with status %{status}." + connection_disabled: "The connection is configured but switched off, so no feature will use it." + connection_error: "The server could not be reached. Please ensure it is running and reachable from OpenProject." + default_model_missing: "%{model} is no longer offered by the server." + default_model_unset: "No default model has been chosen, so features fall back to their own selection." + empty_completion: "%{model} answered with an empty message." + feature_flag_off: "The LLM connection feature is switched off, so the connection is not used yet." + features_incapable: "%{features} cannot run, because the chosen model does not provide: %{capabilities}." + features_model_missing: "%{features} point at a model the server no longer offers." + features_unbound: "%{features} have no model chosen yet." + invalid_api_key: "The server rejected the stored API key." + locked_model_missing: "%{features} indexed data with a model the server no longer offers. The index cannot be extended until it is rebuilt." + no_model_to_test: "No model is available to test with. Add a model or choose a default first." + no_models: "The server does not publish a model list. Either it does not offer one, or the endpoint is missing its API version segment. Models can be added manually; the inference check is what proves the endpoint is right." + no_models_endpoint: "The server does not offer a model list, so the API key could not be verified. Run the inference check to verify it." + not_configured: "No endpoint has been configured." + not_openai_compatible: "The endpoint did not return a valid model list. Please ensure it points at an OpenAI-API-compatible endpoint, including the API version segment." + request_timed_out: "The server did not respond in time." + server_error: "The server responded with status %{status}." + ssrf_filtered: "The endpoint resolves to a blocked address." + unsupported_api_format: "%{api_format} needs credentials that OpenProject cannot store." + features: + bindings_resolvable: "Every feature has a usable model" + header: "AI features" + locked_bindings_intact: "Indexed data still has its model" + inference: + chat_round_trip: "The server answers a request" + header: "Inference" + models: + catalogue_fresh: "Model list is up to date" + catalogue_present: "Models are available" + default_chat_model: "Default chat model" + default_embedding_model: "Default embedding model" + header: "Models" + server: + credentials_accepted: "The server accepts the API key" + header: "Server" + reachable: "The server can be reached" model_kinds: chat: "Chat" embedding: "Embedding" @@ -4360,6 +4418,13 @@ en: unknown: "Not verified" unsupported: "Not supported" llm_connections: + side_panel: + health_status_component: + last_check: "Last checked %{datetime}." + never_checked: "This connection has not been checked yet." + open_report: "Open full health report" + run_checks: "Run checks now" + title: "Health status" macro_execution_error: "Error executing the macro %{macro_name}" macro_unavailable: "Macro %{macro_name} cannot be displayed." macro_unknown: "Unknown or unsupported macro." diff --git a/config/routes.rb b/config/routes.rb index f85a16056e5c..d113c9c7d130 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -749,6 +749,10 @@ get :disconnect_dialog post :disconnect get :search_models, defaults: { format: :turbo_stream } + + resource :health_status_report, only: %i[show create], controller: "admin/llm_health_status" do + post :create_health_status_report + end end # Manual entries only; discovered models are managed by the sync. diff --git a/db/migrate/20260813090000_add_created_at_index_to_health_reports.rb b/db/migrate/20260813090000_add_created_at_index_to_health_reports.rb new file mode 100644 index 000000000000..dfcba13aa170 --- /dev/null +++ b/db/migrate/20260813090000_add_created_at_index_to_health_reports.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Every consumer of a health report reads the newest one for a subject +# (`health_reports.order(created_at: :asc).last`), and the table has only a +# [subject_type, subject_id] index, so that read sorts. Until now the table grew +# a row per manual "Run checks" click; the LLM connection adds a scheduled check, +# which makes it grow unattended and adds a pruning delete that filters on age. +# +# Storages and wikis benefit from this too. +class AddCreatedAtIndexToHealthReports < ActiveRecord::Migration[8.1] + disable_ddl_transaction! + + def change + add_index :health_reports, + %i[subject_type subject_id created_at], + algorithm: :concurrently, + if_not_exists: true + end +end diff --git a/spec/features/admin/llm_connection_spec.rb b/spec/features/admin/llm_connection_spec.rb index a1203322b941..79dc4100d1d1 100644 --- a/spec/features/admin/llm_connection_spec.rb +++ b/spec/features/admin/llm_connection_spec.rb @@ -122,6 +122,24 @@ def choose_action(item) # The point of disconnecting rather than deleting. expect(connection.models.count).to eq(2) end + + describe "the health report" do + before { mock_llm_chat_response(base_url) } + + it "runs the checks and renders the report accessibly" do + visit llm_connection_path + + find_test_selector("llm-connection--run-health-checks").click + + wait_for { connection.health_reports.count }.to eq(1) + + find_test_selector("llm-connection--open-health-report").click + + expect(page).to have_current_path(llm_connection_health_status_report_path) + expect(page).to have_text("Configuration") + expect(page).to be_axe_clean.within("#content") + end + end end describe "the AI models page" do diff --git a/spec/requests/admin/llm_health_status_spec.rb b/spec/requests/admin/llm_health_status_spec.rb new file mode 100644 index 000000000000..ad04b17ffa8f --- /dev/null +++ b/spec/requests/admin/llm_health_status_spec.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe "LLM connection health status", :llm_server_helpers, :skip_csrf, :webmock, + type: :rails_request, with_flag: { llm_connection: true } do + shared_let(:admin) { create(:admin) } + + let(:base_url) { "https://example.com/v1" } + + before { login_as(admin) } + + describe "GET /admin/llm_connection/health_status_report" do + it "redirects to the settings page when nothing is configured yet" do + get llm_connection_health_status_report_path + + expect(response).to redirect_to(llm_connection_path) + end + + context "with a connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + it "offers to run the checks when none have run" do + get llm_connection_health_status_report_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("has not been checked yet") + end + + it "renders the report once it exists" do + mock_llm_models_response(base_url) + mock_llm_chat_response(base_url) + post llm_connection_health_status_report_path + + get llm_connection_health_status_report_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Configuration") + end + end + end + + describe "POST /admin/llm_connection/health_status_report" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:, default_chat_model_id: "qwen3.6-27b") } + + before do + mock_llm_models_response(base_url) + mock_llm_chat_response(base_url) + end + + it "stores a report and sends the completion an administrator asked for" do + expect { post llm_connection_health_status_report_path } + .to change(connection.health_reports, :count).by(1) + + expect(response).to redirect_to(llm_connection_health_status_report_path) + expect(WebMock).to have_requested(:post, "#{base_url}/chat/completions").once + end + end + + describe "GET /admin/llm_connection/health_status_report.txt" do + let!(:connection) do + create(:llm_connection, :with_models, :enabled, base_url:, + api_key: "sk-super-secret", + custom_headers: { "apikey" => "gateway-secret" }) + end + + before do + mock_llm_models_response(base_url) + mock_llm_chat_response(base_url) + post llm_connection_health_status_report_path + end + + it "downloads the report" do + get llm_connection_health_status_report_path(format: :txt) + + expect(response).to have_http_status(:ok) + expect(response.body).to include("base_url") + end + + # The report is pasted into support tickets. A gateway header routinely + # carries a second credential, so neither it nor the API key may appear. + it "contains no credentials" do + get llm_connection_health_status_report_path(format: :txt) + + expect(response.body).not_to include("sk-super-secret") + expect(response.body).not_to include("gateway-secret") + end + end + + describe "authorisation" do + let!(:connection) { create(:llm_connection, :enabled, base_url:) } + + it "is refused to a non-admin" do + login_as(create(:user)) + + get llm_connection_health_status_report_path + + expect(response).not_to have_http_status(:ok) + end + end +end diff --git a/spec/services/llm/validators/connection_validator_spec.rb b/spec/services/llm/validators/connection_validator_spec.rb new file mode 100644 index 000000000000..43ba395a2d8e --- /dev/null +++ b/spec/services/llm/validators/connection_validator_spec.rb @@ -0,0 +1,179 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Validators::ConnectionValidator, :llm_server_helpers, :webmock, + with_flag: { llm_connection: true } do + subject(:report) { described_class.new(connection).call } + + let(:base_url) { "https://example.com/v1" } + let(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + def result_for(group, key) + report.group(group)&.result_for(key) + end + + context "with a healthy connection" do + before do + mock_llm_models_response(base_url) + connection.update!(default_chat_model_id: "qwen3.6-27b") + end + + it "passes the configuration and server groups" do + expect(result_for(:configuration, :base_url_present).state).to eq(:success) + expect(result_for(:server, :reachable).state).to eq(:success) + expect(result_for(:server, :credentials_accepted).state).to eq(:success) + end + + # The billed request only runs when an administrator asks for it. + it "skips inference unless a deep check was asked for" do + expect(report.group(:inference)).to be_nil + expect(WebMock).not_to have_requested(:post, "#{base_url}/chat/completions") + end + + context "when a deep check is asked for" do + before do + connection.deep_health_check = true + mock_llm_chat_response(base_url, content: "pong") + end + + it "sends a completion and reports the round trip" do + expect(result_for(:inference, :chat_round_trip).state).to eq(:success) + expect(WebMock).to have_requested(:post, "#{base_url}/chat/completions").once + end + end + end + + context "when the server publishes no model list" do + before { mock_llm_models_response(base_url, response_code: 404) } + + # The case that motivated manual model entry: a gateway exposing only chat. + it "is still reachable, and says the key could not be verified" do + expect(result_for(:server, :reachable).state).to eq(:success) + expect(result_for(:server, :credentials_accepted).state).to eq(:warning) + expect(result_for(:server, :credentials_accepted).code).to eq(:no_models_endpoint) + end + + it "can still prove the connection works through inference" do + connection.deep_health_check = true + connection.update!(default_chat_model_id: "qwen3.6-27b") + mock_llm_chat_response(base_url) + + expect(result_for(:inference, :chat_round_trip).state).to eq(:success) + end + end + + context "when the key is rejected" do + before { mock_llm_models_response(base_url, response_code: 401) } + + it "separates reachability from authentication" do + expect(result_for(:server, :reachable).state).to eq(:success) + expect(result_for(:server, :credentials_accepted).state).to eq(:failure) + expect(report).to be_unhealthy + end + end + + context "when the server cannot be reached" do + before { mock_llm_models_response(base_url, timeout: true) } + + it "fails reachability and skips the credential check" do + expect(result_for(:server, :reachable).state).to eq(:failure) + expect(result_for(:server, :credentials_accepted).state).to eq(:skipped) + end + end + + context "without a base URL" do + let(:connection) { create(:llm_connection, :enabled).tap { |c| c.update_column(:base_url, "") } } + + it "fails and does not ask the server anything" do + expect(result_for(:configuration, :base_url_present).state).to eq(:failure) + expect(report.group(:server)).to be_nil + expect(WebMock).not_to have_requested(:get, "#{base_url}/models") + end + end + + describe "the features group" do + before { mock_llm_models_response(base_url) } + + it "warns about features with no model chosen" do + result = result_for(:features, :bindings_resolvable) + + expect(result.state).to eq(:warning) + expect(result.code).to eq(:features_unbound) + end + + it "fails when a binding points at a model the server no longer offers" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "vanished") + + result = result_for(:features, :bindings_resolvable) + + expect(result.state).to eq(:failure) + expect(result.code).to eq(:features_model_missing) + end + end + + # HealthReports::ResultComponent resolves a check's label from its key and its + # explanation from its code, so a key with no translation renders as + # "translation missing" rather than failing anywhere a spec would notice. + describe "translations" do + let(:scenarios) do + [ + -> { mock_llm_models_response(base_url) }, + -> { mock_llm_models_response(base_url, response_code: 401) }, + -> { mock_llm_models_response(base_url, response_code: 404) }, + -> { mock_llm_models_response(base_url, timeout: true) } + ] + end + + it "exist for every check and every code the validator can emit" do + # A dangling binding, so the features group emits its failure codes too. + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "vanished") + + scenarios.each do |setup| + WebMock.reset! + setup.call + + described_class.new(connection).call.results.each do |group| + expect(I18n.t("#{group.key}.header", scope: "llm.health_checks", raise: true)).to be_present + + group.results.each do |result| + expect(I18n.t("#{group.key}.#{result.key}", scope: "llm.health_checks", raise: true)).to be_present + next if result.code.nil? + + context_vars = result.context&.symbolize_keys || {} + expect(I18n.t("errors.#{result.code}", scope: "llm.health_checks", raise: true, **context_vars)) + .to be_present + end + end + end + end + end +end diff --git a/spec/workers/llm/health_check_job_spec.rb b/spec/workers/llm/health_check_job_spec.rb new file mode 100644 index 000000000000..92603af70ff2 --- /dev/null +++ b/spec/workers/llm/health_check_job_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::HealthCheckJob, :llm_server_helpers, :webmock, with_flag: { llm_connection: true } do + let(:base_url) { "https://example.com/v1" } + + describe "#perform" do + context "with an enabled connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before { mock_llm_models_response(base_url) } + + it "stores a report" do + expect { described_class.perform_now }.to change(connection.health_reports, :count).by(1) + end + + # The whole reason the inference group is gated: an unattended job must not + # run up a bill on a hosted provider four times a day. + # Group keys come back from jsonb as strings, so a stored report is + # queried by string where an in-memory one is queried by symbol. + it "does not spend a completion" do + described_class.perform_now + + expect(WebMock).not_to have_requested(:post, "#{base_url}/chat/completions") + expect(connection.latest_health_report.results.map(&:key)).not_to include("inference") + end + + it "still reports what the free checks found" do + described_class.perform_now + + expect(connection.latest_health_report.group("server").result_for("reachable").state).to eq(:success) + end + end + + it "does nothing without a connection" do + expect { described_class.perform_now }.not_to change(HealthReport, :count) + end + + it "does nothing while the connection is switched off" do + create(:llm_connection, :with_models, enabled: false, base_url:) + + expect { described_class.perform_now }.not_to change(HealthReport, :count) + end + + context "with the feature flag off", with_flag: { llm_connection: false } do + it "does nothing" do + create(:llm_connection, :with_models, :enabled, base_url:) + + expect { described_class.perform_now }.not_to change(HealthReport, :count) + end + end + end + + describe ".toggle_cron_job" do + it "enables the cron once a connection is usable" do + create(:llm_connection, :with_models, :enabled, base_url:) + + described_class.toggle_cron_job + + expect(GoodJob::Setting.cron_key_enabled?(described_class::CRON_JOB_KEY)).to be(true) + end + + it "disables the cron while there is nothing to check" do + GoodJob::Setting.cron_key_enable(described_class::CRON_JOB_KEY) + + described_class.toggle_cron_job + + expect(GoodJob::Setting.cron_key_enabled?(described_class::CRON_JOB_KEY)).to be(false) + end + end +end diff --git a/spec/workers/llm/prune_health_reports_job_spec.rb b/spec/workers/llm/prune_health_reports_job_spec.rb new file mode 100644 index 000000000000..7d3af3097978 --- /dev/null +++ b/spec/workers/llm/prune_health_reports_job_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::PruneHealthReportsJob do + shared_let(:connection) { create(:llm_connection) } + + def report(created_at:, subject: connection) + subject.health_reports.create!( + results: [HealthReport::ResultGroup.new(key: :configuration, results: [])], + created_at: + ) + end + + it "deletes reports that are both old and beyond the retention count" do + stub_const("#{described_class}::KEEP", 2) + oldest = report(created_at: 300.days.ago) + report(created_at: 200.days.ago) + report(created_at: 100.days.ago) + + expect { described_class.perform_now }.to change { HealthReport.exists?(oldest.id) }.from(true).to(false) + end + + it "keeps recent reports however many there are" do + recent = report(created_at: 1.day.ago) + + described_class.perform_now + + expect(HealthReport.exists?(recent.id)).to be(true) + end + + # Age alone is not enough: a connection checked once a year ago should still + # show that check rather than nothing at all. + it "keeps the newest reports even when they are old" do + oldest = report(created_at: 200.days.ago) + + described_class.perform_now + + expect(HealthReport.exists?(oldest.id)).to be(true) + end + + it "leaves other subjects' reports alone" do + storage = create(:nextcloud_storage) + foreign = report(created_at: 200.days.ago, subject: storage) + + described_class.perform_now + + expect(HealthReport.exists?(foreign.id)).to be(true) + end +end