[AI-3] Store the model catalogue and capability verdicts

app/components/llm_connections/delete_api_key_dialog_component.html.erb

@@ -10,7 +10,13 @@
   ) do |dialog|
     dialog.with_confirmation_message do |message|
       message.with_heading(tag: :h2) { t("admin.llm_connections.delete_api_key.heading") }
-      message.with_description_content(t("admin.llm_connections.delete_api_key.description"))
+      message.with_description_content(
+        if loses_admin_verdicts?
+          t("admin.llm_connections.delete_api_key.description_verdicts")
+        else
+          t("admin.llm_connections.delete_api_key.description")
+        end
+      )
     end
   end
 %>

app/components/llm_connections/delete_api_key_dialog_component.rb

@@ -32,7 +32,7 @@ module LlmConnections
   # Confirms removing the stored API key.
   #
   # No confirmation checkbox: the key itself can simply be pasted again. The
-  # dialog exists for what is *not* recoverable.
+  # dialog exists for what is *not* recoverable -- see #loses_admin_verdicts?.
   class DeleteApiKeyDialogComponent < ApplicationComponent
     include OpTurbo::Streamable
     include OpPrimer::ComponentHelpers
@@ -44,5 +44,14 @@ class DeleteApiKeyDialogComponent < ApplicationComponent
     def form_arguments
       { action: url_helpers.api_key_llm_connection_path, method: :delete }
     end
+
+    # The catalogue sync fingerprints base_url and api_key together, so the next
+    # refresh after the key changes treats the endpoint as a different deployment
+    # and discards every capability verdict -- including the ones an
+    # administrator asserted by hand, which nothing else in the system throws
+    # away. Worth saying out loud before the key goes.
+    def loses_admin_verdicts?
+      connection.capability_verdicts.exists?(source: "admin")
+    end
   end
 end

app/components/llm_connections/disconnect_dialog_component.html.erb

@@ -19,7 +19,8 @@
           content_tag(:ul) do
             safe_join(
               [
-                content_tag(:li, t("admin.llm_connections.disconnect.keeps_settings"))
+                content_tag(:li, t("admin.llm_connections.disconnect.keeps_settings")),
+                content_tag(:li, t("admin.llm_connections.disconnect.keeps_models"))
               ].compact
             )
           end

app/controllers/admin/llm_connections_controller.rb

@@ -51,6 +51,16 @@ def update
       result.on_failure { render_form_with_errors }
     end
 
+    def refresh_models
+      result = ::LlmConnections::SyncModelsService.new(@connection).call
+
+      if result.success?
+        redirect_with_notice(t(".success"))
+      else
+        redirect_with_error(t(".failure"))
+      end
+    end
+
     def disconnect_dialog
       respond_with_dialog LlmConnections::DisconnectDialogComponent.new(@connection)
     end
@@ -85,8 +95,17 @@ def require_feature
       render_404 unless OpenProject::FeatureDecisions.llm_connection_active?
     end
 
+    # A connection can be perfectly usable without offering a model list, so the
+    # save succeeds either way; the administrator is told what to do next rather
+    # than being left with an empty table and no explanation.
     def redirect_after_save
-      redirect_with_notice(t(".success"))
+      if @connection.reload.models.none?
+        flash[:warning] = t(".no_models")
+      else
+        flash[:notice] = t(".success")
+      end
+
+      redirect_to llm_connection_path, status: :see_other
     end
 
     def render_form_with_errors

app/models/llm_capability_verdict.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# See COPYRIGHT and LICENSE files for more details.
+#++
+
+# What we know about one capability of one model on one server.
+#
+# Three states rather than a boolean, because the honest answer is very often
+# "we cannot tell": the OpenAI model list carries no capability information at
+# all, and only some servers offer a non-standard endpoint that does.
+#
+# The rule the rest of the system depends on: only :unsupported blocks. An
+# :unknown verdict warns and lets the administrator proceed, because refusing
+# on "we could not tell" would make most self-hosted servers unusable.
+class LlmCapabilityVerdict < ApplicationRecord
+  belongs_to :llm_connection
+
+  enum :state, { supported: "supported", unsupported: "unsupported", unknown: "unknown" }, validate: true
+
+  # Where the verdict came from. Orthogonal to the state, so that an
+  # administrator's assertion can be shown as such without adding a fourth state
+  # that every caller would have to handle.
+  enum :source,
+       { metadata: "metadata", probe: "probe", admin: "admin", observed: "observed" },
+       prefix: true,
+       validate: true
+
+  validates :model_id, presence: true
+  validates :capability, presence: true, uniqueness: { scope: %i[llm_connection_id model_id] }
+
+  scope :for_model, ->(model_id) { where(model_id:) }
+  scope :for_capability, ->(capability) { where(capability: capability.to_s) }
+
+  # An administrator's assertion survives re-detection: they know something about
+  # their deployment that we could not determine.
+  scope :sticky, -> { where(source: "admin") }
+
+  def blocking?
+    unsupported?
+  end
+
+  def dimensions
+    detail["dimensions"]
+  end
+end

app/models/llm_connection.rb

@@ -38,9 +38,9 @@ class LlmConnection < ApplicationRecord
   include Redmine::Ciphering
 
   SINGLETON_NAME = "default"
-
   has_many :health_reports, as: :subject, dependent: :delete_all
-
+  has_many :models, class_name: "LlmModel", dependent: :delete_all
+  has_many :capability_verdicts, class_name: "LlmCapabilityVerdict", dependent: :delete_all
   validates :base_url, presence: true
   validate :only_one_connection, on: :create
 
@@ -82,6 +82,16 @@ def configured?
     base_url.present?
   end
 
+  # Every model that can be addressed today: discovered and still offered, plus
+  # anything an administrator entered by hand.
+  #
+  # Deliberately includes models an administrator has deactivated. This is what
+  # Llm::Runtime resolves against, and hiding a model from the pickers must not
+  # break a feature that is already bound to it.
+  def available_model_ids
+    models.active.by_identifier.pluck(:external_id)
+  end
+
   def server_flavour
     options["server_flavour"].presence&.to_sym
   end

app/models/llm_model.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# See COPYRIGHT and LICENSE files for more details.
+#++
+
+# A model this connection can address.
+#
+# Rows come from two places: discovered from the server's model list, or entered
+# by an administrator. Both are addressed by +external_id+, which is whatever the
+# deployment calls the model -- provider-specific and not comparable across
+# vendors, which is why it is never used as a lookup key into a public catalogue.
+class LlmModel < ApplicationRecord
+  belongs_to :llm_connection
+
+  validates :external_id, presence: true, uniqueness: { scope: :llm_connection_id }
+
+  scope :active, -> { where(active: true) }
+  scope :discovered, -> { where(manual: false) }
+  scope :manual, -> { where(manual: true) }
+  scope :by_identifier, -> { order(:external_id) }
+
+  def name = display_name.presence || external_id
+
+  # Precedence: what an administrator set, then what the server reported (vLLM
+  # and SGLang publish the operator's actual --max-model-len), then what a
+  # registry believes about the model in general.
+  def context_window
+    raw_metadata["admin_context_window"] ||
+      raw_metadata["max_model_len"] ||
+      raw_metadata["context_window"]
+  end
+
+  def context_window_source
+    return :admin if raw_metadata["admin_context_window"].present?
+    return :server if raw_metadata["max_model_len"].present?
+    return :registry if raw_metadata["context_window"].present?
+
+    nil
+  end
+
+  def verdict_for(capability)
+    llm_connection.capability_verdicts
+                  .for_model(external_id)
+                  .for_capability(capability)
+                  .first
+  end
+
+  # Discovered models that the server stopped offering are deactivated rather
+  # than deleted, so a binding or verdict pointing at one still has something to
+  # name. Manual entries are never deactivated by a refresh: nothing confirms
+  # them, so nothing can un-confirm them either.
+  def withdrawn? = !active? && !manual?
+end

app/services/llm/capabilities.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# See COPYRIGHT and LICENSE files for more details.
+#++
+
+module Llm
+  # The capabilities a model may have, and how to read the published ones.
+  module Capabilities
+    ALL = %i[embeddings function_calling structured_output vision reasoning].freeze
+
+    # Capabilities a chat model can be asked about. +embeddings+ is deliberately
+    # absent: it is a different kind of model, not a feature of a chat one.
+    CHAT = (ALL - %i[embeddings]).freeze
+    EMBEDDING = %i[embeddings].freeze
+
+    module_function
+
+    # What a public registry says about this model id.
+    #
+    # Advisory only. The registry describes a model as some vendor deploys it,
+    # which is not the same as this deployment: the same weights are catalogued
+    # with contradictory capability flags and context windows an order of
+    # magnitude apart across providers. Verdicts derived from it are therefore
+    # recorded with source "metadata", and an administrator can overrule them.
+    #
+    # @return [Hash{Symbol => Symbol}, nil] capability => :supported / :unsupported,
+    #   or nil when the registry does not know the model -- the normal case for a
+    #   self-hosted server.
+    def published_for(model_id)
+      info = RubyLLM.models.find(model_id)
+
+      { states: states_from(info), context_window: info.context_window, display_name: info.name }
+    rescue RubyLLM::ModelNotFoundError
+      nil
+    rescue StandardError => e
+      # Registry lookup is an enrichment; it must never break a model sync.
+      Rails.logger.info { "LLM capability lookup for #{model_id} failed: #{e.class} #{e.message}" }
+      nil
+    end
+
+    def states_from(info)
+      embedding = info.type.to_s == "embedding"
+      published = Array(info.capabilities).map(&:to_sym)
+      relevant = embedding ? EMBEDDING : CHAT
+
+      states = relevant.index_with { |capability| published.include?(capability) ? :supported : :unsupported }
+      states.merge(embeddings: embedding ? :supported : :unsupported)
+    end
+
+    def label(capability)
+      I18n.t("llm.capabilities.#{capability}.label")
+    end
+  end
+end