diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs index 320aeacda..5762e4951 100644 --- a/.dialyzer_ignore.exs +++ b/.dialyzer_ignore.exs @@ -142,5 +142,9 @@ # ExUnit internal functions — false positives when test/support is compiled in MIX_ENV=test # Dialyzer cannot resolve ExUnit private macros expanded at compile time {"test/support/conn_case.ex", :unknown_function}, - {"test/support/data_case.ex", :unknown_function} + {"test/support/data_case.ex", :unknown_function}, + + # LLMText module — Publishing is an external package, guarded by Code.ensure_loaded? + {"lib/modules/llm_text/sources/publishing.ex", :unknown_function}, + {"lib/modules/llm_text/publishing_subscriber.ex", :unknown_function} ] diff --git a/lib/modules/llm_text/file_storage.ex b/lib/modules/llm_text/file_storage.ex new file mode 100644 index 000000000..149405c73 --- /dev/null +++ b/lib/modules/llm_text/file_storage.ex @@ -0,0 +1,170 @@ +defmodule PhoenixKit.Modules.LLMText.FileStorage do + @moduledoc """ + File-based storage for LLM text files. + + Stores files under the host app's `priv/static/llms/` directory: + + priv/static/llms/llms.txt -> index file + priv/static/llms/page.md -> individual page file + priv/static/llms/posts/article.md -> nested page file + + ## Test Override + + Set `Application.put_env(:phoenix_kit, :llm_text_test_storage_dir, "/tmp/...")` to + override the storage directory in tests. + """ + + require Logger + + @index_filename "llms.txt" + + @doc """ + Returns the root storage directory for LLM text files. + + Uses test override if configured, otherwise resolves to the host app's + `priv/static/llms/` directory. + """ + @spec storage_dir() :: String.t() + def storage_dir do + case Application.get_env(:phoenix_kit, :llm_text_test_storage_dir) do + nil -> resolve_storage_dir() + dir -> dir + end + end + + @doc """ + Returns the path to the llms.txt index file. + """ + @spec index_path() :: String.t() + def index_path do + Path.join(storage_dir(), @index_filename) + end + + @doc """ + Returns the full path for a relative file path within the storage directory. + """ + @spec file_path(String.t()) :: String.t() + def file_path(relative_path) when is_binary(relative_path) do + Path.join(storage_dir(), relative_path) + end + + @doc """ + Writes content to a file at the given relative path, creating directories as needed. + """ + @spec write(String.t(), String.t()) :: :ok | {:error, term()} + def write(relative_path, content) when is_binary(relative_path) and is_binary(content) do + path = file_path(relative_path) + + with :ok <- ensure_directory_exists(path), + :ok <- File.write(path, content) do + Logger.debug("LLMText.FileStorage: Wrote #{relative_path} (#{byte_size(content)} bytes)") + :ok + else + {:error, reason} = error -> + Logger.warning( + "LLMText.FileStorage: Failed to write #{relative_path}: #{inspect(reason)}" + ) + + error + end + end + + @doc """ + Writes the llms.txt index file. + """ + @spec write_index(String.t()) :: :ok | {:error, term()} + def write_index(content) when is_binary(content) do + path = index_path() + + with :ok <- ensure_directory_exists(path), + :ok <- File.write(path, content) do + Logger.debug("LLMText.FileStorage: Wrote llms.txt (#{byte_size(content)} bytes)") + :ok + else + {:error, reason} = error -> + Logger.warning("LLMText.FileStorage: Failed to write llms.txt: #{inspect(reason)}") + error + end + end + + @doc """ + Deletes a file at the given relative path. Returns :ok if file does not exist. + """ + @spec delete(String.t()) :: :ok + def delete(relative_path) when is_binary(relative_path) do + path = file_path(relative_path) + + case File.rm(path) do + :ok -> + Logger.debug("LLMText.FileStorage: Deleted #{relative_path}") + :ok + + {:error, :enoent} -> + :ok + + {:error, reason} -> + Logger.warning( + "LLMText.FileStorage: Failed to delete #{relative_path}: #{inspect(reason)}" + ) + + :ok + end + end + + @doc """ + Checks if a file at the given relative path exists. + """ + @spec exists?(String.t()) :: boolean() + def exists?(relative_path) when is_binary(relative_path) do + relative_path |> file_path() |> File.exists?() + end + + @doc """ + Deletes the entire storage directory and all its contents. + """ + @spec delete_all() :: :ok + def delete_all do + dir = storage_dir() + + case File.rm_rf(dir) do + {:ok, _} -> + Logger.debug("LLMText.FileStorage: Deleted all files in #{dir}") + :ok + + {:error, reason, _path} -> + Logger.warning("LLMText.FileStorage: Failed to delete storage dir: #{inspect(reason)}") + :ok + end + rescue + _ -> :ok + end + + # Private helpers + + defp resolve_storage_dir do + otp_app = PhoenixKit.Config.get(:otp_app, :phoenix_kit) + + priv_dir = + case :code.priv_dir(otp_app) do + {:error, :bad_name} -> + case :code.priv_dir(:phoenix_kit) do + {:error, :bad_name} -> "priv" + dir -> to_string(dir) + end + + dir -> + to_string(dir) + end + + Path.join([priv_dir, "static", "llms"]) + end + + defp ensure_directory_exists(file_path) do + dir = Path.dirname(file_path) + + case File.mkdir_p(dir) do + :ok -> :ok + {:error, reason} -> {:error, {:mkdir_failed, reason}} + end + end +end diff --git a/lib/modules/llm_text/generator.ex b/lib/modules/llm_text/generator.ex new file mode 100644 index 000000000..d14405143 --- /dev/null +++ b/lib/modules/llm_text/generator.ex @@ -0,0 +1,173 @@ +defmodule PhoenixKit.Modules.LLMText.Generator do + @moduledoc """ + Generator for LLM-friendly text files. + + Produces: + - `llms.txt` - Index file linking to all LLM-readable pages + - Individual page `.md` files per source + + ## Usage + + # Regenerate all sources and rebuild index + Generator.run_all() + + # Regenerate a single source and rebuild index + Generator.run_source(MyApp.LLMText.BlogSource) + + # Only rebuild the index from all sources + Generator.rebuild_index() + """ + + require Logger + + alias PhoenixKit.Modules.LLMText.FileStorage + alias PhoenixKit.Modules.LLMText.Sources.Source + + @doc """ + Regenerates files for one source and rebuilds the llms.txt index. + """ + @spec run_source(module()) :: :ok + def run_source(source_module) do + Logger.info("LLMText.Generator: Running source #{inspect(source_module)}") + + files = Source.safe_collect_page_files(source_module) + + Enum.each(files, fn {path, content} -> + case FileStorage.write(path, content) do + :ok -> + :ok + + {:error, reason} -> + Logger.warning("LLMText.Generator: Failed to write #{path}: #{inspect(reason)}") + end + end) + + rebuild_index() + end + + @doc """ + Regenerates all sources and rebuilds the llms.txt index. + """ + @spec run_all() :: :ok + def run_all do + sources = get_sources() + Logger.info("LLMText.Generator: Running all #{length(sources)} sources") + + Enum.each(sources, fn source_module -> + files = Source.safe_collect_page_files(source_module) + + Enum.each(files, fn {path, content} -> + case FileStorage.write(path, content) do + :ok -> + :ok + + {:error, reason} -> + Logger.warning("LLMText.Generator: Failed to write #{path}: #{inspect(reason)}") + end + end) + end) + + rebuild_index() + end + + @doc """ + Rebuilds the llms.txt index from all sources without regenerating page files. + """ + @spec rebuild_index() :: :ok + def rebuild_index do + sources = get_sources() + Logger.debug("LLMText.Generator: Rebuilding index from #{length(sources)} sources") + + entries = + sources + |> Enum.flat_map(&Source.safe_collect_index_entries/1) + + content = build_index_content(entries) + + case FileStorage.write_index(content) do + :ok -> + :ok + + {:error, reason} -> + Logger.warning("LLMText.Generator: Failed to write index: #{inspect(reason)}") + :ok + end + end + + @doc """ + Builds the llms.txt markdown content from a list of index entries. + + Groups entries by their `:group` field. Group order follows first-seen order. + Within each group, entries appear in the order they were provided. + """ + @spec build_index_content([Source.index_entry()]) :: String.t() + def build_index_content(entries) do + site_name = get_site_name() + site_description = get_site_description() + + # Build ordered groups (first-seen order) using prepend + reverse for O(n) performance + {groups_reversed, groups_map} = + Enum.reduce(entries, {[], %{}}, fn entry, {order, map} -> + group = Map.get(entry, :group, "General") + + if Map.has_key?(map, group) do + {order, Map.update!(map, group, &[entry | &1])} + else + {[group | order], Map.put(map, group, [entry])} + end + end) + + groups_ordered = Enum.reverse(groups_reversed) + + header = + if site_description && site_description != "" do + "# #{site_name}\n\n> #{site_description}\n\n" + else + "# #{site_name}\n\n" + end + + sections = + Enum.map_join(groups_ordered, "\n\n", fn group -> + group_entries = Map.get(groups_map, group, []) |> Enum.reverse() + + links = + Enum.map_join(group_entries, "\n", fn entry -> + title = Map.get(entry, :title, "") + url = Map.get(entry, :url, "") + description = Map.get(entry, :description, "") + + if description && description != "" do + "- [#{title}](#{url}): #{description}" + else + "- [#{title}](#{url})" + end + end) + + "## #{group}\n\n#{links}" + end) + + header <> sections + end + + @doc """ + Returns the configured LLM text sources. + """ + @spec get_sources() :: [module()] + def get_sources do + Application.get_env(:phoenix_kit, :llm_text_sources, []) + end + + # Private helpers + + defp get_site_name do + PhoenixKit.Settings.get_setting("site_name", "Site") + rescue + _ -> "Site" + end + + defp get_site_description do + PhoenixKit.Settings.get_setting("site_description", "") + rescue + _ -> "" + end +end diff --git a/lib/modules/llm_text/llm_text.ex b/lib/modules/llm_text/llm_text.ex new file mode 100644 index 000000000..26d641cc3 --- /dev/null +++ b/lib/modules/llm_text/llm_text.ex @@ -0,0 +1,114 @@ +defmodule PhoenixKit.Modules.LLMText do + @moduledoc """ + LLM Text module for PhoenixKit. + + Generates LLM-friendly text files (llms.txt + per-page .txt files) + from configured sources, served at `/llms.txt` and `/llms/*path`. + + ## Settings keys + + - `llm_text_enabled` — enable/disable the module (boolean, default: false) + + ## Configuration + + Configure sources in the host app: + + config :phoenix_kit, :llm_text_sources, [ + PhoenixKit.Modules.LLMText.Sources.Publishing + ] + + ## Usage + + # Check if enabled + PhoenixKit.Modules.LLMText.enabled?() + + # Regenerate all files + PhoenixKit.Modules.LLMText.Generator.run_all() + """ + + use PhoenixKit.Module + + alias PhoenixKit.Dashboard.Tab + alias PhoenixKit.Modules.LLMText.Generator + alias PhoenixKit.Modules.LLMText.PublishingSubscriber + + @enabled_key "llm_text_enabled" + + # ── Required Module callbacks ────────────────────────────────────── + + @impl PhoenixKit.Module + def module_key, do: "llm_text" + + @impl PhoenixKit.Module + def module_name, do: "LLM Text" + + @impl PhoenixKit.Module + def enabled? do + settings_call(:get_boolean_setting, [@enabled_key, false]) + end + + @impl PhoenixKit.Module + def enable_system do + settings_call(:update_boolean_setting, [@enabled_key, true]) + end + + @impl PhoenixKit.Module + def disable_system do + settings_call(:update_boolean_setting, [@enabled_key, false]) + end + + # ── Optional Module callbacks ────────────────────────────────────── + + @impl PhoenixKit.Module + def permission_metadata do + %{ + key: "llm_text", + label: "LLM Text", + icon: "hero-document-text", + description: "Generate LLM-friendly text files (llms.txt) for AI consumption" + } + end + + @impl PhoenixKit.Module + def get_config do + %{ + enabled: enabled?(), + sources: Generator.get_sources() + } + end + + @impl PhoenixKit.Module + def settings_tabs do + [ + Tab.new!( + id: :admin_settings_llm_text, + label: "LLM Text", + icon: "hero-document-text", + path: "llm-text", + priority: 935, + level: :admin, + parent: :admin_settings, + permission: "llm_text" + ) + ] + end + + @impl PhoenixKit.Module + def children do + if enabled?() do + [PublishingSubscriber] + else + [] + end + end + + # ── Private helpers ──────────────────────────────────────────────── + + defp settings_module do + PhoenixKit.Config.get(:llm_text_settings_module, PhoenixKit.Settings) + end + + defp settings_call(fun, args) do + apply(settings_module(), fun, args) + end +end diff --git a/lib/modules/llm_text/publishing_subscriber.ex b/lib/modules/llm_text/publishing_subscriber.ex new file mode 100644 index 000000000..702d4f04e --- /dev/null +++ b/lib/modules/llm_text/publishing_subscriber.ex @@ -0,0 +1,205 @@ +defmodule PhoenixKit.Modules.LLMText.PublishingSubscriber do + @moduledoc """ + GenServer that subscribes to Publishing PubSub events and triggers + incremental LLM text file regeneration. + + ## Events handled + + - `{:post_status_changed, post}` — if published: enqueue file job; otherwise delete file + - `{:post_updated, post}` — if published: enqueue file job; otherwise skip + - `{:post_deleted, post_identifier}` — delete file + enqueue source rebuild + - `{:group_created, group}` — subscribe to new group's posts topic + - `{:group_deleted, group_slug}` — enqueue source rebuild + + ## Topics + + Subscribes to: + - `"publishing:groups"` — group lifecycle events + - `"publishing:{group_slug}:posts"` — per-group post events for each existing group + + All Publishing calls are guarded with `Code.ensure_loaded?`. + """ + + use GenServer + + require Logger + + alias PhoenixKit.Modules.LLMText.FileStorage + alias PhoenixKit.Modules.LLMText.Sources.Publishing, as: PublishingSource + alias PhoenixKit.Modules.LLMText.Workers.GenerateLLMTextJob + alias PhoenixKit.PubSub.Manager, as: PubSubManager + + @compile {:no_warn_undefined, + [ + {PhoenixKit.Modules.Publishing, :enabled?, 0}, + {PhoenixKit.Modules.Publishing, :list_groups, 0}, + {PhoenixKit.Modules.Publishing.PubSub, :groups_topic, 0}, + {PhoenixKit.Modules.Publishing.PubSub, :posts_topic, 1} + ]} + + @publishing_mod PhoenixKit.Modules.Publishing + @publishing_pubsub PhoenixKit.Modules.Publishing.PubSub + + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @impl GenServer + def init(_opts) do + subscribe_to_groups() + subscribe_to_existing_groups() + {:ok, %{}} + end + + @impl GenServer + def handle_info({:post_status_changed, post}, state) do + handle_post_event(post) + {:noreply, state} + end + + def handle_info({:post_updated, post}, state) do + if published?(post) do + handle_post_event(post) + end + + {:noreply, state} + end + + def handle_info({:post_deleted, post_identifier}, state) do + handle_post_removed(post_identifier) + {:noreply, state} + end + + def handle_info({:group_created, group}, state) do + subscribe_to_group(group["slug"]) + {:noreply, state} + end + + def handle_info({:group_deleted, group_slug}, state) do + enqueue_source_rebuild(group_slug) + {:noreply, state} + end + + def handle_info(_msg, state) do + {:noreply, state} + end + + # Private helpers + + defp handle_post_event(post) do + group_slug = Map.get(post, :group, "") + post_slug = extract_post_slug(post) + + if published?(post) do + changeset = + GenerateLLMTextJob.enqueue_for_file(:publishing, "#{group_slug}/#{post_slug}.txt") + + insert_job(changeset) + else + path = PublishingSource.build_file_path(group_slug, post_slug) + FileStorage.delete(path) + enqueue_source_rebuild(group_slug) + end + rescue + error -> + Logger.warning("LLMText PublishingSubscriber: handle_post_event failed: #{inspect(error)}") + end + + defp handle_post_removed(post_identifier) do + # post_identifier may be a string slug or map + {group_slug, post_slug} = + case post_identifier do + %{group: g, slug: s} -> {to_string(g), to_string(s)} + %{"group" => g, "slug" => s} -> {to_string(g), to_string(s)} + _ -> {"unknown", to_string(post_identifier)} + end + + path = PublishingSource.build_file_path(group_slug, post_slug) + FileStorage.delete(path) + enqueue_source_rebuild(group_slug) + rescue + error -> + Logger.warning( + "LLMText PublishingSubscriber: handle_post_removed failed: #{inspect(error)}" + ) + end + + defp enqueue_source_rebuild(_group_slug) do + changeset = GenerateLLMTextJob.enqueue_for_source(:publishing) + insert_job(changeset) + end + + defp insert_job(changeset) do + Oban.insert(changeset) + rescue + error -> + Logger.warning("LLMText PublishingSubscriber: failed to insert job: #{inspect(error)}") + {:error, error} + end + + defp subscribe_to_groups do + if publishing_pubsub_available?() do + topic = @publishing_pubsub.groups_topic() + PubSubManager.subscribe(topic) + end + rescue + _ -> :ok + end + + defp subscribe_to_existing_groups do + if publishing_available?() do + groups = @publishing_mod.list_groups() + + Enum.each(groups, fn group -> + subscribe_to_group(group["slug"]) + end) + end + rescue + _ -> :ok + end + + defp subscribe_to_group(group_slug) when is_binary(group_slug) do + if publishing_pubsub_available?() do + topic = @publishing_pubsub.posts_topic(group_slug) + PubSubManager.subscribe(topic) + end + rescue + _ -> :ok + end + + defp publishing_available? do + Code.ensure_loaded?(@publishing_mod) and + function_exported?(@publishing_mod, :list_groups, 0) + end + + defp publishing_pubsub_available? do + Code.ensure_loaded?(@publishing_pubsub) and + function_exported?(@publishing_pubsub, :groups_topic, 0) + end + + defp published?(post) do + case post do + %{metadata: %{status: "published"}} -> true + %{metadata: %{"status" => "published"}} -> true + _ -> false + end + end + + defp extract_post_slug(post) do + case Map.get(post, :mode) do + :timestamp -> + date = Map.get(post, :date) + time = Map.get(post, :time) + + if date && time do + time_str = time |> Time.to_string() |> String.slice(0..4) |> String.replace(":", "-") + "#{Date.to_iso8601(date)}-#{time_str}" + else + Map.get(post, :slug, "post") || "post" + end + + _ -> + Map.get(post, :url_slug) || Map.get(post, :slug, "post") || "post" + end + end +end diff --git a/lib/modules/llm_text/sources/publishing.ex b/lib/modules/llm_text/sources/publishing.ex new file mode 100644 index 000000000..8323aa213 --- /dev/null +++ b/lib/modules/llm_text/sources/publishing.ex @@ -0,0 +1,240 @@ +defmodule PhoenixKit.Modules.LLMText.Sources.Publishing do + @moduledoc """ + LLM text source for PhoenixKit Publishing module. + + Generates: + - Index entries (one per published post) for llms.txt + - Individual `.txt` files per published post at `{group_slug}/{post_slug}.txt` + + Only active when the Publishing module is loaded and enabled. + """ + + @behaviour PhoenixKit.Modules.LLMText.Sources.Source + + @compile {:no_warn_undefined, + [ + {PhoenixKit.Modules.Publishing, :enabled?, 0}, + {PhoenixKit.Modules.Publishing, :list_groups, 0}, + {PhoenixKit.Modules.Publishing, :list_posts, 2} + ]} + + require Logger + + alias PhoenixKit.Utils.Routes + + @publishing_mod PhoenixKit.Modules.Publishing + + @impl true + def source_name, do: :publishing + + @impl true + def enabled? do + Code.ensure_loaded?(@publishing_mod) and + function_exported?(@publishing_mod, :enabled?, 0) and + @publishing_mod.enabled?() + rescue + _ -> false + end + + @impl true + def collect_index_entries do + if enabled?() do + language = get_default_language() + groups = @publishing_mod.list_groups() + + Enum.flat_map(groups, fn group -> + group_slug = group["slug"] + group_name = group["name"] + + group_slug + |> @publishing_mod.list_posts(language) + |> Enum.filter(&published?/1) + |> Enum.map(fn post -> + %{ + title: get_title(post), + url: build_post_url(post, group_slug), + description: extract_description(post), + group: group_name + } + end) + end) + else + [] + end + rescue + error -> + Logger.warning( + "LLMText PublishingSource failed to collect index entries: #{inspect(error)}" + ) + + [] + end + + @impl true + def collect_page_files do + if enabled?() do + language = get_default_language() + groups = @publishing_mod.list_groups() + + Enum.flat_map(groups, fn group -> + group_slug = group["slug"] + + group_slug + |> @publishing_mod.list_posts(language) + |> Enum.filter(&published?/1) + |> Enum.map(fn post -> + path = build_file_path(group_slug, get_post_slug(post)) + content = build_post_content(post, group_slug, get_title(post)) + {path, content} + end) + end) + else + [] + end + rescue + error -> + Logger.warning("LLMText PublishingSource failed to collect page files: #{inspect(error)}") + + [] + end + + @doc """ + Builds the file path for a post's LLM text file. + + iex> build_file_path("blog", "hello-world") + "blog/hello-world.txt" + """ + @spec build_file_path(String.t(), String.t()) :: String.t() + def build_file_path(group_slug, post_slug) do + "#{group_slug}/#{post_slug}.txt" + end + + @doc """ + Builds markdown content for a post's LLM text file. + """ + @spec build_post_content(map(), String.t(), String.t()) :: String.t() + def build_post_content(post, group_slug, title) do + url = build_post_url(post, group_slug) + description = extract_description(post) + body = Map.get(post, :content, "") + + url_line = if url != "", do: "> Source: #{url}\n\n", else: "" + desc_line = if description != "", do: "#{description}\n\n", else: "" + body_part = if is_binary(body), do: body, else: "" + + "# #{title}\n\n#{url_line}#{desc_line}#{body_part}" + end + + @doc """ + Extracts a description for a post. + + Uses metadata.description if available, otherwise falls back to the first + 160 characters of the post content. + """ + @spec extract_description(map()) :: String.t() + def extract_description(post) do + case post do + %{metadata: %{description: desc}} when is_binary(desc) and desc != "" -> + desc + + %{metadata: %{"description" => desc}} when is_binary(desc) and desc != "" -> + desc + + _ -> + content = Map.get(post, :content, "") + + if is_binary(content) and content != "" do + content + |> String.replace(~r/\s+/, " ") + |> String.trim() + |> String.slice(0, 160) + else + "" + end + end + end + + # Private helpers + + defp published?(post) do + case post do + %{metadata: %{status: "published"}} -> true + %{metadata: %{"status" => "published"}} -> true + _ -> false + end + end + + defp get_title(post) do + case post do + %{metadata: %{title: title}} when is_binary(title) and title != "" -> title + %{metadata: %{"title" => title}} when is_binary(title) and title != "" -> title + %{slug: slug} when is_binary(slug) -> format_slug(slug) + _ -> "Post" + end + end + + defp get_post_slug(post) do + case Map.get(post, :mode) do + :timestamp -> + date = Map.get(post, :date) + time = Map.get(post, :time) + + if date && time do + time_str = time |> Time.to_string() |> String.slice(0..4) |> String.replace(":", "-") + "#{Date.to_iso8601(date)}-#{time_str}" + else + Map.get(post, :slug, "post") || "post" + end + + _ -> + Map.get(post, :url_slug) || Map.get(post, :slug, "post") || "post" + end + end + + defp build_post_url(post, group_slug) do + site_url = get_site_url() + prefix = get_url_prefix() + post_slug = get_post_slug(post) + + path_parts = + [prefix, group_slug, post_slug] + |> Enum.reject(&(&1 in [nil, "", "/"])) + + path = "/" <> Enum.join(path_parts, "/") + + if site_url != "" do + String.trim_trailing(site_url, "/") <> path + else + path + end + end + + defp get_default_language do + PhoenixKit.Settings.get_content_language() + rescue + _ -> nil + end + + defp get_site_url do + PhoenixKit.Settings.get_setting("site_url", "") + rescue + _ -> "" + end + + defp get_url_prefix do + case Routes.url_prefix() do + "/" -> "" + prefix -> String.trim(prefix, "/") + end + rescue + _ -> "" + end + + defp format_slug(slug) do + slug + |> String.replace("-", " ") + |> String.replace("_", " ") + |> String.split() + |> Enum.map_join(" ", &String.capitalize/1) + end +end diff --git a/lib/modules/llm_text/sources/source.ex b/lib/modules/llm_text/sources/source.ex new file mode 100644 index 000000000..5e9490317 --- /dev/null +++ b/lib/modules/llm_text/sources/source.ex @@ -0,0 +1,112 @@ +defmodule PhoenixKit.Modules.LLMText.Sources.Source do + @moduledoc """ + Behaviour for LLM text data sources. + + Each source module must implement this behaviour to provide content + for LLM-friendly text files (llms.txt index and individual page files). + + ## Required Callbacks + + - `source_name/0` - Unique atom identifier for the source + - `enabled?/0` - Whether this source is active + - `collect_index_entries/0` - Collect index entries for llms.txt + - `collect_page_files/0` - Collect individual page files + + ## Index Entry Format + + Each index entry is a map with: + - `:title` - Page title (string) + - `:url` - Full URL to the page (string) + - `:description` - Brief description (string) + - `:group` - Group name for organizing entries (string) + """ + + require Logger + + @type index_entry :: %{ + title: String.t(), + url: String.t(), + description: String.t(), + group: String.t() + } + + @doc """ + Returns the unique name/identifier for this source. + """ + @callback source_name() :: atom() + + @doc """ + Checks if this source is enabled and should be included. + """ + @callback enabled?() :: boolean() + + @doc """ + Collects index entries for llms.txt from this source. + """ + @callback collect_index_entries() :: [index_entry()] + + @doc """ + Collects individual page files from this source. + + Returns a list of `{relative_path, content}` tuples. + The relative_path is relative to the llms storage directory. + """ + @callback collect_page_files() :: [{path :: String.t(), content :: String.t()}] + + @doc """ + Checks if a source module implements all required callbacks. + """ + @spec valid_source?(module()) :: boolean() + def valid_source?(module) when is_atom(module) do + case Code.ensure_loaded(module) do + {:module, _} -> + function_exported?(module, :source_name, 0) and + function_exported?(module, :enabled?, 0) and + function_exported?(module, :collect_index_entries, 0) and + function_exported?(module, :collect_page_files, 0) + + {:error, _} -> + false + end + end + + def valid_source?(_), do: false + + @doc """ + Safely collects index entries from a source, returning [] if disabled or on error. + """ + @spec safe_collect_index_entries(module()) :: [index_entry()] + def safe_collect_index_entries(source_module) do + if valid_source?(source_module) and source_module.enabled?() do + source_module.collect_index_entries() + else + [] + end + rescue + error -> + Logger.warning( + "LLMText source #{inspect(source_module)} failed to collect index entries: #{inspect(error)}" + ) + + [] + end + + @doc """ + Safely collects page files from a source, returning [] if disabled or on error. + """ + @spec safe_collect_page_files(module()) :: [{String.t(), String.t()}] + def safe_collect_page_files(source_module) do + if valid_source?(source_module) and source_module.enabled?() do + source_module.collect_page_files() + else + [] + end + rescue + error -> + Logger.warning( + "LLMText source #{inspect(source_module)} failed to collect page files: #{inspect(error)}" + ) + + [] + end +end diff --git a/lib/modules/llm_text/web/controller.ex b/lib/modules/llm_text/web/controller.ex new file mode 100644 index 000000000..56c6bd5f3 --- /dev/null +++ b/lib/modules/llm_text/web/controller.ex @@ -0,0 +1,82 @@ +defmodule PhoenixKit.Modules.LLMText.Web.Controller do + @moduledoc """ + Controller for serving LLM-friendly text files. + + ## Endpoints + + - GET /{prefix}/llms.txt — serves the index file listing all LLM-readable pages + - GET /{prefix}/llms/*path — serves individual LLM text files + + Files are served directly from `priv/static/llms/` with `text/plain` content type. + Returns 404 for files that do not exist. + """ + + use PhoenixKitWeb, :controller + + require Logger + + alias PhoenixKit.Modules.LLMText.FileStorage + + @doc """ + Serves the llms.txt index file. + """ + def index(conn, _params) do + path = FileStorage.index_path() + + case File.read(path) do + {:ok, content} -> + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(200, content) + + {:error, :enoent} -> + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(404, "Not found") + + {:error, reason} -> + Logger.warning("LLMText controller: failed to read llms.txt: #{inspect(reason)}") + + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(500, "Internal error") + end + end + + @doc """ + Serves an individual LLM text file at `/llms/*path`. + """ + def show(conn, %{"path" => path_parts}) when is_list(path_parts) do + relative_path = Path.join(path_parts) + + # Prevent path traversal + if String.contains?(relative_path, "..") do + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(400, "Bad request") + else + full_path = FileStorage.file_path(relative_path) + + case File.read(full_path) do + {:ok, content} -> + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(200, content) + + {:error, :enoent} -> + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(404, "Not found") + + {:error, reason} -> + Logger.warning( + "LLMText controller: failed to read #{relative_path}: #{inspect(reason)}" + ) + + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(500, "Internal error") + end + end + end +end diff --git a/lib/modules/llm_text/web/settings.ex b/lib/modules/llm_text/web/settings.ex new file mode 100644 index 000000000..2b556f021 --- /dev/null +++ b/lib/modules/llm_text/web/settings.ex @@ -0,0 +1,29 @@ +defmodule PhoenixKit.Modules.LLMText.Web.Settings do + @moduledoc """ + Admin settings LiveView for the LLM Text module. + + Placeholder — full UI to be implemented in a future iteration. + """ + + use PhoenixKitWeb, :live_view + + alias PhoenixKit.Modules.LLMText + + @impl true + def mount(_params, _session, socket) do + {:ok, assign(socket, page_title: "LLM Text Settings", enabled: LLMText.enabled?())} + end + + @impl true + def render(assigns) do + ~H""" +
LLM Text settings coming soon.
++ Module enabled: {@enabled} +
+