diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs index 320aeacda..5762e4951 100644 --- a/.dialyzer_ignore.exs +++ b/.dialyzer_ignore.exs @@ -142,5 +142,9 @@ # ExUnit internal functions — false positives when test/support is compiled in MIX_ENV=test # Dialyzer cannot resolve ExUnit private macros expanded at compile time {"test/support/conn_case.ex", :unknown_function}, - {"test/support/data_case.ex", :unknown_function} + {"test/support/data_case.ex", :unknown_function}, + + # LLMText module — Publishing is an external package, guarded by Code.ensure_loaded? + {"lib/modules/llm_text/sources/publishing.ex", :unknown_function}, + {"lib/modules/llm_text/publishing_subscriber.ex", :unknown_function} ] diff --git a/lib/modules/llm_text/file_storage.ex b/lib/modules/llm_text/file_storage.ex new file mode 100644 index 000000000..149405c73 --- /dev/null +++ b/lib/modules/llm_text/file_storage.ex @@ -0,0 +1,170 @@ +defmodule PhoenixKit.Modules.LLMText.FileStorage do + @moduledoc """ + File-based storage for LLM text files. + + Stores files under the host app's `priv/static/llms/` directory: + + priv/static/llms/llms.txt -> index file + priv/static/llms/page.md -> individual page file + priv/static/llms/posts/article.md -> nested page file + + ## Test Override + + Set `Application.put_env(:phoenix_kit, :llm_text_test_storage_dir, "/tmp/...")` to + override the storage directory in tests. + """ + + require Logger + + @index_filename "llms.txt" + + @doc """ + Returns the root storage directory for LLM text files. + + Uses test override if configured, otherwise resolves to the host app's + `priv/static/llms/` directory. + """ + @spec storage_dir() :: String.t() + def storage_dir do + case Application.get_env(:phoenix_kit, :llm_text_test_storage_dir) do + nil -> resolve_storage_dir() + dir -> dir + end + end + + @doc """ + Returns the path to the llms.txt index file. + """ + @spec index_path() :: String.t() + def index_path do + Path.join(storage_dir(), @index_filename) + end + + @doc """ + Returns the full path for a relative file path within the storage directory. + """ + @spec file_path(String.t()) :: String.t() + def file_path(relative_path) when is_binary(relative_path) do + Path.join(storage_dir(), relative_path) + end + + @doc """ + Writes content to a file at the given relative path, creating directories as needed. + """ + @spec write(String.t(), String.t()) :: :ok | {:error, term()} + def write(relative_path, content) when is_binary(relative_path) and is_binary(content) do + path = file_path(relative_path) + + with :ok <- ensure_directory_exists(path), + :ok <- File.write(path, content) do + Logger.debug("LLMText.FileStorage: Wrote #{relative_path} (#{byte_size(content)} bytes)") + :ok + else + {:error, reason} = error -> + Logger.warning( + "LLMText.FileStorage: Failed to write #{relative_path}: #{inspect(reason)}" + ) + + error + end + end + + @doc """ + Writes the llms.txt index file. + """ + @spec write_index(String.t()) :: :ok | {:error, term()} + def write_index(content) when is_binary(content) do + path = index_path() + + with :ok <- ensure_directory_exists(path), + :ok <- File.write(path, content) do + Logger.debug("LLMText.FileStorage: Wrote llms.txt (#{byte_size(content)} bytes)") + :ok + else + {:error, reason} = error -> + Logger.warning("LLMText.FileStorage: Failed to write llms.txt: #{inspect(reason)}") + error + end + end + + @doc """ + Deletes a file at the given relative path. Returns :ok if file does not exist. + """ + @spec delete(String.t()) :: :ok + def delete(relative_path) when is_binary(relative_path) do + path = file_path(relative_path) + + case File.rm(path) do + :ok -> + Logger.debug("LLMText.FileStorage: Deleted #{relative_path}") + :ok + + {:error, :enoent} -> + :ok + + {:error, reason} -> + Logger.warning( + "LLMText.FileStorage: Failed to delete #{relative_path}: #{inspect(reason)}" + ) + + :ok + end + end + + @doc """ + Checks if a file at the given relative path exists. + """ + @spec exists?(String.t()) :: boolean() + def exists?(relative_path) when is_binary(relative_path) do + relative_path |> file_path() |> File.exists?() + end + + @doc """ + Deletes the entire storage directory and all its contents. + """ + @spec delete_all() :: :ok + def delete_all do + dir = storage_dir() + + case File.rm_rf(dir) do + {:ok, _} -> + Logger.debug("LLMText.FileStorage: Deleted all files in #{dir}") + :ok + + {:error, reason, _path} -> + Logger.warning("LLMText.FileStorage: Failed to delete storage dir: #{inspect(reason)}") + :ok + end + rescue + _ -> :ok + end + + # Private helpers + + defp resolve_storage_dir do + otp_app = PhoenixKit.Config.get(:otp_app, :phoenix_kit) + + priv_dir = + case :code.priv_dir(otp_app) do + {:error, :bad_name} -> + case :code.priv_dir(:phoenix_kit) do + {:error, :bad_name} -> "priv" + dir -> to_string(dir) + end + + dir -> + to_string(dir) + end + + Path.join([priv_dir, "static", "llms"]) + end + + defp ensure_directory_exists(file_path) do + dir = Path.dirname(file_path) + + case File.mkdir_p(dir) do + :ok -> :ok + {:error, reason} -> {:error, {:mkdir_failed, reason}} + end + end +end diff --git a/lib/modules/llm_text/generator.ex b/lib/modules/llm_text/generator.ex new file mode 100644 index 000000000..d14405143 --- /dev/null +++ b/lib/modules/llm_text/generator.ex @@ -0,0 +1,173 @@ +defmodule PhoenixKit.Modules.LLMText.Generator do + @moduledoc """ + Generator for LLM-friendly text files. + + Produces: + - `llms.txt` - Index file linking to all LLM-readable pages + - Individual page `.md` files per source + + ## Usage + + # Regenerate all sources and rebuild index + Generator.run_all() + + # Regenerate a single source and rebuild index + Generator.run_source(MyApp.LLMText.BlogSource) + + # Only rebuild the index from all sources + Generator.rebuild_index() + """ + + require Logger + + alias PhoenixKit.Modules.LLMText.FileStorage + alias PhoenixKit.Modules.LLMText.Sources.Source + + @doc """ + Regenerates files for one source and rebuilds the llms.txt index. + """ + @spec run_source(module()) :: :ok + def run_source(source_module) do + Logger.info("LLMText.Generator: Running source #{inspect(source_module)}") + + files = Source.safe_collect_page_files(source_module) + + Enum.each(files, fn {path, content} -> + case FileStorage.write(path, content) do + :ok -> + :ok + + {:error, reason} -> + Logger.warning("LLMText.Generator: Failed to write #{path}: #{inspect(reason)}") + end + end) + + rebuild_index() + end + + @doc """ + Regenerates all sources and rebuilds the llms.txt index. + """ + @spec run_all() :: :ok + def run_all do + sources = get_sources() + Logger.info("LLMText.Generator: Running all #{length(sources)} sources") + + Enum.each(sources, fn source_module -> + files = Source.safe_collect_page_files(source_module) + + Enum.each(files, fn {path, content} -> + case FileStorage.write(path, content) do + :ok -> + :ok + + {:error, reason} -> + Logger.warning("LLMText.Generator: Failed to write #{path}: #{inspect(reason)}") + end + end) + end) + + rebuild_index() + end + + @doc """ + Rebuilds the llms.txt index from all sources without regenerating page files. + """ + @spec rebuild_index() :: :ok + def rebuild_index do + sources = get_sources() + Logger.debug("LLMText.Generator: Rebuilding index from #{length(sources)} sources") + + entries = + sources + |> Enum.flat_map(&Source.safe_collect_index_entries/1) + + content = build_index_content(entries) + + case FileStorage.write_index(content) do + :ok -> + :ok + + {:error, reason} -> + Logger.warning("LLMText.Generator: Failed to write index: #{inspect(reason)}") + :ok + end + end + + @doc """ + Builds the llms.txt markdown content from a list of index entries. + + Groups entries by their `:group` field. Group order follows first-seen order. + Within each group, entries appear in the order they were provided. + """ + @spec build_index_content([Source.index_entry()]) :: String.t() + def build_index_content(entries) do + site_name = get_site_name() + site_description = get_site_description() + + # Build ordered groups (first-seen order) using prepend + reverse for O(n) performance + {groups_reversed, groups_map} = + Enum.reduce(entries, {[], %{}}, fn entry, {order, map} -> + group = Map.get(entry, :group, "General") + + if Map.has_key?(map, group) do + {order, Map.update!(map, group, &[entry | &1])} + else + {[group | order], Map.put(map, group, [entry])} + end + end) + + groups_ordered = Enum.reverse(groups_reversed) + + header = + if site_description && site_description != "" do + "# #{site_name}\n\n> #{site_description}\n\n" + else + "# #{site_name}\n\n" + end + + sections = + Enum.map_join(groups_ordered, "\n\n", fn group -> + group_entries = Map.get(groups_map, group, []) |> Enum.reverse() + + links = + Enum.map_join(group_entries, "\n", fn entry -> + title = Map.get(entry, :title, "") + url = Map.get(entry, :url, "") + description = Map.get(entry, :description, "") + + if description && description != "" do + "- [#{title}](#{url}): #{description}" + else + "- [#{title}](#{url})" + end + end) + + "## #{group}\n\n#{links}" + end) + + header <> sections + end + + @doc """ + Returns the configured LLM text sources. + """ + @spec get_sources() :: [module()] + def get_sources do + Application.get_env(:phoenix_kit, :llm_text_sources, []) + end + + # Private helpers + + defp get_site_name do + PhoenixKit.Settings.get_setting("site_name", "Site") + rescue + _ -> "Site" + end + + defp get_site_description do + PhoenixKit.Settings.get_setting("site_description", "") + rescue + _ -> "" + end +end diff --git a/lib/modules/llm_text/llm_text.ex b/lib/modules/llm_text/llm_text.ex new file mode 100644 index 000000000..26d641cc3 --- /dev/null +++ b/lib/modules/llm_text/llm_text.ex @@ -0,0 +1,114 @@ +defmodule PhoenixKit.Modules.LLMText do + @moduledoc """ + LLM Text module for PhoenixKit. + + Generates LLM-friendly text files (llms.txt + per-page .txt files) + from configured sources, served at `/llms.txt` and `/llms/*path`. + + ## Settings keys + + - `llm_text_enabled` — enable/disable the module (boolean, default: false) + + ## Configuration + + Configure sources in the host app: + + config :phoenix_kit, :llm_text_sources, [ + PhoenixKit.Modules.LLMText.Sources.Publishing + ] + + ## Usage + + # Check if enabled + PhoenixKit.Modules.LLMText.enabled?() + + # Regenerate all files + PhoenixKit.Modules.LLMText.Generator.run_all() + """ + + use PhoenixKit.Module + + alias PhoenixKit.Dashboard.Tab + alias PhoenixKit.Modules.LLMText.Generator + alias PhoenixKit.Modules.LLMText.PublishingSubscriber + + @enabled_key "llm_text_enabled" + + # ── Required Module callbacks ────────────────────────────────────── + + @impl PhoenixKit.Module + def module_key, do: "llm_text" + + @impl PhoenixKit.Module + def module_name, do: "LLM Text" + + @impl PhoenixKit.Module + def enabled? do + settings_call(:get_boolean_setting, [@enabled_key, false]) + end + + @impl PhoenixKit.Module + def enable_system do + settings_call(:update_boolean_setting, [@enabled_key, true]) + end + + @impl PhoenixKit.Module + def disable_system do + settings_call(:update_boolean_setting, [@enabled_key, false]) + end + + # ── Optional Module callbacks ────────────────────────────────────── + + @impl PhoenixKit.Module + def permission_metadata do + %{ + key: "llm_text", + label: "LLM Text", + icon: "hero-document-text", + description: "Generate LLM-friendly text files (llms.txt) for AI consumption" + } + end + + @impl PhoenixKit.Module + def get_config do + %{ + enabled: enabled?(), + sources: Generator.get_sources() + } + end + + @impl PhoenixKit.Module + def settings_tabs do + [ + Tab.new!( + id: :admin_settings_llm_text, + label: "LLM Text", + icon: "hero-document-text", + path: "llm-text", + priority: 935, + level: :admin, + parent: :admin_settings, + permission: "llm_text" + ) + ] + end + + @impl PhoenixKit.Module + def children do + if enabled?() do + [PublishingSubscriber] + else + [] + end + end + + # ── Private helpers ──────────────────────────────────────────────── + + defp settings_module do + PhoenixKit.Config.get(:llm_text_settings_module, PhoenixKit.Settings) + end + + defp settings_call(fun, args) do + apply(settings_module(), fun, args) + end +end diff --git a/lib/modules/llm_text/publishing_subscriber.ex b/lib/modules/llm_text/publishing_subscriber.ex new file mode 100644 index 000000000..702d4f04e --- /dev/null +++ b/lib/modules/llm_text/publishing_subscriber.ex @@ -0,0 +1,205 @@ +defmodule PhoenixKit.Modules.LLMText.PublishingSubscriber do + @moduledoc """ + GenServer that subscribes to Publishing PubSub events and triggers + incremental LLM text file regeneration. + + ## Events handled + + - `{:post_status_changed, post}` — if published: enqueue file job; otherwise delete file + - `{:post_updated, post}` — if published: enqueue file job; otherwise skip + - `{:post_deleted, post_identifier}` — delete file + enqueue source rebuild + - `{:group_created, group}` — subscribe to new group's posts topic + - `{:group_deleted, group_slug}` — enqueue source rebuild + + ## Topics + + Subscribes to: + - `"publishing:groups"` — group lifecycle events + - `"publishing:{group_slug}:posts"` — per-group post events for each existing group + + All Publishing calls are guarded with `Code.ensure_loaded?`. + """ + + use GenServer + + require Logger + + alias PhoenixKit.Modules.LLMText.FileStorage + alias PhoenixKit.Modules.LLMText.Sources.Publishing, as: PublishingSource + alias PhoenixKit.Modules.LLMText.Workers.GenerateLLMTextJob + alias PhoenixKit.PubSub.Manager, as: PubSubManager + + @compile {:no_warn_undefined, + [ + {PhoenixKit.Modules.Publishing, :enabled?, 0}, + {PhoenixKit.Modules.Publishing, :list_groups, 0}, + {PhoenixKit.Modules.Publishing.PubSub, :groups_topic, 0}, + {PhoenixKit.Modules.Publishing.PubSub, :posts_topic, 1} + ]} + + @publishing_mod PhoenixKit.Modules.Publishing + @publishing_pubsub PhoenixKit.Modules.Publishing.PubSub + + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @impl GenServer + def init(_opts) do + subscribe_to_groups() + subscribe_to_existing_groups() + {:ok, %{}} + end + + @impl GenServer + def handle_info({:post_status_changed, post}, state) do + handle_post_event(post) + {:noreply, state} + end + + def handle_info({:post_updated, post}, state) do + if published?(post) do + handle_post_event(post) + end + + {:noreply, state} + end + + def handle_info({:post_deleted, post_identifier}, state) do + handle_post_removed(post_identifier) + {:noreply, state} + end + + def handle_info({:group_created, group}, state) do + subscribe_to_group(group["slug"]) + {:noreply, state} + end + + def handle_info({:group_deleted, group_slug}, state) do + enqueue_source_rebuild(group_slug) + {:noreply, state} + end + + def handle_info(_msg, state) do + {:noreply, state} + end + + # Private helpers + + defp handle_post_event(post) do + group_slug = Map.get(post, :group, "") + post_slug = extract_post_slug(post) + + if published?(post) do + changeset = + GenerateLLMTextJob.enqueue_for_file(:publishing, "#{group_slug}/#{post_slug}.txt") + + insert_job(changeset) + else + path = PublishingSource.build_file_path(group_slug, post_slug) + FileStorage.delete(path) + enqueue_source_rebuild(group_slug) + end + rescue + error -> + Logger.warning("LLMText PublishingSubscriber: handle_post_event failed: #{inspect(error)}") + end + + defp handle_post_removed(post_identifier) do + # post_identifier may be a string slug or map + {group_slug, post_slug} = + case post_identifier do + %{group: g, slug: s} -> {to_string(g), to_string(s)} + %{"group" => g, "slug" => s} -> {to_string(g), to_string(s)} + _ -> {"unknown", to_string(post_identifier)} + end + + path = PublishingSource.build_file_path(group_slug, post_slug) + FileStorage.delete(path) + enqueue_source_rebuild(group_slug) + rescue + error -> + Logger.warning( + "LLMText PublishingSubscriber: handle_post_removed failed: #{inspect(error)}" + ) + end + + defp enqueue_source_rebuild(_group_slug) do + changeset = GenerateLLMTextJob.enqueue_for_source(:publishing) + insert_job(changeset) + end + + defp insert_job(changeset) do + Oban.insert(changeset) + rescue + error -> + Logger.warning("LLMText PublishingSubscriber: failed to insert job: #{inspect(error)}") + {:error, error} + end + + defp subscribe_to_groups do + if publishing_pubsub_available?() do + topic = @publishing_pubsub.groups_topic() + PubSubManager.subscribe(topic) + end + rescue + _ -> :ok + end + + defp subscribe_to_existing_groups do + if publishing_available?() do + groups = @publishing_mod.list_groups() + + Enum.each(groups, fn group -> + subscribe_to_group(group["slug"]) + end) + end + rescue + _ -> :ok + end + + defp subscribe_to_group(group_slug) when is_binary(group_slug) do + if publishing_pubsub_available?() do + topic = @publishing_pubsub.posts_topic(group_slug) + PubSubManager.subscribe(topic) + end + rescue + _ -> :ok + end + + defp publishing_available? do + Code.ensure_loaded?(@publishing_mod) and + function_exported?(@publishing_mod, :list_groups, 0) + end + + defp publishing_pubsub_available? do + Code.ensure_loaded?(@publishing_pubsub) and + function_exported?(@publishing_pubsub, :groups_topic, 0) + end + + defp published?(post) do + case post do + %{metadata: %{status: "published"}} -> true + %{metadata: %{"status" => "published"}} -> true + _ -> false + end + end + + defp extract_post_slug(post) do + case Map.get(post, :mode) do + :timestamp -> + date = Map.get(post, :date) + time = Map.get(post, :time) + + if date && time do + time_str = time |> Time.to_string() |> String.slice(0..4) |> String.replace(":", "-") + "#{Date.to_iso8601(date)}-#{time_str}" + else + Map.get(post, :slug, "post") || "post" + end + + _ -> + Map.get(post, :url_slug) || Map.get(post, :slug, "post") || "post" + end + end +end diff --git a/lib/modules/llm_text/sources/publishing.ex b/lib/modules/llm_text/sources/publishing.ex new file mode 100644 index 000000000..8323aa213 --- /dev/null +++ b/lib/modules/llm_text/sources/publishing.ex @@ -0,0 +1,240 @@ +defmodule PhoenixKit.Modules.LLMText.Sources.Publishing do + @moduledoc """ + LLM text source for PhoenixKit Publishing module. + + Generates: + - Index entries (one per published post) for llms.txt + - Individual `.txt` files per published post at `{group_slug}/{post_slug}.txt` + + Only active when the Publishing module is loaded and enabled. + """ + + @behaviour PhoenixKit.Modules.LLMText.Sources.Source + + @compile {:no_warn_undefined, + [ + {PhoenixKit.Modules.Publishing, :enabled?, 0}, + {PhoenixKit.Modules.Publishing, :list_groups, 0}, + {PhoenixKit.Modules.Publishing, :list_posts, 2} + ]} + + require Logger + + alias PhoenixKit.Utils.Routes + + @publishing_mod PhoenixKit.Modules.Publishing + + @impl true + def source_name, do: :publishing + + @impl true + def enabled? do + Code.ensure_loaded?(@publishing_mod) and + function_exported?(@publishing_mod, :enabled?, 0) and + @publishing_mod.enabled?() + rescue + _ -> false + end + + @impl true + def collect_index_entries do + if enabled?() do + language = get_default_language() + groups = @publishing_mod.list_groups() + + Enum.flat_map(groups, fn group -> + group_slug = group["slug"] + group_name = group["name"] + + group_slug + |> @publishing_mod.list_posts(language) + |> Enum.filter(&published?/1) + |> Enum.map(fn post -> + %{ + title: get_title(post), + url: build_post_url(post, group_slug), + description: extract_description(post), + group: group_name + } + end) + end) + else + [] + end + rescue + error -> + Logger.warning( + "LLMText PublishingSource failed to collect index entries: #{inspect(error)}" + ) + + [] + end + + @impl true + def collect_page_files do + if enabled?() do + language = get_default_language() + groups = @publishing_mod.list_groups() + + Enum.flat_map(groups, fn group -> + group_slug = group["slug"] + + group_slug + |> @publishing_mod.list_posts(language) + |> Enum.filter(&published?/1) + |> Enum.map(fn post -> + path = build_file_path(group_slug, get_post_slug(post)) + content = build_post_content(post, group_slug, get_title(post)) + {path, content} + end) + end) + else + [] + end + rescue + error -> + Logger.warning("LLMText PublishingSource failed to collect page files: #{inspect(error)}") + + [] + end + + @doc """ + Builds the file path for a post's LLM text file. + + iex> build_file_path("blog", "hello-world") + "blog/hello-world.txt" + """ + @spec build_file_path(String.t(), String.t()) :: String.t() + def build_file_path(group_slug, post_slug) do + "#{group_slug}/#{post_slug}.txt" + end + + @doc """ + Builds markdown content for a post's LLM text file. + """ + @spec build_post_content(map(), String.t(), String.t()) :: String.t() + def build_post_content(post, group_slug, title) do + url = build_post_url(post, group_slug) + description = extract_description(post) + body = Map.get(post, :content, "") + + url_line = if url != "", do: "> Source: #{url}\n\n", else: "" + desc_line = if description != "", do: "#{description}\n\n", else: "" + body_part = if is_binary(body), do: body, else: "" + + "# #{title}\n\n#{url_line}#{desc_line}#{body_part}" + end + + @doc """ + Extracts a description for a post. + + Uses metadata.description if available, otherwise falls back to the first + 160 characters of the post content. + """ + @spec extract_description(map()) :: String.t() + def extract_description(post) do + case post do + %{metadata: %{description: desc}} when is_binary(desc) and desc != "" -> + desc + + %{metadata: %{"description" => desc}} when is_binary(desc) and desc != "" -> + desc + + _ -> + content = Map.get(post, :content, "") + + if is_binary(content) and content != "" do + content + |> String.replace(~r/\s+/, " ") + |> String.trim() + |> String.slice(0, 160) + else + "" + end + end + end + + # Private helpers + + defp published?(post) do + case post do + %{metadata: %{status: "published"}} -> true + %{metadata: %{"status" => "published"}} -> true + _ -> false + end + end + + defp get_title(post) do + case post do + %{metadata: %{title: title}} when is_binary(title) and title != "" -> title + %{metadata: %{"title" => title}} when is_binary(title) and title != "" -> title + %{slug: slug} when is_binary(slug) -> format_slug(slug) + _ -> "Post" + end + end + + defp get_post_slug(post) do + case Map.get(post, :mode) do + :timestamp -> + date = Map.get(post, :date) + time = Map.get(post, :time) + + if date && time do + time_str = time |> Time.to_string() |> String.slice(0..4) |> String.replace(":", "-") + "#{Date.to_iso8601(date)}-#{time_str}" + else + Map.get(post, :slug, "post") || "post" + end + + _ -> + Map.get(post, :url_slug) || Map.get(post, :slug, "post") || "post" + end + end + + defp build_post_url(post, group_slug) do + site_url = get_site_url() + prefix = get_url_prefix() + post_slug = get_post_slug(post) + + path_parts = + [prefix, group_slug, post_slug] + |> Enum.reject(&(&1 in [nil, "", "/"])) + + path = "/" <> Enum.join(path_parts, "/") + + if site_url != "" do + String.trim_trailing(site_url, "/") <> path + else + path + end + end + + defp get_default_language do + PhoenixKit.Settings.get_content_language() + rescue + _ -> nil + end + + defp get_site_url do + PhoenixKit.Settings.get_setting("site_url", "") + rescue + _ -> "" + end + + defp get_url_prefix do + case Routes.url_prefix() do + "/" -> "" + prefix -> String.trim(prefix, "/") + end + rescue + _ -> "" + end + + defp format_slug(slug) do + slug + |> String.replace("-", " ") + |> String.replace("_", " ") + |> String.split() + |> Enum.map_join(" ", &String.capitalize/1) + end +end diff --git a/lib/modules/llm_text/sources/source.ex b/lib/modules/llm_text/sources/source.ex new file mode 100644 index 000000000..5e9490317 --- /dev/null +++ b/lib/modules/llm_text/sources/source.ex @@ -0,0 +1,112 @@ +defmodule PhoenixKit.Modules.LLMText.Sources.Source do + @moduledoc """ + Behaviour for LLM text data sources. + + Each source module must implement this behaviour to provide content + for LLM-friendly text files (llms.txt index and individual page files). + + ## Required Callbacks + + - `source_name/0` - Unique atom identifier for the source + - `enabled?/0` - Whether this source is active + - `collect_index_entries/0` - Collect index entries for llms.txt + - `collect_page_files/0` - Collect individual page files + + ## Index Entry Format + + Each index entry is a map with: + - `:title` - Page title (string) + - `:url` - Full URL to the page (string) + - `:description` - Brief description (string) + - `:group` - Group name for organizing entries (string) + """ + + require Logger + + @type index_entry :: %{ + title: String.t(), + url: String.t(), + description: String.t(), + group: String.t() + } + + @doc """ + Returns the unique name/identifier for this source. + """ + @callback source_name() :: atom() + + @doc """ + Checks if this source is enabled and should be included. + """ + @callback enabled?() :: boolean() + + @doc """ + Collects index entries for llms.txt from this source. + """ + @callback collect_index_entries() :: [index_entry()] + + @doc """ + Collects individual page files from this source. + + Returns a list of `{relative_path, content}` tuples. + The relative_path is relative to the llms storage directory. + """ + @callback collect_page_files() :: [{path :: String.t(), content :: String.t()}] + + @doc """ + Checks if a source module implements all required callbacks. + """ + @spec valid_source?(module()) :: boolean() + def valid_source?(module) when is_atom(module) do + case Code.ensure_loaded(module) do + {:module, _} -> + function_exported?(module, :source_name, 0) and + function_exported?(module, :enabled?, 0) and + function_exported?(module, :collect_index_entries, 0) and + function_exported?(module, :collect_page_files, 0) + + {:error, _} -> + false + end + end + + def valid_source?(_), do: false + + @doc """ + Safely collects index entries from a source, returning [] if disabled or on error. + """ + @spec safe_collect_index_entries(module()) :: [index_entry()] + def safe_collect_index_entries(source_module) do + if valid_source?(source_module) and source_module.enabled?() do + source_module.collect_index_entries() + else + [] + end + rescue + error -> + Logger.warning( + "LLMText source #{inspect(source_module)} failed to collect index entries: #{inspect(error)}" + ) + + [] + end + + @doc """ + Safely collects page files from a source, returning [] if disabled or on error. + """ + @spec safe_collect_page_files(module()) :: [{String.t(), String.t()}] + def safe_collect_page_files(source_module) do + if valid_source?(source_module) and source_module.enabled?() do + source_module.collect_page_files() + else + [] + end + rescue + error -> + Logger.warning( + "LLMText source #{inspect(source_module)} failed to collect page files: #{inspect(error)}" + ) + + [] + end +end diff --git a/lib/modules/llm_text/web/controller.ex b/lib/modules/llm_text/web/controller.ex new file mode 100644 index 000000000..56c6bd5f3 --- /dev/null +++ b/lib/modules/llm_text/web/controller.ex @@ -0,0 +1,82 @@ +defmodule PhoenixKit.Modules.LLMText.Web.Controller do + @moduledoc """ + Controller for serving LLM-friendly text files. + + ## Endpoints + + - GET /{prefix}/llms.txt — serves the index file listing all LLM-readable pages + - GET /{prefix}/llms/*path — serves individual LLM text files + + Files are served directly from `priv/static/llms/` with `text/plain` content type. + Returns 404 for files that do not exist. + """ + + use PhoenixKitWeb, :controller + + require Logger + + alias PhoenixKit.Modules.LLMText.FileStorage + + @doc """ + Serves the llms.txt index file. + """ + def index(conn, _params) do + path = FileStorage.index_path() + + case File.read(path) do + {:ok, content} -> + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(200, content) + + {:error, :enoent} -> + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(404, "Not found") + + {:error, reason} -> + Logger.warning("LLMText controller: failed to read llms.txt: #{inspect(reason)}") + + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(500, "Internal error") + end + end + + @doc """ + Serves an individual LLM text file at `/llms/*path`. + """ + def show(conn, %{"path" => path_parts}) when is_list(path_parts) do + relative_path = Path.join(path_parts) + + # Prevent path traversal + if String.contains?(relative_path, "..") do + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(400, "Bad request") + else + full_path = FileStorage.file_path(relative_path) + + case File.read(full_path) do + {:ok, content} -> + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(200, content) + + {:error, :enoent} -> + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(404, "Not found") + + {:error, reason} -> + Logger.warning( + "LLMText controller: failed to read #{relative_path}: #{inspect(reason)}" + ) + + conn + |> put_resp_content_type("text/plain; charset=utf-8") + |> send_resp(500, "Internal error") + end + end + end +end diff --git a/lib/modules/llm_text/web/settings.ex b/lib/modules/llm_text/web/settings.ex new file mode 100644 index 000000000..2b556f021 --- /dev/null +++ b/lib/modules/llm_text/web/settings.ex @@ -0,0 +1,29 @@ +defmodule PhoenixKit.Modules.LLMText.Web.Settings do + @moduledoc """ + Admin settings LiveView for the LLM Text module. + + Placeholder — full UI to be implemented in a future iteration. + """ + + use PhoenixKitWeb, :live_view + + alias PhoenixKit.Modules.LLMText + + @impl true + def mount(_params, _session, socket) do + {:ok, assign(socket, page_title: "LLM Text Settings", enabled: LLMText.enabled?())} + end + + @impl true + def render(assigns) do + ~H""" +
+

LLM Text

+

LLM Text settings coming soon.

+

+ Module enabled: {@enabled} +

+
+ """ + end +end diff --git a/lib/modules/llm_text/workers/generate_llm_text_job.ex b/lib/modules/llm_text/workers/generate_llm_text_job.ex new file mode 100644 index 000000000..0312f599c --- /dev/null +++ b/lib/modules/llm_text/workers/generate_llm_text_job.ex @@ -0,0 +1,131 @@ +defmodule PhoenixKit.Modules.LLMText.Workers.GenerateLLMTextJob do + @moduledoc """ + Oban worker for generating LLM text files. + + ## Scopes + + - `"all"` - Regenerate all sources and rebuild index + - `"source"` - Regenerate a single source (by source_name) and rebuild index + - `"file"` - Regenerate a single file (by source_name + path) and rebuild index + + ## Enqueueing + + Use the helper functions to build changesets; the caller inserts them: + + changeset = GenerateLLMTextJob.enqueue_all() + {:ok, job} = Oban.insert(changeset) + + changeset = GenerateLLMTextJob.enqueue_for_source(:blog) + {:ok, job} = Oban.insert(changeset) + + changeset = GenerateLLMTextJob.enqueue_for_file(:blog, "posts/article.md") + {:ok, job} = Oban.insert(changeset) + """ + + use Oban.Worker, + queue: :llm_text, + max_attempts: 3, + unique: [period: 5, fields: [:args], keys: [:scope, :source]] + + require Logger + + alias PhoenixKit.Modules.LLMText.Generator + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"scope" => "all"}}) do + Logger.info("GenerateLLMTextJob: Running all sources") + Generator.run_all() + end + + def perform(%Oban.Job{args: %{"scope" => "source", "source" => source_name}}) do + Logger.info("GenerateLLMTextJob: Running source #{source_name}") + + case resolve_source(source_name) do + nil -> + Logger.warning("GenerateLLMTextJob: Source not found: #{source_name}") + {:error, {:source_not_found, source_name}} + + source_module -> + Generator.run_source(source_module) + end + end + + def perform(%Oban.Job{args: %{"scope" => "file", "source" => source_name, "path" => path}}) do + Logger.info("GenerateLLMTextJob: Running file #{path} from source #{source_name}") + + case resolve_source(source_name) do + nil -> + Logger.warning("GenerateLLMTextJob: Source not found: #{source_name}") + {:error, {:source_not_found, source_name}} + + source_module -> + alias PhoenixKit.Modules.LLMText.FileStorage + alias PhoenixKit.Modules.LLMText.Sources.Source + + files = Source.safe_collect_page_files(source_module) + + case Enum.find(files, fn {p, _} -> p == path end) do + nil -> + Logger.warning("GenerateLLMTextJob: File not found in source: #{path}") + {:error, {:file_not_found, path}} + + {_, content} -> + FileStorage.write(path, content) + Generator.rebuild_index() + end + end + end + + @impl Oban.Worker + def timeout(_job), do: :timer.minutes(5) + + @doc """ + Returns an Oban changeset that regenerates all sources. Caller inserts it. + """ + @spec enqueue_all() :: Ecto.Changeset.t() + def enqueue_all do + new(%{"scope" => "all"}) + end + + @doc """ + Returns an Oban changeset that regenerates a specific source. Caller inserts it. + """ + @spec enqueue_for_source(atom() | String.t()) :: Ecto.Changeset.t() + def enqueue_for_source(source_name) when is_atom(source_name) do + enqueue_for_source(Atom.to_string(source_name)) + end + + def enqueue_for_source(source_name) when is_binary(source_name) do + new(%{"scope" => "source", "source" => source_name}) + end + + @doc """ + Returns an Oban changeset that regenerates a single file. Caller inserts it. + """ + @spec enqueue_for_file(atom() | String.t(), String.t()) :: Ecto.Changeset.t() + def enqueue_for_file(source_name, path) when is_atom(source_name) do + enqueue_for_file(Atom.to_string(source_name), path) + end + + def enqueue_for_file(source_name, path) when is_binary(source_name) and is_binary(path) do + new(%{"scope" => "file", "source" => source_name, "path" => path}) + end + + @doc """ + Finds a source module whose `source_name/0` matches the given string. + """ + @spec resolve_source(String.t()) :: module() | nil + def resolve_source(source_name) when is_binary(source_name) do + Generator.get_sources() + |> Enum.find(fn mod -> + case Code.ensure_loaded(mod) do + {:module, _} -> + function_exported?(mod, :source_name, 0) and + to_string(mod.source_name()) == source_name + + _ -> + false + end + end) + end +end diff --git a/lib/phoenix_kit_web/integration.ex b/lib/phoenix_kit_web/integration.ex index e57a3e0f3..a0049c2e4 100644 --- a/lib/phoenix_kit_web/integration.ex +++ b/lib/phoenix_kit_web/integration.ex @@ -287,6 +287,12 @@ defmodule PhoenixKitWeb.Integration do :razorpay end + # LLM text routes - public plain text endpoints for AI/LLM consumption + scope unquote(url_prefix) do + get "/llms.txt", PhoenixKit.Modules.LLMText.Web.Controller, :index + get "/llms/*path", PhoenixKit.Modules.LLMText.Web.Controller, :show + end + # Shop public routes are generated via generate_shop_public_routes/1 helper # This supports locale-prefixed URLs (/:locale/shop/...) with language switching # Shop user dashboard routes are now in phoenix_kit_authenticated_routes/1. @@ -555,6 +561,12 @@ defmodule PhoenixKitWeb.Integration do :index, as: :billing_provider_settings + # LLM Text settings + live "/admin/settings/llm-text", + PhoenixKit.Modules.LLMText.Web.Settings, + :index, + as: :llm_text_settings + # DB Explorer routes live "/admin/db", PhoenixKit.Modules.DB.Web.Index, :index, as: :db_index diff --git a/test/modules/llm_text/file_storage_test.exs b/test/modules/llm_text/file_storage_test.exs new file mode 100644 index 000000000..f1457db99 --- /dev/null +++ b/test/modules/llm_text/file_storage_test.exs @@ -0,0 +1,93 @@ +defmodule PhoenixKit.Modules.LLMText.FileStorageTest do + use ExUnit.Case, async: false + + alias PhoenixKit.Modules.LLMText.FileStorage + + setup do + tmp_dir = System.tmp_dir!() |> Path.join("llm_text_test_#{:rand.uniform(1_000_000)}") + Application.put_env(:phoenix_kit, :llm_text_test_storage_dir, tmp_dir) + + on_exit(fn -> + Application.delete_env(:phoenix_kit, :llm_text_test_storage_dir) + File.rm_rf(tmp_dir) + end) + + {:ok, tmp_dir: tmp_dir} + end + + describe "storage_dir/0" do + test "returns the test override directory", %{tmp_dir: tmp_dir} do + assert FileStorage.storage_dir() == tmp_dir + end + end + + describe "index_path/0" do + test "returns llms.txt inside storage dir", %{tmp_dir: tmp_dir} do + assert FileStorage.index_path() == Path.join(tmp_dir, "llms.txt") + end + end + + describe "file_path/1" do + test "returns full path for relative path", %{tmp_dir: tmp_dir} do + assert FileStorage.file_path("posts/article.md") == + Path.join(tmp_dir, "posts/article.md") + end + end + + describe "write/2" do + test "writes content and creates parent dirs" do + assert :ok = FileStorage.write("nested/dir/file.md", "# Hello") + assert File.read!(FileStorage.file_path("nested/dir/file.md")) == "# Hello" + end + + test "overwrites existing file" do + :ok = FileStorage.write("page.md", "v1") + :ok = FileStorage.write("page.md", "v2") + assert File.read!(FileStorage.file_path("page.md")) == "v2" + end + end + + describe "write_index/1" do + test "writes llms.txt" do + assert :ok = FileStorage.write_index("# Index\n- [Page](/page)") + assert File.read!(FileStorage.index_path()) == "# Index\n- [Page](/page)" + end + end + + describe "delete/1" do + test "deletes an existing file" do + :ok = FileStorage.write("to_delete.md", "content") + assert :ok = FileStorage.delete("to_delete.md") + refute File.exists?(FileStorage.file_path("to_delete.md")) + end + + test "returns :ok when file does not exist" do + assert :ok = FileStorage.delete("nonexistent.md") + end + end + + describe "exists?/1" do + test "returns true for existing file" do + :ok = FileStorage.write("exists.md", "yes") + assert FileStorage.exists?("exists.md") == true + end + + test "returns false for missing file" do + assert FileStorage.exists?("missing.md") == false + end + end + + describe "delete_all/0" do + test "removes the entire storage directory", %{tmp_dir: tmp_dir} do + :ok = FileStorage.write("a.md", "a") + :ok = FileStorage.write("b/c.md", "c") + assert :ok = FileStorage.delete_all() + refute File.exists?(tmp_dir) + end + + test "returns :ok if directory does not exist" do + FileStorage.delete_all() + assert :ok = FileStorage.delete_all() + end + end +end diff --git a/test/modules/llm_text/generator_test.exs b/test/modules/llm_text/generator_test.exs new file mode 100644 index 000000000..8fb942abc --- /dev/null +++ b/test/modules/llm_text/generator_test.exs @@ -0,0 +1,152 @@ +defmodule PhoenixKit.Modules.LLMText.GeneratorTest do + use ExUnit.Case, async: false + + alias PhoenixKit.Modules.LLMText.FileStorage + alias PhoenixKit.Modules.LLMText.Generator + + defmodule StubSource do + @behaviour PhoenixKit.Modules.LLMText.Sources.Source + + def source_name, do: :stub + def enabled?, do: true + + def collect_index_entries do + [ + %{title: "Home", url: "/", description: "Home page", group: "General"}, + %{title: "About", url: "/about", description: "About us", group: "General"}, + %{title: "Blog", url: "/blog", description: "Latest posts", group: "Posts"} + ] + end + + def collect_page_files do + [ + {"home.md", "# Home\nWelcome"}, + {"about.md", "# About\nAbout us"} + ] + end + end + + defmodule DisabledSource do + @behaviour PhoenixKit.Modules.LLMText.Sources.Source + + def source_name, do: :disabled_stub + def enabled?, do: false + + def collect_index_entries, + do: [%{title: "Hidden", url: "/hidden", description: "Hidden", group: "Hidden"}] + + def collect_page_files, do: [{"hidden.md", "# Hidden"}] + end + + setup do + tmp_dir = System.tmp_dir!() |> Path.join("llm_text_gen_test_#{:rand.uniform(1_000_000)}") + Application.put_env(:phoenix_kit, :llm_text_test_storage_dir, tmp_dir) + Application.put_env(:phoenix_kit, :llm_text_sources, [StubSource]) + + on_exit(fn -> + Application.delete_env(:phoenix_kit, :llm_text_test_storage_dir) + Application.delete_env(:phoenix_kit, :llm_text_sources) + File.rm_rf(tmp_dir) + end) + + {:ok, tmp_dir: tmp_dir} + end + + describe "build_index_content/1" do + test "builds markdown with header and grouped entries" do + entries = [ + %{title: "Home", url: "/", description: "Home page", group: "General"}, + %{title: "Blog", url: "/blog", description: "Latest posts", group: "Posts"}, + %{title: "About", url: "/about", description: "About us", group: "General"} + ] + + content = Generator.build_index_content(entries) + + assert content =~ "## General" + assert content =~ "## Posts" + assert content =~ "[Home](/)" + assert content =~ "[Blog](/blog)" + assert content =~ "[About](/about)" + end + + test "group order follows first-seen order" do + entries = [ + %{title: "A", url: "/a", description: "", group: "Second"}, + %{title: "B", url: "/b", description: "", group: "First"}, + %{title: "C", url: "/c", description: "", group: "Second"} + ] + + content = Generator.build_index_content(entries) + second_pos = :binary.match(content, "## Second") |> elem(0) + first_pos = :binary.match(content, "## First") |> elem(0) + + assert second_pos < first_pos + end + + test "entries without description omit the colon" do + entries = [%{title: "Page", url: "/page", description: "", group: "General"}] + content = Generator.build_index_content(entries) + assert content =~ "[Page](/page)" + refute content =~ "[Page](/page):" + end + + test "handles empty entries list" do + content = Generator.build_index_content([]) + assert is_binary(content) + end + end + + describe "run_all/0" do + test "writes page files from all enabled sources" do + :ok = Generator.run_all() + assert FileStorage.exists?("home.md") + assert FileStorage.exists?("about.md") + end + + test "writes llms.txt index" do + :ok = Generator.run_all() + assert File.exists?(FileStorage.index_path()) + content = File.read!(FileStorage.index_path()) + assert content =~ "Home" + end + + test "skips disabled sources" do + Application.put_env(:phoenix_kit, :llm_text_sources, [StubSource, DisabledSource]) + :ok = Generator.run_all() + refute FileStorage.exists?("hidden.md") + end + end + + describe "run_source/1" do + test "writes files for the given source" do + :ok = Generator.run_source(StubSource) + assert FileStorage.exists?("home.md") + assert FileStorage.exists?("about.md") + end + + test "rebuilds index after running source" do + :ok = Generator.run_source(StubSource) + assert File.exists?(FileStorage.index_path()) + end + end + + describe "rebuild_index/0" do + test "writes llms.txt with entries from all enabled sources" do + :ok = Generator.rebuild_index() + content = File.read!(FileStorage.index_path()) + assert content =~ "Home" + assert content =~ "Blog" + end + end + + describe "get_sources/0" do + test "returns configured sources" do + assert Generator.get_sources() == [StubSource] + end + + test "returns [] when not configured" do + Application.delete_env(:phoenix_kit, :llm_text_sources) + assert Generator.get_sources() == [] + end + end +end diff --git a/test/modules/llm_text/source_test.exs b/test/modules/llm_text/source_test.exs new file mode 100644 index 000000000..c9622d669 --- /dev/null +++ b/test/modules/llm_text/source_test.exs @@ -0,0 +1,103 @@ +defmodule PhoenixKit.Modules.LLMText.Sources.SourceTest do + use ExUnit.Case, async: true + + alias PhoenixKit.Modules.LLMText.Sources.Source + + # A valid stub source + defmodule ValidSource do + @behaviour PhoenixKit.Modules.LLMText.Sources.Source + + def source_name, do: :valid_stub + def enabled?, do: true + + def collect_index_entries, + do: [%{title: "Page", url: "/page", description: "Desc", group: "General"}] + + def collect_page_files, do: [{"page.md", "# Page\nContent"}] + end + + # A disabled source + defmodule DisabledSource do + @behaviour PhoenixKit.Modules.LLMText.Sources.Source + + def source_name, do: :disabled_stub + def enabled?, do: false + + def collect_index_entries, + do: [%{title: "Hidden", url: "/hidden", description: "Hidden", group: "Hidden"}] + + def collect_page_files, do: [{"hidden.md", "# Hidden"}] + end + + # A crashing source + defmodule CrashingSource do + @behaviour PhoenixKit.Modules.LLMText.Sources.Source + + def source_name, do: :crashing_stub + def enabled?, do: true + def collect_index_entries, do: raise("collect_index_entries crash") + def collect_page_files, do: raise("collect_page_files crash") + end + + # An invalid module (missing callbacks) + defmodule InvalidSource do + def source_name, do: :invalid + end + + describe "valid_source?/1" do + test "returns true for a module with all 4 callbacks" do + assert Source.valid_source?(ValidSource) == true + end + + test "returns false for a module missing callbacks" do + assert Source.valid_source?(InvalidSource) == false + end + + test "returns false for a non-existent module" do + assert Source.valid_source?(NonExistentModule.Foo) == false + end + + test "returns false for non-atom" do + assert Source.valid_source?("not_a_module") == false + end + end + + describe "safe_collect_page_files/1" do + test "returns files when source is valid and enabled" do + result = Source.safe_collect_page_files(ValidSource) + assert result == [{"page.md", "# Page\nContent"}] + end + + test "returns [] when source is disabled" do + result = Source.safe_collect_page_files(DisabledSource) + assert result == [] + end + + test "returns [] when source crashes" do + result = Source.safe_collect_page_files(CrashingSource) + assert result == [] + end + + test "returns [] for invalid module" do + result = Source.safe_collect_page_files(InvalidSource) + assert result == [] + end + end + + describe "safe_collect_index_entries/1" do + test "returns entries when source is valid and enabled" do + result = Source.safe_collect_index_entries(ValidSource) + assert [%{title: "Page", url: "/page"}] = result + end + + test "returns [] when source is disabled" do + result = Source.safe_collect_index_entries(DisabledSource) + assert result == [] + end + + test "returns [] when source crashes" do + result = Source.safe_collect_index_entries(CrashingSource) + assert result == [] + end + end +end diff --git a/test/modules/llm_text/sources/publishing_test.exs b/test/modules/llm_text/sources/publishing_test.exs new file mode 100644 index 000000000..e32acf576 --- /dev/null +++ b/test/modules/llm_text/sources/publishing_test.exs @@ -0,0 +1,132 @@ +defmodule PhoenixKit.Modules.LLMText.Sources.PublishingTest do + use ExUnit.Case, async: true + + alias PhoenixKit.Modules.LLMText.Sources.Publishing + + describe "source_name/0" do + test "returns :publishing" do + assert Publishing.source_name() == :publishing + end + end + + describe "enabled?/0" do + test "returns false when Publishing module is not available" do + assert Publishing.enabled?() == false + end + end + + describe "build_file_path/2" do + test "builds path as group_slug/post_slug.txt" do + assert Publishing.build_file_path("blog", "hello-world") == "blog/hello-world.txt" + end + + test "works with any group and slug" do + assert Publishing.build_file_path("news", "2024-01-15-10-30") == + "news/2024-01-15-10-30.txt" + end + end + + describe "extract_description/1" do + test "returns metadata.description atom key when present" do + post = %{metadata: %{description: "A great post", status: "published"}, content: "body"} + assert Publishing.extract_description(post) == "A great post" + end + + test "returns metadata description string key when present" do + post = %{metadata: %{"description" => "A great post"}, content: "body"} + assert Publishing.extract_description(post) == "A great post" + end + + test "falls back to first 160 chars of content when description is absent" do + long_content = String.duplicate("word ", 50) + post = %{metadata: %{status: "published"}, content: long_content} + result = Publishing.extract_description(post) + assert String.length(result) <= 160 + assert String.starts_with?(result, "word") + end + + test "returns empty string when description is empty and no content" do + post = %{metadata: %{description: ""}, content: ""} + assert Publishing.extract_description(post) == "" + end + + test "returns empty string when metadata has no description and content is nil" do + post = %{metadata: %{status: "published"}} + assert Publishing.extract_description(post) == "" + end + + test "collapses whitespace in content fallback" do + post = %{metadata: %{}, content: "Hello\n\nWorld more text"} + result = Publishing.extract_description(post) + refute result =~ "\n" + assert result =~ "Hello" + end + end + + describe "build_post_content/3" do + test "includes title as h1 heading" do + post = %{ + metadata: %{description: "desc", status: "published"}, + content: "Post body.", + mode: :slug, + url_slug: "my-post", + slug: "my-post" + } + + result = Publishing.build_post_content(post, "blog", "My Post") + assert result =~ "# My Post" + end + + test "includes source URL line" do + post = %{ + metadata: %{description: "", status: "published"}, + content: "Body.", + mode: :slug, + url_slug: "my-post", + slug: "my-post" + } + + result = Publishing.build_post_content(post, "blog", "Title") + assert result =~ "> Source:" + end + + test "includes post body content" do + post = %{ + metadata: %{description: "", status: "published"}, + content: "This is the body.", + mode: :slug, + url_slug: "my-post", + slug: "my-post" + } + + result = Publishing.build_post_content(post, "blog", "Title") + assert result =~ "This is the body." + end + + test "includes description when present" do + post = %{ + metadata: %{description: "A great description", status: "published"}, + content: "Body.", + mode: :slug, + url_slug: "my-post", + slug: "my-post" + } + + result = Publishing.build_post_content(post, "blog", "Title") + assert result =~ "A great description" + end + + test "handles missing content gracefully" do + post = %{ + metadata: %{description: "", status: "published"}, + mode: :slug, + url_slug: "my-post", + slug: "my-post" + } + + result = Publishing.build_post_content(post, "blog", "Title") + assert is_binary(result) + assert result =~ "# Title" + end + end +end diff --git a/test/modules/llm_text/workers/generate_llm_text_job_test.exs b/test/modules/llm_text/workers/generate_llm_text_job_test.exs new file mode 100644 index 000000000..472d96e39 --- /dev/null +++ b/test/modules/llm_text/workers/generate_llm_text_job_test.exs @@ -0,0 +1,80 @@ +defmodule PhoenixKit.Modules.LLMText.Workers.GenerateLLMTextJobTest do + use ExUnit.Case, async: false + + alias PhoenixKit.Modules.LLMText.Workers.GenerateLLMTextJob + + defmodule BlogSource do + @behaviour PhoenixKit.Modules.LLMText.Sources.Source + + def source_name, do: :blog + def enabled?, do: true + def collect_index_entries, do: [] + def collect_page_files, do: [] + end + + setup do + Application.put_env(:phoenix_kit, :llm_text_sources, [BlogSource]) + + on_exit(fn -> + Application.delete_env(:phoenix_kit, :llm_text_sources) + end) + + :ok + end + + describe "enqueue_all/0" do + test "returns a changeset with scope 'all'" do + changeset = GenerateLLMTextJob.enqueue_all() + assert changeset.valid? + assert changeset.changes.args == %{"scope" => "all"} + end + end + + describe "enqueue_for_source/1" do + test "accepts atom source name" do + changeset = GenerateLLMTextJob.enqueue_for_source(:blog) + assert changeset.valid? + assert changeset.changes.args == %{"scope" => "source", "source" => "blog"} + end + + test "accepts string source name" do + changeset = GenerateLLMTextJob.enqueue_for_source("blog") + assert changeset.valid? + assert changeset.changes.args == %{"scope" => "source", "source" => "blog"} + end + end + + describe "enqueue_for_file/2" do + test "accepts atom source name and path" do + changeset = GenerateLLMTextJob.enqueue_for_file(:blog, "posts/article.md") + assert changeset.valid? + + assert changeset.changes.args == %{ + "scope" => "file", + "source" => "blog", + "path" => "posts/article.md" + } + end + + test "accepts string source name and path" do + changeset = GenerateLLMTextJob.enqueue_for_file("blog", "posts/article.md") + assert changeset.valid? + + assert changeset.changes.args == %{ + "scope" => "file", + "source" => "blog", + "path" => "posts/article.md" + } + end + end + + describe "resolve_source/1" do + test "finds a source module by name string" do + assert GenerateLLMTextJob.resolve_source("blog") == BlogSource + end + + test "returns nil for unknown source" do + assert GenerateLLMTextJob.resolve_source("nonexistent") == nil + end + end +end