Add LLMText module for AI/LLM-friendly content generation

.dialyzer_ignore.exs

@@ -142,5 +142,9 @@
   # ExUnit internal functions — false positives when test/support is compiled in MIX_ENV=test
   # Dialyzer cannot resolve ExUnit private macros expanded at compile time
   {"test/support/conn_case.ex", :unknown_function},
-  {"test/support/data_case.ex", :unknown_function}
+  {"test/support/data_case.ex", :unknown_function},
+
+  # LLMText module — Publishing is an external package, guarded by Code.ensure_loaded?
+  {"lib/modules/llm_text/sources/publishing.ex", :unknown_function},
+  {"lib/modules/llm_text/publishing_subscriber.ex", :unknown_function}
 ]

lib/modules/llm_text/file_storage.ex

@@ -0,0 +1,170 @@
+defmodule PhoenixKit.Modules.LLMText.FileStorage do
+  @moduledoc """
+  File-based storage for LLM text files.
+
+  Stores files under the host app's `priv/static/llms/` directory:
+
+      priv/static/llms/llms.txt          -> index file
+      priv/static/llms/page.md           -> individual page file
+      priv/static/llms/posts/article.md  -> nested page file
+
+  ## Test Override
+
+  Set `Application.put_env(:phoenix_kit, :llm_text_test_storage_dir, "/tmp/...")` to
+  override the storage directory in tests.
+  """
+
+  require Logger
+
+  @index_filename "llms.txt"
+
+  @doc """
+  Returns the root storage directory for LLM text files.
+
+  Uses test override if configured, otherwise resolves to the host app's
+  `priv/static/llms/` directory.
+  """
+  @spec storage_dir() :: String.t()
+  def storage_dir do
+    case Application.get_env(:phoenix_kit, :llm_text_test_storage_dir) do
+      nil -> resolve_storage_dir()
+      dir -> dir
+    end
+  end
+
+  @doc """
+  Returns the path to the llms.txt index file.
+  """
+  @spec index_path() :: String.t()
+  def index_path do
+    Path.join(storage_dir(), @index_filename)
+  end
+
+  @doc """
+  Returns the full path for a relative file path within the storage directory.
+  """
+  @spec file_path(String.t()) :: String.t()
+  def file_path(relative_path) when is_binary(relative_path) do
+    Path.join(storage_dir(), relative_path)
+  end
+
+  @doc """
+  Writes content to a file at the given relative path, creating directories as needed.
+  """
+  @spec write(String.t(), String.t()) :: :ok | {:error, term()}
+  def write(relative_path, content) when is_binary(relative_path) and is_binary(content) do
+    path = file_path(relative_path)
+
+    with :ok <- ensure_directory_exists(path),
+         :ok <- File.write(path, content) do
+      Logger.debug("LLMText.FileStorage: Wrote #{relative_path} (#{byte_size(content)} bytes)")
+      :ok
+    else
+      {:error, reason} = error ->
+        Logger.warning(
+          "LLMText.FileStorage: Failed to write #{relative_path}: #{inspect(reason)}"
+        )
+
+        error
+    end
+  end
+
+  @doc """
+  Writes the llms.txt index file.
+  """
+  @spec write_index(String.t()) :: :ok | {:error, term()}
+  def write_index(content) when is_binary(content) do
+    path = index_path()
+
+    with :ok <- ensure_directory_exists(path),
+         :ok <- File.write(path, content) do
+      Logger.debug("LLMText.FileStorage: Wrote llms.txt (#{byte_size(content)} bytes)")
+      :ok
+    else
+      {:error, reason} = error ->
+        Logger.warning("LLMText.FileStorage: Failed to write llms.txt: #{inspect(reason)}")
+        error
+    end
+  end
+
+  @doc """
+  Deletes a file at the given relative path. Returns :ok if file does not exist.
+  """
+  @spec delete(String.t()) :: :ok
+  def delete(relative_path) when is_binary(relative_path) do
+    path = file_path(relative_path)
+
+    case File.rm(path) do
+      :ok ->
+        Logger.debug("LLMText.FileStorage: Deleted #{relative_path}")
+        :ok
+
+      {:error, :enoent} ->
+        :ok
+
+      {:error, reason} ->
+        Logger.warning(
+          "LLMText.FileStorage: Failed to delete #{relative_path}: #{inspect(reason)}"
+        )
+
+        :ok
+    end
+  end
+
+  @doc """
+  Checks if a file at the given relative path exists.
+  """
+  @spec exists?(String.t()) :: boolean()
+  def exists?(relative_path) when is_binary(relative_path) do
+    relative_path |> file_path() |> File.exists?()
+  end
+
+  @doc """
+  Deletes the entire storage directory and all its contents.
+  """
+  @spec delete_all() :: :ok
+  def delete_all do
+    dir = storage_dir()
+
+    case File.rm_rf(dir) do
+      {:ok, _} ->
+        Logger.debug("LLMText.FileStorage: Deleted all files in #{dir}")
+        :ok
+
+      {:error, reason, _path} ->
+        Logger.warning("LLMText.FileStorage: Failed to delete storage dir: #{inspect(reason)}")
+        :ok
+    end
+  rescue
+    _ -> :ok
+  end
+
+  # Private helpers
+
+  defp resolve_storage_dir do
+    otp_app = PhoenixKit.Config.get(:otp_app, :phoenix_kit)
+
+    priv_dir =
+      case :code.priv_dir(otp_app) do
+        {:error, :bad_name} ->
+          case :code.priv_dir(:phoenix_kit) do
+            {:error, :bad_name} -> "priv"
+            dir -> to_string(dir)
+          end
+
+        dir ->
+          to_string(dir)
+      end
+
+    Path.join([priv_dir, "static", "llms"])
+  end
+
+  defp ensure_directory_exists(file_path) do
+    dir = Path.dirname(file_path)
+
+    case File.mkdir_p(dir) do
+      :ok -> :ok
+      {:error, reason} -> {:error, {:mkdir_failed, reason}}
+    end
+  end
+end

lib/modules/llm_text/generator.ex

@@ -0,0 +1,173 @@
+defmodule PhoenixKit.Modules.LLMText.Generator do
+  @moduledoc """
+  Generator for LLM-friendly text files.
+
+  Produces:
+  - `llms.txt` - Index file linking to all LLM-readable pages
+  - Individual page `.md` files per source
+
+  ## Usage
+
+      # Regenerate all sources and rebuild index
+      Generator.run_all()
+
+      # Regenerate a single source and rebuild index
+      Generator.run_source(MyApp.LLMText.BlogSource)
+
+      # Only rebuild the index from all sources
+      Generator.rebuild_index()
+  """
+
+  require Logger
+
+  alias PhoenixKit.Modules.LLMText.FileStorage
+  alias PhoenixKit.Modules.LLMText.Sources.Source
+
+  @doc """
+  Regenerates files for one source and rebuilds the llms.txt index.
+  """
+  @spec run_source(module()) :: :ok
+  def run_source(source_module) do
+    Logger.info("LLMText.Generator: Running source #{inspect(source_module)}")
+
+    files = Source.safe_collect_page_files(source_module)
+
+    Enum.each(files, fn {path, content} ->
+      case FileStorage.write(path, content) do
+        :ok ->
+          :ok
+
+        {:error, reason} ->
+          Logger.warning("LLMText.Generator: Failed to write #{path}: #{inspect(reason)}")
+      end
+    end)
+
+    rebuild_index()
+  end
+
+  @doc """
+  Regenerates all sources and rebuilds the llms.txt index.
+  """
+  @spec run_all() :: :ok
+  def run_all do
+    sources = get_sources()
+    Logger.info("LLMText.Generator: Running all #{length(sources)} sources")
+
+    Enum.each(sources, fn source_module ->
+      files = Source.safe_collect_page_files(source_module)
+
+      Enum.each(files, fn {path, content} ->
+        case FileStorage.write(path, content) do
+          :ok ->
+            :ok
+
+          {:error, reason} ->
+            Logger.warning("LLMText.Generator: Failed to write #{path}: #{inspect(reason)}")
+        end
+      end)
+    end)
+
+    rebuild_index()
+  end
+
+  @doc """
+  Rebuilds the llms.txt index from all sources without regenerating page files.
+  """
+  @spec rebuild_index() :: :ok
+  def rebuild_index do
+    sources = get_sources()
+    Logger.debug("LLMText.Generator: Rebuilding index from #{length(sources)} sources")
+
+    entries =
+      sources
+      |> Enum.flat_map(&Source.safe_collect_index_entries/1)
+
+    content = build_index_content(entries)
+
+    case FileStorage.write_index(content) do
+      :ok ->
+        :ok
+
+      {:error, reason} ->
+        Logger.warning("LLMText.Generator: Failed to write index: #{inspect(reason)}")
+        :ok
+    end
+  end
+
+  @doc """
+  Builds the llms.txt markdown content from a list of index entries.
+
+  Groups entries by their `:group` field. Group order follows first-seen order.
+  Within each group, entries appear in the order they were provided.
+  """
+  @spec build_index_content([Source.index_entry()]) :: String.t()
+  def build_index_content(entries) do
+    site_name = get_site_name()
+    site_description = get_site_description()
+
+    # Build ordered groups (first-seen order) using prepend + reverse for O(n) performance
+    {groups_reversed, groups_map} =
+      Enum.reduce(entries, {[], %{}}, fn entry, {order, map} ->
+        group = Map.get(entry, :group, "General")
+
+        if Map.has_key?(map, group) do
+          {order, Map.update!(map, group, &[entry | &1])}
+        else
+          {[group | order], Map.put(map, group, [entry])}
+        end
+      end)
+
+    groups_ordered = Enum.reverse(groups_reversed)
+
+    header =
+      if site_description && site_description != "" do
+        "# #{site_name}\n\n> #{site_description}\n\n"
+      else
+        "# #{site_name}\n\n"
+      end
+
+    sections =
+      Enum.map_join(groups_ordered, "\n\n", fn group ->
+        group_entries = Map.get(groups_map, group, []) |> Enum.reverse()
+
+        links =
+          Enum.map_join(group_entries, "\n", fn entry ->
+            title = Map.get(entry, :title, "")
+            url = Map.get(entry, :url, "")
+            description = Map.get(entry, :description, "")
+
+            if description && description != "" do
+              "- [#{title}](#{url}): #{description}"
+            else
+              "- [#{title}](#{url})"
+            end
+          end)
+
+        "## #{group}\n\n#{links}"
+      end)
+
+    header <> sections
+  end
+
+  @doc """
+  Returns the configured LLM text sources.
+  """
+  @spec get_sources() :: [module()]
+  def get_sources do
+    Application.get_env(:phoenix_kit, :llm_text_sources, [])
+  end
+
+  # Private helpers
+
+  defp get_site_name do
+    PhoenixKit.Settings.get_setting("site_name", "Site")
+  rescue
+    _ -> "Site"
+  end
+
+  defp get_site_description do
+    PhoenixKit.Settings.get_setting("site_description", "")
+  rescue
+    _ -> ""
+  end
+end