Clean up

.github/workflows/test.yml

@@ -0,0 +1,53 @@
+name: Test
+
+on:
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: OTP ${{ matrix.otp }} / Elixir ${{ matrix.elixir }}
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        ports:
+        - 5432:5432
+        env:
+          POSTGRES_DB: chunkr_test
+          POSTGRES_PASSWORD: postgres
+
+    strategy:
+      matrix:
+        elixir: ["1.14.1"]
+        otp: ["25.1.1"]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Elixir
+      uses: erlef/setup-beam@v1
+      with:
+        otp-version: ${{ matrix.otp }}
+        elixir-version: ${{ matrix.elixir }}
+    - name: Restore dependencies cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          _build
+          deps
+        key: ${{ runner.os }}-otp-${{ matrix.otp }}-elixir-${{ matrix.elixir }}-mix-${{ hashFiles('**/mix.lock') }}
+        restore-keys: ${{ runner.os }}-otp-${{ matrix.otp }}-elixir-${{ matrix.elixir }}-
+    - name: Install dependencies
+      run: mix deps.get
+    - name: Remove compiled application files
+      run: mix clean
+    - name: Compile dependencies
+      run: mix compile
+      env:
+        MIX_ENV: test
+    - name: Run tests
+      run: mix test test/chunkr/chunkr_test.exs:45

.gitignore

@@ -23,3 +23,4 @@ erl_crash.dump
 chunkr-*.tar
 
 .iex.exs
+.vscode/

.tool-versions

@@ -1,2 +1,2 @@
-elixir 1.12.0-otp-24
-erlang 24.0.5
+elixir 1.14.1-otp-25
+erlang 25.1.1

README.md

@@ -1,7 +1,9 @@
-<img alt="Chunkr" width="500px" src="assets/logo_o.svg">
+## <img alt="Chunkr" width="500px" src="assets/logo_o.svg">
 
 <!-- MDOC !-->
 
+[![Test](https://github.com/goodpixel/chunkr/actions/workflows/test.yml/badge.svg)](https://github.com/goodpixel/chunkr/actions/workflows/test.yml)
+
 Keyset-based query pagination for Elixir and Ecto.
 
 ## Use cases
@@ -76,12 +78,10 @@ Chunkr took inspiration from both [Paginator](https://github.com/duffelhq/pagina
 
 Quarto already addressed the deal-breaking need to reliably sort by columns that might contain
 `NULL` values. However, other limitations remained. E.g. it wasn't easy to paginate in reverse
-from the end of a result set to the beginning. Also, pagination was broken when sorting by fields on
-associations that might be missing (say, for example, sorting users by their preferred
-address—specifically when not all users have specified a "preferred" address). Furthermore, the
-existing libraries didn't allow for sorting by Ecto fragments, which is problematic because it’s
-often desirable to sort by calculated values—e.g. to provide case-insensitive sorts of people's
-names via an Ecto fragment such as `lower(last_name)`.
+from the end of a result set to the beginning. Also, the existing libraries didn't allow for
+sorting by Ecto fragments, which is problematic because it’s often desirable to sort by
+calculated values—e.g. to provide case-insensitive sorts of people's names via an Ecto fragment
+such as `lower(last_name)`.
 
 Chunkr:
 * provides a simple DSL to declare your pagination strategies
@@ -92,6 +92,11 @@ Chunkr:
 * allows custom encoding of individual cursor value types
 * allows generation of fully custom cursors (e.g. signed cursors)
 
+Limitations of Chunkr:
+* requires Ecto
+* requires pagination strategies to be declared up front
+* doesn't support custom selection of fields (it always retrieves all fields)
+
 ## Installation
 
 Add `chunkr` to your list of dependencies in `mix.exs`:

lib/chunkr.ex

@@ -6,23 +6,27 @@ defmodule Chunkr do
              |> Enum.fetch!(1)
 
   @default_max_limit 100
-  @default_opts [cursor_mod: Chunkr.Cursor.Base64, max_limit: @default_max_limit]
+  @default_cursor_mod Chunkr.Cursor.Base64
+  @defaults [cursor_mod: @default_cursor_mod, max_limit: @default_max_limit]
 
   @doc false
   defmacro __using__(config) do
     quote do
+      @chunkr_default_opts unquote(config) ++ [{:repo, __MODULE__} | unquote(@defaults)]
+
       def paginate!(queryable, strategy, sort_dir, opts) do
-        default_opts = unquote(config) ++ [{:repo, __MODULE__} | unquote(@default_opts)]
-        Chunkr.Pagination.paginate!(queryable, strategy, sort_dir, opts ++ default_opts)
+        Chunkr.Pagination.paginate!(queryable, strategy, sort_dir, opts ++ @chunkr_default_opts)
       end
 
       def paginate(queryable, strategy, sort_dir, opts) do
-        default_opts = unquote(config) ++ [{:repo, __MODULE__} | unquote(@default_opts)]
-        Chunkr.Pagination.paginate(queryable, strategy, sort_dir, opts ++ default_opts)
+        Chunkr.Pagination.paginate(queryable, strategy, sort_dir, opts ++ @chunkr_default_opts)
       end
     end
   end
 
   @doc false
   def default_max_limit(), do: @default_max_limit
+
+  @doc false
+  def default_cursor_mod(), do: @default_cursor_mod
 end

lib/chunkr/cursor/base64.ex

@@ -10,8 +10,8 @@ defmodule Chunkr.Cursor.Base64 do
 
   ## Example
 
-      iex> Chunkr.Cursor.Base64.to_cursor(["some", "value", 123])
-      {:ok, "g2wAAAADbQAAAARzb21lbQAAAAV2YWx1ZWF7ag=="}
+      iex> Chunkr.Cursor.Base64.to_cursor(["some", :value, 123])
+      {:ok, "g2wAAAADbQAAAARzb21lZAAFdmFsdWVhe2o="}
   """
   @impl true
   def to_cursor(values) do
@@ -28,8 +28,8 @@ defmodule Chunkr.Cursor.Base64 do
 
   ## Example
 
-      iex> Chunkr.Cursor.Base64.to_values("g2wAAAADbQAAAARzb21lbQAAAAV2YWx1ZWF7ag==")
-      {:ok, ["some", "value", 123]}
+      iex> Chunkr.Cursor.Base64.to_values("g2wAAAADbQAAAARzb21lZAAFdmFsdWVhe2o=")
+      {:ok, ["some", :value, 123]}
   """
   @impl true
   def to_values(opaque_cursor) do
@@ -40,27 +40,21 @@ defmodule Chunkr.Cursor.Base64 do
       else
         {:error, "Expected a list of values but got #{inspect(cursor_values)}"}
       end
-    else
-      {:error, :invalid_base64_value} ->
-        {:error, "Error decoding base64-encoded string: '#{inspect(opaque_cursor)}'"}
-
-      {:error, :invalid_term} ->
-        {:error, "Unable to translate binary to an Elixir term: '#{inspect(opaque_cursor)}'"}
     end
   end
 
-  defp base64_decode(string) do
-    case Base.url_decode64(string) do
+  defp base64_decode(opaque_cursor) do
+    case Base.url_decode64(opaque_cursor) do
       {:ok, value} -> {:ok, value}
-      :error -> {:error, :invalid_base64_value}
+      :error -> {:error, "Error decoding base64-encoded string: '#{inspect(opaque_cursor)}'"}
     end
   end
 
   defp binary_to_term(binary) do
     try do
       {:ok, :erlang.binary_to_term(binary, [:safe])}
     rescue
-      _ -> {:error, :invalid_term}
+      _ -> {:error, "Unable to translate binary to an Elixir term"}
     end
   end
 end

lib/chunkr/pagination.ex

@@ -31,7 +31,7 @@ defmodule Chunkr.Pagination do
     * `:max_limit` — Maximum number of results the user can request for this query.
       Default is #{Chunkr.default_max_limit()}.
     * `:cursor_mod` — Specifies the cursor module to use for encoding values as a cursor.
-      Defaults to `Chunkr.Cursor.Base64`.
+      Defaults to `#{Chunkr.default_cursor_mod()}`.
     * `:repo` — Repo to use for querying (automatically passed when calling either of
       the paginate convenience functions on your Repo).
     * `:planner` — The module implementing your pagination strategy (automatically passed
@@ -42,6 +42,16 @@ defmodule Chunkr.Pagination do
   def paginate(queryable, strategy, sort_dir, options) do
     with {:ok, opts} <- Opts.new(queryable, strategy, sort_dir, options),
          {:ok, queryable} <- validate_queryable(queryable) do
+
+          query =
+      queryable
+      |> apply_where(opts)
+      |> apply_order(opts)
+      |> apply_select(opts)
+      |> apply_limit(opts.limit + 1, opts)
+      Ecto.Adapters.SQL.to_sql(:all, Chunkr.TestRepo, query)
+      |> IO.inspect(label: "THE SQL")
+
       extended_rows =
         queryable
         |> apply_where(opts)
@@ -61,8 +71,8 @@ defmodule Chunkr.Pagination do
       {:ok,
        %Page{
          raw_results: rows_to_return,
-         has_previous_page: has_previous_page?(opts, extended_rows, requested_rows),
-         has_next_page: has_next_page?(opts, extended_rows, requested_rows),
+         has_previous_page: prev_page?(opts, extended_rows, requested_rows),
+         has_next_page: next_page?(opts, extended_rows, requested_rows),
          start_cursor: List.first(rows_to_return) |> row_to_cursor(opts),
          end_cursor: List.last(rows_to_return) |> row_to_cursor(opts),
          opts: opts
@@ -90,13 +100,11 @@ defmodule Chunkr.Pagination do
     end
   end
 
-  defp has_previous_page?(%{paging_dir: :forward} = opts, _, _), do: !!opts.cursor
-
-  defp has_previous_page?(%{paging_dir: :backward}, rows, requested_rows),
-    do: rows != requested_rows
+  defp prev_page?(%{paging_dir: :forward} = opts, _, _), do: !!opts.cursor
+  defp prev_page?(%{paging_dir: :backward}, extended, requested), do: extended != requested
 
-  defp has_next_page?(%{paging_dir: :forward}, rows, requested_rows), do: rows != requested_rows
-  defp has_next_page?(%{paging_dir: :backward} = opts, _, _), do: !!opts.cursor
+  defp next_page?(%{paging_dir: :forward}, extended, requested), do: extended != requested
+  defp next_page?(%{paging_dir: :backward} = opts, _, _), do: !!opts.cursor
 
   defp row_to_cursor(nil, _opts), do: nil
   defp row_to_cursor({cursor_values, _}, opts), do: Cursor.encode!(cursor_values, opts.cursor_mod)

lib/chunkr/pagination_planner.ex

@@ -55,9 +55,18 @@ defmodule Chunkr.PaginationPlanner do
 
   Because these sort clauses must reference bindings that have not yet been established,
   we use [`:as`](https://hexdocs.pm/ecto/Ecto.Query.html#module-named-bindings)
-  to take advantage of Ecto's late binding. The column referenced by `:as` must then be
+  to take advantage of Ecto's late binding feature. The column referenced by `:as` must then be
   explicitly provided within your query or it fail.
 
+  For example, with the following pagination strategy established…
+
+      paginate_by :username do
+        sort :asc, as(:user).username
+      end
+
+  …the query `from u in User, as: :user` would be valid, but `from u in User` would not (because
+  Ecto would not know where to find the `username` field).
+
   ## Always coalesce `NULL` values!
 
   SQL cannot reasonably compare `NULL` to a non-`NULL` value using operators like `<` and `>`.
@@ -129,10 +138,10 @@ defmodule Chunkr.PaginationPlanner do
   def parse_sorts([dir, field, [type: type]]), do: {dir, field, type}
 
   @doc false
-  def with_cursor_fields_func(query_name, fields) do
+  def with_cursor_fields_func(query_name, cursor_fields) do
     quote do
       def apply_select(query, unquote(query_name)) do
-        Ecto.Query.select(query, [record], {unquote(fields), record})
+        Ecto.Query.select(query, [record], {unquote(cursor_fields), record})
       end
     end
   end