From 8988a1299ca95b2d12a2cd9d40da385661904202 Mon Sep 17 00:00:00 2001 From: Ben Coppock Date: Fri, 13 May 2022 22:00:19 -0400 Subject: [PATCH 01/10] Add vscode to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e31b243..c714733 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ erl_crash.dump chunkr-*.tar .iex.exs +.vscode/ From 812d7f14e6c9ab684b1e0b1142236f3108c90822 Mon Sep 17 00:00:00 2001 From: Ben Coppock Date: Fri, 13 May 2022 22:01:39 -0400 Subject: [PATCH 02/10] Clean up Chunkr defaults --- lib/chunkr.ex | 14 +++++++++----- lib/chunkr/pagination.ex | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/chunkr.ex b/lib/chunkr.ex index 2ce892c..a4a3d35 100644 --- a/lib/chunkr.ex +++ b/lib/chunkr.ex @@ -6,23 +6,27 @@ defmodule Chunkr do |> Enum.fetch!(1) @default_max_limit 100 - @default_opts [cursor_mod: Chunkr.Cursor.Base64, max_limit: @default_max_limit] + @default_cursor_mod Chunkr.Cursor.Base64 + @defaults [cursor_mod: @default_cursor_mod, max_limit: @default_max_limit] @doc false defmacro __using__(config) do quote do + @chunkr_default_opts unquote(config) ++ [{:repo, __MODULE__} | unquote(@defaults)] + def paginate!(queryable, strategy, sort_dir, opts) do - default_opts = unquote(config) ++ [{:repo, __MODULE__} | unquote(@default_opts)] - Chunkr.Pagination.paginate!(queryable, strategy, sort_dir, opts ++ default_opts) + Chunkr.Pagination.paginate!(queryable, strategy, sort_dir, opts ++ @chunkr_default_opts) end def paginate(queryable, strategy, sort_dir, opts) do - default_opts = unquote(config) ++ [{:repo, __MODULE__} | unquote(@default_opts)] - Chunkr.Pagination.paginate(queryable, strategy, sort_dir, opts ++ default_opts) + Chunkr.Pagination.paginate(queryable, strategy, sort_dir, opts ++ @chunkr_default_opts) end end end @doc false def default_max_limit(), do: @default_max_limit + + @doc false + def default_cursor_mod(), do: @default_cursor_mod end diff --git a/lib/chunkr/pagination.ex b/lib/chunkr/pagination.ex index 2ddf29b..8a9c800 100644 --- a/lib/chunkr/pagination.ex +++ b/lib/chunkr/pagination.ex @@ -31,7 +31,7 @@ defmodule Chunkr.Pagination do * `:max_limit` — Maximum number of results the user can request for this query. Default is #{Chunkr.default_max_limit()}. * `:cursor_mod` — Specifies the cursor module to use for encoding values as a cursor. - Defaults to `Chunkr.Cursor.Base64`. + Defaults to `#{Chunkr.default_cursor_mod()}`. * `:repo` — Repo to use for querying (automatically passed when calling either of the paginate convenience functions on your Repo). * `:planner` — The module implementing your pagination strategy (automatically passed From 2d811ec29c4adc03aa4532cdf24bb59a354b0237 Mon Sep 17 00:00:00 2001 From: Ben Coppock Date: Wed, 12 Oct 2022 17:18:26 -0500 Subject: [PATCH 03/10] Improve docs --- README.md | 15 +++++++++------ lib/chunkr/cursor/base64.ex | 8 ++++---- lib/chunkr/pagination_planner.ex | 11 ++++++++++- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index abec728..1747dff 100644 --- a/README.md +++ b/README.md @@ -76,12 +76,10 @@ Chunkr took inspiration from both [Paginator](https://github.com/duffelhq/pagina Quarto already addressed the deal-breaking need to reliably sort by columns that might contain `NULL` values. However, other limitations remained. E.g. it wasn't easy to paginate in reverse -from the end of a result set to the beginning. Also, pagination was broken when sorting by fields on -associations that might be missing (say, for example, sorting users by their preferred -address—specifically when not all users have specified a "preferred" address). Furthermore, the -existing libraries didn't allow for sorting by Ecto fragments, which is problematic because it’s -often desirable to sort by calculated values—e.g. to provide case-insensitive sorts of people's -names via an Ecto fragment such as `lower(last_name)`. +from the end of a result set to the beginning. Also, the existing libraries didn't allow for +sorting by Ecto fragments, which is problematic because it’s often desirable to sort by +calculated values—e.g. to provide case-insensitive sorts of people's names via an Ecto fragment +such as `lower(last_name)`. Chunkr: * provides a simple DSL to declare your pagination strategies @@ -92,6 +90,11 @@ Chunkr: * allows custom encoding of individual cursor value types * allows generation of fully custom cursors (e.g. signed cursors) +Limitations of Chunkr: +* requires Ecto +* requires pagination strategies to be declared up front +* doesn't support custom selection of fields (it always retrieves all fields) + ## Installation Add `chunkr` to your list of dependencies in `mix.exs`: diff --git a/lib/chunkr/cursor/base64.ex b/lib/chunkr/cursor/base64.ex index e3fed15..baa4fcc 100644 --- a/lib/chunkr/cursor/base64.ex +++ b/lib/chunkr/cursor/base64.ex @@ -10,8 +10,8 @@ defmodule Chunkr.Cursor.Base64 do ## Example - iex> Chunkr.Cursor.Base64.to_cursor(["some", "value", 123]) - {:ok, "g2wAAAADbQAAAARzb21lbQAAAAV2YWx1ZWF7ag=="} + iex> Chunkr.Cursor.Base64.to_cursor(["some", :value, 123]) + {:ok, "g2wAAAADbQAAAARzb21lZAAFdmFsdWVhe2o="} """ @impl true def to_cursor(values) do @@ -28,8 +28,8 @@ defmodule Chunkr.Cursor.Base64 do ## Example - iex> Chunkr.Cursor.Base64.to_values("g2wAAAADbQAAAARzb21lbQAAAAV2YWx1ZWF7ag==") - {:ok, ["some", "value", 123]} + iex> Chunkr.Cursor.Base64.to_values("g2wAAAADbQAAAARzb21lZAAFdmFsdWVhe2o=") + {:ok, ["some", :value, 123]} """ @impl true def to_values(opaque_cursor) do diff --git a/lib/chunkr/pagination_planner.ex b/lib/chunkr/pagination_planner.ex index 3fbefb6..9f99db1 100644 --- a/lib/chunkr/pagination_planner.ex +++ b/lib/chunkr/pagination_planner.ex @@ -55,9 +55,18 @@ defmodule Chunkr.PaginationPlanner do Because these sort clauses must reference bindings that have not yet been established, we use [`:as`](https://hexdocs.pm/ecto/Ecto.Query.html#module-named-bindings) - to take advantage of Ecto's late binding. The column referenced by `:as` must then be + to take advantage of Ecto's late binding feature. The column referenced by `:as` must then be explicitly provided within your query or it fail. + For example, with the following pagination strategy established… + + paginate_by :username do + sort :asc, as(:user).username + end + + …the query `from u in User, as: :user` would be valid, but `from u in User` would not (because + Ecto would not know where to find the `username` field). + ## Always coalesce `NULL` values! SQL cannot reasonably compare `NULL` to a non-`NULL` value using operators like `<` and `>`. From 3336cdbd9266bfb62bf107226e518172f06f532b Mon Sep 17 00:00:00 2001 From: Ben Coppock Date: Wed, 12 Oct 2022 17:20:38 -0500 Subject: [PATCH 04/10] Clean up error handling --- lib/chunkr/cursor/base64.ex | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/chunkr/cursor/base64.ex b/lib/chunkr/cursor/base64.ex index baa4fcc..8a5238b 100644 --- a/lib/chunkr/cursor/base64.ex +++ b/lib/chunkr/cursor/base64.ex @@ -40,19 +40,13 @@ defmodule Chunkr.Cursor.Base64 do else {:error, "Expected a list of values but got #{inspect(cursor_values)}"} end - else - {:error, :invalid_base64_value} -> - {:error, "Error decoding base64-encoded string: '#{inspect(opaque_cursor)}'"} - - {:error, :invalid_term} -> - {:error, "Unable to translate binary to an Elixir term: '#{inspect(opaque_cursor)}'"} end end - defp base64_decode(string) do - case Base.url_decode64(string) do + defp base64_decode(opaque_cursor) do + case Base.url_decode64(opaque_cursor) do {:ok, value} -> {:ok, value} - :error -> {:error, :invalid_base64_value} + :error -> {:error, "Error decoding base64-encoded string: '#{inspect(opaque_cursor)}'"} end end @@ -60,7 +54,7 @@ defmodule Chunkr.Cursor.Base64 do try do {:ok, :erlang.binary_to_term(binary, [:safe])} rescue - _ -> {:error, :invalid_term} + _ -> {:error, "Unable to translate binary to an Elixir term"} end end end From 881fc95bc5c9748850b4f060f890a4e99c2fe80c Mon Sep 17 00:00:00 2001 From: Ben Coppock Date: Wed, 12 Oct 2022 21:06:37 -0500 Subject: [PATCH 05/10] Clean up naming of functions & variables --- lib/chunkr/pagination.ex | 14 ++++++-------- lib/chunkr/pagination_planner.ex | 4 ++-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/chunkr/pagination.ex b/lib/chunkr/pagination.ex index 8a9c800..2af656f 100644 --- a/lib/chunkr/pagination.ex +++ b/lib/chunkr/pagination.ex @@ -61,8 +61,8 @@ defmodule Chunkr.Pagination do {:ok, %Page{ raw_results: rows_to_return, - has_previous_page: has_previous_page?(opts, extended_rows, requested_rows), - has_next_page: has_next_page?(opts, extended_rows, requested_rows), + has_previous_page: prev_page?(opts, extended_rows, requested_rows), + has_next_page: next_page?(opts, extended_rows, requested_rows), start_cursor: List.first(rows_to_return) |> row_to_cursor(opts), end_cursor: List.last(rows_to_return) |> row_to_cursor(opts), opts: opts @@ -90,13 +90,11 @@ defmodule Chunkr.Pagination do end end - defp has_previous_page?(%{paging_dir: :forward} = opts, _, _), do: !!opts.cursor + defp prev_page?(%{paging_dir: :forward} = opts, _, _), do: !!opts.cursor + defp prev_page?(%{paging_dir: :backward}, extended, requested), do: extended != requested - defp has_previous_page?(%{paging_dir: :backward}, rows, requested_rows), - do: rows != requested_rows - - defp has_next_page?(%{paging_dir: :forward}, rows, requested_rows), do: rows != requested_rows - defp has_next_page?(%{paging_dir: :backward} = opts, _, _), do: !!opts.cursor + defp next_page?(%{paging_dir: :forward}, extended, requested), do: extended != requested + defp next_page?(%{paging_dir: :backward} = opts, _, _), do: !!opts.cursor defp row_to_cursor(nil, _opts), do: nil defp row_to_cursor({cursor_values, _}, opts), do: Cursor.encode!(cursor_values, opts.cursor_mod) diff --git a/lib/chunkr/pagination_planner.ex b/lib/chunkr/pagination_planner.ex index 9f99db1..71e3c75 100644 --- a/lib/chunkr/pagination_planner.ex +++ b/lib/chunkr/pagination_planner.ex @@ -138,10 +138,10 @@ defmodule Chunkr.PaginationPlanner do def parse_sorts([dir, field, [type: type]]), do: {dir, field, type} @doc false - def with_cursor_fields_func(query_name, fields) do + def with_cursor_fields_func(query_name, cursor_fields) do quote do def apply_select(query, unquote(query_name)) do - Ecto.Query.select(query, [record], {unquote(fields), record}) + Ecto.Query.select(query, [record], {unquote(cursor_fields), record}) end end end From 9be8252b870df89a66d62f234dfc84a5d193755f Mon Sep 17 00:00:00 2001 From: Ben Coppock Date: Wed, 12 Oct 2022 21:09:35 -0500 Subject: [PATCH 06/10] Add tests for custom encode/decode --- test/chunkr/cursor_value/decode_test.exs | 20 ++++++++++++++++++++ test/chunkr/cursor_value/encode_test.exs | 20 ++++++++++++++++++++ test/support/custom_encoding.ex | 2 +- 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 test/chunkr/cursor_value/decode_test.exs create mode 100644 test/chunkr/cursor_value/encode_test.exs diff --git a/test/chunkr/cursor_value/decode_test.exs b/test/chunkr/cursor_value/decode_test.exs new file mode 100644 index 0000000..8fc3d18 --- /dev/null +++ b/test/chunkr/cursor_value/decode_test.exs @@ -0,0 +1,20 @@ +defmodule Chunkr.CursorValue.DecodeTest do + use ExUnit.Case, async: true + alias Chunkr.CursorValue.Decode + + doctest Chunkr.CursorValue.Decode + + describe "Chunkr.CursorValue.Decode.convert/1" do + test "returns the initial value unchanged by default" do + assert :foo = Decode.convert(:foo) + assert "bar" = Decode.convert("bar") + assert 123 = Decode.convert(123) + assert %{foo: :bar} = Decode.convert(%{foo: :bar}) + end + + # See /test/support/custom_encoding.ex + test "honors custom encoding of cursor values" do + assert ~U[2022-10-12 22:48:33.437848Z] = Decode.convert({:dt, 1_665_614_913_437_848}) + end + end +end diff --git a/test/chunkr/cursor_value/encode_test.exs b/test/chunkr/cursor_value/encode_test.exs new file mode 100644 index 0000000..5100b67 --- /dev/null +++ b/test/chunkr/cursor_value/encode_test.exs @@ -0,0 +1,20 @@ +defmodule Chunkr.CursorValue.EncodeTest do + use ExUnit.Case, async: true + alias Chunkr.CursorValue.Encode + + doctest Chunkr.CursorValue.Encode + + describe "Chunkr.CursorValue.Encode.convert/1" do + test "returns the initial value unchanged by default" do + assert :foo = Encode.convert(:foo) + assert "bar" = Encode.convert("bar") + assert 123 = Encode.convert(123) + assert %{foo: :bar} = Encode.convert(%{foo: :bar}) + end + + # See /test/support/custom_encoding.ex + test "honors custom encoding of cursor values" do + assert {:dt, 1_665_614_913_437_848} = Encode.convert(~U[2022-10-12 22:48:33.437848Z]) + end + end +end diff --git a/test/support/custom_encoding.ex b/test/support/custom_encoding.ex index 5a23818..4f74565 100644 --- a/test/support/custom_encoding.ex +++ b/test/support/custom_encoding.ex @@ -5,5 +5,5 @@ defimpl Chunkr.CursorValue.Encode, for: DateTime do end defimpl Chunkr.CursorValue.Decode, for: Tuple do - def convert({:dt, unix_timestamp}), do: DateTime.from_unix!(unix_timestamp, :microsecond) + def convert({:dt, usec}) when is_integer(usec), do: DateTime.from_unix!(usec, :microsecond) end From fbb177a31f2dd6cf5ee4d81e6ee246291ccb82b0 Mon Sep 17 00:00:00 2001 From: Ben Coppock Date: Thu, 13 Oct 2022 10:14:34 -0500 Subject: [PATCH 07/10] Update Erlang/Elixir versions --- .tool-versions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.tool-versions b/.tool-versions index 51e363f..e4f68f1 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -elixir 1.12.0-otp-24 -erlang 24.0.5 +elixir 1.14.1-otp-25 +erlang 25.1.1 From 7acb4443d18883c7b739f0c5345d11dc769a45ff Mon Sep 17 00:00:00 2001 From: Ben Coppock Date: Thu, 13 Oct 2022 10:16:42 -0500 Subject: [PATCH 08/10] Add CI --- .github/workflows/test.yml | 53 ++++++++++++++++++++++++++++++++++++++ README.md | 4 ++- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..f2420ea --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,53 @@ +name: Test + +on: + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +jobs: + test: + name: OTP ${{ matrix.otp }} / Elixir ${{ matrix.elixir }} + runs-on: ubuntu-latest + services: + postgres: + image: postgres + ports: + - 5432:5432 + env: + POSTGRES_DB: chunkr_test + POSTGRES_PASSWORD: postgres + + strategy: + matrix: + elixir: ["1.14.1"] + otp: ["25.1.1"] + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Elixir + uses: erlef/setup-beam@v1 + with: + otp-version: ${{ matrix.otp }} + elixir-version: ${{ matrix.elixir }} + - name: Restore dependencies cache + uses: actions/cache@v3 + with: + path: | + _build + deps + key: ${{ runner.os }}-otp-${{ matrix.otp }}-elixir-${{ matrix.elixir }}-mix-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-otp-${{ matrix.otp }}-elixir-${{ matrix.elixir }}- + - name: Install dependencies + run: mix deps.get + - name: Remove compiled application files + run: mix clean + - name: Compile dependencies + run: mix compile + env: + MIX_ENV: test + - name: Run tests + run: mix test diff --git a/README.md b/README.md index 1747dff..1bb61ba 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ -Chunkr +## Chunkr +[![Test](https://github.com/goodpixel/chunkr/actions/workflows/test.yml/badge.svg)](https://github.com/goodpixel/chunkr/actions/workflows/test.yml) + Keyset-based query pagination for Elixir and Ecto. ## Use cases From 12f97d5d7c2f8625b297065ce614cddd48f147c9 Mon Sep 17 00:00:00 2001 From: Ben Coppock Date: Thu, 13 Oct 2022 12:07:28 -0500 Subject: [PATCH 09/10] WIP [Temp] --- test/chunkr/chunkr_test.exs | 254 ++++++++++++++++++------------------ 1 file changed, 127 insertions(+), 127 deletions(-) diff --git a/test/chunkr/chunkr_test.exs b/test/chunkr/chunkr_test.exs index 89c3a33..8b93cab 100644 --- a/test/chunkr/chunkr_test.exs +++ b/test/chunkr/chunkr_test.exs @@ -1,5 +1,5 @@ defmodule ChunkrTest do - use ExUnit.Case, async: true + use ExUnit.Case, async: false import Ecto.Query import Chunkr.PaginationHelpers import Chunkr.TestDataGenerators @@ -33,96 +33,96 @@ defmodule ChunkrTest do verify_pagination(TestRepo, query, :two_fields, :asc, expected, @count) end - test "inverting a two field sort" do - {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) - query = from(u in User, as: :user) - expected = TestRepo.all(from u in User, order_by: [desc_nulls_first: u.last_name, asc: u.id]) - verify_pagination(TestRepo, query, :two_fields, :desc, expected, @count) - end - - test "paginating by three fields" do - {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) - query = from(u in User, as: :user) - - expected_results = - TestRepo.all( - from(u in User, - order_by: [ - asc_nulls_last: u.last_name, - asc_nulls_last: u.first_name, - desc: u.id - ] - ) - ) - - verify_pagination(TestRepo, query, :three_fields, :asc, expected_results, @count) - end - - test "inverting a three field sort" do - {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) - query = from(u in User, as: :user) - - expected_results = - TestRepo.all( - from(u in User, - order_by: [ - desc_nulls_first: u.last_name, - desc_nulls_first: u.first_name, - asc: u.id - ] - ) - ) - - verify_pagination(TestRepo, query, :three_fields, :desc, expected_results, @count) - end - - test "paginating by four fields" do - {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) - query = from(u in User, as: :user) - - expected_results = - TestRepo.all( - from(u in User, - order_by: [ - desc_nulls_first: u.last_name, - desc_nulls_first: u.first_name, - desc_nulls_first: u.middle_name, - asc: u.id - ] - ) - ) - - verify_pagination(TestRepo, query, :four_fields, :desc, expected_results, @count) - end - - test "inverting a four field sort" do - {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) - query = from(u in User, as: :user) - - expected_results = - TestRepo.all( - from(u in User, - order_by: [ - asc_nulls_last: u.last_name, - asc_nulls_last: u.first_name, - asc_nulls_last: u.middle_name, - desc: u.id - ] - ) - ) - - verify_pagination(TestRepo, query, :four_fields, :asc, expected_results, @count) - end - - test "paginating by UUID" do - {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) - query = from(u in User, as: :user) - - expected_results = - TestRepo.all(from u in User, order_by: [asc_nulls_last: u.last_name, desc: u.public_id]) - - verify_pagination(TestRepo, query, :uuid, :asc, expected_results, @count) - end + # test "inverting a two field sort" do + # {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) + # query = from(u in User, as: :user) + # expected = TestRepo.all(from u in User, order_by: [desc_nulls_first: u.last_name, asc: u.id]) + # verify_pagination(TestRepo, query, :two_fields, :desc, expected, @count) + # end + + # test "paginating by three fields" do + # {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) + # query = from(u in User, as: :user) + + # expected_results = + # TestRepo.all( + # from(u in User, + # order_by: [ + # asc_nulls_last: u.last_name, + # asc_nulls_last: u.first_name, + # desc: u.id + # ] + # ) + # ) + + # verify_pagination(TestRepo, query, :three_fields, :asc, expected_results, @count) + # end + + # test "inverting a three field sort" do + # {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) + # query = from(u in User, as: :user) + + # expected_results = + # TestRepo.all( + # from(u in User, + # order_by: [ + # desc_nulls_first: u.last_name, + # desc_nulls_first: u.first_name, + # asc: u.id + # ] + # ) + # ) + + # verify_pagination(TestRepo, query, :three_fields, :desc, expected_results, @count) + # end + + # test "paginating by four fields" do + # {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) + # query = from(u in User, as: :user) + + # expected_results = + # TestRepo.all( + # from(u in User, + # order_by: [ + # desc_nulls_first: u.last_name, + # desc_nulls_first: u.first_name, + # desc_nulls_first: u.middle_name, + # asc: u.id + # ] + # ) + # ) + + # verify_pagination(TestRepo, query, :four_fields, :desc, expected_results, @count) + # end + + # test "inverting a four field sort" do + # {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) + # query = from(u in User, as: :user) + + # expected_results = + # TestRepo.all( + # from(u in User, + # order_by: [ + # asc_nulls_last: u.last_name, + # asc_nulls_last: u.first_name, + # asc_nulls_last: u.middle_name, + # desc: u.id + # ] + # ) + # ) + + # verify_pagination(TestRepo, query, :four_fields, :asc, expected_results, @count) + # end + + # test "paginating by UUID" do + # {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) + # query = from(u in User, as: :user) + + # expected_results = + # TestRepo.all(from u in User, order_by: [asc_nulls_last: u.last_name, desc: u.public_id]) + + # verify_pagination(TestRepo, query, :uuid, :asc, expected_results, @count) + # end test "paginating with a subquery" do TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) @@ -169,42 +169,42 @@ defmodule ChunkrTest do verify_pagination(TestRepo, query, :computed_value, :desc, expected_results, @count) end - test "sorting by a potentially-missing association" do - user_attrs = Enum.take(user_attrs(), @count) - {_, users} = TestRepo.insert_all(User, user_attrs, returning: true) - user_ids = users |> Enum.map(& &1.id) - - phone_attrs = Enum.take(phone_attrs(), @count) - phone_attrs = maybe_assign_user_ids(phone_attrs, user_ids) - TestRepo.insert_all(PhoneNumber, phone_attrs) - - query = - from p in PhoneNumber, - as: :phone, - left_join: u in assoc(p, :user), - as: :user, - preload: [user: u] - - expected_results = - TestRepo.all( - from p in PhoneNumber, - left_join: u in assoc(p, :user), - order_by: [ - asc_nulls_last: u.first_name, - asc: p.id - ], - preload: [user: u] - ) - - verify_pagination( - TestRepo, - query, - :by_possibly_null_association, - :asc, - expected_results, - @count - ) - end + # test "sorting by a potentially-missing association" do + # user_attrs = Enum.take(user_attrs(), @count) + # {_, users} = TestRepo.insert_all(User, user_attrs, returning: true) + # user_ids = users |> Enum.map(& &1.id) + + # phone_attrs = Enum.take(phone_attrs(), @count) + # phone_attrs = maybe_assign_user_ids(phone_attrs, user_ids) + # TestRepo.insert_all(PhoneNumber, phone_attrs) + + # query = + # from p in PhoneNumber, + # as: :phone, + # left_join: u in assoc(p, :user), + # as: :user, + # preload: [user: u] + + # expected_results = + # TestRepo.all( + # from p in PhoneNumber, + # left_join: u in assoc(p, :user), + # order_by: [ + # asc_nulls_last: u.first_name, + # asc: p.id + # ], + # preload: [user: u] + # ) + + # verify_pagination( + # TestRepo, + # query, + # :by_possibly_null_association, + # :asc, + # expected_results, + # @count + # ) + # end # # HELPERS From 14c16c2d4512fa3c2f4d858da3f6edb909e11402 Mon Sep 17 00:00:00 2001 From: Ben Coppock Date: Thu, 13 Oct 2022 14:40:10 -0500 Subject: [PATCH 10/10] WIP --- .github/workflows/test.yml | 2 +- lib/chunkr/pagination.ex | 10 +++++++ mix.lock | 28 +++++++++--------- test/chunkr/chunkr_test.exs | 57 ++++++++++++++++++++++++++++++++++--- 4 files changed, 78 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f2420ea..35024f4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,4 +50,4 @@ jobs: env: MIX_ENV: test - name: Run tests - run: mix test + run: mix test test/chunkr/chunkr_test.exs:45 diff --git a/lib/chunkr/pagination.ex b/lib/chunkr/pagination.ex index 2af656f..026fc61 100644 --- a/lib/chunkr/pagination.ex +++ b/lib/chunkr/pagination.ex @@ -42,6 +42,16 @@ defmodule Chunkr.Pagination do def paginate(queryable, strategy, sort_dir, options) do with {:ok, opts} <- Opts.new(queryable, strategy, sort_dir, options), {:ok, queryable} <- validate_queryable(queryable) do + + query = + queryable + |> apply_where(opts) + |> apply_order(opts) + |> apply_select(opts) + |> apply_limit(opts.limit + 1, opts) + Ecto.Adapters.SQL.to_sql(:all, Chunkr.TestRepo, query) + |> IO.inspect(label: "THE SQL") + extended_rows = queryable |> apply_where(opts) diff --git a/mix.lock b/mix.lock index bda1e81..9044cf4 100644 --- a/mix.lock +++ b/mix.lock @@ -1,22 +1,22 @@ %{ - "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, + "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"}, "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, - "credo": {:hex, :credo, "1.5.6", "e04cc0fdc236fefbb578e0c04bd01a471081616e741d386909e527ac146016c6", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "4b52a3e558bd64e30de62a648518a5ea2b6e3e5d2b164ef5296244753fc7eb17"}, - "db_connection": {:hex, :db_connection, "2.4.0", "d04b1b73795dae60cead94189f1b8a51cc9e1f911c234cc23074017c43c031e5", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad416c21ad9f61b3103d254a71b63696ecadb6a917b36f563921e0de00d7d7c8"}, + "credo": {:hex, :credo, "1.6.7", "323f5734350fd23a456f2688b9430e7d517afb313fbd38671b8a4449798a7854", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "41e110bfb007f7eda7f897c10bf019ceab9a0b269ce79f015d54b0dcf4fc7dd3"}, + "db_connection": {:hex, :db_connection, "2.4.2", "f92e79aff2375299a16bcb069a14ee8615c3414863a6fef93156aee8e86c2ff3", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4fe53ca91b99f55ea249693a0229356a08f4d1a7931d8ffa79289b145fe83668"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, - "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.15", "b29e8e729f4aa4a00436580dcc2c9c5c51890613457c193cc8525c388ccb2f06", [:mix], [], "hexpm", "044523d6438ea19c1b8ec877ec221b008661d3c27e3b848f4c879f500421ca5c"}, - "ecto": {:hex, :ecto, "3.7.1", "a20598862351b29f80f285b21ec5297da1181c0442687f9b8329f0445d228892", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d36e5b39fc479e654cffd4dbe1865d9716e4a9b6311faff799b6f90ab81b8638"}, - "ecto_sql": {:hex, :ecto_sql, "3.7.0", "2fcaad4ab0c8d76a5afbef078162806adbe709c04160aca58400d5cbbe8eeac6", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.7.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.4.0 or ~> 0.5.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a26135dfa1d99bf87a928c464cfa25bba6535a4fe761eefa56077a4febc60f70"}, + "dialyxir": {:hex, :dialyxir, "1.2.0", "58344b3e87c2e7095304c81a9ae65cb68b613e28340690dfe1a5597fd08dec37", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "61072136427a851674cab81762be4dbeae7679f85b1272b6d25c3a839aff8463"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.28", "0bf6546eb7cd6185ae086cbc5d20cd6dbb4b428aad14c02c49f7b554484b4586", [:mix], [], "hexpm", "501cef12286a3231dc80c81352a9453decf9586977f917a96e619293132743fb"}, + "ecto": {:hex, :ecto, "3.9.1", "67173b1687afeb68ce805ee7420b4261649d5e2deed8fe5550df23bab0bc4396", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c80bb3d736648df790f7f92f81b36c922d9dd3203ca65be4ff01d067f54eb304"}, + "ecto_sql": {:hex, :ecto_sql, "3.9.0", "2bb21210a2a13317e098a420a8c1cc58b0c3421ab8e3acfa96417dab7817918c", [:mix], [{:db_connection, "~> 2.5 or ~> 2.4.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a8f3f720073b8b1ac4c978be25fa7960ed7fd44997420c304a4a2e200b596453"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "ex_doc": {:hex, :ex_doc, "0.25.2", "4f1cae793c4d132e06674b282f1d9ea3bf409bcca027ddb2fe177c4eed6a253f", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5b0c172e87ac27f14dfd152d52a145238ec71a95efbf29849550278c58a393d6"}, + "ex_doc": {:hex, :ex_doc, "0.28.6", "2bbd7a143d3014fc26de9056793e97600ae8978af2ced82c2575f130b7c0d7d7", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bca1441614654710ba37a0e173079273d619f9160cbcc8cd04e6bd59f1ad0e29"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, - "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"}, - "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, + "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, + "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, - "postgrex": {:hex, :postgrex, "0.15.10", "2809dee1b1d76f7cbabe570b2a9285c2e7b41be60cf792f5f2804a54b838a067", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "1560ca427542f6b213f8e281633ae1a3b31cdbcd84ebd7f50628765b8f6132be"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, + "postgrex": {:hex, :postgrex, "0.16.5", "fcc4035cc90e23933c5d69a9cd686e329469446ef7abba2cf70f08e2c4b69810", [:mix], [{:connection, "~> 1.1", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "edead639dc6e882618c01d8fc891214c481ab9a3788dfe38dd5e37fd1d5fb2e8"}, "stream_data": {:hex, :stream_data, "0.5.0", "b27641e58941685c75b353577dc602c9d2c12292dd84babf506c2033cd97893e", [:mix], [], "hexpm", "012bd2eec069ada4db3411f9115ccafa38540a3c78c4c0349f151fc761b9e271"}, - "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"}, + "telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"}, } diff --git a/test/chunkr/chunkr_test.exs b/test/chunkr/chunkr_test.exs index 8b93cab..d34ec1a 100644 --- a/test/chunkr/chunkr_test.exs +++ b/test/chunkr/chunkr_test.exs @@ -12,25 +12,74 @@ defmodule ChunkrTest do @count 30 + defp generate_users(count \\ @count) do + {^count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), count)) + end + test "paginating by a single field" do - {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) + generate_users() query = from(u in User, as: :user) expected_results = TestRepo.all(from(u in User, order_by: [asc: u.id])) verify_pagination(TestRepo, query, :single_field, :asc, expected_results, @count) end test "inverting a single field sort" do - {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) + generate_users() query = from(u in User, as: :user) expected_results = TestRepo.all(from(u in User, order_by: [desc: u.id])) verify_pagination(TestRepo, query, :single_field, :desc, expected_results, @count) end test "paginating by two fields" do - {@count, _records} = TestRepo.insert_all(User, Enum.take(user_attrs(), @count)) + # generate_users() + params = [ + %{ + updated_at: ~U[1506-07-12 09:51:32.000000Z], + inserted_at: ~U[1038-12-03 17:10:10.000000Z], + last_name: "", + middle_name: nil, + first_name: nil, + public_id: "d1728d8a-34b6-46f9-b57a-137fe40b5266", + id: 12, + }, + %{ + updated_at: ~U[1148-05-22 06:55:26.000000Z], + inserted_at: ~U[2299-03-18 05:02:19.000000Z], + last_name: "{", + middle_name: "", + first_name: "B", + public_id: "7c0defad-ed6a-47f2-ad71-9d7fab0113f8", + id: 1, + }, + %{ + updated_at: ~U[2789-10-10 18:44:47.000000Z], + inserted_at: ~U[1181-02-24 13:36:53.000000Z], + last_name: nil, + middle_name: "z;,SO(K@ew6_l Enum.map(&(&1.last_name)) + |> IO.inspect(label: "expected") + verify_pagination(TestRepo, query, :two_fields, :asc, expected, 4) end # test "inverting a two field sort" do