From 656f8cb8b3e44811b8fee537b2db9f207e99cbaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=A4rtschi?= Date: Fri, 6 Mar 2026 16:16:48 +0100 Subject: [PATCH 1/2] feat: add raw_rows option for high-performance worksheet generation Bypass the Cell protocol and XmlStream when `raw_rows: true` is passed to `Worksheet.new/4`, generating SpreadsheetML XML directly as iodata. Rows are processed in parallel batches via Task.async_stream, yielding ~4x speedup for large exports. Co-Authored-By: Claude Opus 4.6 --- README.md | 15 +++ lib/exceed/file.ex | 7 +- lib/exceed/worksheet.ex | 210 ++++++++++++++++++++++++++++++++- test/exceed/worksheet_test.exs | 55 +++++++++ test/exceed_test.exs | 51 ++++++++ 5 files changed, 334 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f4e55e8..3bb624e 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,21 @@ rows = Exceed.Worksheet.new("Sheet Name", ["Heading 1", "Heading 2"], rows) ``` +### Raw rows mode + +For large data sets, the `raw_rows: true` option bypasses the `Cell` protocol +and `XmlStream`, generating SpreadsheetML XML directly as iodata. Rows are +processed in parallel batches using `Task.async_stream`. + +``` elixir +Exceed.Worksheet.new("Sheet Name", ["Heading 1", "Heading 2"], rows, raw_rows: true) +``` + +This is significantly faster for worksheets with many rows, at the cost of not +supporting custom `Cell` protocol implementations. Supported value types: +strings, integers, floats, booleans, atoms, `nil`, `Date`, `DateTime`, and +`NaiveDateTime`. + ## Alternatives & References This library is inspired by and learns from other great libraries. One might diff --git a/lib/exceed/file.ex b/lib/exceed/file.ex index c9d3c08..9d666b2 100644 --- a/lib/exceed/file.ex +++ b/lib/exceed/file.ex @@ -4,12 +4,17 @@ defmodule Exceed.File do @buffer_size_bytes 128 * 1024 @accumulator {<<>>, 0} - def file(content, filename, opts) do + def file(content, filename, opts) when is_list(content) do stream = XmlStream.stream!(content, printer: Exceed.Xml) stream = if Keyword.get(opts, :buffer, true), do: buffer(stream), else: stream Zstream.entry(filename, stream) end + def file(content, filename, opts) do + stream = if Keyword.get(opts, :buffer, true), do: buffer(content), else: content + Zstream.entry(filename, stream) + end + defp buffer(stream) do stream |> Stream.chunk_while( diff --git a/lib/exceed/worksheet.ex b/lib/exceed/worksheet.ex index 18bf726..cd14e2a 100644 --- a/lib/exceed/worksheet.ex +++ b/lib/exceed/worksheet.ex @@ -65,6 +65,26 @@ defmodule Exceed.Worksheet do the [OpenXML.Spreadsheet Column docs](https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.column?view=openxml-3.0.1) for more info). + ## Raw rows mode + + For large data sets, the `raw_rows: true` option bypasses the `Exceed.Worksheet.Cell` + protocol and `XmlStream`, generating SpreadsheetML XML directly as iodata. This can + be significantly faster for worksheets with many rows. + + ``` elixir + iex> headers = ["Name", "Age"] + iex> rows = [["Alice", 30], ["Bob", 25]] + iex> Exceed.Worksheet.new("Sheet", headers, rows, raw_rows: true) + #Exceed.Worksheet + ``` + + Row data is processed in batches using `Task.async_stream` for parallel + iodata generation. Supported value types: strings, integers, floats, booleans, + atoms, `nil`, `Date`, `DateTime`, and `NaiveDateTime`. + + Note: because the `Cell` protocol is bypassed, custom `Cell` implementations + are not available in raw rows mode. + """ alias Exceed.Worksheet.Cell alias XmlStream, as: Xs @@ -80,7 +100,8 @@ defmodule Exceed.Worksheet do content: Enum.t(), headers: headers(), name: String.t(), - opts: spreadsheet_options() + opts: spreadsheet_options(), + raw_rows: boolean() } @derive {Inspect, only: [:name]} @@ -89,12 +110,19 @@ defmodule Exceed.Worksheet do headers name opts + raw_rows )a @doc """ Initialize a new worksheet to be added to a workbook. See `Exceed.Workbook.add_worksheet/2`. For worksheet options, see the module docs for `m:Exceed.Worksheet#module-sheet-options`. + ## Options + + - `:raw_rows` - when `true`, bypasses the `Cell` protocol and generates XML + directly as iodata for better performance. See the module docs for details. + - `:cols` - column configuration. See the module docs for details. + ## Examples ``` elixir @@ -111,10 +139,18 @@ defmodule Exceed.Worksheet do def new(_name, [], _content, _opts), do: raise(Exceed.Error, "Worksheet headers must be a list of items or nil") - def new(name, headers, content, opts), - do: __struct__(name: name, headers: headers, content: content, opts: opts) + def new(name, headers, content, opts) do + {raw_rows, opts} = Keyword.pop(opts, :raw_rows, false) + __struct__(name: name, headers: headers, content: content, opts: opts, raw_rows: raw_rows) + end + + @raw_batch_size 5_000 @doc false + def to_xml(%__MODULE__{raw_rows: true} = ws) do + raw_xml_stream(ws) + end + def to_xml(%__MODULE__{headers: headers, content: content, opts: opts}) do %{ col_padding: col_padding, @@ -229,4 +265,172 @@ defmodule Exceed.Worksheet do identifier = to_string(row_idx) {[Xs.element("row", %{"r" => identifier}, to_cells(items, identifier))], row_idx + 1} end + + # # # Raw rows fast path # # # + + defp raw_xml_stream(%__MODULE__{headers: headers, content: content, opts: opts}) do + %{col_padding: col_padding, col_widths: col_widths} = normalize_opts(opts) + col_letters = column_letters(headers || hd(Enum.take(content, 1))) + + header_xml = raw_header_xml(headers, col_padding, col_widths) + footer_xml = raw_footer_xml() + + row_stream = + content + |> prepend_headers(headers) + |> Stream.with_index(1) + |> Stream.chunk_every(@raw_batch_size) + |> Task.async_stream( + fn batch -> + Enum.map(batch, fn {row, row_idx} -> + raw_row(row, row_idx, col_letters) + end) + end, + ordered: true, + max_concurrency: System.schedulers_online() + ) + |> Stream.map(fn {:ok, rows} -> rows end) + + Stream.concat([ + [header_xml], + row_stream, + [footer_xml] + ]) + end + + defp column_letters(items) do + items + |> Enum.with_index() + |> Enum.map(fn {_, i} -> + col_idx_to_letter(i + 1) + end) + end + + defp col_idx_to_letter(n) when n <= 26, do: <> + + defp col_idx_to_letter(n) do + col_idx_to_letter(div(n - 1, 26)) <> <> + end + + defp raw_header_xml(headers, col_padding, col_widths) do + cols_xml = raw_cols_xml(headers, col_padding, col_widths) + + [ + ~s(), + ~s(), + ~s(), + ~s(), + ~s(), + cols_xml, + ~s() + ] + end + + defp raw_cols_xml(nil, _padding, _widths), do: "" + + defp raw_cols_xml(headers, padding, widths) do + cols = + headers + |> Enum.with_index(1) + |> Enum.map(fn {header, i} -> + width = Map.get(widths, i, String.length(to_string(header)) + padding) + ~s() + end) + + ["", cols, ""] + end + + defp raw_footer_xml do + [ + "", + ~s(), + ~s(), + ~s(), + "", + "", + "" + ] + end + + defp raw_row(cells, row_idx, col_letters) do + row_str = Integer.to_string(row_idx) + + [ + "", + raw_cells(cells, row_str, col_letters), + "" + ] + end + + defp raw_cells(cells, row_str, col_letters) do + cells + |> Enum.zip(col_letters) + |> Enum.map(fn {value, letter} -> + raw_cell(value, letter, row_str) + end) + end + + defp raw_cell(value, letter, row_str) when is_integer(value) do + ["", Integer.to_string(value), ""] + end + + defp raw_cell(value, letter, row_str) when is_float(value) do + ["", Float.to_string(value), ""] + end + + defp raw_cell(value, letter, row_str) when is_binary(value) do + ["", escape_xml(value), ""] + end + + defp raw_cell(nil, letter, row_str) do + [""] + end + + defp raw_cell(true, letter, row_str) do + ["1"] + end + + defp raw_cell(false, letter, row_str) do + ["0"] + end + + defp raw_cell(value, letter, row_str) when is_atom(value) do + raw_cell(Atom.to_string(value), letter, row_str) + end + + defp raw_cell(%Date{year: year} = date, letter, row_str) when year >= 1900 do + value = Exceed.Util.to_excel_datetime(date) + ["", Float.to_string(value), ""] + end + + defp raw_cell(%Date{} = date, letter, row_str) do + raw_cell(Date.to_iso8601(date), letter, row_str) + end + + defp raw_cell(%DateTime{year: year} = dt, letter, row_str) when year >= 1900 do + value = Exceed.Util.to_excel_datetime(dt) + ["", Float.to_string(value), ""] + end + + defp raw_cell(%DateTime{} = dt, letter, row_str) do + raw_cell(DateTime.to_iso8601(dt), letter, row_str) + end + + defp raw_cell(%NaiveDateTime{year: year} = ndt, letter, row_str) when year >= 1900 do + value = Exceed.Util.to_excel_datetime(ndt) + ["", Float.to_string(value), ""] + end + + defp raw_cell(%NaiveDateTime{} = ndt, letter, row_str) do + raw_cell(NaiveDateTime.to_iso8601(ndt), letter, row_str) + end + + defp escape_xml(string) do + string + |> String.replace("&", "&") + |> String.replace("<", "<") + |> String.replace(">", ">") + end end diff --git a/test/exceed/worksheet_test.exs b/test/exceed/worksheet_test.exs index af49643..87ffae1 100644 --- a/test/exceed/worksheet_test.exs +++ b/test/exceed/worksheet_test.exs @@ -200,6 +200,61 @@ defmodule Exceed.WorksheetTest do end end + describe "to_xml with raw_rows" do + test "generates rows for the headers and each member of the stream", %{headers: headers, stream: stream} do + ws = Worksheet.new("sheet", headers, Enum.take(stream, 2), raw_rows: true) + xml = Worksheet.to_xml(ws) |> Enum.to_list() |> IO.iodata_to_binary() + xml = XmlQuery.parse(xml) + + assert [header_row, row_1, row_2] = Xq.all(xml, "/worksheet/sheetData/row") + + assert header_row |> Xq.attr("r") == "1" + + header_row + |> extract_cells() + |> assert_eq([ + %{type: "inlineStr", text: "inline strings", children: "is/t", cell: "A1"}, + %{type: "inlineStr", text: "integers", children: "is/t", cell: "B1"}, + %{type: "inlineStr", text: "floats", children: "is/t", cell: "C1"} + ]) + + row_1 + |> extract_cells() + |> assert_eq([ + %{type: "inlineStr", text: "row 1 cell 1", children: "is/t", cell: "A2"}, + %{type: "n", text: "1", children: "v", cell: "B2"}, + %{type: "n", text: "1.5", children: "v", cell: "C2"} + ]) + + row_2 + |> extract_cells() + |> assert_eq([ + %{type: "inlineStr", text: "row 2 cell 1", children: "is/t", cell: "A3"}, + %{type: "n", text: "2", children: "v", cell: "B3"}, + %{type: "n", text: "3.0", children: "v", cell: "C3"} + ]) + end + + test "generates correct column widths from headers", %{headers: headers, stream: stream} do + ws = Worksheet.new("sheet", headers, Enum.take(stream, 0), raw_rows: true) + xml = Worksheet.to_xml(ws) |> Enum.to_list() |> IO.iodata_to_binary() + xml = XmlQuery.parse(xml) + + assert [col1, col2, col3] = Xq.all(xml, "/worksheet/cols/col") + + assert Xq.attr(col1, "width") == "18.25" + assert Xq.attr(col2, "width") == "12.25" + assert Xq.attr(col3, "width") == "10.25" + end + + test "escapes XML special characters in strings" do + ws = Worksheet.new("sheet", ["h"], [["bold & \"quotes\""]], raw_rows: true) + xml = Worksheet.to_xml(ws) |> Enum.to_list() |> IO.iodata_to_binary() + + assert xml =~ "<b>bold</b> & \"quotes\"" + end + end + # # # defp extract_cells(row) do diff --git a/test/exceed_test.exs b/test/exceed_test.exs index 6d9fc3d..40589c6 100644 --- a/test/exceed_test.exs +++ b/test/exceed_test.exs @@ -144,6 +144,57 @@ defmodule ExceedTest do end end + describe "raw_rows roundtrip" do + @describetag :tmp_dir + + test "produces valid XLSX with correct data", %{tmp_dir: tmpdir} do + rows = [ + ["Alice", 30, 1.75], + ["Bob", 25, 1.80], + ["Charlie", 35, 1.65] + ] + + filename = + Exceed.Workbook.new("me") + |> Exceed.Workbook.add_worksheet( + Exceed.Worksheet.new("People", ["Name", "Age", "Height"], rows, raw_rows: true) + ) + |> stream_to_file(tmpdir) + + assert {:ok, wb} = XlsxReader.open(to_string(filename)) + assert XlsxReader.sheet_names(wb) == ["People"] + assert {:ok, sheet_rows} = XlsxReader.sheet(wb, "People") + + assert sheet_rows == [ + ["Name", "Age", "Height"], + ["Alice", 30, 1.75], + ["Bob", 25, 1.8], + ["Charlie", 35, 1.65] + ] + end + + test "handles dates", %{tmp_dir: tmpdir} do + today = Date.utc_today() + rows = [[today], [Date.add(today, -1)]] + + filename = + Exceed.Workbook.new("me") + |> Exceed.Workbook.add_worksheet( + Exceed.Worksheet.new("Dates", ["Date"], rows, raw_rows: true) + ) + |> stream_to_file(tmpdir) + + assert {:ok, wb} = XlsxReader.open(to_string(filename)) + assert {:ok, sheet_rows} = XlsxReader.sheet(wb, "Dates") + + assert sheet_rows == [ + ["Date"], + [today], + [Date.add(today, -1)] + ] + end + end + describe "strings" do @describetag :tmp_dir test "can be parsed", %{tmp_dir: tmpdir} do From 7f6a20f5fde262bb1dd3ca3a61200fd8c65343ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=A4rtschi?= Date: Fri, 27 Mar 2026 08:30:59 +0100 Subject: [PATCH 2/2] refactor: address PR feedback on raw rows implementation - Extract raw XML generation into Exceed.Worksheet.Raw module - Change raw_rows: true option to raw: true within opts (no separate struct field) - Add Benchee benchmark comparing normal vs raw: true modes --- README.md | 4 +- benchmark/exceed.exs | 9 ++ lib/exceed/worksheet.ex | 204 +++------------------------------ lib/exceed/worksheet/raw.ex | 179 +++++++++++++++++++++++++++++ test/exceed/worksheet_test.exs | 8 +- test/exceed_test.exs | 10 +- 6 files changed, 215 insertions(+), 199 deletions(-) create mode 100644 lib/exceed/worksheet/raw.ex diff --git a/README.md b/README.md index 3bb624e..1972533 100644 --- a/README.md +++ b/README.md @@ -76,12 +76,12 @@ Exceed.Worksheet.new("Sheet Name", ["Heading 1", "Heading 2"], rows) ### Raw rows mode -For large data sets, the `raw_rows: true` option bypasses the `Cell` protocol +For large data sets, the `raw: true` option bypasses the `Cell` protocol and `XmlStream`, generating SpreadsheetML XML directly as iodata. Rows are processed in parallel batches using `Task.async_stream`. ``` elixir -Exceed.Worksheet.new("Sheet Name", ["Heading 1", "Heading 2"], rows, raw_rows: true) +Exceed.Worksheet.new("Sheet Name", ["Heading 1", "Heading 2"], rows, raw: true) ``` This is significantly faster for worksheets with many rows, at the cost of not diff --git a/benchmark/exceed.exs b/benchmark/exceed.exs index 20ba109..10c200a 100644 --- a/benchmark/exceed.exs +++ b/benchmark/exceed.exs @@ -11,6 +11,15 @@ defmodule Benchmark do |> Zstream.zip() |> Stream.run() end) + + benchmark(column_count, row_count, fn -> + Exceed.Worksheet.new("Sheet Name", headers, stream, raw: true) + |> Exceed.Worksheet.to_xml() + |> Exceed.File.file("xl/worksheets/sheet1.xml", opts) + |> List.wrap() + |> Zstream.zip() + |> Stream.run() + end) end defp headers(column_count) do diff --git a/lib/exceed/worksheet.ex b/lib/exceed/worksheet.ex index cd14e2a..4c965f0 100644 --- a/lib/exceed/worksheet.ex +++ b/lib/exceed/worksheet.ex @@ -67,14 +67,14 @@ defmodule Exceed.Worksheet do ## Raw rows mode - For large data sets, the `raw_rows: true` option bypasses the `Exceed.Worksheet.Cell` + For large data sets, the `raw: true` option bypasses the `Exceed.Worksheet.Cell` protocol and `XmlStream`, generating SpreadsheetML XML directly as iodata. This can be significantly faster for worksheets with many rows. ``` elixir iex> headers = ["Name", "Age"] iex> rows = [["Alice", 30], ["Bob", 25]] - iex> Exceed.Worksheet.new("Sheet", headers, rows, raw_rows: true) + iex> Exceed.Worksheet.new("Sheet", headers, rows, raw: true) #Exceed.Worksheet ``` @@ -100,8 +100,7 @@ defmodule Exceed.Worksheet do content: Enum.t(), headers: headers(), name: String.t(), - opts: spreadsheet_options(), - raw_rows: boolean() + opts: spreadsheet_options() } @derive {Inspect, only: [:name]} @@ -110,7 +109,6 @@ defmodule Exceed.Worksheet do headers name opts - raw_rows )a @doc """ @@ -119,7 +117,7 @@ defmodule Exceed.Worksheet do ## Options - - `:raw_rows` - when `true`, bypasses the `Cell` protocol and generates XML + - `:raw` - when `true`, bypasses the `Cell` protocol and generates XML directly as iodata for better performance. See the module docs for details. - `:cols` - column configuration. See the module docs for details. @@ -139,19 +137,19 @@ defmodule Exceed.Worksheet do def new(_name, [], _content, _opts), do: raise(Exceed.Error, "Worksheet headers must be a list of items or nil") - def new(name, headers, content, opts) do - {raw_rows, opts} = Keyword.pop(opts, :raw_rows, false) - __struct__(name: name, headers: headers, content: content, opts: opts, raw_rows: raw_rows) - end - - @raw_batch_size 5_000 + def new(name, headers, content, opts), + do: __struct__(name: name, headers: headers, content: content, opts: opts) @doc false - def to_xml(%__MODULE__{raw_rows: true} = ws) do - raw_xml_stream(ws) + def to_xml(%__MODULE__{opts: opts} = ws) do + if Keyword.get(opts, :raw, false) do + Exceed.Worksheet.Raw.stream(ws) + else + xml_stream(ws) + end end - def to_xml(%__MODULE__{headers: headers, content: content, opts: opts}) do + defp xml_stream(%__MODULE__{headers: headers, content: content, opts: opts}) do %{ col_padding: col_padding, col_widths: col_widths @@ -228,7 +226,8 @@ defmodule Exceed.Worksheet do ) end - defp normalize_opts(opts) do + @doc false + def normalize_opts(opts) do %{ col_padding: get_in(opts, [:cols, :padding]) || 4.25, col_widths: get_in(opts, [:cols, :widths]) || %{} @@ -239,8 +238,9 @@ defmodule Exceed.Worksheet do defp next_alphabet([]), do: [?A] defp next_alphabet([x | rest]) when x == ?Z, do: [?A | next_alphabet(rest)] - defp prepend_headers(stream, nil), do: stream - defp prepend_headers(stream, headers), do: Stream.concat([headers], stream) + @doc false + def prepend_headers(stream, nil), do: stream + def prepend_headers(stream, headers), do: Stream.concat([headers], stream) defp sheet_data(stream, headers) do stream @@ -265,172 +265,4 @@ defmodule Exceed.Worksheet do identifier = to_string(row_idx) {[Xs.element("row", %{"r" => identifier}, to_cells(items, identifier))], row_idx + 1} end - - # # # Raw rows fast path # # # - - defp raw_xml_stream(%__MODULE__{headers: headers, content: content, opts: opts}) do - %{col_padding: col_padding, col_widths: col_widths} = normalize_opts(opts) - col_letters = column_letters(headers || hd(Enum.take(content, 1))) - - header_xml = raw_header_xml(headers, col_padding, col_widths) - footer_xml = raw_footer_xml() - - row_stream = - content - |> prepend_headers(headers) - |> Stream.with_index(1) - |> Stream.chunk_every(@raw_batch_size) - |> Task.async_stream( - fn batch -> - Enum.map(batch, fn {row, row_idx} -> - raw_row(row, row_idx, col_letters) - end) - end, - ordered: true, - max_concurrency: System.schedulers_online() - ) - |> Stream.map(fn {:ok, rows} -> rows end) - - Stream.concat([ - [header_xml], - row_stream, - [footer_xml] - ]) - end - - defp column_letters(items) do - items - |> Enum.with_index() - |> Enum.map(fn {_, i} -> - col_idx_to_letter(i + 1) - end) - end - - defp col_idx_to_letter(n) when n <= 26, do: <> - - defp col_idx_to_letter(n) do - col_idx_to_letter(div(n - 1, 26)) <> <> - end - - defp raw_header_xml(headers, col_padding, col_widths) do - cols_xml = raw_cols_xml(headers, col_padding, col_widths) - - [ - ~s(), - ~s(), - ~s(), - ~s(), - ~s(), - cols_xml, - ~s() - ] - end - - defp raw_cols_xml(nil, _padding, _widths), do: "" - - defp raw_cols_xml(headers, padding, widths) do - cols = - headers - |> Enum.with_index(1) - |> Enum.map(fn {header, i} -> - width = Map.get(widths, i, String.length(to_string(header)) + padding) - ~s() - end) - - ["", cols, ""] - end - - defp raw_footer_xml do - [ - "", - ~s(), - ~s(), - ~s(), - "", - "", - "" - ] - end - - defp raw_row(cells, row_idx, col_letters) do - row_str = Integer.to_string(row_idx) - - [ - "", - raw_cells(cells, row_str, col_letters), - "" - ] - end - - defp raw_cells(cells, row_str, col_letters) do - cells - |> Enum.zip(col_letters) - |> Enum.map(fn {value, letter} -> - raw_cell(value, letter, row_str) - end) - end - - defp raw_cell(value, letter, row_str) when is_integer(value) do - ["", Integer.to_string(value), ""] - end - - defp raw_cell(value, letter, row_str) when is_float(value) do - ["", Float.to_string(value), ""] - end - - defp raw_cell(value, letter, row_str) when is_binary(value) do - ["", escape_xml(value), ""] - end - - defp raw_cell(nil, letter, row_str) do - [""] - end - - defp raw_cell(true, letter, row_str) do - ["1"] - end - - defp raw_cell(false, letter, row_str) do - ["0"] - end - - defp raw_cell(value, letter, row_str) when is_atom(value) do - raw_cell(Atom.to_string(value), letter, row_str) - end - - defp raw_cell(%Date{year: year} = date, letter, row_str) when year >= 1900 do - value = Exceed.Util.to_excel_datetime(date) - ["", Float.to_string(value), ""] - end - - defp raw_cell(%Date{} = date, letter, row_str) do - raw_cell(Date.to_iso8601(date), letter, row_str) - end - - defp raw_cell(%DateTime{year: year} = dt, letter, row_str) when year >= 1900 do - value = Exceed.Util.to_excel_datetime(dt) - ["", Float.to_string(value), ""] - end - - defp raw_cell(%DateTime{} = dt, letter, row_str) do - raw_cell(DateTime.to_iso8601(dt), letter, row_str) - end - - defp raw_cell(%NaiveDateTime{year: year} = ndt, letter, row_str) when year >= 1900 do - value = Exceed.Util.to_excel_datetime(ndt) - ["", Float.to_string(value), ""] - end - - defp raw_cell(%NaiveDateTime{} = ndt, letter, row_str) do - raw_cell(NaiveDateTime.to_iso8601(ndt), letter, row_str) - end - - defp escape_xml(string) do - string - |> String.replace("&", "&") - |> String.replace("<", "<") - |> String.replace(">", ">") - end end diff --git a/lib/exceed/worksheet/raw.ex b/lib/exceed/worksheet/raw.ex new file mode 100644 index 0000000..0408989 --- /dev/null +++ b/lib/exceed/worksheet/raw.ex @@ -0,0 +1,179 @@ +defmodule Exceed.Worksheet.Raw do + @moduledoc """ + Generates SpreadsheetML XML directly as iodata, bypassing the + `Exceed.Worksheet.Cell` protocol and `XmlStream` for improved performance + on large data sets. + + Used internally by `Exceed.Worksheet.to_xml/1` when the `raw: true` option + is set. Row data is processed in parallel batches via `Task.async_stream`. + + Supported value types: strings, integers, floats, booleans, atoms, `nil`, + `Date`, `DateTime`, and `NaiveDateTime`. + """ + + @raw_batch_size 5_000 + + def stream(%Exceed.Worksheet{headers: headers, content: content, opts: opts}) do + %{col_padding: col_padding, col_widths: col_widths} = Exceed.Worksheet.normalize_opts(opts) + col_letters = column_letters(headers || Enum.at(content, 0)) + + header_xml = header_xml(headers, col_padding, col_widths) + footer_xml = footer_xml() + + row_stream = + content + |> Exceed.Worksheet.prepend_headers(headers) + |> Stream.with_index(1) + |> Stream.chunk_every(@raw_batch_size) + |> Task.async_stream( + fn batch -> + Enum.map(batch, fn {row, row_idx} -> + row(row, row_idx, col_letters) + end) + end, + ordered: true, + max_concurrency: System.schedulers_online() + ) + |> Stream.map(fn {:ok, rows} -> rows end) + + Stream.concat([ + [header_xml], + row_stream, + [footer_xml] + ]) + end + + # # # + + defp column_letters(items) do + items + |> Enum.with_index() + |> Enum.map(fn {_, i} -> col_idx_to_letter(i + 1) end) + end + + defp col_idx_to_letter(n) when n <= 26, do: <> + + defp col_idx_to_letter(n) do + col_idx_to_letter(div(n - 1, 26)) <> <> + end + + defp header_xml(headers, col_padding, col_widths) do + cols_xml = cols_xml(headers, col_padding, col_widths) + + [ + ~s(), + ~s(), + ~s(), + ~s(), + ~s(), + cols_xml, + ~s() + ] + end + + defp cols_xml(nil, _padding, _widths), do: "" + + defp cols_xml(headers, padding, widths) do + cols = + headers + |> Enum.with_index(1) + |> Enum.map(fn {header, i} -> + width = Map.get(widths, i, String.length(to_string(header)) + padding) + ~s() + end) + + ["", cols, ""] + end + + defp footer_xml do + [ + "", + ~s(), + ~s(), + ~s(), + "", + "", + "" + ] + end + + defp row(cells, row_idx, col_letters) do + row_str = Integer.to_string(row_idx) + + [ + "", + cells(cells, row_str, col_letters), + "" + ] + end + + defp cells(cells, row_str, col_letters) do + cells + |> Enum.zip(col_letters) + |> Enum.map(fn {value, letter} -> cell(value, letter, row_str) end) + end + + defp cell(value, letter, row_str) when is_integer(value) do + ["", Integer.to_string(value), ""] + end + + defp cell(value, letter, row_str) when is_float(value) do + ["", Float.to_string(value), ""] + end + + defp cell(value, letter, row_str) when is_binary(value) do + ["", escape_xml(value), ""] + end + + defp cell(nil, letter, row_str) do + [""] + end + + defp cell(true, letter, row_str) do + ["1"] + end + + defp cell(false, letter, row_str) do + ["0"] + end + + defp cell(value, letter, row_str) when is_atom(value) do + cell(Atom.to_string(value), letter, row_str) + end + + defp cell(%Date{year: year} = date, letter, row_str) when year >= 1900 do + value = Exceed.Util.to_excel_datetime(date) + ["", Float.to_string(value), ""] + end + + defp cell(%Date{} = date, letter, row_str) do + cell(Date.to_iso8601(date), letter, row_str) + end + + defp cell(%DateTime{year: year} = dt, letter, row_str) when year >= 1900 do + value = Exceed.Util.to_excel_datetime(dt) + ["", Float.to_string(value), ""] + end + + defp cell(%DateTime{} = dt, letter, row_str) do + cell(DateTime.to_iso8601(dt), letter, row_str) + end + + defp cell(%NaiveDateTime{year: year} = ndt, letter, row_str) when year >= 1900 do + value = Exceed.Util.to_excel_datetime(ndt) + ["", Float.to_string(value), ""] + end + + defp cell(%NaiveDateTime{} = ndt, letter, row_str) do + cell(NaiveDateTime.to_iso8601(ndt), letter, row_str) + end + + defp escape_xml(string) do + string + |> String.replace("&", "&") + |> String.replace("<", "<") + |> String.replace(">", ">") + end +end diff --git a/test/exceed/worksheet_test.exs b/test/exceed/worksheet_test.exs index 87ffae1..b7fcbf6 100644 --- a/test/exceed/worksheet_test.exs +++ b/test/exceed/worksheet_test.exs @@ -200,9 +200,9 @@ defmodule Exceed.WorksheetTest do end end - describe "to_xml with raw_rows" do + describe "to_xml with raw: true" do test "generates rows for the headers and each member of the stream", %{headers: headers, stream: stream} do - ws = Worksheet.new("sheet", headers, Enum.take(stream, 2), raw_rows: true) + ws = Worksheet.new("sheet", headers, Enum.take(stream, 2), raw: true) xml = Worksheet.to_xml(ws) |> Enum.to_list() |> IO.iodata_to_binary() xml = XmlQuery.parse(xml) @@ -236,7 +236,7 @@ defmodule Exceed.WorksheetTest do end test "generates correct column widths from headers", %{headers: headers, stream: stream} do - ws = Worksheet.new("sheet", headers, Enum.take(stream, 0), raw_rows: true) + ws = Worksheet.new("sheet", headers, Enum.take(stream, 0), raw: true) xml = Worksheet.to_xml(ws) |> Enum.to_list() |> IO.iodata_to_binary() xml = XmlQuery.parse(xml) @@ -248,7 +248,7 @@ defmodule Exceed.WorksheetTest do end test "escapes XML special characters in strings" do - ws = Worksheet.new("sheet", ["h"], [["bold & \"quotes\""]], raw_rows: true) + ws = Worksheet.new("sheet", ["h"], [["bold & \"quotes\""]], raw: true) xml = Worksheet.to_xml(ws) |> Enum.to_list() |> IO.iodata_to_binary() assert xml =~ "<b>bold</b> & \"quotes\"" diff --git a/test/exceed_test.exs b/test/exceed_test.exs index 40589c6..303003d 100644 --- a/test/exceed_test.exs +++ b/test/exceed_test.exs @@ -144,7 +144,7 @@ defmodule ExceedTest do end end - describe "raw_rows roundtrip" do + describe "raw: true roundtrip" do @describetag :tmp_dir test "produces valid XLSX with correct data", %{tmp_dir: tmpdir} do @@ -156,9 +156,7 @@ defmodule ExceedTest do filename = Exceed.Workbook.new("me") - |> Exceed.Workbook.add_worksheet( - Exceed.Worksheet.new("People", ["Name", "Age", "Height"], rows, raw_rows: true) - ) + |> Exceed.Workbook.add_worksheet(Exceed.Worksheet.new("People", ["Name", "Age", "Height"], rows, raw: true)) |> stream_to_file(tmpdir) assert {:ok, wb} = XlsxReader.open(to_string(filename)) @@ -179,9 +177,7 @@ defmodule ExceedTest do filename = Exceed.Workbook.new("me") - |> Exceed.Workbook.add_worksheet( - Exceed.Worksheet.new("Dates", ["Date"], rows, raw_rows: true) - ) + |> Exceed.Workbook.add_worksheet(Exceed.Worksheet.new("Dates", ["Date"], rows, raw: true)) |> stream_to_file(tmpdir) assert {:ok, wb} = XlsxReader.open(to_string(filename))