diff --git a/README.md b/README.md index f4e55e8..1972533 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,21 @@ rows = Exceed.Worksheet.new("Sheet Name", ["Heading 1", "Heading 2"], rows) ``` +### Raw rows mode + +For large data sets, the `raw: true` option bypasses the `Cell` protocol +and `XmlStream`, generating SpreadsheetML XML directly as iodata. Rows are +processed in parallel batches using `Task.async_stream`. + +``` elixir +Exceed.Worksheet.new("Sheet Name", ["Heading 1", "Heading 2"], rows, raw: true) +``` + +This is significantly faster for worksheets with many rows, at the cost of not +supporting custom `Cell` protocol implementations. Supported value types: +strings, integers, floats, booleans, atoms, `nil`, `Date`, `DateTime`, and +`NaiveDateTime`. + ## Alternatives & References This library is inspired by and learns from other great libraries. One might diff --git a/benchmark/exceed.exs b/benchmark/exceed.exs index 20ba109..10c200a 100644 --- a/benchmark/exceed.exs +++ b/benchmark/exceed.exs @@ -11,6 +11,15 @@ defmodule Benchmark do |> Zstream.zip() |> Stream.run() end) + + benchmark(column_count, row_count, fn -> + Exceed.Worksheet.new("Sheet Name", headers, stream, raw: true) + |> Exceed.Worksheet.to_xml() + |> Exceed.File.file("xl/worksheets/sheet1.xml", opts) + |> List.wrap() + |> Zstream.zip() + |> Stream.run() + end) end defp headers(column_count) do diff --git a/lib/exceed/file.ex b/lib/exceed/file.ex index c9d3c08..9d666b2 100644 --- a/lib/exceed/file.ex +++ b/lib/exceed/file.ex @@ -4,12 +4,17 @@ defmodule Exceed.File do @buffer_size_bytes 128 * 1024 @accumulator {<<>>, 0} - def file(content, filename, opts) do + def file(content, filename, opts) when is_list(content) do stream = XmlStream.stream!(content, printer: Exceed.Xml) stream = if Keyword.get(opts, :buffer, true), do: buffer(stream), else: stream Zstream.entry(filename, stream) end + def file(content, filename, opts) do + stream = if Keyword.get(opts, :buffer, true), do: buffer(content), else: content + Zstream.entry(filename, stream) + end + defp buffer(stream) do stream |> Stream.chunk_while( diff --git a/lib/exceed/worksheet.ex b/lib/exceed/worksheet.ex index 18bf726..4c965f0 100644 --- a/lib/exceed/worksheet.ex +++ b/lib/exceed/worksheet.ex @@ -65,6 +65,26 @@ defmodule Exceed.Worksheet do the [OpenXML.Spreadsheet Column docs](https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.column?view=openxml-3.0.1) for more info). + ## Raw rows mode + + For large data sets, the `raw: true` option bypasses the `Exceed.Worksheet.Cell` + protocol and `XmlStream`, generating SpreadsheetML XML directly as iodata. This can + be significantly faster for worksheets with many rows. + + ``` elixir + iex> headers = ["Name", "Age"] + iex> rows = [["Alice", 30], ["Bob", 25]] + iex> Exceed.Worksheet.new("Sheet", headers, rows, raw: true) + #Exceed.Worksheet + ``` + + Row data is processed in batches using `Task.async_stream` for parallel + iodata generation. Supported value types: strings, integers, floats, booleans, + atoms, `nil`, `Date`, `DateTime`, and `NaiveDateTime`. + + Note: because the `Cell` protocol is bypassed, custom `Cell` implementations + are not available in raw rows mode. + """ alias Exceed.Worksheet.Cell alias XmlStream, as: Xs @@ -95,6 +115,12 @@ defmodule Exceed.Worksheet do Initialize a new worksheet to be added to a workbook. See `Exceed.Workbook.add_worksheet/2`. For worksheet options, see the module docs for `m:Exceed.Worksheet#module-sheet-options`. + ## Options + + - `:raw` - when `true`, bypasses the `Cell` protocol and generates XML + directly as iodata for better performance. See the module docs for details. + - `:cols` - column configuration. See the module docs for details. + ## Examples ``` elixir @@ -115,7 +141,15 @@ defmodule Exceed.Worksheet do do: __struct__(name: name, headers: headers, content: content, opts: opts) @doc false - def to_xml(%__MODULE__{headers: headers, content: content, opts: opts}) do + def to_xml(%__MODULE__{opts: opts} = ws) do + if Keyword.get(opts, :raw, false) do + Exceed.Worksheet.Raw.stream(ws) + else + xml_stream(ws) + end + end + + defp xml_stream(%__MODULE__{headers: headers, content: content, opts: opts}) do %{ col_padding: col_padding, col_widths: col_widths @@ -192,7 +226,8 @@ defmodule Exceed.Worksheet do ) end - defp normalize_opts(opts) do + @doc false + def normalize_opts(opts) do %{ col_padding: get_in(opts, [:cols, :padding]) || 4.25, col_widths: get_in(opts, [:cols, :widths]) || %{} @@ -203,8 +238,9 @@ defmodule Exceed.Worksheet do defp next_alphabet([]), do: [?A] defp next_alphabet([x | rest]) when x == ?Z, do: [?A | next_alphabet(rest)] - defp prepend_headers(stream, nil), do: stream - defp prepend_headers(stream, headers), do: Stream.concat([headers], stream) + @doc false + def prepend_headers(stream, nil), do: stream + def prepend_headers(stream, headers), do: Stream.concat([headers], stream) defp sheet_data(stream, headers) do stream diff --git a/lib/exceed/worksheet/raw.ex b/lib/exceed/worksheet/raw.ex new file mode 100644 index 0000000..0408989 --- /dev/null +++ b/lib/exceed/worksheet/raw.ex @@ -0,0 +1,179 @@ +defmodule Exceed.Worksheet.Raw do + @moduledoc """ + Generates SpreadsheetML XML directly as iodata, bypassing the + `Exceed.Worksheet.Cell` protocol and `XmlStream` for improved performance + on large data sets. + + Used internally by `Exceed.Worksheet.to_xml/1` when the `raw: true` option + is set. Row data is processed in parallel batches via `Task.async_stream`. + + Supported value types: strings, integers, floats, booleans, atoms, `nil`, + `Date`, `DateTime`, and `NaiveDateTime`. + """ + + @raw_batch_size 5_000 + + def stream(%Exceed.Worksheet{headers: headers, content: content, opts: opts}) do + %{col_padding: col_padding, col_widths: col_widths} = Exceed.Worksheet.normalize_opts(opts) + col_letters = column_letters(headers || Enum.at(content, 0)) + + header_xml = header_xml(headers, col_padding, col_widths) + footer_xml = footer_xml() + + row_stream = + content + |> Exceed.Worksheet.prepend_headers(headers) + |> Stream.with_index(1) + |> Stream.chunk_every(@raw_batch_size) + |> Task.async_stream( + fn batch -> + Enum.map(batch, fn {row, row_idx} -> + row(row, row_idx, col_letters) + end) + end, + ordered: true, + max_concurrency: System.schedulers_online() + ) + |> Stream.map(fn {:ok, rows} -> rows end) + + Stream.concat([ + [header_xml], + row_stream, + [footer_xml] + ]) + end + + # # # + + defp column_letters(items) do + items + |> Enum.with_index() + |> Enum.map(fn {_, i} -> col_idx_to_letter(i + 1) end) + end + + defp col_idx_to_letter(n) when n <= 26, do: <> + + defp col_idx_to_letter(n) do + col_idx_to_letter(div(n - 1, 26)) <> <> + end + + defp header_xml(headers, col_padding, col_widths) do + cols_xml = cols_xml(headers, col_padding, col_widths) + + [ + ~s(), + ~s(), + ~s(), + ~s(), + ~s(), + cols_xml, + ~s() + ] + end + + defp cols_xml(nil, _padding, _widths), do: "" + + defp cols_xml(headers, padding, widths) do + cols = + headers + |> Enum.with_index(1) + |> Enum.map(fn {header, i} -> + width = Map.get(widths, i, String.length(to_string(header)) + padding) + ~s() + end) + + ["", cols, ""] + end + + defp footer_xml do + [ + "", + ~s(), + ~s(), + ~s(), + "", + "", + "" + ] + end + + defp row(cells, row_idx, col_letters) do + row_str = Integer.to_string(row_idx) + + [ + "", + cells(cells, row_str, col_letters), + "" + ] + end + + defp cells(cells, row_str, col_letters) do + cells + |> Enum.zip(col_letters) + |> Enum.map(fn {value, letter} -> cell(value, letter, row_str) end) + end + + defp cell(value, letter, row_str) when is_integer(value) do + ["", Integer.to_string(value), ""] + end + + defp cell(value, letter, row_str) when is_float(value) do + ["", Float.to_string(value), ""] + end + + defp cell(value, letter, row_str) when is_binary(value) do + ["", escape_xml(value), ""] + end + + defp cell(nil, letter, row_str) do + [""] + end + + defp cell(true, letter, row_str) do + ["1"] + end + + defp cell(false, letter, row_str) do + ["0"] + end + + defp cell(value, letter, row_str) when is_atom(value) do + cell(Atom.to_string(value), letter, row_str) + end + + defp cell(%Date{year: year} = date, letter, row_str) when year >= 1900 do + value = Exceed.Util.to_excel_datetime(date) + ["", Float.to_string(value), ""] + end + + defp cell(%Date{} = date, letter, row_str) do + cell(Date.to_iso8601(date), letter, row_str) + end + + defp cell(%DateTime{year: year} = dt, letter, row_str) when year >= 1900 do + value = Exceed.Util.to_excel_datetime(dt) + ["", Float.to_string(value), ""] + end + + defp cell(%DateTime{} = dt, letter, row_str) do + cell(DateTime.to_iso8601(dt), letter, row_str) + end + + defp cell(%NaiveDateTime{year: year} = ndt, letter, row_str) when year >= 1900 do + value = Exceed.Util.to_excel_datetime(ndt) + ["", Float.to_string(value), ""] + end + + defp cell(%NaiveDateTime{} = ndt, letter, row_str) do + cell(NaiveDateTime.to_iso8601(ndt), letter, row_str) + end + + defp escape_xml(string) do + string + |> String.replace("&", "&") + |> String.replace("<", "<") + |> String.replace(">", ">") + end +end diff --git a/test/exceed/worksheet_test.exs b/test/exceed/worksheet_test.exs index af49643..b7fcbf6 100644 --- a/test/exceed/worksheet_test.exs +++ b/test/exceed/worksheet_test.exs @@ -200,6 +200,61 @@ defmodule Exceed.WorksheetTest do end end + describe "to_xml with raw: true" do + test "generates rows for the headers and each member of the stream", %{headers: headers, stream: stream} do + ws = Worksheet.new("sheet", headers, Enum.take(stream, 2), raw: true) + xml = Worksheet.to_xml(ws) |> Enum.to_list() |> IO.iodata_to_binary() + xml = XmlQuery.parse(xml) + + assert [header_row, row_1, row_2] = Xq.all(xml, "/worksheet/sheetData/row") + + assert header_row |> Xq.attr("r") == "1" + + header_row + |> extract_cells() + |> assert_eq([ + %{type: "inlineStr", text: "inline strings", children: "is/t", cell: "A1"}, + %{type: "inlineStr", text: "integers", children: "is/t", cell: "B1"}, + %{type: "inlineStr", text: "floats", children: "is/t", cell: "C1"} + ]) + + row_1 + |> extract_cells() + |> assert_eq([ + %{type: "inlineStr", text: "row 1 cell 1", children: "is/t", cell: "A2"}, + %{type: "n", text: "1", children: "v", cell: "B2"}, + %{type: "n", text: "1.5", children: "v", cell: "C2"} + ]) + + row_2 + |> extract_cells() + |> assert_eq([ + %{type: "inlineStr", text: "row 2 cell 1", children: "is/t", cell: "A3"}, + %{type: "n", text: "2", children: "v", cell: "B3"}, + %{type: "n", text: "3.0", children: "v", cell: "C3"} + ]) + end + + test "generates correct column widths from headers", %{headers: headers, stream: stream} do + ws = Worksheet.new("sheet", headers, Enum.take(stream, 0), raw: true) + xml = Worksheet.to_xml(ws) |> Enum.to_list() |> IO.iodata_to_binary() + xml = XmlQuery.parse(xml) + + assert [col1, col2, col3] = Xq.all(xml, "/worksheet/cols/col") + + assert Xq.attr(col1, "width") == "18.25" + assert Xq.attr(col2, "width") == "12.25" + assert Xq.attr(col3, "width") == "10.25" + end + + test "escapes XML special characters in strings" do + ws = Worksheet.new("sheet", ["h"], [["bold & \"quotes\""]], raw: true) + xml = Worksheet.to_xml(ws) |> Enum.to_list() |> IO.iodata_to_binary() + + assert xml =~ "<b>bold</b> & \"quotes\"" + end + end + # # # defp extract_cells(row) do diff --git a/test/exceed_test.exs b/test/exceed_test.exs index 6d9fc3d..303003d 100644 --- a/test/exceed_test.exs +++ b/test/exceed_test.exs @@ -144,6 +144,53 @@ defmodule ExceedTest do end end + describe "raw: true roundtrip" do + @describetag :tmp_dir + + test "produces valid XLSX with correct data", %{tmp_dir: tmpdir} do + rows = [ + ["Alice", 30, 1.75], + ["Bob", 25, 1.80], + ["Charlie", 35, 1.65] + ] + + filename = + Exceed.Workbook.new("me") + |> Exceed.Workbook.add_worksheet(Exceed.Worksheet.new("People", ["Name", "Age", "Height"], rows, raw: true)) + |> stream_to_file(tmpdir) + + assert {:ok, wb} = XlsxReader.open(to_string(filename)) + assert XlsxReader.sheet_names(wb) == ["People"] + assert {:ok, sheet_rows} = XlsxReader.sheet(wb, "People") + + assert sheet_rows == [ + ["Name", "Age", "Height"], + ["Alice", 30, 1.75], + ["Bob", 25, 1.8], + ["Charlie", 35, 1.65] + ] + end + + test "handles dates", %{tmp_dir: tmpdir} do + today = Date.utc_today() + rows = [[today], [Date.add(today, -1)]] + + filename = + Exceed.Workbook.new("me") + |> Exceed.Workbook.add_worksheet(Exceed.Worksheet.new("Dates", ["Date"], rows, raw: true)) + |> stream_to_file(tmpdir) + + assert {:ok, wb} = XlsxReader.open(to_string(filename)) + assert {:ok, sheet_rows} = XlsxReader.sheet(wb, "Dates") + + assert sheet_rows == [ + ["Date"], + [today], + [Date.add(today, -1)] + ] + end + end + describe "strings" do @describetag :tmp_dir test "can be parsed", %{tmp_dir: tmpdir} do