From f9b873faa4930049e46eea0f9516e39c9b622150 Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Thu, 25 Jun 2026 12:02:33 +0000 Subject: [PATCH 01/23] feat: implement bulk_load API for all ART tree modes (#636) Add bulk_load(first, last) and bulk_load(std::execution::par, first, last) to db, mutex_db, and olc_db. Builds optimal-size internal nodes directly from sorted input, avoiding incremental insert overhead. Parallel execution dispatches one std::async task per root-level partition; subtree construction within each partition is sequential. API: tree.bulk_load(first, last); // sequential tree.bulk_load(std::execution::par, first, last); // parallel Performance (96-core, 1M random uint64 keys, Release): db: seq 11M/s, par 67M/s (6x) mutex_db: seq 11M/s, par 68M/s (6x) olc_db: seq 11M/s, par 65M/s (6x) Implementation: - art_bulk_load_detail.hpp: shared recursive algorithm template - art_internal_impl.hpp: create_bulk factories for inode_4/16/48/256 - art.hpp/olc_art.hpp/mutex_art.hpp: thin public API wrappers Tests: 33 new tests (boundary, structural, operational) across all modes. Benchmarks: BM_BulkLoad_seq/par for all tree types vs BM_Insert baseline. --- art.hpp | 101 ++++++ art_bulk_load_detail.hpp | 402 +++++++++++++++++++++ art_internal_impl.hpp | 183 ++++++++++ benchmark/CMakeLists.txt | 13 +- benchmark/micro_benchmark_bulk_load.cpp | 460 ++++++++++++++++++++++++ examples/CMakeLists.txt | 9 +- examples/example_bulk_load.cpp | 72 ++++ mutex_art.hpp | 18 + olc_art.hpp | 86 +++++ test/CMakeLists.txt | 4 + test/test_art_bulk_load.cpp | 171 +++++++++ test/test_art_bulk_load_ops.cpp | 209 +++++++++++ test/test_art_bulk_load_structural.cpp | 284 +++++++++++++++ 13 files changed, 2007 insertions(+), 5 deletions(-) create mode 100644 art_bulk_load_detail.hpp create mode 100644 benchmark/micro_benchmark_bulk_load.cpp create mode 100644 examples/example_bulk_load.cpp create mode 100644 test/test_art_bulk_load.cpp create mode 100644 test/test_art_bulk_load_ops.cpp create mode 100644 test/test_art_bulk_load_structural.cpp diff --git a/art.hpp b/art.hpp index f6af4d77..292a3cf3 100644 --- a/art.hpp +++ b/art.hpp @@ -17,9 +17,13 @@ #include #include #include +#include +#include #include +#include #include #include +#include #include #include @@ -111,6 +115,11 @@ using leaf_type = basic_leaf, node_header>; template class inode : public inode_base {}; +/// Forward declaration of shared bulk_load algorithm. +template +void bulk_load_impl(Db& self, ExecutionPolicy&& policy, RandomIt first, + RandomIt last); + } // namespace detail template @@ -195,6 +204,53 @@ class db final { /// tree and the associated index entry was removed). [[nodiscard]] bool remove_internal(art_key_type remove_key); + // ─── Bulk Load Infrastructure ───────────────────────────────────────── + + /// Tagged return from build_subtree: pointer + whether it's a VIS packed + /// value (not a real heap pointer — must not be passed to delete_subtree). + struct build_result { + detail::node_ptr ptr{nullptr}; + bool is_packed_value{false}; + }; + + /// RAII guard for subtrees during bulk construction. + /// Skips deallocation for VIS packed values (not heap-allocated). + struct bulk_subtree_guard { + db_type& db_; + detail::node_ptr ptr{nullptr}; + bool is_packed_value{false}; + + constexpr explicit bulk_subtree_guard(db_type& db + UNODB_DETAIL_LIFETIMEBOUND) noexcept + : db_{db} {} + + ~bulk_subtree_guard() noexcept { + if (ptr != nullptr && !is_packed_value) { + art_policy::delete_subtree(ptr, db_); + } + } + + void release() noexcept { ptr = nullptr; } + + bulk_subtree_guard(const bulk_subtree_guard&) = delete; + bulk_subtree_guard(bulk_subtree_guard&& other) noexcept + : db_{other.db_}, + ptr{other.ptr}, + is_packed_value{other.is_packed_value} { + other.ptr = nullptr; + } + auto& operator=(const bulk_subtree_guard&) = delete; + auto& operator=(bulk_subtree_guard&&) = delete; + }; + + /// Extract byte at \a depth from a key. Handles big-endian uint64_t + /// (via art_key_type bswap) and key_view (direct byte access). + [[nodiscard]] static constexpr std::byte key_byte_at( + const key_type& key, detail::tree_depth depth) noexcept { + const art_key_type ak{key}; + return ak[depth]; + } + /// Two-pass removal with chain cleanup for variable-length keys. [[nodiscard]] bool remove_internal_key_view(art_key_type remove_key); @@ -304,6 +360,24 @@ class db final { /// nodes is freed. Node growth/shrink counters are preserved. void clear() noexcept; + /// Bulk-load an ART tree from a pre-sorted range of key-value pairs. + /// + /// \tparam RandomIt Random-access iterator to std::pair + /// \param first, last Sorted range (ART byte order). No duplicates. + /// \param parallelism Ignored for db (allocator not thread-safe). + /// \pre empty() == true + /// \throws std::invalid_argument if tree is non-empty. + /// \throws std::bad_alloc (strong guarantee — tree left empty). + template + void bulk_load(ExecutionPolicy&& policy, RandomIt first, RandomIt last); + + /// Convenience overload: sequential execution. + template + void bulk_load(RandomIt first, RandomIt last) { + bulk_load(std::execution::seq, first, last); + } + /// Internal iterator for tree traversal. /// /// Iterator is an internal API. Use scan() for the public API. @@ -870,6 +944,15 @@ class db final { /// Node type with 4 children. using inode_4 = detail::inode_4; + /// Node type with 16 children. + using inode_16 = detail::inode_16; + + /// Node type with 48 children. + using inode_48 = detail::inode_48; + + /// Node type with 256 children. + using inode_256 = detail::inode_256; + /// Tree depth tracking type. using tree_depth_type = detail::tree_depth; @@ -973,6 +1056,10 @@ class db final { friend auto detail::make_db_leaf_ptr(art_key_type, value_type, db&); + /// detail::bulk_load_impl + template + friend void detail::bulk_load_impl(Db2&, Ep2&&, It2, It2); + /// detail::basic_db_leaf_deleter template friend class detail::basic_db_leaf_deleter; @@ -2745,6 +2832,20 @@ bool db::upsert(Key k, value_type v, FN fn) { } } +template +template +void db::bulk_load(ExecutionPolicy&& policy, RandomIt first, + RandomIt last) { + if (!empty()) { + throw std::invalid_argument("bulk_load requires empty tree"); + } + if (first == last) return; + detail::bulk_load_impl(*this, std::forward(policy), first, + last); +} + } // namespace unodb +#include "art_bulk_load_detail.hpp" + #endif // UNODB_DETAIL_ART_HPP diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp new file mode 100644 index 00000000..d9bb788e --- /dev/null +++ b/art_bulk_load_detail.hpp @@ -0,0 +1,402 @@ +// Copyright 2026 UnoDB contributors +// +// Shared implementation of the bulk_load algorithm for all ART variants. +// Included at the bottom of art.hpp and olc_art.hpp after class definitions. +// +// Template parameter Db: must expose art_key_type, art_policy, tree_depth_type, +// inode_4, inode_16, inode_48, inode_256, build_result, bulk_subtree_guard, +// build_chain(), account_growing_inode<>(), and root member. + +#ifndef UNODB_DETAIL_ART_BULK_LOAD_DETAIL_HPP +#define UNODB_DETAIL_ART_BULK_LOAD_DETAIL_HPP + +/// \cond UNODB_DETAIL_INTERNAL +namespace unodb::detail { + +/// Core bulk_load algorithm parameterized on database type. +/// +/// \pre Tree must be empty (caller validates). +/// \pre [first, last) is sorted and non-empty (caller validates). +template +void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, + RandomIt last) { + using art_key_type = typename Db::art_key_type; + using art_policy = typename Db::art_policy; + using tree_depth_type = typename Db::tree_depth_type; + using inode_4 = typename Db::inode_4; + using inode_16 = typename Db::inode_16; + using inode_48 = typename Db::inode_48; + using inode_256 = typename Db::inode_256; + using build_result_t = typename Db::build_result; + using guard_t = typename Db::bulk_subtree_guard; + using node_ptr_t = decltype(build_result_t{}.ptr); + using bulk_child_t = bulk_child; + constexpr std::size_t prefix_cap = key_prefix_capacity; + + auto common_prefix_length = [](RandomIt f, RandomIt l, + tree_depth_type depth) -> std::size_t { + const art_key_type first_ak{f->first}; + const art_key_type last_ak{std::prev(l)->first}; + const auto fk = first_ak.get_key_view(); + const auto lk = last_ak.get_key_view(); + const auto d = static_cast(depth); + const auto max_len = std::min(fk.size(), lk.size()); + std::size_t len = 0; + while (d + len < max_len && fk[d + len] == lk[d + len]) ++len; + return len; + }; + + struct partition_entry { + RandomIt begin; + std::byte key_byte; + }; + + auto partition_by_byte = [](RandomIt f, RandomIt l, + tree_depth_type dispatch_depth) { + boost::container::small_vector parts; + const auto dd = static_cast(dispatch_depth); + auto cur = f; + while (cur != l) { + const art_key_type ak{cur->first}; + const auto kv = ak.get_key_view(); + const auto byte = kv[dd]; + parts.push_back({cur, byte}); + ++cur; + while (cur != l) { + const art_key_type ak2{cur->first}; + const auto kv2 = ak2.get_key_view(); + if (kv2[dd] != byte) break; + ++cur; + } + } + return std::move(parts); // NOLINT(performance-move-const-arg) + }; + + auto build_prefix_chain = [&self](art_key_type k, node_ptr_t child_inode, + tree_depth_type start_depth, + std::size_t end_depth) -> node_ptr_t { + const auto full_key = k.get_key_view(); + const auto start = static_cast(start_depth); + auto current = child_inode; + std::size_t pos = end_depth; + while (pos > start + prefix_cap) { + const auto depth = pos - prefix_cap - 1; + const auto dispatch = full_key[pos - 1]; + auto remaining = k; + remaining.shift_right(depth); + auto chain{ + inode_4::create(self, full_key, remaining, + tree_depth_type{static_cast(depth)}, + dispatch, current)}; + current = node_ptr_t{chain.release(), node_type::I4}; +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + pos = depth; + } + if (pos > start) { + const auto dispatch = full_key[pos - 1]; + auto chain{inode_4::create( + self, full_key, tree_depth_type{static_cast(start)}, + static_cast(pos - start - 1), dispatch, current)}; + current = node_ptr_t{chain.release(), node_type::I4}; +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + } + return current; + }; + + auto build_single_leaf = [&self](RandomIt it, + tree_depth_type depth) -> build_result_t { + const art_key_type ak{it->first}; + if constexpr (art_policy::can_eliminate_leaf) { + const auto packed = art_policy::pack_value(it->second); + if (static_cast(depth) >= ak.size()) { + return {packed, true}; + } + return {self.build_chain(ak, packed, depth), false}; + } else if constexpr (art_policy::can_eliminate_key_in_leaf) { + auto leaf = art_policy::make_db_leaf_ptr(ak, it->second, self); + auto leaf_ptr = node_ptr_t{leaf.release(), node_type::LEAF}; + if (static_cast(depth) >= ak.size()) { + return {leaf_ptr, false}; + } + return {self.build_chain(ak, leaf_ptr, depth), false}; + } else { + auto leaf = art_policy::make_db_leaf_ptr(ak, it->second, self); + auto leaf_ptr = node_ptr_t{leaf.release(), node_type::LEAF}; + return {leaf_ptr, false}; + } + }; + + // ─── Recursive subtree builder ───────────────────────────────────────── + + struct subtree_builder { + Db& self; + decltype(common_prefix_length)& cpl; + decltype(partition_by_byte)& pbb; + decltype(build_prefix_chain)& bpc; + decltype(build_single_leaf)& bsl; + + build_result_t operator()(RandomIt f, RandomIt l, + tree_depth_type depth) const { + const auto n = std::distance(f, l); + if (n == 0) return {node_ptr_t{nullptr}, false}; + if (n == 1) return bsl(f, depth); + + const auto prefix_len = cpl(f, l, depth); + const auto dispatch_depth = tree_depth_type{static_cast( + static_cast(depth) + prefix_len)}; + auto parts = pbb(f, l, dispatch_depth); + const auto child_count = parts.size(); + + boost::container::small_vector guards; + guards.reserve(child_count); + boost::container::small_vector children; + children.reserve(child_count); + + const auto next_depth = tree_depth_type{static_cast( + static_cast(dispatch_depth) + 1)}; + + for (std::size_t i = 0; i < child_count; ++i) { + const auto part_begin = parts[i].begin; + const auto part_end = (i + 1 < child_count) ? parts[i + 1].begin : l; + auto result = (*this)(part_begin, part_end, next_depth); + guards.emplace_back(self); + guards.back().ptr = result.ptr; + guards.back().is_packed_value = result.is_packed_value; + children.push_back({parts[i].key_byte, result.ptr}); + } + + const std::size_t chain_consumed = + (prefix_len > prefix_cap) ? (prefix_len / 8) * 8 : 0; + const auto inode_prefix_len = + static_cast(prefix_len - chain_consumed); + const auto inode_depth = tree_depth_type{static_cast( + static_cast(depth) + chain_consumed)}; + + const art_key_type prefix_key{f->first}; + const auto prefix_kv = prefix_key.get_key_view(); + + auto make_inode = [&]() -> node_ptr_t { + const auto cs = + std::span{children.data(), children.size()}; + if (child_count <= 4) { + std::uint8_t vmask = 0; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask |= static_cast(1U << i); + } + } + auto ptr = inode_4::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I4}; + } + if (child_count <= 16) { + std::uint16_t vmask = 0; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask |= static_cast(1U << i); + } + } + auto ptr = inode_16::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I16}; + } + if (child_count <= 48) { + std::array vmask{}; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask[i / 8] |= static_cast(1U << (i % 8)); + } + } + auto ptr = inode_48::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I48}; + } + std::array vmask{}; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) { + const auto kb = static_cast(children[i].key_byte); + vmask[kb / 8] |= static_cast(1U << (kb % 8)); + } + } + } + auto ptr = inode_256::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I256}; + }; + + auto inode_ptr = make_inode(); + for (auto& g : guards) g.release(); + + if (chain_consumed > 0) { + return {bpc(prefix_key, inode_ptr, depth, + chain_consumed + static_cast(depth)), + false}; + } + return {inode_ptr, false}; + } + }; + + subtree_builder builder{self, common_prefix_length, partition_by_byte, + build_prefix_chain, build_single_leaf}; + + using policy_t = std::remove_cvref_t; + constexpr bool is_parallel = + std::is_same_v || + std::is_same_v; + + if constexpr (!is_parallel) { + auto result = builder(first, last, tree_depth_type{0}); + self.root = result.ptr; + } else { + const auto n = std::distance(first, last); + if (n <= 1) { + auto result = builder(first, last, tree_depth_type{0}); + self.root = result.ptr; + return; + } + + const auto prefix_len = + common_prefix_length(first, last, tree_depth_type{0}); + const auto dispatch_depth = + tree_depth_type{static_cast(prefix_len)}; + auto parts = partition_by_byte(first, last, dispatch_depth); + const auto child_count = parts.size(); + + const auto next_depth = tree_depth_type{static_cast( + static_cast(dispatch_depth) + 1)}; + + std::vector> futures; + futures.reserve(child_count); + for (std::size_t i = 0; i < child_count; ++i) { + const auto part_begin = parts[i].begin; + const auto part_end = (i + 1 < child_count) ? parts[i + 1].begin : last; + futures.push_back(std::async( + std::launch::async, [&builder, part_begin, part_end, next_depth] { + return builder(part_begin, part_end, next_depth); + })); + } + + boost::container::small_vector guards; + guards.reserve(child_count); + boost::container::small_vector children; + children.reserve(child_count); + + for (std::size_t i = 0; i < child_count; ++i) { + auto result = futures[i].get(); + guards.emplace_back(self); + guards.back().ptr = result.ptr; + guards.back().is_packed_value = result.is_packed_value; + children.push_back({parts[i].key_byte, result.ptr}); + } + + const std::size_t chain_consumed = + (prefix_len > prefix_cap) ? (prefix_len / 8) * 8 : 0; + const auto inode_prefix_len = + static_cast(prefix_len - chain_consumed); + const auto inode_depth = + tree_depth_type{static_cast(chain_consumed)}; + + const art_key_type prefix_key{first->first}; + const auto prefix_kv = prefix_key.get_key_view(); + + auto make_root_inode = [&]() -> node_ptr_t { + const auto cs = + std::span{children.data(), children.size()}; + if (child_count <= 4) { + std::uint8_t vmask = 0; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask |= static_cast(1U << i); + } + } + auto ptr = inode_4::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I4}; + } + if (child_count <= 16) { + std::uint16_t vmask = 0; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask |= static_cast(1U << i); + } + } + auto ptr = inode_16::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I16}; + } + if (child_count <= 48) { + std::array vmask{}; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask[i / 8] |= static_cast(1U << (i % 8)); + } + } + auto ptr = inode_48::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I48}; + } + std::array vmask{}; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) { + const auto kb = static_cast(children[i].key_byte); + vmask[kb / 8] |= static_cast(1U << (kb % 8)); + } + } + } + auto ptr = inode_256::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I256}; + }; + + auto inode_ptr = make_root_inode(); + for (auto& g : guards) g.release(); + + if (chain_consumed > 0) { + self.root = build_prefix_chain(prefix_key, inode_ptr, tree_depth_type{0}, + chain_consumed); + } else { + self.root = inode_ptr; + } + } +} + +} // namespace unodb::detail +/// \endcond + +#endif // UNODB_DETAIL_ART_BULK_LOAD_DETAIL_HPP diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index 40755b64..f366c15f 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -149,6 +150,14 @@ struct value_bitmask_field, CritSec> { static constexpr void insert_at(std::uint8_t) noexcept {} }; +/// A child entry for bulk inode construction. +/// Pre-sorted by key_byte; used by create_bulk factories. +template +struct bulk_child { + std::byte key_byte; + NodePtr child; +}; + #ifdef UNODB_DETAIL_X86_64 /// Compare packed unsigned 8-bit integers for less-than-or-equal. @@ -2193,6 +2202,16 @@ class [[nodiscard]] basic_inode : public basic_inode_impl { tree_depth depth, single_child_tag) noexcept : parent{1, prefix_len, k1, depth} {} + + /// Tag type to select the bulk-load constructor. + struct bulk_load_tag {}; + + /// Construct with arbitrary children count and explicit prefix. + /// Used by create_bulk for bulk-loading sorted data. + constexpr basic_inode(unsigned n_children, detail::key_prefix_size prefix_len, + unodb::key_view k1, tree_depth depth, + bulk_load_tag) noexcept + : parent{n_children, prefix_len, k1, depth} {} }; /// Type alias for basic_inode_4 parent class. @@ -2380,6 +2399,14 @@ class basic_inode_4 init(key_byte, child); } + /// Construct for bulk load with explicit children count and prefix. + constexpr basic_inode_4(db_type&, unsigned n_children, + detail::key_prefix_size prefix_len, key_view k1, + // cppcheck-suppress passedByValue + tree_depth_type depth, + typename parent_class::bulk_load_tag tag) noexcept + : parent_class{n_children, prefix_len, k1, depth, tag} {} + /// \} /// \name Initialization methods @@ -3037,6 +3064,44 @@ class basic_inode_4 template friend class basic_inode_16; friend class basic_inode_impl; + + public: + /// Bulk-construct an inode4 with pre-sorted children. + /// + /// \pre children.size() in [2, 4] + /// \pre children sorted by key_byte + /// \param db_instance Database for allocation + /// \param prefix_len Prefix bytes to store + /// \param prefix_key Key view for prefix extraction + /// \param depth Depth at which prefix starts + /// \param children_span Span of sorted (key_byte, child_ptr) pairs + /// \param value_mask Precomputed VIS bitmask + UNODB_DETAIL_DISABLE_CLANG_21_WARNING("-Wnrvo") + [[nodiscard]] static auto create_bulk( + db_type& db_instance, detail::key_prefix_size prefix_len, + unodb::key_view prefix_key, tree_depth_type depth, + std::span> children_span, + std::uint8_t value_mask = 0) { + const auto n = static_cast(children_span.size()); + UNODB_DETAIL_ASSERT(n >= 2 && n <= 4); + auto result = + inode4_type::create(db_instance, n, prefix_len, prefix_key, depth, + typename parent_class::bulk_load_tag{}); + auto* node = result.get(); + for (unsigned i = 0; i < n; ++i) { + node->keys.byte_array[i] = children_span[i].key_byte; + node->children[i] = children_span[i].child; + } +#ifndef UNODB_DETAIL_X86_64 + for (unsigned i = n; i < 4; ++i) node->keys.byte_array[i] = unused_key_byte; +#endif + if constexpr (ArtPolicy::can_eliminate_leaf) { + node->bitmask_base::bits = value_mask; + } + return result; + } + UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() + }; // class basic_inode_4 /// Type alias for basic_inode_16 parent class. @@ -3123,6 +3188,14 @@ class basic_inode_16 init(db_instance, source_node, child_to_delete); } + /// Construct for bulk load with explicit children count and prefix. + constexpr basic_inode_16(db_type&, unsigned n_children, + detail::key_prefix_size prefix_len, key_view k1, + // cppcheck-suppress passedByValue + tree_depth_type depth, + typename parent_class::bulk_load_tag tag) noexcept + : parent_class{n_children, prefix_len, k1, depth, tag} {} + /// Initialize by growing from basic_inode_4 and adding a child. /// /// \param db_instance Database instance @@ -3659,6 +3732,35 @@ class basic_inode_16 friend class basic_inode_impl; template friend class basic_inode_48; + + public: + /// Bulk-construct an inode16 with pre-sorted children. + /// + /// \pre children_span.size() in [5, 16] + /// \pre children sorted by key_byte + UNODB_DETAIL_DISABLE_CLANG_21_WARNING("-Wnrvo") + [[nodiscard]] static auto create_bulk( + db_type& db_instance, detail::key_prefix_size prefix_len, + unodb::key_view prefix_key, tree_depth_type depth, + std::span> children_span, + std::uint16_t value_mask = 0) { + const auto n = static_cast(children_span.size()); + UNODB_DETAIL_ASSERT(n >= 5 && n <= 16); + auto result = + inode16_type::create(db_instance, n, prefix_len, prefix_key, depth, + typename parent_class::bulk_load_tag{}); + auto* node = result.get(); + for (unsigned i = 0; i < n; ++i) { + node->keys.byte_array[i] = children_span[i].key_byte; + node->children[i] = children_span[i].child; + } + if constexpr (ArtPolicy::can_eliminate_leaf) { + node->bitmask_base::bits = value_mask; + } + return result; + } + UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() + }; // class basic_inode_16 /// Type alias for basic_inode_48 parent class. @@ -3749,6 +3851,14 @@ class basic_inode_48 init(db_instance, source_node, child_to_delete); } + /// Construct for bulk load with explicit children count and prefix. + constexpr basic_inode_48(db_type&, unsigned n_children, + detail::key_prefix_size prefix_len, key_view k1, + // cppcheck-suppress passedByValue + tree_depth_type depth, + typename parent_class::bulk_load_tag tag) noexcept + : parent_class{n_children, prefix_len, k1, depth, tag} {} + /// Initialize by growing from basic_inode_16 and adding a child. /// /// \param db_instance Database instance @@ -4436,6 +4546,39 @@ class basic_inode_48 friend class basic_inode_impl; template friend class basic_inode_256; + + public: + /// Bulk-construct an inode48 with pre-sorted children. + /// + /// \pre children_span.size() in [17, 48] + /// \pre children sorted by key_byte + UNODB_DETAIL_DISABLE_CLANG_21_WARNING("-Wnrvo") + [[nodiscard]] static auto create_bulk( + db_type& db_instance, detail::key_prefix_size prefix_len, + unodb::key_view prefix_key, tree_depth_type depth, + std::span> children_span, + std::array value_mask = {}) { + const auto n = static_cast(children_span.size()); + UNODB_DETAIL_ASSERT(n >= 17 && n <= 48); + auto result = + inode48_type::create(db_instance, n, prefix_len, prefix_key, depth, + typename parent_class::bulk_load_tag{}); + auto* node = result.get(); + for (unsigned i = 0; i < n; ++i) { + const auto kb = static_cast(children_span[i].key_byte); + node->child_indexes[kb] = static_cast(i); + node->children.pointer_array[i] = children_span[i].child; + } + for (unsigned i = n; i < 48; ++i) { + node->children.pointer_array[i] = node_ptr{nullptr}; + } + if constexpr (ArtPolicy::can_eliminate_leaf) { + node->bitmask_base::bits = value_mask; + } + return result; + } + UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() + }; // class basic_inode_48 /// Type alias for basic_inode_256 parent class. @@ -4508,6 +4651,14 @@ class basic_inode_256 init(db_instance, source_node, packed_value, depth, key_byte); } + /// Construct for bulk load with explicit children count and prefix. + constexpr basic_inode_256(db_type&, unsigned n_children, + detail::key_prefix_size prefix_len, key_view k1, + // cppcheck-suppress passedByValue + tree_depth_type depth, + typename parent_class::bulk_load_tag tag) noexcept + : parent_class{n_children, prefix_len, k1, depth, tag} {} + /// Initialize by growing from basic_inode_48 and adding a child. /// /// \param db_instance Database for memory tracking @@ -4881,6 +5032,38 @@ class basic_inode_256 template friend class basic_inode_48; friend class basic_inode_impl; + + public: + /// Bulk-construct an inode256 with pre-sorted children. + /// + /// \pre children_span.size() in [49, 256] + /// \pre children sorted by key_byte + UNODB_DETAIL_DISABLE_CLANG_21_WARNING("-Wnrvo") + [[nodiscard]] static auto create_bulk( + db_type& db_instance, detail::key_prefix_size prefix_len, + unodb::key_view prefix_key, tree_depth_type depth, + std::span> children_span, + std::array value_mask = {}) { + const auto n = static_cast(children_span.size()); + UNODB_DETAIL_ASSERT(n >= 49 && n <= 256); + auto result = + parent_class::create(db_instance, n, prefix_len, prefix_key, depth, + typename parent_class::bulk_load_tag{}); + auto* node = result.get(); + for (unsigned i = 0; i < 256; ++i) { + node->children[i] = node_ptr{nullptr}; + } + for (unsigned i = 0; i < n; ++i) { + const auto kb = static_cast(children_span[i].key_byte); + node->children[kb] = children_span[i].child; + } + if constexpr (ArtPolicy::can_eliminate_leaf) { + node->bitmask_base::bits = value_mask; + } + return result; + } + UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() + }; // class basic_inode_256 } // namespace unodb::detail diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index a6b12656..fa4e7fb3 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -10,6 +10,7 @@ set(micro_benchmark_quick_arg "--benchmark_filter=\".*/100$$|.*/1000/.*:800$$|.*/100/.*:0$$\"") set(micro_benchmark_mutex_quick_arg "--benchmark_filter=\"/4/70000/\"") set(micro_benchmark_olc_quick_arg "--benchmark_filter=\"/4/70000/\"") +set(micro_benchmark_bulk_load_quick_arg "--benchmark_filter=\"/1024\"") add_custom_target(benchmarks env ${SANITIZER_ENV} ./micro_benchmark_key_prefix @@ -20,7 +21,8 @@ add_custom_target(benchmarks COMMAND env ${SANITIZER_ENV} ./micro_benchmark_n256 COMMAND env ${SANITIZER_ENV} ./micro_benchmark COMMAND env ${SANITIZER_ENV} ./micro_benchmark_mutex - COMMAND env ${SANITIZER_ENV} ./micro_benchmark_olc) + COMMAND env ${SANITIZER_ENV} ./micro_benchmark_olc + COMMAND env ${SANITIZER_ENV} ./micro_benchmark_bulk_load) add_custom_target(quick_benchmarks env ${SANITIZER_ENV} @@ -40,7 +42,9 @@ add_custom_target(quick_benchmarks COMMAND env ${SANITIZER_ENV} ./micro_benchmark_mutex ${micro_benchmark_mutex_quick_arg} COMMAND env ${SANITIZER_ENV} - ./micro_benchmark_olc ${micro_benchmark_olc_quick_arg}) + ./micro_benchmark_olc ${micro_benchmark_olc_quick_arg} + COMMAND env ${SANITIZER_ENV} + ./micro_benchmark_bulk_load ${micro_benchmark_bulk_load_quick_arg}) add_custom_target(valgrind_benchmarks COMMAND ${VALGRIND_COMMAND} ./micro_benchmark_key_prefix @@ -59,7 +63,9 @@ add_custom_target(valgrind_benchmarks COMMAND ${VALGRIND_COMMAND} ./micro_benchmark_mutex ${micro_benchmark_mutex_quick_arg} COMMAND ${VALGRIND_COMMAND} ./micro_benchmark_olc - ${micro_benchmark_olc_quick_arg}) + ${micro_benchmark_olc_quick_arg} + COMMAND ${VALGRIND_COMMAND} ./micro_benchmark_bulk_load + ${micro_benchmark_bulk_load_quick_arg}) add_library(micro_benchmark_utils STATIC micro_benchmark_utils.cpp micro_benchmark_utils.hpp) @@ -102,3 +108,4 @@ add_node_benchmark_target(micro_benchmark_n256) add_node_benchmark_target(micro_benchmark) add_concurrent_benchmark_target(micro_benchmark_mutex) add_concurrent_benchmark_target(micro_benchmark_olc) +add_benchmark_target(micro_benchmark_bulk_load) diff --git a/benchmark/micro_benchmark_bulk_load.cpp b/benchmark/micro_benchmark_bulk_load.cpp new file mode 100644 index 00000000..16f7da62 --- /dev/null +++ b/benchmark/micro_benchmark_bulk_load.cpp @@ -0,0 +1,460 @@ +// Copyright 2026 UnoDB contributors + +// Should be the first include +#include "global.hpp" // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "art.hpp" +#include "art_common.hpp" +#include "micro_benchmark_utils.hpp" +#include "mutex_art.hpp" +#include "olc_art.hpp" + +namespace { + +using db = unodb::benchmark::db; +using mutex_db = unodb::benchmark::mutex_db; +using olc_db = unodb::benchmark::olc_db; + +using kv_db = unodb::benchmark::kv_db; +using kv_mutex_db = unodb::mutex_db; +using kv_olc_db = unodb::benchmark::kv_olc_db; + +constexpr auto val8 = std::array{}; +const auto value = unodb::value_view{val8}; + +// =================================================================== +// Helpers +// =================================================================== + +/// Generate sorted sequential keys [0..n). +[[nodiscard]] auto sequential_keys(std::int64_t n) { + std::vector> kv; + kv.reserve(static_cast(n)); + for (std::int64_t i = 0; i < n; ++i) + kv.emplace_back(static_cast(i), value); + return kv; +} + +/// Generate sorted random unique keys. +[[nodiscard]] auto random_keys(std::int64_t n) { + std::mt19937_64 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) + std::vector keys(static_cast(n)); + std::generate(keys.begin(), keys.end(), rng); + std::sort(keys.begin(), keys.end()); + keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); + std::vector> kv; + kv.reserve(keys.size()); + for (auto k : keys) kv.emplace_back(k, value); + return kv; +} + +// =================================================================== +// BM01: bulk_load sequential uint64 keys (all tree modes) +// =================================================================== + +template +void BM_BulkLoad_seq(benchmark::State& state) { + const auto kv = sequential_keys(state.range(0)); + for (const auto _ : state) { + Db test_db; + test_db.bulk_load(kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +template +void BM_Insert_seq(benchmark::State& state) { + const auto kv = sequential_keys(state.range(0)); + for (const auto _ : state) { + Db test_db; + for (const auto& [k, v] : kv) + static_cast(test_db.insert(k, v)); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +// =================================================================== +// BM02: bulk_load random uint64 keys (all tree modes) +// =================================================================== + +template +void BM_BulkLoad_rand(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + Db test_db; + test_db.bulk_load(kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +template +void BM_Insert_rand(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + Db test_db; + for (const auto& [k, v] : kv) + static_cast(test_db.insert(k, v)); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +// =================================================================== +// BM03: bulk_load key_view compound keys (all key_view tree modes) +// =================================================================== + +template +void BM_BulkLoad_kv(benchmark::State& state) { + const auto n = static_cast(state.range(0)); + auto keys = unodb::benchmark::key_view_set::compound(0x01, n); + std::vector> kv; + kv.reserve(n); + for (std::size_t i = 0; i < n; ++i) kv.emplace_back(keys[i], value); + + for (const auto _ : state) { + Db test_db; + test_db.bulk_load(kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} + +template +void BM_Insert_kv(benchmark::State& state) { + const auto n = static_cast(state.range(0)); + auto keys = unodb::benchmark::key_view_set::compound(0x01, n); + std::vector> kv; + kv.reserve(n); + for (std::size_t i = 0; i < n; ++i) kv.emplace_back(keys[i], value); + + for (const auto _ : state) { + Db test_db; + for (const auto& [k, v] : kv) + static_cast(test_db.insert(k, v)); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} + +// =================================================================== +// BM04: bulk_load parallel scaling (olc_db) +// =================================================================== + +void BM_BulkLoad_par_db(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + db test_db; + test_db.bulk_load(std::execution::par, kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +void BM_BulkLoad_par_mutex(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + mutex_db test_db; + test_db.bulk_load(std::execution::par, kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +void BM_BulkLoad_par_olc(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + olc_db test_db; + test_db.bulk_load(std::execution::par, kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +// =================================================================== +// BM05: bulk_load scaling (N varies, all tree modes) +// =================================================================== + +template +void BM_BulkLoad_scaling(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + Db test_db; + test_db.bulk_load(kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +// =================================================================== +// BM07: memory comparison (requires STATS build) +// =================================================================== + +#ifdef UNODB_DETAIL_WITH_STATS + +template +void BM_BulkLoad_memory(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + std::size_t bulk_mem = 0; + std::size_t insert_mem = 0; + + for (const auto _ : state) { + { + Db bulk_db; + bulk_db.bulk_load(kv.begin(), kv.end()); + bulk_mem = bulk_db.get_current_memory_use(); + benchmark::DoNotOptimize(bulk_mem); + state.PauseTiming(); + bulk_db.clear(); + state.ResumeTiming(); + } + { + state.PauseTiming(); + Db insert_db; + for (const auto& [k, v] : kv) + static_cast(insert_db.insert(k, v)); + insert_mem = insert_db.get_current_memory_use(); + insert_db.clear(); + state.ResumeTiming(); + } + } + state.counters["bulk_bytes"] = benchmark::Counter( + static_cast(bulk_mem), benchmark::Counter::kDefaults, + benchmark::Counter::OneK::kIs1024); + state.counters["insert_bytes"] = benchmark::Counter( + static_cast(insert_mem), benchmark::Counter::kDefaults, + benchmark::Counter::OneK::kIs1024); +} + +#endif // UNODB_DETAIL_WITH_STATS + +// =================================================================== +// BM08: first-scan penalty after bulk_load (all tree modes) +// =================================================================== + +template +void BM_BulkLoad_first_scan(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + state.PauseTiming(); + Db test_db; + test_db.bulk_load(kv.begin(), kv.end()); + state.ResumeTiming(); + + std::size_t count = 0; + test_db.scan([&count](const auto& /*visitor*/) { + ++count; + return false; // continue + }); + benchmark::DoNotOptimize(count); + + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +template +void BM_Insert_first_scan(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + state.PauseTiming(); + Db test_db; + for (const auto& [k, v] : kv) + static_cast(test_db.insert(k, v)); + state.ResumeTiming(); + + std::size_t count = 0; + test_db.scan([&count](const auto& /*visitor*/) { + ++count; + return false; // continue + }); + benchmark::DoNotOptimize(count); + + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +} // namespace + +// =================================================================== +// Registration — all tree modes +// =================================================================== + +UNODB_START_BENCHMARKS() + +// BM01: Sequential bulk_load vs insert (db, mutex_db, olc_db) +BENCHMARK_TEMPLATE(BM_BulkLoad_seq, db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_seq, mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_seq, olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_seq, db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_seq, mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_seq, olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); + +// BM02: Random bulk_load vs insert (db, mutex_db, olc_db) +BENCHMARK_TEMPLATE(BM_BulkLoad_rand, db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_rand, mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_rand, olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_rand, db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_rand, mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_rand, olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); + +// BM03: key_view bulk_load vs insert (kv_db, kv_mutex_db, kv_olc_db) +BENCHMARK_TEMPLATE(BM_BulkLoad_kv, kv_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_kv, kv_mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_kv, kv_olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_kv, kv_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_kv, kv_mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_kv, kv_olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); + +// BM04: Parallel execution (all tree modes, par vs seq) +BENCHMARK(BM_BulkLoad_par_db)->Arg(1 << 20)->Unit(benchmark::kMillisecond); +BENCHMARK(BM_BulkLoad_par_mutex)->Arg(1 << 20)->Unit(benchmark::kMillisecond); +BENCHMARK(BM_BulkLoad_par_olc)->Arg(1 << 20)->Unit(benchmark::kMillisecond); + +// BM05: Size scaling (db, mutex_db, olc_db) +BENCHMARK_TEMPLATE(BM_BulkLoad_scaling, db) + ->Arg(1 << 10) + ->Arg(1 << 14) + ->Arg(1 << 17) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_scaling, mutex_db) + ->Arg(1 << 10) + ->Arg(1 << 14) + ->Arg(1 << 17) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_scaling, olc_db) + ->Arg(1 << 10) + ->Arg(1 << 14) + ->Arg(1 << 17) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); + +// BM07: Memory comparison (stats build only; db, mutex_db, olc_db) +#ifdef UNODB_DETAIL_WITH_STATS +BENCHMARK_TEMPLATE(BM_BulkLoad_memory, db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_memory, mutex_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_memory, olc_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +#endif + +// BM08: First-scan penalty (db, mutex_db, olc_db) +BENCHMARK_TEMPLATE(BM_BulkLoad_first_scan, db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_first_scan, mutex_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_first_scan, olc_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_first_scan, db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_first_scan, mutex_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_first_scan, olc_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); + +UNODB_BENCHMARK_MAIN(); diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index cc5fd84b..85767e4c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -39,12 +39,16 @@ target_link_libraries(example_olc_art PRIVATE unodb) add_executable(example_upsert example_upsert.cpp) target_link_libraries(example_upsert PRIVATE unodb) +add_executable(example_bulk_load example_bulk_load.cpp) +target_link_libraries(example_bulk_load PRIVATE unodb) + add_custom_target(examples ./example_art COMMAND ./example_art_stats COMMAND ./example_olc_art COMMAND ./example_upsert - DEPENDS example_art example_art_stats example_olc_art example_upsert) + COMMAND ./example_bulk_load + DEPENDS example_art example_art_stats example_olc_art example_upsert example_bulk_load) set(VALGRIND_COMMAND "valgrind" "--error-exitcode=1" "--leak-check=full" "--trace-children=yes" "-v") @@ -54,4 +58,5 @@ add_custom_target(valgrind_examples COMMAND ${VALGRIND_COMMAND} ./example_art_stats COMMAND ${VALGRIND_COMMAND} ./example_olc_art COMMAND ${VALGRIND_COMMAND} ./example_upsert - DEPENDS example_art example_art_stats example_olc_art example_upsert) + COMMAND ${VALGRIND_COMMAND} ./example_bulk_load + DEPENDS example_art example_art_stats example_olc_art example_upsert example_bulk_load) diff --git a/examples/example_bulk_load.cpp b/examples/example_bulk_load.cpp new file mode 100644 index 00000000..7f45e5bb --- /dev/null +++ b/examples/example_bulk_load.cpp @@ -0,0 +1,72 @@ +// Copyright 2026 UnoDB contributors + +// Example: bulk_load with sequential and parallel execution policies. + +#include "global.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "art.hpp" +#include "art_common.hpp" +#include "mutex_art.hpp" + +namespace { + +constexpr std::size_t key_count = 100'000; + +/// Build sorted key-value pairs. The caller is responsible for +/// pre-encoding and pre-sorting keys before calling bulk_load. +[[nodiscard]] auto make_sorted_data() { + constexpr std::array val{}; + const auto value = unodb::value_view{val}; + std::vector> kv; + kv.reserve(key_count); + for (std::size_t i = 0; i < key_count; ++i) + kv.emplace_back(static_cast(i), value); + return kv; +} + +} // namespace + +int main() { + const auto data = make_sorted_data(); + + // ─── Sequential bulk_load (default) ──────────────────────────────────────── + { + unodb::db tree; + tree.bulk_load(data.begin(), data.end()); // default: std::execution::seq + std::cerr << "Sequential bulk_load: " << key_count << " keys loaded\n"; + std::cerr << " get(42) found: " << tree.get(42).has_value() << '\n'; + tree.clear(); + } + + // ─── Parallel bulk_load ──────────────────────────────────────────────────── + // std::execution::par enables concurrent subtree construction. + // The implementation partitions at the root level and builds each + // subtree on a separate thread. Safe for all tree modes because + // bulk_load operates on an unpublished tree (no concurrent readers). + { + unodb::db tree; + tree.bulk_load(std::execution::par, data.begin(), data.end()); + std::cerr << "Parallel bulk_load: " << key_count << " keys loaded\n"; + std::cerr << " get(99999) found: " << tree.get(99999).has_value() << '\n'; + tree.clear(); + } + + // ─── mutex_db: same API ──────────────────────────────────────────────────── + { + unodb::mutex_db tree; + tree.bulk_load(std::execution::par, data.begin(), data.end()); + std::cerr << "mutex_db parallel bulk_load: " << key_count + << " keys loaded\n"; + tree.clear(); + } + + std::cerr << "Done.\n"; +} diff --git a/mutex_art.hpp b/mutex_art.hpp index 8aaaf223..497a5849 100644 --- a/mutex_art.hpp +++ b/mutex_art.hpp @@ -147,6 +147,24 @@ class mutex_db final { db_.clear(); } + /// Bulk-load sorted key-value pairs into an empty tree. + /// + /// \tparam RandomIt Random access iterator over pairs of (key, value) + /// \param first Start of sorted range + /// \param last End of sorted range + /// \param policy Execution policy (std::execution::seq or par) + template + void bulk_load(ExecutionPolicy&& policy, RandomIt first, RandomIt last) { + const std::lock_guard guard{mutex}; + db_.bulk_load(std::forward(policy), first, last); + } + + /// Convenience overload: sequential execution. + template + void bulk_load(RandomIt first, RandomIt last) { + bulk_load(std::execution::seq, first, last); + } + // // scan API. // diff --git a/olc_art.hpp b/olc_art.hpp index 0e207ef0..b814887b 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include #include @@ -167,6 +169,11 @@ template using olc_leaf_unique_ptr = basic_db_leaf_unique_ptr; +/// Forward declaration of shared bulk_load algorithm. +template +void bulk_load_impl(Db& self, ExecutionPolicy&& policy, RandomIt first, + RandomIt last); + } // namespace detail /// A thread-safe Adaptive Radix Tree that is synchronized using optimistic lock @@ -296,6 +303,22 @@ class olc_db final { /// \note Only legal in single-threaded context, as destructor void clear() noexcept; + /// Bulk-load sorted key-value pairs into an empty tree. + /// + /// \note Only legal in single-threaded context (same as clear). + /// \tparam RandomIt Random access iterator over pairs of (key, value) + /// \param first Start of sorted range + /// \param last End of sorted range + /// \param parallelism Parallelism hint (0=auto, 1=sequential) + template + void bulk_load(ExecutionPolicy&& policy, RandomIt first, RandomIt last); + + /// Convenience overload: sequential execution. + template + void bulk_load(RandomIt first, RandomIt last) { + bulk_load(std::execution::seq, first, last); + } + // // iterator (the iterator is an internal API, the public API is scan()). // @@ -854,6 +877,42 @@ class olc_db final { using visitor_type = visitor; using olc_db_leaf_unique_ptr_type = detail::olc_db_leaf_unique_ptr; + + /// Result of subtree construction during bulk_load. + struct build_result { + detail::olc_node_ptr ptr{nullptr}; + bool is_packed_value{false}; + }; + + /// RAII guard for subtrees during bulk construction. + struct bulk_subtree_guard { + olc_db& db_; + detail::olc_node_ptr ptr{nullptr}; + bool is_packed_value{false}; + + constexpr explicit bulk_subtree_guard(olc_db& db + UNODB_DETAIL_LIFETIMEBOUND) noexcept + : db_{db} {} + + ~bulk_subtree_guard() noexcept { + if (ptr != nullptr && !is_packed_value) { + art_policy::delete_subtree(ptr, db_); + } + } + + void release() noexcept { ptr = nullptr; } + + bulk_subtree_guard(const bulk_subtree_guard&) = delete; + bulk_subtree_guard(bulk_subtree_guard&& other) noexcept + : db_{other.db_}, + ptr{other.ptr}, + is_packed_value{other.is_packed_value} { + other.ptr = nullptr; + } + auto& operator=(const bulk_subtree_guard&) = delete; + auto& operator=(bulk_subtree_guard&&) = delete; + }; + // If get_result is not present, the search was interrupted. Yes, this // resolves to std::optional>, but IMHO both // levels of std::optional are clear here @@ -1023,6 +1082,10 @@ class olc_db final { friend auto detail::make_db_leaf_ptr(art_key_type, value_type, olc_db&); + /// detail::bulk_load_impl + template + friend void detail::bulk_load_impl(Db2&, Ep2&&, It2, It2); + template friend class detail::basic_db_leaf_deleter; @@ -4968,4 +5031,27 @@ olc_db::try_chain_cut( } } // namespace unodb +// ─── olc_db::bulk_load ─────────────────────────────────────────────────────── + +namespace unodb { + +template +template +void olc_db::bulk_load(ExecutionPolicy&& policy, RandomIt first, + RandomIt last) { + UNODB_DETAIL_QSBR_ASSERT( + qsbr_state::single_thread_mode(qsbr::instance().get_state())); + + if (!empty()) { + throw std::invalid_argument("bulk_load requires empty tree"); + } + if (first == last) return; + detail::bulk_load_impl(*this, std::forward(policy), first, + last); +} + +} // namespace unodb + +#include "art_bulk_load_detail.hpp" + #endif // UNODB_DETAIL_OLC_ART_HPP diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e58704a4..aa7c3114 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,10 @@ add_db_test_target(test_key_encode_decode) add_db_test_target(test_art) target_compile_options(test_art PRIVATE "$<${is_msvc}:/bigobj>") +add_db_test_target(test_art_bulk_load) +add_db_test_target(test_art_bulk_load_structural) +add_db_test_target(test_art_bulk_load_ops) + add_db_test_target(test_art_key_view) add_db_test_target(test_art_key_view_full_chain) add_db_test_target(test_art_iter) diff --git a/test/test_art_bulk_load.cpp b/test/test_art_bulk_load.cpp new file mode 100644 index 00000000..b6131c90 --- /dev/null +++ b/test/test_art_bulk_load.cpp @@ -0,0 +1,171 @@ +// Copyright 2026 UnoDB contributors + +// Should be the first include +#include "global.hpp" // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include + +#include + +#include "art_common.hpp" +#include "db_test_utils.hpp" +#include "gtest_utils.hpp" +#include "node_type.hpp" + +#ifdef UNODB_DETAIL_WITH_STATS + +namespace { + +using unodb::as_i; +using unodb::node_type; +using unodb::value_view; +using unodb::test::u64_db; + +constexpr auto val_bytes = + std::array{std::byte{0x68}, std::byte{0x65}, std::byte{0x6C}, + std::byte{0x6C}, std::byte{0x6F}}; +constexpr auto val = value_view{val_bytes}; + +/// Generate N keys that differ at byte 0 (big-endian uint64_t). +/// For N <= 256, all keys differ only at byte 0 (single root node). +/// For N > 256, extra keys share byte 0 with key 0 but differ at byte 1. +[[nodiscard]] std::vector> make_keys( + std::size_t n) { + std::vector> kv; + kv.reserve(n); + for (std::size_t i = 0; i < n && i < 256; ++i) { + const auto key = static_cast(i) << 56U; + kv.emplace_back(key, val); + } + // Overflow keys: share byte 0 == 0x00, differ at byte 1 + for (std::size_t i = 256; i < n; ++i) { + const auto key = static_cast(i - 255) << 48U; + kv.emplace_back(key, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + return kv; +} + +// T01 +TEST(BulkLoad, Empty) { + u64_db db; + std::vector> kv; + db.bulk_load(kv.begin(), kv.end()); + UNODB_ASSERT_TRUE(db.empty()); + const auto counts = db.get_node_counts(); + for (const auto c : counts) { + UNODB_ASSERT_EQ(c, 0U); + } +} + +// T02 +TEST(BulkLoad, Single) { + u64_db db; + const std::uint64_t key = 1; + std::vector> kv{{key, val}}; + db.bulk_load(kv.begin(), kv.end()); + const auto result = db.get(key); + ASSERT_TRUE(result.has_value()); + UNODB_ASSERT_EQ( // NOLINT(bugprone-unchecked-optional-access) + result.value().size(), val.size()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +// T03 +TEST(BulkLoad, Small4Keys) { + u64_db db; + auto kv = make_keys(4); + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()); + } + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 4U); + UNODB_ASSERT_EQ(counts[as_i], 1U); +} + +// T04 +TEST(BulkLoad, Boundary4) { + u64_db db; + auto kv = make_keys(4); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); + UNODB_ASSERT_EQ(counts[as_i], 0U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +// T05 +TEST(BulkLoad, Boundary5) { + u64_db db; + auto kv = make_keys(5); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +// T06 +TEST(BulkLoad, Boundary16) { + u64_db db; + auto kv = make_keys(16); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +// T07 +TEST(BulkLoad, Boundary17) { + u64_db db; + auto kv = make_keys(17); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +// T08 +TEST(BulkLoad, Boundary48) { + u64_db db; + auto kv = make_keys(48); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +// T09 +TEST(BulkLoad, Boundary49) { + u64_db db; + auto kv = make_keys(49); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +// T10 +TEST(BulkLoad, Growth260) { + u64_db db; + auto kv = make_keys(260); + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()); + } + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 260U); +} + +} // namespace + +#endif // UNODB_DETAIL_WITH_STATS diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp new file mode 100644 index 00000000..61b39b46 --- /dev/null +++ b/test/test_art_bulk_load_ops.cpp @@ -0,0 +1,209 @@ +// Copyright 2026 UnoDB contributors + +// Should be the first include +#include "global.hpp" // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "art_common.hpp" +#include "db_test_utils.hpp" +#include "gtest_utils.hpp" +#include "node_type.hpp" + +namespace { + +#ifdef UNODB_DETAIL_WITH_STATS +using unodb::as_i; +using unodb::node_type; +#endif +using unodb::value_view; +using unodb::test::u64_db; +using unodb::test::u64_mutex_db; +using unodb::test::u64_olc_db; + +constexpr auto val_bytes = + std::array{std::byte{0x68}, std::byte{0x65}, std::byte{0x6C}, + std::byte{0x6C}, std::byte{0x6F}}; +constexpr auto val = value_view{val_bytes}; + +// ─── Error Tests ───────────────────────────────────────────────────────────── + +// T26: bulk_load on non-empty tree throws +TEST(BulkLoadError, NonEmpty) { + u64_db db; + const std::uint64_t key = 42; + ASSERT_TRUE(db.insert(key, val)); + std::vector> kv{{100, val}}; + EXPECT_THROW(db.bulk_load(kv.begin(), kv.end()), std::invalid_argument); + // Original key still present + const auto result = db.get(key); + ASSERT_TRUE(result.has_value()); +} + +// T36: db ignores parallelism parameter +TEST(BulkLoadError, DbIgnoresParallelism) { + u64_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + const auto key = i << 56U; + kv.emplace_back(key, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + // std::execution::par should enable parallel subtree construction + db.bulk_load(std::execution::par, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +// ─── Operational Tests ─────────────────────────────────────────────────────── + +// T38: Operations work correctly after bulk_load +TEST(BulkLoadOps, ThenOperations) { + u64_db db; + std::vector> kv; + kv.reserve(10); + for (std::uint64_t i = 0; i < 10; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + + // get works for all keys + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()); + } + + // insert new key works + const std::uint64_t new_key = 0xFFULL << 56U; + ASSERT_TRUE(db.insert(new_key, val)); + ASSERT_TRUE(db.get(new_key).has_value()); + + // insert duplicate fails + ASSERT_FALSE(db.insert(kv[0].first, val)); + + // remove works + ASSERT_TRUE(db.remove(kv[0].first)); + ASSERT_FALSE(db.get(kv[0].first).has_value()); + + // scan works + std::size_t count = 0; + db.scan([&count](auto&) { + ++count; + return false; // continue + }); + UNODB_ASSERT_EQ(count, 10U); // 10 original - 1 removed + 1 new = 10 +} + +// T39: Stats are correct after bulk_load +#ifdef UNODB_DETAIL_WITH_STATS +TEST(BulkLoadOps, Stats) { + u64_db db; + // 17 keys differing at byte 0 → should produce 1 inode48 + std::vector> kv; + kv.reserve(17); + for (std::uint64_t i = 0; i < 17; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 17U); + UNODB_ASSERT_EQ(counts[as_i], 0U); + UNODB_ASSERT_EQ(counts[as_i], 0U); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} +#endif // UNODB_DETAIL_WITH_STATS + +// T40: No growth events during bulk_load (all right-sized at allocation) +TEST(BulkLoadOps, NoGrowthEvents) { + u64_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + + // In bulk_load, no inode should ever grow (they're right-sized at creation). + // The growing_inode_counts track create events, not grow events — but the + // key insight is that no shrink events should exist. + // Just verify all keys are present: + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()) << "key " << k << " not found"; + } +} + +// T42: mutex_db bulk_load works +TEST(BulkLoadOps, MutexDb) { + u64_mutex_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.first.has_value()) << "key " << k << " not found"; + } +} + +// T38 extended: clear after bulk_load then re-bulk_load +TEST(BulkLoadOps, ClearAndReload) { + u64_db db; + std::vector> kv; + kv.reserve(10); + for (std::uint64_t i = 0; i < 10; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + db.clear(); + ASSERT_TRUE(db.empty()); + // Re-load different data + std::vector> kv2; + kv2.reserve(10); + for (std::uint64_t i = 100; i < 110; ++i) { + kv2.emplace_back(i << 56U, val); + } + std::ranges::sort(kv2, {}, &decltype(kv2)::value_type::first); + db.bulk_load(kv2.begin(), kv2.end()); + for (const auto& [k, v] : kv2) { + ASSERT_TRUE(db.get(k).has_value()); + } +} + +// T43: olc_db parallel bulk_load +TEST(BulkLoadOps, OlcDbParallel) { + u64_olc_db db; + std::vector> kv; + kv.reserve(1000); + for (std::uint64_t i = 0; i < 1000; ++i) { + kv.emplace_back(i << 48U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(std::execution::par, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +} // namespace diff --git a/test/test_art_bulk_load_structural.cpp b/test/test_art_bulk_load_structural.cpp new file mode 100644 index 00000000..69b78197 --- /dev/null +++ b/test/test_art_bulk_load_structural.cpp @@ -0,0 +1,284 @@ +// Copyright 2026 UnoDB contributors + +#include "global.hpp" // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "art_common.hpp" +#include "db_test_utils.hpp" +#include "gtest_utils.hpp" +#include "node_type.hpp" + +#ifdef UNODB_DETAIL_WITH_STATS + +namespace { + +using unodb::as_i; +using unodb::key_view; +using unodb::node_type; +using unodb::value_view; +using unodb::test::key_view_db; +using unodb::test::key_view_u64val_db; +using unodb::test::u64_db; + +constexpr auto val16 = std::array{}; +constexpr auto large_val = value_view{val16}; +constexpr auto sval_bytes = + std::array{std::byte{0x68}, std::byte{0x65}, std::byte{0x6C}, + std::byte{0x6C}, std::byte{0x6F}}; +constexpr auto sval = value_view{sval_bytes}; + +// T11 +TEST(BulkLoadStructural, BulkLoadPrefix7) { + u64_db db; + std::vector> kv{ + {0x0102030405060700ULL, sval}, {0x0102030405060701ULL, sval}}; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} +// T12 +TEST(BulkLoadStructural, BulkLoadPrefix8) { + std::vector> s{ + {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}, std::byte{5}, + std::byte{6}, std::byte{7}, std::byte{8}, std::byte{0}}, + {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}, std::byte{5}, + std::byte{6}, std::byte{7}, std::byte{8}, std::byte{1}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 2U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} +// T13 +TEST(BulkLoadStructural, BulkLoadPrefix15) { + std::vector> s( + 2, std::vector(16, std::byte{0xAA})); + s[0][15] = std::byte{0}; + s[1][15] = std::byte{1}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 2U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} +// T14 +TEST(BulkLoadStructural, BulkLoadPrefix16) { + std::vector> s( + 2, std::vector(17, std::byte{0xBB})); + s[0][16] = std::byte{0}; + s[1][16] = std::byte{1}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 3U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} +// T15 +TEST(BulkLoadStructural, BulkLoadVIS) { + std::vector> s{{std::byte{1}}, {std::byte{2}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} +// T16 +TEST(BulkLoadStructural, BulkLoadVISWithChain) { + std::vector> s{{std::byte{1}, std::byte{0xAA}}, + {std::byte{1}, std::byte{0xBB}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, 99ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} +// T17 +TEST(BulkLoadStructural, BulkLoadVISLongPrefix) { + std::vector> s( + 2, std::vector(9, std::byte{0xCC})); + s[0][8] = std::byte{0}; + s[1][8] = std::byte{1}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, 77ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 2U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} +// T18 +TEST(BulkLoadStructural, BulkLoadKeylessLeaf) { + std::vector> s{ + {std::byte{1}, std::byte{2}, std::byte{3}}, + {std::byte{4}, std::byte{5}, std::byte{6}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + // Root I4 (dispatch byte 0) + 2 chain I4s wrapping keyless leaves + UNODB_ASSERT_EQ(c[as_i], 3U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} +// T19 +TEST(BulkLoadStructural, BulkLoadKeylessLeafWithChain) { + std::vector> s{ + {std::byte{1}, std::byte{0xAA}, std::byte{0xFF}}, + {std::byte{1}, std::byte{0xBB}, std::byte{0xFF}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 3U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} +// T20 +TEST(BulkLoadStructural, BulkLoadLarge) { + std::mt19937 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) + std::vector> kv; + kv.reserve(100000); + for (int i = 0; i < 100000; ++i) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + (static_cast(rng()) << 32U) | rng(), sval); + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + kv.erase(std::unique( // NOLINT(modernize-use-ranges) + kv.begin(), kv.end(), + [](const auto& a, const auto& b) { return a.first == b.first; }), + kv.end()); + u64_db db; + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) ASSERT_TRUE(db.get(k).has_value()); + std::uint64_t prev = 0; + bool first = true; + UNODB_DETAIL_DISABLE_MSVC_WARNING(26440) + db.scan([&](auto& visitor) { + unodb::key_decoder dec{visitor.get_key()}; + std::uint64_t k{}; + dec.decode(k); + if (!first) EXPECT_GE(k, prev); + prev = k; + first = false; + return false; + }); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +} +// T21 — key_view bulk_load with random fixed-length keys (prefix-free). +TEST(BulkLoadStructural, BulkLoadKeyView) { + constexpr std::size_t key_len = + 10; // Fixed length → no key is prefix of another + std::mt19937 rng(123); // NOLINT(cert-msc32-c,cert-msc51-cpp) + std::vector> storage; + for (int i = 0; i < 1000; ++i) { + std::vector k(key_len); + for (auto& b : k) b = static_cast(rng() & 0xFFU); + storage.push_back(std::move(k)); + } + std::ranges::sort(storage); + // NOLINTNEXTLINE(modernize-use-ranges) + storage.erase(std::unique(storage.begin(), storage.end()), storage.end()); + std::vector> kv; + kv.reserve(storage.size()); + for (auto& v : storage) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) ASSERT_TRUE(db.get(k).has_value()); +} +// T22 +TEST(BulkLoadStructural, BulkLoadVISRootSingle) { + std::vector> s{{std::byte{1}, std::byte{2}}}; + std::vector> kv; + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{s[0]}, 55ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); +} +// T23 +TEST(BulkLoadStructural, BulkLoadKeylessLeafRootSingle) { + std::vector> s{ + {std::byte{1}, std::byte{2}, std::byte{3}}}; + std::vector> kv; + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{s[0]}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); +} +// T24 +TEST(BulkLoadStructural, BulkLoadFullLeafRootSingle) { + u64_db db; + std::vector> kv{{0xDEADBEEFULL, sval}}; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} +// T25 +TEST(BulkLoadStructural, BulkLoadScanOrder) { + std::vector> kv; + for (std::uint64_t i = 0; i < 256; ++i) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + i << 56U, sval); + u64_db db; + db.bulk_load(kv.begin(), kv.end()); + std::uint64_t prev = 0; + std::size_t count = 0; + UNODB_DETAIL_DISABLE_MSVC_WARNING(26440) + db.scan([&](auto& visitor) { + unodb::key_decoder dec{visitor.get_key()}; + std::uint64_t k{}; + dec.decode(k); + if (count > 0) EXPECT_GT(k, prev); + prev = k; + ++count; + return false; + }); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_ASSERT_EQ(count, 256U); +} + +} // namespace + +#endif // UNODB_DETAIL_WITH_STATS From 936eb1602954b5e67704aa570b6df94b67715feb Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Thu, 25 Jun 2026 18:56:40 +0000 Subject: [PATCH 02/23] test: add OOM and concurrent reader tests for bulk_load - Add strong exception guarantee (try-catch + clear) to db::bulk_load and olc_db::bulk_load so tree is left empty on allocation failure - Add 5 OOM typed tests (x3 db types = 15 tests) exercising single key, inode4, inode16, inode48, and nested inode topologies - Add T44 OlcDbConcurrentReaders: after parallel bulk_load, spawn 4 qsbr_threads reading all 10K keys to verify lock initialization - Link test_art_bulk_load_ops against qsbr_test_utils - Fix clang-format-21 whitespace in benchmark file --- art.hpp | 9 ++- benchmark/micro_benchmark_bulk_load.cpp | 15 ++--- olc_art.hpp | 9 ++- test/CMakeLists.txt | 1 + test/test_art_bulk_load_ops.cpp | 38 ++++++++++++ test/test_art_oom.cpp | 82 ++++++++++++++++++++++++- 6 files changed, 139 insertions(+), 15 deletions(-) diff --git a/art.hpp b/art.hpp index 292a3cf3..1c8db281 100644 --- a/art.hpp +++ b/art.hpp @@ -2840,8 +2840,13 @@ void db::bulk_load(ExecutionPolicy&& policy, RandomIt first, throw std::invalid_argument("bulk_load requires empty tree"); } if (first == last) return; - detail::bulk_load_impl(*this, std::forward(policy), first, - last); + try { + detail::bulk_load_impl(*this, std::forward(policy), first, + last); + } catch (...) { + clear(); + throw; + } } } // namespace unodb diff --git a/benchmark/micro_benchmark_bulk_load.cpp b/benchmark/micro_benchmark_bulk_load.cpp index 16f7da62..6392f544 100644 --- a/benchmark/micro_benchmark_bulk_load.cpp +++ b/benchmark/micro_benchmark_bulk_load.cpp @@ -82,8 +82,7 @@ void BM_Insert_seq(benchmark::State& state) { const auto kv = sequential_keys(state.range(0)); for (const auto _ : state) { Db test_db; - for (const auto& [k, v] : kv) - static_cast(test_db.insert(k, v)); + for (const auto& [k, v] : kv) static_cast(test_db.insert(k, v)); benchmark::DoNotOptimize(test_db); state.PauseTiming(); test_db.clear(); @@ -117,8 +116,7 @@ void BM_Insert_rand(benchmark::State& state) { const auto kv = random_keys(state.range(0)); for (const auto _ : state) { Db test_db; - for (const auto& [k, v] : kv) - static_cast(test_db.insert(k, v)); + for (const auto& [k, v] : kv) static_cast(test_db.insert(k, v)); benchmark::DoNotOptimize(test_db); state.PauseTiming(); test_db.clear(); @@ -161,8 +159,7 @@ void BM_Insert_kv(benchmark::State& state) { for (const auto _ : state) { Db test_db; - for (const auto& [k, v] : kv) - static_cast(test_db.insert(k, v)); + for (const auto& [k, v] : kv) static_cast(test_db.insert(k, v)); benchmark::DoNotOptimize(test_db); state.PauseTiming(); test_db.clear(); @@ -261,8 +258,7 @@ void BM_BulkLoad_memory(benchmark::State& state) { { state.PauseTiming(); Db insert_db; - for (const auto& [k, v] : kv) - static_cast(insert_db.insert(k, v)); + for (const auto& [k, v] : kv) static_cast(insert_db.insert(k, v)); insert_mem = insert_db.get_current_memory_use(); insert_db.clear(); state.ResumeTiming(); @@ -312,8 +308,7 @@ void BM_Insert_first_scan(benchmark::State& state) { for (const auto _ : state) { state.PauseTiming(); Db test_db; - for (const auto& [k, v] : kv) - static_cast(test_db.insert(k, v)); + for (const auto& [k, v] : kv) static_cast(test_db.insert(k, v)); state.ResumeTiming(); std::size_t count = 0; diff --git a/olc_art.hpp b/olc_art.hpp index b814887b..b3f40774 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -5046,8 +5046,13 @@ void olc_db::bulk_load(ExecutionPolicy&& policy, RandomIt first, throw std::invalid_argument("bulk_load requires empty tree"); } if (first == last) return; - detail::bulk_load_impl(*this, std::forward(policy), first, - last); + try { + detail::bulk_load_impl(*this, std::forward(policy), first, + last); + } catch (...) { + clear(); + throw; + } } } // namespace unodb diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index aa7c3114..49abe271 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -62,6 +62,7 @@ target_compile_options(test_art PRIVATE "$<${is_msvc}:/bigobj>") add_db_test_target(test_art_bulk_load) add_db_test_target(test_art_bulk_load_structural) add_db_test_target(test_art_bulk_load_ops) +target_link_libraries(test_art_bulk_load_ops PRIVATE qsbr_test_utils) add_db_test_target(test_art_key_view) add_db_test_target(test_art_key_view_full_chain) diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp index 61b39b46..63af822a 100644 --- a/test/test_art_bulk_load_ops.cpp +++ b/test/test_art_bulk_load_ops.cpp @@ -18,6 +18,8 @@ #include "db_test_utils.hpp" #include "gtest_utils.hpp" #include "node_type.hpp" +#include "qsbr.hpp" +#include "qsbr_test_utils.hpp" namespace { @@ -206,4 +208,40 @@ TEST(BulkLoadOps, OlcDbParallel) { } } +// T44: olc_db concurrent readers after parallel bulk_load +// Verifies that optimistic locks are correctly initialized after bulk_load so +// concurrent readers can traverse the tree without deadlock or data corruption. +TEST(BulkLoadOps, OlcDbConcurrentReaders) { + u64_olc_db db; + constexpr std::size_t n_keys = 10000; + std::vector> kv; + kv.reserve(n_keys); + for (std::uint64_t i = 0; i < n_keys; ++i) { + kv.emplace_back(i << 40U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(std::execution::par, kv.begin(), kv.end()); + + // Pause main thread QSBR so reader threads can register + unodb::this_thread().qsbr_pause(); + + constexpr std::size_t n_threads = 4; + std::array threads; + for (std::size_t t = 0; t < n_threads; ++t) { + threads[t] = unodb::qsbr_thread([&kv, &db] { + for (const auto& [k, v] : kv) { + const unodb::quiescent_state_on_scope_exit qsbr{}; + const auto result = db.get(k); + EXPECT_TRUE(result.has_value()) << "key " << k << " not found"; + } + unodb::this_thread().quiescent(); + }); + } + for (auto& t : threads) t.join(); + + unodb::this_thread().qsbr_resume(); + unodb::this_thread().quiescent(); + unodb::test::expect_idle_qsbr(); +} + } // namespace diff --git a/test/test_art_oom.cpp b/test/test_art_oom.cpp index cda64e41..5637f8a1 100644 --- a/test/test_art_oom.cpp +++ b/test/test_art_oom.cpp @@ -1,4 +1,4 @@ -// Copyright 2022-2025 UnoDB contributors +// Copyright 2022-2026 UnoDB contributors #ifndef NDEBUG @@ -11,6 +11,8 @@ #include #include // IWYU pragma: keep +#include +#include #include @@ -644,6 +646,84 @@ UNODB_TYPED_TEST(ARTKeyViewOOMTest, BuildChainMultiNode) { [](unodb::test::tree_verifier&) {}); } +// =================================================================== +// bulk_load OOM tests — strong exception guarantee: tree stays empty +// =================================================================== + +template +void bulk_load_oom_test( + unsigned fail_limit, + std::vector> + kv) { // NOLINT(performance-unnecessary-value-param) + unsigned fail_n; + for (fail_n = 1; fail_n <= fail_limit; ++fail_n) { + TypeParam test_db; + + unodb::test::allocation_failure_injector::fail_on_nth_allocation(fail_n); + try { + test_db.bulk_load(kv.begin(), kv.end()); + // Success: we've found the limit + unodb::test::allocation_failure_injector::reset(); + UNODB_ASSERT_FALSE(test_db.empty()); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(TypeParam::key_found(test_db.get(k))); + } + return; + } catch (const std::bad_alloc&) { + unodb::test::allocation_failure_injector::reset(); + // Strong guarantee: tree must be empty after failed bulk_load + UNODB_ASSERT_TRUE(test_db.empty()); + } + } + FAIL() << "bulk_load did not succeed within " << fail_limit << " allocations"; +} + +// Small tree (single leaf → one allocation) +UNODB_TYPED_TEST(ARTOOMTest, BulkLoadSingleKey) { + constexpr auto val = unodb::test::test_values[0]; + const std::vector> kv{{42, val}}; + bulk_load_oom_test(5, kv); +} + +// Tree that creates one inode4 (4 leaves + 1 inode4) +UNODB_TYPED_TEST(ARTOOMTest, BulkLoadInode4) { + constexpr auto val = unodb::test::test_values[0]; + std::vector> kv; + kv.reserve(4); + for (std::uint64_t i = 0; i < 4; ++i) kv.emplace_back(i << 56U, val); + bulk_load_oom_test(10, kv); +} + +// Tree that creates an inode16 (10 leaves + 1 inode16) +UNODB_TYPED_TEST(ARTOOMTest, BulkLoadInode16) { + constexpr auto val = unodb::test::test_values[0]; + std::vector> kv; + kv.reserve(10); + for (std::uint64_t i = 0; i < 10; ++i) kv.emplace_back(i << 56U, val); + bulk_load_oom_test(15, kv); +} + +// Tree that creates an inode48 (20 leaves + 1 inode48) +UNODB_TYPED_TEST(ARTOOMTest, BulkLoadInode48) { + constexpr auto val = unodb::test::test_values[0]; + std::vector> kv; + kv.reserve(20); + for (std::uint64_t i = 0; i < 20; ++i) kv.emplace_back(i << 56U, val); + bulk_load_oom_test(30, kv); +} + +// Tree with nested inodes (two inode4s under one root inode4) +UNODB_TYPED_TEST(ARTOOMTest, BulkLoadNested) { + constexpr auto val = unodb::test::test_values[0]; + std::vector> kv; + kv.reserve(8); + for (std::uint64_t i = 0; i < 4; ++i) + kv.emplace_back((1ULL << 56U) | (i << 48U), val); + for (std::uint64_t i = 0; i < 4; ++i) + kv.emplace_back((2ULL << 56U) | (i << 48U), val); + bulk_load_oom_test(20, kv); +} + } // namespace #endif // #ifndef NDEBUG From e68c9664d44bac5e59de6eee9fc6bd9640b7822e Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Fri, 26 Jun 2026 00:23:34 +0000 Subject: [PATCH 03/23] fix: portability for macOS and GCC warnings - Add portability_execution.hpp with __cpp_lib_execution detection; provides fallback sequential/parallel policy types when is unavailable (Apple libc++, older libstdc++) - Replace all #include with portability header - Fix GCC -Wuseless-cast by using uint64_t loop variable in test_art_bulk_load.cpp and DISABLE_GCC_WARNING in structural test - Fix GCC -Wdangling-else by adding braces around EXPECT_GE/EXPECT_GT after bare if in test_art_bulk_load_structural.cpp --- art.hpp | 2 +- benchmark/micro_benchmark_bulk_load.cpp | 2 +- examples/example_bulk_load.cpp | 2 +- olc_art.hpp | 2 +- portability_execution.hpp | 45 +++++++++++++++++++++++++ test/test_art_bulk_load.cpp | 8 ++--- test/test_art_bulk_load_ops.cpp | 2 +- test/test_art_bulk_load_structural.cpp | 10 ++++-- 8 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 portability_execution.hpp diff --git a/art.hpp b/art.hpp index 1c8db281..1e131123 100644 --- a/art.hpp +++ b/art.hpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -36,6 +35,7 @@ #include "assert.hpp" #include "in_fake_critical_section.hpp" #include "node_type.hpp" +#include "portability_execution.hpp" namespace unodb { diff --git a/benchmark/micro_benchmark_bulk_load.cpp b/benchmark/micro_benchmark_bulk_load.cpp index 6392f544..e9f4ee00 100644 --- a/benchmark/micro_benchmark_bulk_load.cpp +++ b/benchmark/micro_benchmark_bulk_load.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -18,6 +17,7 @@ #include "micro_benchmark_utils.hpp" #include "mutex_art.hpp" #include "olc_art.hpp" +#include "portability_execution.hpp" namespace { diff --git a/examples/example_bulk_load.cpp b/examples/example_bulk_load.cpp index 7f45e5bb..62f6c49c 100644 --- a/examples/example_bulk_load.cpp +++ b/examples/example_bulk_load.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -15,6 +14,7 @@ #include "art.hpp" #include "art_common.hpp" #include "mutex_art.hpp" +#include "portability_execution.hpp" namespace { diff --git a/olc_art.hpp b/olc_art.hpp index b3f40774..a7ff2f62 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -32,6 +31,7 @@ #include "node_type.hpp" #include "optimistic_lock.hpp" #include "portability_arch.hpp" +#include "portability_execution.hpp" #include "qsbr.hpp" #include "sync.hpp" diff --git a/portability_execution.hpp b/portability_execution.hpp new file mode 100644 index 00000000..233cf7aa --- /dev/null +++ b/portability_execution.hpp @@ -0,0 +1,45 @@ +// Copyright 2026 UnoDB contributors +#ifndef UNODB_DETAIL_PORTABILITY_EXECUTION_HPP +#define UNODB_DETAIL_PORTABILITY_EXECUTION_HPP + +/// \file +/// Portability shim for C++17 execution policies (``). +/// +/// Apple libc++ and some older standard libraries lack execution policy +/// support. This header detects availability and provides a minimal +/// sequential-only fallback when the real header is absent. + +// Should be the first include +#include "global.hpp" // IWYU pragma: keep + +#if __has_include() +#include // IWYU pragma: export +#endif + +// Detect usable execution policies via the library feature-test macro. +// __cpp_lib_execution >= 201603 means parallel_policy is available. +#if defined(__cpp_lib_execution) && __cpp_lib_execution >= 201603L +#define UNODB_DETAIL_HAS_EXECUTION_POLICIES 1 +#else +#define UNODB_DETAIL_HAS_EXECUTION_POLICIES 0 +#endif + +#if !UNODB_DETAIL_HAS_EXECUTION_POLICIES + +/// Minimal fallback: sequential execution tag only. +namespace std::execution { // NOLINT(cert-dcl58-cpp) + +struct sequenced_policy {}; +inline constexpr sequenced_policy seq{}; + +struct parallel_policy {}; +inline constexpr parallel_policy par{}; + +struct parallel_unsequenced_policy {}; +inline constexpr parallel_unsequenced_policy par_unseq{}; + +} // namespace std::execution + +#endif // !UNODB_DETAIL_HAS_EXECUTION_POLICIES + +#endif // UNODB_DETAIL_PORTABILITY_EXECUTION_HPP diff --git a/test/test_art_bulk_load.cpp b/test/test_art_bulk_load.cpp index b6131c90..612a002a 100644 --- a/test/test_art_bulk_load.cpp +++ b/test/test_art_bulk_load.cpp @@ -38,13 +38,13 @@ constexpr auto val = value_view{val_bytes}; std::size_t n) { std::vector> kv; kv.reserve(n); - for (std::size_t i = 0; i < n && i < 256; ++i) { - const auto key = static_cast(i) << 56U; + for (std::uint64_t i = 0; i < n && i < 256; ++i) { + const auto key = i << 56U; kv.emplace_back(key, val); } // Overflow keys: share byte 0 == 0x00, differ at byte 1 - for (std::size_t i = 256; i < n; ++i) { - const auto key = static_cast(i - 255) << 48U; + for (std::uint64_t i = 256; i < n; ++i) { + const auto key = (i - 255) << 48U; kv.emplace_back(key, val); } std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp index 63af822a..0a66507d 100644 --- a/test/test_art_bulk_load_ops.cpp +++ b/test/test_art_bulk_load_ops.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -18,6 +17,7 @@ #include "db_test_utils.hpp" #include "gtest_utils.hpp" #include "node_type.hpp" +#include "portability_execution.hpp" #include "qsbr.hpp" #include "qsbr_test_utils.hpp" diff --git a/test/test_art_bulk_load_structural.cpp b/test/test_art_bulk_load_structural.cpp index 69b78197..9f326fd8 100644 --- a/test/test_art_bulk_load_structural.cpp +++ b/test/test_art_bulk_load_structural.cpp @@ -174,9 +174,11 @@ TEST(BulkLoadStructural, BulkLoadLarge) { std::mt19937 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) std::vector> kv; kv.reserve(100000); + UNODB_DETAIL_DISABLE_GCC_WARNING("-Wuseless-cast") for (int i = 0; i < 100000; ++i) kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) (static_cast(rng()) << 32U) | rng(), sval); + UNODB_DETAIL_RESTORE_GCC_WARNINGS() std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); kv.erase(std::unique( // NOLINT(modernize-use-ranges) kv.begin(), kv.end(), @@ -192,7 +194,9 @@ TEST(BulkLoadStructural, BulkLoadLarge) { unodb::key_decoder dec{visitor.get_key()}; std::uint64_t k{}; dec.decode(k); - if (!first) EXPECT_GE(k, prev); + if (!first) { + EXPECT_GE(k, prev); + } prev = k; first = false; return false; @@ -270,7 +274,9 @@ TEST(BulkLoadStructural, BulkLoadScanOrder) { unodb::key_decoder dec{visitor.get_key()}; std::uint64_t k{}; dec.decode(k); - if (count > 0) EXPECT_GT(k, prev); + if (count > 0) { + EXPECT_GT(k, prev); + } prev = k; ++count; return false; From da66cab608c9b305798dcf23effcfc7959450a4b Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Tue, 30 Jun 2026 01:02:46 +0000 Subject: [PATCH 04/23] fix: address CodeRabbit review feedback (#864) - Add UNODB_DETAIL_ASSERT bounds checks in partition_by_byte - Replace hardcoded 8 with (prefix_cap + 1) in chain_consumed - Extract make_bulk_inode shared lambda (eliminates duplication) - Add exception safety for parallel futures drain - Remove trailing blank lines in inode class closings - Add missing #include in example_bulk_load.cpp - Wire bulk-load tests into coverage and valgrind targets - Strengthen OOM test with memory_use == 0 assertion --- art_bulk_load_detail.hpp | 249 +++++++++++++++------------------ art_internal_impl.hpp | 4 - examples/example_bulk_load.cpp | 1 + test/CMakeLists.txt | 9 +- test/test_art_oom.cpp | 1 + 5 files changed, 117 insertions(+), 147 deletions(-) diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index d9bb788e..cc68bdcf 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -59,12 +59,14 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, while (cur != l) { const art_key_type ak{cur->first}; const auto kv = ak.get_key_view(); + UNODB_DETAIL_ASSERT(dd < kv.size()); const auto byte = kv[dd]; parts.push_back({cur, byte}); ++cur; while (cur != l) { const art_key_type ak2{cur->first}; const auto kv2 = ak2.get_key_view(); + UNODB_DETAIL_ASSERT(dd < kv2.size()); if (kv2[dd] != byte) break; ++cur; } @@ -130,6 +132,77 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, } }; + // ─── Shared inode factory ──────────────────────────────────────────── + + auto make_bulk_inode = + [&self](std::span cs, + const boost::container::small_vector& guards, + const boost::container::small_vector& children, + key_prefix_size inode_prefix_len, key_view prefix_kv, + tree_depth_type inode_depth) -> node_ptr_t { + const auto child_count = cs.size(); + if (child_count <= 4) { + std::uint8_t vmask = 0; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask |= static_cast(1U << i); + } + } + auto ptr = inode_4::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I4}; + } + if (child_count <= 16) { + std::uint16_t vmask = 0; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask |= static_cast(1U << i); + } + } + auto ptr = inode_16::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I16}; + } + if (child_count <= 48) { + std::array vmask{}; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask[i / 8] |= static_cast(1U << (i % 8)); + } + } + auto ptr = inode_48::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I48}; + } + std::array vmask{}; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) { + const auto kb = static_cast(children[i].key_byte); + vmask[kb / 8] |= static_cast(1U << (kb % 8)); + } + } + } + auto ptr = inode_256::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I256}; + }; + // ─── Recursive subtree builder ───────────────────────────────────────── struct subtree_builder { @@ -138,6 +211,7 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, decltype(partition_by_byte)& pbb; decltype(build_prefix_chain)& bpc; decltype(build_single_leaf)& bsl; + decltype(make_bulk_inode)& mbi; build_result_t operator()(RandomIt f, RandomIt l, tree_depth_type depth) const { @@ -170,7 +244,9 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, } const std::size_t chain_consumed = - (prefix_len > prefix_cap) ? (prefix_len / 8) * 8 : 0; + (prefix_len > prefix_cap) + ? (prefix_len / (prefix_cap + 1)) * (prefix_cap + 1) + : 0; const auto inode_prefix_len = static_cast(prefix_len - chain_consumed); const auto inode_depth = tree_depth_type{static_cast( @@ -179,72 +255,10 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, const art_key_type prefix_key{f->first}; const auto prefix_kv = prefix_key.get_key_view(); - auto make_inode = [&]() -> node_ptr_t { - const auto cs = - std::span{children.data(), children.size()}; - if (child_count <= 4) { - std::uint8_t vmask = 0; - if constexpr (art_policy::can_eliminate_leaf) { - for (std::size_t i = 0; i < child_count; ++i) { - if (guards[i].is_packed_value) - vmask |= static_cast(1U << i); - } - } - auto ptr = inode_4::create_bulk(self, inode_prefix_len, prefix_kv, - inode_depth, cs, vmask); -#ifdef UNODB_DETAIL_WITH_STATS - self.template account_growing_inode(); -#endif - return node_ptr_t{ptr.release(), node_type::I4}; - } - if (child_count <= 16) { - std::uint16_t vmask = 0; - if constexpr (art_policy::can_eliminate_leaf) { - for (std::size_t i = 0; i < child_count; ++i) { - if (guards[i].is_packed_value) - vmask |= static_cast(1U << i); - } - } - auto ptr = inode_16::create_bulk(self, inode_prefix_len, prefix_kv, - inode_depth, cs, vmask); -#ifdef UNODB_DETAIL_WITH_STATS - self.template account_growing_inode(); -#endif - return node_ptr_t{ptr.release(), node_type::I16}; - } - if (child_count <= 48) { - std::array vmask{}; - if constexpr (art_policy::can_eliminate_leaf) { - for (std::size_t i = 0; i < child_count; ++i) { - if (guards[i].is_packed_value) - vmask[i / 8] |= static_cast(1U << (i % 8)); - } - } - auto ptr = inode_48::create_bulk(self, inode_prefix_len, prefix_kv, - inode_depth, cs, vmask); -#ifdef UNODB_DETAIL_WITH_STATS - self.template account_growing_inode(); -#endif - return node_ptr_t{ptr.release(), node_type::I48}; - } - std::array vmask{}; - if constexpr (art_policy::can_eliminate_leaf) { - for (std::size_t i = 0; i < child_count; ++i) { - if (guards[i].is_packed_value) { - const auto kb = static_cast(children[i].key_byte); - vmask[kb / 8] |= static_cast(1U << (kb % 8)); - } - } - } - auto ptr = inode_256::create_bulk(self, inode_prefix_len, prefix_kv, - inode_depth, cs, vmask); -#ifdef UNODB_DETAIL_WITH_STATS - self.template account_growing_inode(); -#endif - return node_ptr_t{ptr.release(), node_type::I256}; - }; - - auto inode_ptr = make_inode(); + const auto cs = + std::span{children.data(), children.size()}; + auto inode_ptr = mbi(cs, guards, children, inode_prefix_len, prefix_kv, + inode_depth); for (auto& g : guards) g.release(); if (chain_consumed > 0) { @@ -257,7 +271,8 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, }; subtree_builder builder{self, common_prefix_length, partition_by_byte, - build_prefix_chain, build_single_leaf}; + build_prefix_chain, build_single_leaf, + make_bulk_inode}; using policy_t = std::remove_cvref_t; constexpr bool is_parallel = @@ -302,15 +317,31 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, children.reserve(child_count); for (std::size_t i = 0; i < child_count; ++i) { - auto result = futures[i].get(); - guards.emplace_back(self); - guards.back().ptr = result.ptr; - guards.back().is_packed_value = result.is_packed_value; - children.push_back({parts[i].key_byte, result.ptr}); + try { + auto result = futures[i].get(); + guards.emplace_back(self); + guards.back().ptr = result.ptr; + guards.back().is_packed_value = result.is_packed_value; + children.push_back({parts[i].key_byte, result.ptr}); + } catch (...) { + // Drain remaining futures into guards to prevent leaks. + for (std::size_t j = i + 1; j < child_count; ++j) { + try { + auto r = futures[j].get(); + guards.emplace_back(self); + guards.back().ptr = r.ptr; + guards.back().is_packed_value = r.is_packed_value; + } catch (...) { + } // LCOV_EXCL_LINE + } + throw; + } } const std::size_t chain_consumed = - (prefix_len > prefix_cap) ? (prefix_len / 8) * 8 : 0; + (prefix_len > prefix_cap) + ? (prefix_len / (prefix_cap + 1)) * (prefix_cap + 1) + : 0; const auto inode_prefix_len = static_cast(prefix_len - chain_consumed); const auto inode_depth = @@ -319,72 +350,10 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, const art_key_type prefix_key{first->first}; const auto prefix_kv = prefix_key.get_key_view(); - auto make_root_inode = [&]() -> node_ptr_t { - const auto cs = - std::span{children.data(), children.size()}; - if (child_count <= 4) { - std::uint8_t vmask = 0; - if constexpr (art_policy::can_eliminate_leaf) { - for (std::size_t i = 0; i < child_count; ++i) { - if (guards[i].is_packed_value) - vmask |= static_cast(1U << i); - } - } - auto ptr = inode_4::create_bulk(self, inode_prefix_len, prefix_kv, - inode_depth, cs, vmask); -#ifdef UNODB_DETAIL_WITH_STATS - self.template account_growing_inode(); -#endif - return node_ptr_t{ptr.release(), node_type::I4}; - } - if (child_count <= 16) { - std::uint16_t vmask = 0; - if constexpr (art_policy::can_eliminate_leaf) { - for (std::size_t i = 0; i < child_count; ++i) { - if (guards[i].is_packed_value) - vmask |= static_cast(1U << i); - } - } - auto ptr = inode_16::create_bulk(self, inode_prefix_len, prefix_kv, - inode_depth, cs, vmask); -#ifdef UNODB_DETAIL_WITH_STATS - self.template account_growing_inode(); -#endif - return node_ptr_t{ptr.release(), node_type::I16}; - } - if (child_count <= 48) { - std::array vmask{}; - if constexpr (art_policy::can_eliminate_leaf) { - for (std::size_t i = 0; i < child_count; ++i) { - if (guards[i].is_packed_value) - vmask[i / 8] |= static_cast(1U << (i % 8)); - } - } - auto ptr = inode_48::create_bulk(self, inode_prefix_len, prefix_kv, - inode_depth, cs, vmask); -#ifdef UNODB_DETAIL_WITH_STATS - self.template account_growing_inode(); -#endif - return node_ptr_t{ptr.release(), node_type::I48}; - } - std::array vmask{}; - if constexpr (art_policy::can_eliminate_leaf) { - for (std::size_t i = 0; i < child_count; ++i) { - if (guards[i].is_packed_value) { - const auto kb = static_cast(children[i].key_byte); - vmask[kb / 8] |= static_cast(1U << (kb % 8)); - } - } - } - auto ptr = inode_256::create_bulk(self, inode_prefix_len, prefix_kv, - inode_depth, cs, vmask); -#ifdef UNODB_DETAIL_WITH_STATS - self.template account_growing_inode(); -#endif - return node_ptr_t{ptr.release(), node_type::I256}; - }; - - auto inode_ptr = make_root_inode(); + const auto cs = + std::span{children.data(), children.size()}; + auto inode_ptr = make_bulk_inode(cs, guards, children, inode_prefix_len, + prefix_kv, inode_depth); for (auto& g : guards) g.release(); if (chain_consumed > 0) { diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index f366c15f..d5ca53bf 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -3101,7 +3101,6 @@ class basic_inode_4 return result; } UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() - }; // class basic_inode_4 /// Type alias for basic_inode_16 parent class. @@ -3760,7 +3759,6 @@ class basic_inode_16 return result; } UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() - }; // class basic_inode_16 /// Type alias for basic_inode_48 parent class. @@ -4578,7 +4576,6 @@ class basic_inode_48 return result; } UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() - }; // class basic_inode_48 /// Type alias for basic_inode_256 parent class. @@ -5063,7 +5060,6 @@ class basic_inode_256 return result; } UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() - }; // class basic_inode_256 } // namespace unodb::detail diff --git a/examples/example_bulk_load.cpp b/examples/example_bulk_load.cpp index 62f6c49c..973996eb 100644 --- a/examples/example_bulk_load.cpp +++ b/examples/example_bulk_load.cpp @@ -5,6 +5,7 @@ #include "global.hpp" #include +#include #include #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 49abe271..d8118d28 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -104,7 +104,7 @@ add_test(NAME test_olc_no_qsbr COMMAND test_olc_no_qsbr) if(COVERAGE) add_custom_target(tests_for_coverage ctest DEPENDS test_key_encode_decode test_art test_art_key_view test_art_key_view_full_chain test_art_iter test_art_scan test_art_concurrency test_art_allocator test_art_upsert test_olc_no_qsbr test_qsbr_ptr test_qsbr test_art_oom - test_qsbr_oom) + test_qsbr_oom test_art_bulk_load test_art_bulk_load_structural test_art_bulk_load_ops) add_coverage_target(TARGET coverage DEPENDENCY tests_for_coverage) endif() @@ -120,5 +120,8 @@ add_custom_target(valgrind_tests COMMAND ${VALGRIND_COMMAND} ./test_art_iter; COMMAND ${VALGRIND_COMMAND} ./test_art_scan; COMMAND ${VALGRIND_COMMAND} ./test_art_concurrency; - COMMAND ${VALGRIND_COMMAND} ./test_art_upsert - DEPENDS test_qsbr_ptr test_qsbr test_key_encode_decode test_art test_art_key_view test_art_key_view_full_chain test_art_iter test_art_scan test_art_concurrency test_art_upsert) + COMMAND ${VALGRIND_COMMAND} ./test_art_upsert; + COMMAND ${VALGRIND_COMMAND} ./test_art_bulk_load; + COMMAND ${VALGRIND_COMMAND} ./test_art_bulk_load_structural; + COMMAND ${VALGRIND_COMMAND} ./test_art_bulk_load_ops + DEPENDS test_qsbr_ptr test_qsbr test_key_encode_decode test_art test_art_key_view test_art_key_view_full_chain test_art_iter test_art_scan test_art_concurrency test_art_upsert test_art_bulk_load test_art_bulk_load_structural test_art_bulk_load_ops) diff --git a/test/test_art_oom.cpp b/test/test_art_oom.cpp index 5637f8a1..7a39c67a 100644 --- a/test/test_art_oom.cpp +++ b/test/test_art_oom.cpp @@ -673,6 +673,7 @@ void bulk_load_oom_test( unodb::test::allocation_failure_injector::reset(); // Strong guarantee: tree must be empty after failed bulk_load UNODB_ASSERT_TRUE(test_db.empty()); + UNODB_ASSERT_EQ(test_db.get_current_memory_use(), 0); } } FAIL() << "bulk_load did not succeed within " << fail_limit << " allocations"; From f801b12310bc7b42000cd2bd4b3fa705def4ddde Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Tue, 30 Jun 2026 02:04:55 +0000 Subject: [PATCH 05/23] fix: guard memory_use check with STATS, format --- art_bulk_load_detail.hpp | 11 +++++++---- test/test_art_oom.cpp | 2 ++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index cc68bdcf..1025d216 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -257,8 +257,8 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, const auto cs = std::span{children.data(), children.size()}; - auto inode_ptr = mbi(cs, guards, children, inode_prefix_len, prefix_kv, - inode_depth); + auto inode_ptr = + mbi(cs, guards, children, inode_prefix_len, prefix_kv, inode_depth); for (auto& g : guards) g.release(); if (chain_consumed > 0) { @@ -270,8 +270,11 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, } }; - subtree_builder builder{self, common_prefix_length, partition_by_byte, - build_prefix_chain, build_single_leaf, + subtree_builder builder{self, + common_prefix_length, + partition_by_byte, + build_prefix_chain, + build_single_leaf, make_bulk_inode}; using policy_t = std::remove_cvref_t; diff --git a/test/test_art_oom.cpp b/test/test_art_oom.cpp index 7a39c67a..efb75cdb 100644 --- a/test/test_art_oom.cpp +++ b/test/test_art_oom.cpp @@ -673,7 +673,9 @@ void bulk_load_oom_test( unodb::test::allocation_failure_injector::reset(); // Strong guarantee: tree must be empty after failed bulk_load UNODB_ASSERT_TRUE(test_db.empty()); +#ifdef UNODB_DETAIL_WITH_STATS UNODB_ASSERT_EQ(test_db.get_current_memory_use(), 0); +#endif } } FAIL() << "bulk_load did not succeed within " << fail_limit << " allocations"; From 6a18e574a079c9d3ff3dd69198368493e954736e Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Tue, 30 Jun 2026 19:52:58 +0000 Subject: [PATCH 06/23] fix: element-wise bitmask assignment for TSan/OLC builds The array-based value_bitmask_field wraps elements in in_critical_section, so direct array assignment fails. Use element-by-element assignment for inode_48 and inode_256 bulk_load_from_children. --- art_internal_impl.hpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index d5ca53bf..c72bb5e7 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -4571,7 +4571,9 @@ class basic_inode_48 node->children.pointer_array[i] = node_ptr{nullptr}; } if constexpr (ArtPolicy::can_eliminate_leaf) { - node->bitmask_base::bits = value_mask; + for (std::size_t i = 0; i < value_mask.size(); ++i) { + node->bitmask_base::bits[i] = value_mask[i]; + } } return result; } @@ -5055,7 +5057,9 @@ class basic_inode_256 node->children[kb] = children_span[i].child; } if constexpr (ArtPolicy::can_eliminate_leaf) { - node->bitmask_base::bits = value_mask; + for (std::size_t i = 0; i < value_mask.size(); ++i) { + node->bitmask_base::bits[i] = value_mask[i]; + } } return result; } From 394f1e358b057c4edb424dcc9da6f4cc8475e1ee Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Tue, 30 Jun 2026 21:55:24 +0000 Subject: [PATCH 07/23] fix: suppress MSVC C4100 unreferenced parameter 'depth' In the build_single_leaf lambda, the else branch (classic leaf without key/leaf elimination) does not use depth. Cast to void to silence MSVC warning-as-error. --- art_bulk_load_detail.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index 1025d216..1f9c844b 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -126,6 +126,7 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, } return {self.build_chain(ak, leaf_ptr, depth), false}; } else { + static_cast(depth); auto leaf = art_policy::make_db_leaf_ptr(ak, it->second, self); auto leaf_ptr = node_ptr_t{leaf.release(), node_type::LEAF}; return {leaf_ptr, false}; From ad88317ac78c1354537cc4c6d64e65e1ae04505d Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Tue, 30 Jun 2026 23:01:05 +0000 Subject: [PATCH 08/23] fix: suppress MSVC C4100/C4365 warnings in bulk_load - C4100: guards/children unused when can_eliminate_leaf is false - C4365: signed/unsigned mismatch in vmask index for inode_256 --- art_bulk_load_detail.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index 1f9c844b..2c33ca36 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -141,6 +141,10 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, const boost::container::small_vector& children, key_prefix_size inode_prefix_len, key_view prefix_kv, tree_depth_type inode_depth) -> node_ptr_t { + if constexpr (!art_policy::can_eliminate_leaf) { + static_cast(guards); + static_cast(children); + } const auto child_count = cs.size(); if (child_count <= 4) { std::uint8_t vmask = 0; @@ -192,7 +196,8 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, for (std::size_t i = 0; i < child_count; ++i) { if (guards[i].is_packed_value) { const auto kb = static_cast(children[i].key_byte); - vmask[kb / 8] |= static_cast(1U << (kb % 8)); + vmask[static_cast(kb / 8)] |= + static_cast(1U << (kb % 8)); } } } From d716a6f3229015135922b7b0918b1a13a3ee6f3e Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Wed, 1 Jul 2026 00:04:30 +0000 Subject: [PATCH 09/23] fix: MSVC static analysis and clang-cl documentation warnings - Use UNODB_TEST() macro wrapper instead of raw TEST() to suppress C26426 (non-constexpr global initializer from Google Test). - Fix Doxygen \param names to match function signatures (policy not parallelism, separate first/last params). - Fix C4365 signed/unsigned mismatch in vmask index. - Suppress C4100 for guards/children when can_eliminate_leaf is false. --- art.hpp | 6 ++++-- olc_art.hpp | 2 +- test/test_art_bulk_load.cpp | 20 ++++++++--------- test/test_art_bulk_load_ops.cpp | 18 ++++++++-------- test/test_art_bulk_load_structural.cpp | 30 +++++++++++++------------- 5 files changed, 39 insertions(+), 37 deletions(-) diff --git a/art.hpp b/art.hpp index 1e131123..6e3d7210 100644 --- a/art.hpp +++ b/art.hpp @@ -364,8 +364,10 @@ class db final { /// /// \tparam RandomIt Random-access iterator to std::pair - /// \param first, last Sorted range (ART byte order). No duplicates. - /// \param parallelism Ignored for db (allocator not thread-safe). + /// \param policy Execution policy (ignored for db — allocator not + /// thread-safe). + /// \param first Start of sorted range (ART byte order, no duplicates). + /// \param last End of sorted range. /// \pre empty() == true /// \throws std::invalid_argument if tree is non-empty. /// \throws std::bad_alloc (strong guarantee — tree left empty). diff --git a/olc_art.hpp b/olc_art.hpp index a7ff2f62..a734291f 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -307,9 +307,9 @@ class olc_db final { /// /// \note Only legal in single-threaded context (same as clear). /// \tparam RandomIt Random access iterator over pairs of (key, value) + /// \param policy Execution policy (parallelism hint; 0=auto, 1=sequential) /// \param first Start of sorted range /// \param last End of sorted range - /// \param parallelism Parallelism hint (0=auto, 1=sequential) template void bulk_load(ExecutionPolicy&& policy, RandomIt first, RandomIt last); diff --git a/test/test_art_bulk_load.cpp b/test/test_art_bulk_load.cpp index 612a002a..46d42dae 100644 --- a/test/test_art_bulk_load.cpp +++ b/test/test_art_bulk_load.cpp @@ -52,7 +52,7 @@ constexpr auto val = value_view{val_bytes}; } // T01 -TEST(BulkLoad, Empty) { +UNODB_TEST(BulkLoad, Empty) { u64_db db; std::vector> kv; db.bulk_load(kv.begin(), kv.end()); @@ -64,7 +64,7 @@ TEST(BulkLoad, Empty) { } // T02 -TEST(BulkLoad, Single) { +UNODB_TEST(BulkLoad, Single) { u64_db db; const std::uint64_t key = 1; std::vector> kv{{key, val}}; @@ -79,7 +79,7 @@ TEST(BulkLoad, Single) { } // T03 -TEST(BulkLoad, Small4Keys) { +UNODB_TEST(BulkLoad, Small4Keys) { u64_db db; auto kv = make_keys(4); db.bulk_load(kv.begin(), kv.end()); @@ -92,7 +92,7 @@ TEST(BulkLoad, Small4Keys) { } // T04 -TEST(BulkLoad, Boundary4) { +UNODB_TEST(BulkLoad, Boundary4) { u64_db db; auto kv = make_keys(4); db.bulk_load(kv.begin(), kv.end()); @@ -104,7 +104,7 @@ TEST(BulkLoad, Boundary4) { } // T05 -TEST(BulkLoad, Boundary5) { +UNODB_TEST(BulkLoad, Boundary5) { u64_db db; auto kv = make_keys(5); db.bulk_load(kv.begin(), kv.end()); @@ -114,7 +114,7 @@ TEST(BulkLoad, Boundary5) { } // T06 -TEST(BulkLoad, Boundary16) { +UNODB_TEST(BulkLoad, Boundary16) { u64_db db; auto kv = make_keys(16); db.bulk_load(kv.begin(), kv.end()); @@ -124,7 +124,7 @@ TEST(BulkLoad, Boundary16) { } // T07 -TEST(BulkLoad, Boundary17) { +UNODB_TEST(BulkLoad, Boundary17) { u64_db db; auto kv = make_keys(17); db.bulk_load(kv.begin(), kv.end()); @@ -134,7 +134,7 @@ TEST(BulkLoad, Boundary17) { } // T08 -TEST(BulkLoad, Boundary48) { +UNODB_TEST(BulkLoad, Boundary48) { u64_db db; auto kv = make_keys(48); db.bulk_load(kv.begin(), kv.end()); @@ -144,7 +144,7 @@ TEST(BulkLoad, Boundary48) { } // T09 -TEST(BulkLoad, Boundary49) { +UNODB_TEST(BulkLoad, Boundary49) { u64_db db; auto kv = make_keys(49); db.bulk_load(kv.begin(), kv.end()); @@ -154,7 +154,7 @@ TEST(BulkLoad, Boundary49) { } // T10 -TEST(BulkLoad, Growth260) { +UNODB_TEST(BulkLoad, Growth260) { u64_db db; auto kv = make_keys(260); db.bulk_load(kv.begin(), kv.end()); diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp index 0a66507d..3e481074 100644 --- a/test/test_art_bulk_load_ops.cpp +++ b/test/test_art_bulk_load_ops.cpp @@ -40,7 +40,7 @@ constexpr auto val = value_view{val_bytes}; // ─── Error Tests ───────────────────────────────────────────────────────────── // T26: bulk_load on non-empty tree throws -TEST(BulkLoadError, NonEmpty) { +UNODB_TEST(BulkLoadError, NonEmpty) { u64_db db; const std::uint64_t key = 42; ASSERT_TRUE(db.insert(key, val)); @@ -52,7 +52,7 @@ TEST(BulkLoadError, NonEmpty) { } // T36: db ignores parallelism parameter -TEST(BulkLoadError, DbIgnoresParallelism) { +UNODB_TEST(BulkLoadError, DbIgnoresParallelism) { u64_db db; std::vector> kv; kv.reserve(100); @@ -72,7 +72,7 @@ TEST(BulkLoadError, DbIgnoresParallelism) { // ─── Operational Tests ─────────────────────────────────────────────────────── // T38: Operations work correctly after bulk_load -TEST(BulkLoadOps, ThenOperations) { +UNODB_TEST(BulkLoadOps, ThenOperations) { u64_db db; std::vector> kv; kv.reserve(10); @@ -110,7 +110,7 @@ TEST(BulkLoadOps, ThenOperations) { // T39: Stats are correct after bulk_load #ifdef UNODB_DETAIL_WITH_STATS -TEST(BulkLoadOps, Stats) { +UNODB_TEST(BulkLoadOps, Stats) { u64_db db; // 17 keys differing at byte 0 → should produce 1 inode48 std::vector> kv; @@ -131,7 +131,7 @@ TEST(BulkLoadOps, Stats) { #endif // UNODB_DETAIL_WITH_STATS // T40: No growth events during bulk_load (all right-sized at allocation) -TEST(BulkLoadOps, NoGrowthEvents) { +UNODB_TEST(BulkLoadOps, NoGrowthEvents) { u64_db db; std::vector> kv; kv.reserve(100); @@ -151,7 +151,7 @@ TEST(BulkLoadOps, NoGrowthEvents) { } // T42: mutex_db bulk_load works -TEST(BulkLoadOps, MutexDb) { +UNODB_TEST(BulkLoadOps, MutexDb) { u64_mutex_db db; std::vector> kv; kv.reserve(100); @@ -168,7 +168,7 @@ TEST(BulkLoadOps, MutexDb) { } // T38 extended: clear after bulk_load then re-bulk_load -TEST(BulkLoadOps, ClearAndReload) { +UNODB_TEST(BulkLoadOps, ClearAndReload) { u64_db db; std::vector> kv; kv.reserve(10); @@ -193,7 +193,7 @@ TEST(BulkLoadOps, ClearAndReload) { } // T43: olc_db parallel bulk_load -TEST(BulkLoadOps, OlcDbParallel) { +UNODB_TEST(BulkLoadOps, OlcDbParallel) { u64_olc_db db; std::vector> kv; kv.reserve(1000); @@ -211,7 +211,7 @@ TEST(BulkLoadOps, OlcDbParallel) { // T44: olc_db concurrent readers after parallel bulk_load // Verifies that optimistic locks are correctly initialized after bulk_load so // concurrent readers can traverse the tree without deadlock or data corruption. -TEST(BulkLoadOps, OlcDbConcurrentReaders) { +UNODB_TEST(BulkLoadOps, OlcDbConcurrentReaders) { u64_olc_db db; constexpr std::size_t n_keys = 10000; std::vector> kv; diff --git a/test/test_art_bulk_load_structural.cpp b/test/test_art_bulk_load_structural.cpp index 9f326fd8..82688f69 100644 --- a/test/test_art_bulk_load_structural.cpp +++ b/test/test_art_bulk_load_structural.cpp @@ -37,7 +37,7 @@ constexpr auto sval_bytes = constexpr auto sval = value_view{sval_bytes}; // T11 -TEST(BulkLoadStructural, BulkLoadPrefix7) { +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix7) { u64_db db; std::vector> kv{ {0x0102030405060700ULL, sval}, {0x0102030405060701ULL, sval}}; @@ -47,7 +47,7 @@ TEST(BulkLoadStructural, BulkLoadPrefix7) { UNODB_ASSERT_EQ(c[as_i], 2U); } // T12 -TEST(BulkLoadStructural, BulkLoadPrefix8) { +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix8) { std::vector> s{ {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}, std::byte{5}, std::byte{6}, std::byte{7}, std::byte{8}, std::byte{0}}, @@ -64,7 +64,7 @@ TEST(BulkLoadStructural, BulkLoadPrefix8) { UNODB_ASSERT_EQ(c[as_i], 2U); } // T13 -TEST(BulkLoadStructural, BulkLoadPrefix15) { +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix15) { std::vector> s( 2, std::vector(16, std::byte{0xAA})); s[0][15] = std::byte{0}; @@ -80,7 +80,7 @@ TEST(BulkLoadStructural, BulkLoadPrefix15) { UNODB_ASSERT_EQ(c[as_i], 2U); } // T14 -TEST(BulkLoadStructural, BulkLoadPrefix16) { +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix16) { std::vector> s( 2, std::vector(17, std::byte{0xBB})); s[0][16] = std::byte{0}; @@ -96,7 +96,7 @@ TEST(BulkLoadStructural, BulkLoadPrefix16) { UNODB_ASSERT_EQ(c[as_i], 2U); } // T15 -TEST(BulkLoadStructural, BulkLoadVIS) { +UNODB_TEST(BulkLoadStructural, BulkLoadVIS) { std::vector> s{{std::byte{1}}, {std::byte{2}}}; std::vector> kv; for (auto& v : s) @@ -109,7 +109,7 @@ TEST(BulkLoadStructural, BulkLoadVIS) { UNODB_ASSERT_EQ(c[as_i], 0U); } // T16 -TEST(BulkLoadStructural, BulkLoadVISWithChain) { +UNODB_TEST(BulkLoadStructural, BulkLoadVISWithChain) { std::vector> s{{std::byte{1}, std::byte{0xAA}}, {std::byte{1}, std::byte{0xBB}}}; std::vector> kv; @@ -123,7 +123,7 @@ TEST(BulkLoadStructural, BulkLoadVISWithChain) { UNODB_ASSERT_EQ(c[as_i], 0U); } // T17 -TEST(BulkLoadStructural, BulkLoadVISLongPrefix) { +UNODB_TEST(BulkLoadStructural, BulkLoadVISLongPrefix) { std::vector> s( 2, std::vector(9, std::byte{0xCC})); s[0][8] = std::byte{0}; @@ -139,7 +139,7 @@ TEST(BulkLoadStructural, BulkLoadVISLongPrefix) { UNODB_ASSERT_EQ(c[as_i], 0U); } // T18 -TEST(BulkLoadStructural, BulkLoadKeylessLeaf) { +UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeaf) { std::vector> s{ {std::byte{1}, std::byte{2}, std::byte{3}}, {std::byte{4}, std::byte{5}, std::byte{6}}}; @@ -155,7 +155,7 @@ TEST(BulkLoadStructural, BulkLoadKeylessLeaf) { UNODB_ASSERT_EQ(c[as_i], 2U); } // T19 -TEST(BulkLoadStructural, BulkLoadKeylessLeafWithChain) { +UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeafWithChain) { std::vector> s{ {std::byte{1}, std::byte{0xAA}, std::byte{0xFF}}, {std::byte{1}, std::byte{0xBB}, std::byte{0xFF}}}; @@ -170,7 +170,7 @@ TEST(BulkLoadStructural, BulkLoadKeylessLeafWithChain) { UNODB_ASSERT_EQ(c[as_i], 2U); } // T20 -TEST(BulkLoadStructural, BulkLoadLarge) { +UNODB_TEST(BulkLoadStructural, BulkLoadLarge) { std::mt19937 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) std::vector> kv; kv.reserve(100000); @@ -204,7 +204,7 @@ TEST(BulkLoadStructural, BulkLoadLarge) { UNODB_DETAIL_RESTORE_MSVC_WARNINGS() } // T21 — key_view bulk_load with random fixed-length keys (prefix-free). -TEST(BulkLoadStructural, BulkLoadKeyView) { +UNODB_TEST(BulkLoadStructural, BulkLoadKeyView) { constexpr std::size_t key_len = 10; // Fixed length → no key is prefix of another std::mt19937 rng(123); // NOLINT(cert-msc32-c,cert-msc51-cpp) @@ -227,7 +227,7 @@ TEST(BulkLoadStructural, BulkLoadKeyView) { for (const auto& [k, v] : kv) ASSERT_TRUE(db.get(k).has_value()); } // T22 -TEST(BulkLoadStructural, BulkLoadVISRootSingle) { +UNODB_TEST(BulkLoadStructural, BulkLoadVISRootSingle) { std::vector> s{{std::byte{1}, std::byte{2}}}; std::vector> kv; kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) @@ -240,7 +240,7 @@ TEST(BulkLoadStructural, BulkLoadVISRootSingle) { ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); } // T23 -TEST(BulkLoadStructural, BulkLoadKeylessLeafRootSingle) { +UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeafRootSingle) { std::vector> s{ {std::byte{1}, std::byte{2}, std::byte{3}}}; std::vector> kv; @@ -251,7 +251,7 @@ TEST(BulkLoadStructural, BulkLoadKeylessLeafRootSingle) { ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); } // T24 -TEST(BulkLoadStructural, BulkLoadFullLeafRootSingle) { +UNODB_TEST(BulkLoadStructural, BulkLoadFullLeafRootSingle) { u64_db db; std::vector> kv{{0xDEADBEEFULL, sval}}; db.bulk_load(kv.begin(), kv.end()); @@ -260,7 +260,7 @@ TEST(BulkLoadStructural, BulkLoadFullLeafRootSingle) { UNODB_ASSERT_EQ(c[as_i], 0U); } // T25 -TEST(BulkLoadStructural, BulkLoadScanOrder) { +UNODB_TEST(BulkLoadStructural, BulkLoadScanOrder) { std::vector> kv; for (std::uint64_t i = 0; i < 256; ++i) kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) From 886c7cdfa4f04f3ff39ed37c69c810f4604b834c Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Wed, 1 Jul 2026 01:23:17 +0000 Subject: [PATCH 10/23] fix: MSVC static analysis warnings in bulk_load tests - Use UNODB_TEST/UNODB_ASSERT_* wrappers for C26426/C6326/C26818 - File-level C6326/C26818 suppression in ops test (streaming << syntax) - File-level C26445 suppression in structural test (span structured bind) - constexpr for compile-time constant (C26814) - Fix Doxygen param names (policy not parallelism, separate first/last) --- test/test_art_bulk_load.cpp | 8 +++---- test/test_art_bulk_load_ops.cpp | 30 +++++++++++++++++--------- test/test_art_bulk_load_structural.cpp | 18 +++++++++++----- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/test/test_art_bulk_load.cpp b/test/test_art_bulk_load.cpp index 46d42dae..6c427b98 100644 --- a/test/test_art_bulk_load.cpp +++ b/test/test_art_bulk_load.cpp @@ -66,11 +66,11 @@ UNODB_TEST(BulkLoad, Empty) { // T02 UNODB_TEST(BulkLoad, Single) { u64_db db; - const std::uint64_t key = 1; + constexpr std::uint64_t key = 1; std::vector> kv{{key, val}}; db.bulk_load(kv.begin(), kv.end()); const auto result = db.get(key); - ASSERT_TRUE(result.has_value()); + UNODB_ASSERT_TRUE(result.has_value()); UNODB_ASSERT_EQ( // NOLINT(bugprone-unchecked-optional-access) result.value().size(), val.size()); const auto counts = db.get_node_counts(); @@ -84,7 +84,7 @@ UNODB_TEST(BulkLoad, Small4Keys) { auto kv = make_keys(4); db.bulk_load(kv.begin(), kv.end()); for (const auto& [k, v] : kv) { - ASSERT_TRUE(db.get(k).has_value()); + UNODB_ASSERT_TRUE(db.get(k).has_value()); } const auto counts = db.get_node_counts(); UNODB_ASSERT_EQ(counts[as_i], 4U); @@ -159,7 +159,7 @@ UNODB_TEST(BulkLoad, Growth260) { auto kv = make_keys(260); db.bulk_load(kv.begin(), kv.end()); for (const auto& [k, v] : kv) { - ASSERT_TRUE(db.get(k).has_value()); + UNODB_ASSERT_TRUE(db.get(k).has_value()); } const auto counts = db.get_node_counts(); UNODB_ASSERT_EQ(counts[as_i], 1U); diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp index 3e481074..c764e601 100644 --- a/test/test_art_bulk_load_ops.cpp +++ b/test/test_art_bulk_load_ops.cpp @@ -21,6 +21,9 @@ #include "qsbr.hpp" #include "qsbr_test_utils.hpp" +UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) +UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) + namespace { #ifdef UNODB_DETAIL_WITH_STATS @@ -43,12 +46,16 @@ constexpr auto val = value_view{val_bytes}; UNODB_TEST(BulkLoadError, NonEmpty) { u64_db db; const std::uint64_t key = 42; - ASSERT_TRUE(db.insert(key, val)); + UNODB_ASSERT_TRUE(db.insert(key, val)); std::vector> kv{{100, val}}; + UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) + UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) EXPECT_THROW(db.bulk_load(kv.begin(), kv.end()), std::invalid_argument); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() // Original key still present const auto result = db.get(key); - ASSERT_TRUE(result.has_value()); + UNODB_ASSERT_TRUE(result.has_value()); } // T36: db ignores parallelism parameter @@ -84,20 +91,20 @@ UNODB_TEST(BulkLoadOps, ThenOperations) { // get works for all keys for (const auto& [k, v] : kv) { - ASSERT_TRUE(db.get(k).has_value()); + UNODB_ASSERT_TRUE(db.get(k).has_value()); } // insert new key works const std::uint64_t new_key = 0xFFULL << 56U; - ASSERT_TRUE(db.insert(new_key, val)); - ASSERT_TRUE(db.get(new_key).has_value()); + UNODB_ASSERT_TRUE(db.insert(new_key, val)); + UNODB_ASSERT_TRUE(db.get(new_key).has_value()); // insert duplicate fails - ASSERT_FALSE(db.insert(kv[0].first, val)); + UNODB_ASSERT_FALSE(db.insert(kv[0].first, val)); // remove works - ASSERT_TRUE(db.remove(kv[0].first)); - ASSERT_FALSE(db.get(kv[0].first).has_value()); + UNODB_ASSERT_TRUE(db.remove(kv[0].first)); + UNODB_ASSERT_FALSE(db.get(kv[0].first).has_value()); // scan works std::size_t count = 0; @@ -178,7 +185,7 @@ UNODB_TEST(BulkLoadOps, ClearAndReload) { std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); db.bulk_load(kv.begin(), kv.end()); db.clear(); - ASSERT_TRUE(db.empty()); + UNODB_ASSERT_TRUE(db.empty()); // Re-load different data std::vector> kv2; kv2.reserve(10); @@ -188,7 +195,7 @@ UNODB_TEST(BulkLoadOps, ClearAndReload) { std::ranges::sort(kv2, {}, &decltype(kv2)::value_type::first); db.bulk_load(kv2.begin(), kv2.end()); for (const auto& [k, v] : kv2) { - ASSERT_TRUE(db.get(k).has_value()); + UNODB_ASSERT_TRUE(db.get(k).has_value()); } } @@ -245,3 +252,6 @@ UNODB_TEST(BulkLoadOps, OlcDbConcurrentReaders) { } } // namespace + +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() diff --git a/test/test_art_bulk_load_structural.cpp b/test/test_art_bulk_load_structural.cpp index 82688f69..b12856cd 100644 --- a/test/test_art_bulk_load_structural.cpp +++ b/test/test_art_bulk_load_structural.cpp @@ -19,6 +19,8 @@ #ifdef UNODB_DETAIL_WITH_STATS +UNODB_DETAIL_DISABLE_MSVC_WARNING(26445) + namespace { using unodb::as_i; @@ -186,7 +188,7 @@ UNODB_TEST(BulkLoadStructural, BulkLoadLarge) { kv.end()); u64_db db; db.bulk_load(kv.begin(), kv.end()); - for (const auto& [k, v] : kv) ASSERT_TRUE(db.get(k).has_value()); + for (const auto& [k, v] : kv) UNODB_ASSERT_TRUE(db.get(k).has_value()); std::uint64_t prev = 0; bool first = true; UNODB_DETAIL_DISABLE_MSVC_WARNING(26440) @@ -195,7 +197,11 @@ UNODB_TEST(BulkLoadStructural, BulkLoadLarge) { std::uint64_t k{}; dec.decode(k); if (!first) { + UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) + UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) EXPECT_GE(k, prev); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() } prev = k; first = false; @@ -224,7 +230,7 @@ UNODB_TEST(BulkLoadStructural, BulkLoadKeyView) { key_view{v}, large_val); key_view_db db; db.bulk_load(kv.begin(), kv.end()); - for (const auto& [k, v] : kv) ASSERT_TRUE(db.get(k).has_value()); + for (const auto& [k, v] : kv) UNODB_ASSERT_TRUE(db.get(k).has_value()); } // T22 UNODB_TEST(BulkLoadStructural, BulkLoadVISRootSingle) { @@ -237,7 +243,7 @@ UNODB_TEST(BulkLoadStructural, BulkLoadVISRootSingle) { const auto c = db.get_node_counts(); UNODB_ASSERT_EQ(c[as_i], 1U); UNODB_ASSERT_EQ(c[as_i], 0U); - ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); + UNODB_ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); } // T23 UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeafRootSingle) { @@ -248,7 +254,7 @@ UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeafRootSingle) { key_view{s[0]}, large_val); key_view_db db; db.bulk_load(kv.begin(), kv.end()); - ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); + UNODB_ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); } // T24 UNODB_TEST(BulkLoadStructural, BulkLoadFullLeafRootSingle) { @@ -275,7 +281,7 @@ UNODB_TEST(BulkLoadStructural, BulkLoadScanOrder) { std::uint64_t k{}; dec.decode(k); if (count > 0) { - EXPECT_GT(k, prev); + UNODB_EXPECT_GT(k, prev); } prev = k; ++count; @@ -287,4 +293,6 @@ UNODB_TEST(BulkLoadStructural, BulkLoadScanOrder) { } // namespace +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + #endif // UNODB_DETAIL_WITH_STATS From a3ddafd4472c080a9866f653c530b4a79140e49a Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Wed, 1 Jul 2026 02:04:46 +0000 Subject: [PATCH 11/23] fix: remaining MSVC static analysis warnings - C26479: remove unnecessary std::move on local return (NRVO applies) - C26814: constexpr for compile-time key constant - C26445: suppress for structured binding of span in range-for - C26496: const for rng in benchmark --- art_bulk_load_detail.hpp | 2 +- benchmark/micro_benchmark_bulk_load.cpp | 2 +- test/test_art_bulk_load.cpp | 4 ++++ test/test_art_bulk_load_ops.cpp | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index 2c33ca36..b3d4c3c3 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -71,7 +71,7 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, ++cur; } } - return std::move(parts); // NOLINT(performance-move-const-arg) + return parts; // NOLINT(performance-move-const-arg) }; auto build_prefix_chain = [&self](art_key_type k, node_ptr_t child_inode, diff --git a/benchmark/micro_benchmark_bulk_load.cpp b/benchmark/micro_benchmark_bulk_load.cpp index e9f4ee00..44b48d3a 100644 --- a/benchmark/micro_benchmark_bulk_load.cpp +++ b/benchmark/micro_benchmark_bulk_load.cpp @@ -47,7 +47,7 @@ const auto value = unodb::value_view{val8}; /// Generate sorted random unique keys. [[nodiscard]] auto random_keys(std::int64_t n) { - std::mt19937_64 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) + const std::mt19937_64 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) std::vector keys(static_cast(n)); std::generate(keys.begin(), keys.end(), rng); std::sort(keys.begin(), keys.end()); diff --git a/test/test_art_bulk_load.cpp b/test/test_art_bulk_load.cpp index 6c427b98..d8a3e40c 100644 --- a/test/test_art_bulk_load.cpp +++ b/test/test_art_bulk_load.cpp @@ -19,6 +19,8 @@ #ifdef UNODB_DETAIL_WITH_STATS +UNODB_DETAIL_DISABLE_MSVC_WARNING(26445) + namespace { using unodb::as_i; @@ -168,4 +170,6 @@ UNODB_TEST(BulkLoad, Growth260) { } // namespace +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + #endif // UNODB_DETAIL_WITH_STATS diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp index c764e601..31a8810b 100644 --- a/test/test_art_bulk_load_ops.cpp +++ b/test/test_art_bulk_load_ops.cpp @@ -45,7 +45,7 @@ constexpr auto val = value_view{val_bytes}; // T26: bulk_load on non-empty tree throws UNODB_TEST(BulkLoadError, NonEmpty) { u64_db db; - const std::uint64_t key = 42; + constexpr std::uint64_t key = 42; UNODB_ASSERT_TRUE(db.insert(key, val)); std::vector> kv{{100, val}}; UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) From 5585da9e62f919a19fb96dea9ab070da38a5cf76 Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Wed, 1 Jul 2026 02:49:21 +0000 Subject: [PATCH 12/23] fix: MSVC static analysis C26815/C26445/C26496 suppressions - C26815: false positive dangling pointer in build_prefix_chain - C26445: suppress for structured bindings of span in range-for - C26496: rng cannot be const (generate mutates); suppress instead --- art_bulk_load_detail.hpp | 6 ++++++ benchmark/micro_benchmark_bulk_load.cpp | 8 +++++++- test/test_art_bulk_load_ops.cpp | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index b3d4c3c3..21cdb28f 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -10,6 +10,10 @@ #ifndef UNODB_DETAIL_ART_BULK_LOAD_DETAIL_HPP #define UNODB_DETAIL_ART_BULK_LOAD_DETAIL_HPP +// MSVC static analysis false positive: claims pointers dangle after +// smart-pointer release() + reassignment. Suppressed file-wide. +UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) + /// \cond UNODB_DETAIL_INTERNAL namespace unodb::detail { @@ -377,4 +381,6 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, } // namespace unodb::detail /// \endcond +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + #endif // UNODB_DETAIL_ART_BULK_LOAD_DETAIL_HPP diff --git a/benchmark/micro_benchmark_bulk_load.cpp b/benchmark/micro_benchmark_bulk_load.cpp index 44b48d3a..3f09cd48 100644 --- a/benchmark/micro_benchmark_bulk_load.cpp +++ b/benchmark/micro_benchmark_bulk_load.cpp @@ -19,6 +19,9 @@ #include "olc_art.hpp" #include "portability_execution.hpp" +UNODB_DETAIL_DISABLE_MSVC_WARNING(26445) +UNODB_DETAIL_DISABLE_MSVC_WARNING(26496) + namespace { using db = unodb::benchmark::db; @@ -47,7 +50,7 @@ const auto value = unodb::value_view{val8}; /// Generate sorted random unique keys. [[nodiscard]] auto random_keys(std::int64_t n) { - const std::mt19937_64 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) + std::mt19937_64 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) std::vector keys(static_cast(n)); std::generate(keys.begin(), keys.end(), rng); std::sort(keys.begin(), keys.end()); @@ -452,4 +455,7 @@ BENCHMARK_TEMPLATE(BM_Insert_first_scan, olc_db) ->Arg(1 << 20) ->Unit(benchmark::kMillisecond); +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_BENCHMARK_MAIN(); diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp index 31a8810b..370d599e 100644 --- a/test/test_art_bulk_load_ops.cpp +++ b/test/test_art_bulk_load_ops.cpp @@ -23,6 +23,7 @@ UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) +UNODB_DETAIL_DISABLE_MSVC_WARNING(26445) namespace { @@ -255,3 +256,4 @@ UNODB_TEST(BulkLoadOps, OlcDbConcurrentReaders) { UNODB_DETAIL_RESTORE_MSVC_WARNINGS() UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() From 9601a9dac3769c6b6d74e8ff6694222a0dd13e03 Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Wed, 1 Jul 2026 13:12:45 +0000 Subject: [PATCH 13/23] fix: MSVC/clang-cl NRVO, C26495, C26496, C26814 warnings - Restore std::move on partition return (clang -Wnrvo requires it), suppress MSVC C26479 inline since the compilers contradict. - Initialize children{} in inode_256 bulk_load ctor (C26495). - constexpr new_key (C26814), const leaf_ptr (C26496). --- art_bulk_load_detail.hpp | 6 ++++-- art_internal_impl.hpp | 2 +- test/test_art_bulk_load_ops.cpp | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index 21cdb28f..99b0b438 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -75,7 +75,9 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, ++cur; } } - return parts; // NOLINT(performance-move-const-arg) + UNODB_DETAIL_DISABLE_MSVC_WARNING(26479) + return std::move(parts); // NOLINT(performance-move-const-arg) + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() }; auto build_prefix_chain = [&self](art_key_type k, node_ptr_t child_inode, @@ -132,7 +134,7 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, } else { static_cast(depth); auto leaf = art_policy::make_db_leaf_ptr(ak, it->second, self); - auto leaf_ptr = node_ptr_t{leaf.release(), node_type::LEAF}; + const auto leaf_ptr = node_ptr_t{leaf.release(), node_type::LEAF}; return {leaf_ptr, false}; } }; diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index c72bb5e7..b2c0fd01 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -4656,7 +4656,7 @@ class basic_inode_256 // cppcheck-suppress passedByValue tree_depth_type depth, typename parent_class::bulk_load_tag tag) noexcept - : parent_class{n_children, prefix_len, k1, depth, tag} {} + : parent_class{n_children, prefix_len, k1, depth, tag}, children{} {} /// Initialize by growing from basic_inode_48 and adding a child. /// diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp index 370d599e..f014d3c2 100644 --- a/test/test_art_bulk_load_ops.cpp +++ b/test/test_art_bulk_load_ops.cpp @@ -96,7 +96,7 @@ UNODB_TEST(BulkLoadOps, ThenOperations) { } // insert new key works - const std::uint64_t new_key = 0xFFULL << 56U; + constexpr std::uint64_t new_key = 0xFFULL << 56U; UNODB_ASSERT_TRUE(db.insert(new_key, val)); UNODB_ASSERT_TRUE(db.get(new_key).has_value()); From 08500f2f29e35e63f6155c2002023e6d27d904d2 Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Wed, 1 Jul 2026 15:21:33 +0000 Subject: [PATCH 14/23] fix: suppress MSVC C26496 for vmask variables in if-constexpr branches --- art_bulk_load_detail.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index 99b0b438..f9bd1c91 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -13,6 +13,9 @@ // MSVC static analysis false positive: claims pointers dangle after // smart-pointer release() + reassignment. Suppressed file-wide. UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) +// C26496: vmask variables are mutated in if-constexpr branches that +// MSVC SA doesn't track; cannot be const in the general case. +UNODB_DETAIL_DISABLE_MSVC_WARNING(26496) /// \cond UNODB_DETAIL_INTERNAL namespace unodb::detail { @@ -383,6 +386,7 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, } // namespace unodb::detail /// \endcond +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() UNODB_DETAIL_RESTORE_MSVC_WARNINGS() #endif // UNODB_DETAIL_ART_BULK_LOAD_DETAIL_HPP From b226d5aaee812be145ede9265db656e07ff19382 Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Wed, 1 Jul 2026 16:17:35 +0000 Subject: [PATCH 15/23] fix: initialize children{} in all inode bulk_load constructors (C26495) MSVC static analysis requires member initialization even when create_bulk fills them immediately after construction. --- art_internal_impl.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index b2c0fd01..1b05860e 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -2405,7 +2405,7 @@ class basic_inode_4 // cppcheck-suppress passedByValue tree_depth_type depth, typename parent_class::bulk_load_tag tag) noexcept - : parent_class{n_children, prefix_len, k1, depth, tag} {} + : parent_class{n_children, prefix_len, k1, depth, tag}, children{} {} /// \} @@ -3193,7 +3193,7 @@ class basic_inode_16 // cppcheck-suppress passedByValue tree_depth_type depth, typename parent_class::bulk_load_tag tag) noexcept - : parent_class{n_children, prefix_len, k1, depth, tag} {} + : parent_class{n_children, prefix_len, k1, depth, tag}, children{} {} /// Initialize by growing from basic_inode_4 and adding a child. /// @@ -3855,7 +3855,9 @@ class basic_inode_48 // cppcheck-suppress passedByValue tree_depth_type depth, typename parent_class::bulk_load_tag tag) noexcept - : parent_class{n_children, prefix_len, k1, depth, tag} {} + : parent_class{n_children, prefix_len, k1, depth, tag}, + child_indexes{}, + children{} {} /// Initialize by growing from basic_inode_16 and adding a child. /// From a7f222d69dce35bb5b4ba5494ae2f1a0ff8ff16e Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Fri, 3 Jul 2026 18:44:33 +0000 Subject: [PATCH 16/23] fix: disable parallel bulk_load for non-thread-safe db types unodb::db and mutex_db use non-atomic stats counters (in_fake_critical_section). Running std::async subtree construction on these types causes data races (TSan) and assertion failures (growing_inode_counts invariant). Add supports_parallel_bulk_load constant to each db class and condition is_parallel on it. Only olc_db (with real atomic counters) executes parallel subtrees. Also suppress misc-include-cleaner false positive on gtest.h in bulk_load test files (included transitively via gtest_utils.hpp but UNODB_TEST macro requires the direct include). --- art.hpp | 4 ++++ art_bulk_load_detail.hpp | 5 +++-- mutex_art.hpp | 4 ++++ olc_art.hpp | 3 +++ test/test_art_bulk_load.cpp | 2 +- test/test_art_bulk_load_ops.cpp | 2 +- test/test_art_bulk_load_structural.cpp | 2 +- 7 files changed, 17 insertions(+), 5 deletions(-) diff --git a/art.hpp b/art.hpp index 6e3d7210..a4e71a69 100644 --- a/art.hpp +++ b/art.hpp @@ -134,6 +134,10 @@ class db final { friend class mutex_db; public: + /// Whether this tree type supports parallel bulk_load execution. + /// Only olc_db has thread-safe internal counters. + static constexpr bool supports_parallel_bulk_load = false; + /// The type of the keys in the index. using key_type = Key; diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index f9bd1c91..a12783b9 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -294,8 +294,9 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, using policy_t = std::remove_cvref_t; constexpr bool is_parallel = - std::is_same_v || - std::is_same_v; + Db::supports_parallel_bulk_load && + (std::is_same_v || + std::is_same_v); if constexpr (!is_parallel) { auto result = builder(first, last, tree_depth_type{0}); diff --git a/mutex_art.hpp b/mutex_art.hpp index 497a5849..c05c5412 100644 --- a/mutex_art.hpp +++ b/mutex_art.hpp @@ -22,6 +22,10 @@ namespace unodb { template class mutex_db final { public: + /// Whether this tree type supports parallel bulk_load execution. + /// mutex_db wraps db which uses non-atomic counters. + static constexpr bool supports_parallel_bulk_load = false; + /// The type of the keys in the index. using key_type = Key; /// The type of the value associated with the keys in the index. diff --git a/olc_art.hpp b/olc_art.hpp index a734291f..7b5a2a2e 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -185,6 +185,9 @@ void bulk_load_impl(Db& self, ExecutionPolicy&& policy, RandomIt first, template class olc_db final { public: + /// Whether this tree type supports parallel bulk_load execution. + static constexpr bool supports_parallel_bulk_load = true; + /// The type of the keys in the index. using key_type = Key; /// The type of the value associated with the key in the index. diff --git a/test/test_art_bulk_load.cpp b/test/test_art_bulk_load.cpp index d8a3e40c..388ce8a9 100644 --- a/test/test_art_bulk_load.cpp +++ b/test/test_art_bulk_load.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include // NOLINT(misc-include-cleaner) #include "art_common.hpp" #include "db_test_utils.hpp" diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp index f014d3c2..a4bb2655 100644 --- a/test/test_art_bulk_load_ops.cpp +++ b/test/test_art_bulk_load_ops.cpp @@ -11,7 +11,7 @@ #include #include -#include +#include // NOLINT(misc-include-cleaner) #include "art_common.hpp" #include "db_test_utils.hpp" diff --git a/test/test_art_bulk_load_structural.cpp b/test/test_art_bulk_load_structural.cpp index b12856cd..074c4d6a 100644 --- a/test/test_art_bulk_load_structural.cpp +++ b/test/test_art_bulk_load_structural.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include // NOLINT(misc-include-cleaner) #include "art_common.hpp" #include "db_test_utils.hpp" From fb7f6363e73e15e3cc53a44b0dd9c9d7d6f2ad60 Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Mon, 6 Jul 2026 17:01:04 +0000 Subject: [PATCH 17/23] feat: replace ExecutionPolicy with caller-provided fork callable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bulk_load parallel API now takes a fork callable + max_tasks instead of std::execution::par. The library never spawns threads internally — the caller owns all threading decisions. API: tree.bulk_load(first, last); // sequential tree.bulk_load(fork, max_tasks, first, last); // parallel Where fork(Callable) -> Future with .get(). Works with std::async, thread pools, fiber pools, QSBR threads, etc. The implementation forks up to min(partitions, max_tasks) subtrees via the callable; remaining partitions run inline on the calling thread. Thread-local stats accumulators ensure correctness with STATS=ON regardless of the executor type. Also removes portability_execution.hpp (now unused) and dead includes from library headers. --- art.hpp | 78 ++++++++---- art_bulk_load_detail.hpp | 105 ++++++++++++---- benchmark/micro_benchmark_bulk_load.cpp | 20 +++- examples/example_bulk_load.cpp | 17 +-- mutex_art.hpp | 17 ++- node_type.hpp | 22 ++++ olc_art.hpp | 41 ++++--- portability_execution.hpp | 45 ------- test/test_art_bulk_load.cpp | 53 +++++++++ test/test_art_bulk_load_ops.cpp | 151 +++++++++++++++++++++++- test/test_art_bulk_load_structural.cpp | 63 ++++++++++ 11 files changed, 482 insertions(+), 130 deletions(-) delete mode 100644 portability_execution.hpp diff --git a/art.hpp b/art.hpp index a4e71a69..0f5b48ac 100644 --- a/art.hpp +++ b/art.hpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -35,7 +34,6 @@ #include "assert.hpp" #include "in_fake_critical_section.hpp" #include "node_type.hpp" -#include "portability_execution.hpp" namespace unodb { @@ -116,9 +114,9 @@ template class inode : public inode_base {}; /// Forward declaration of shared bulk_load algorithm. -template -void bulk_load_impl(Db& self, ExecutionPolicy&& policy, RandomIt first, - RandomIt last); +template +void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last); } // namespace detail @@ -134,10 +132,6 @@ class db final { friend class mutex_db; public: - /// Whether this tree type supports parallel bulk_load execution. - /// Only olc_db has thread-safe internal counters. - static constexpr bool supports_parallel_bulk_load = false; - /// The type of the keys in the index. using key_type = Key; @@ -237,12 +231,14 @@ class db final { void release() noexcept { ptr = nullptr; } bulk_subtree_guard(const bulk_subtree_guard&) = delete; + // LCOV_EXCL_START — move ctor only called on vector reallocation bulk_subtree_guard(bulk_subtree_guard&& other) noexcept : db_{other.db_}, ptr{other.ptr}, is_packed_value{other.is_packed_value} { other.ptr = nullptr; } + // LCOV_EXCL_STOP auto& operator=(const bulk_subtree_guard&) = delete; auto& operator=(bulk_subtree_guard&&) = delete; }; @@ -366,22 +362,30 @@ class db final { /// Bulk-load an ART tree from a pre-sorted range of key-value pairs. /// + /// Bulk-load the tree from a sorted range, with caller-provided parallelism. + /// + /// \tparam Fork Callable satisfying: fork(Callable) -> Future where + /// Future has .get(). The callable is invoked to submit + /// subtree-building tasks; the library never spawns + /// threads. /// \tparam RandomIt Random-access iterator to std::pair - /// \param policy Execution policy (ignored for db — allocator not - /// thread-safe). + /// value_type> + /// \param fork Callable to submit parallel tasks. + /// \param max_tasks Maximum number of tasks to fork (remaining partitions + /// run inline on the calling thread). /// \param first Start of sorted range (ART byte order, no duplicates). /// \param last End of sorted range. /// \pre empty() == true /// \throws std::invalid_argument if tree is non-empty. /// \throws std::bad_alloc (strong guarantee — tree left empty). - template - void bulk_load(ExecutionPolicy&& policy, RandomIt first, RandomIt last); + template + void bulk_load(Fork&& fork, std::size_t max_tasks, RandomIt first, + RandomIt last); - /// Convenience overload: sequential execution. + /// Convenience overload: sequential execution (no parallelism). template void bulk_load(RandomIt first, RandomIt last) { - bulk_load(std::execution::seq, first, last); + bulk_load(nullptr, 0, first, last); } /// Internal iterator for tree traversal. @@ -976,6 +980,11 @@ class db final { /// Increase tracked memory usage by \a delta bytes. constexpr void increase_memory_use(std::size_t delta) noexcept { UNODB_DETAIL_ASSERT(delta > 0); + + if (auto* const acc = detail::tls_bulk_load_stats; acc != nullptr) { + acc->current_memory_use += delta; // LCOV_EXCL_LINE — exercised by + return; // LCOV_EXCL_LINE — parallel tests + } UNODB_DETAIL_ASSERT( std::numeric_limits::max() - delta >= current_memory_use); @@ -993,6 +1002,11 @@ class db final { /// Increment leaf node count and bump memory usage by \a leaf_size bytes. constexpr void increment_leaf_count(std::size_t leaf_size) noexcept { + if (auto* const acc = detail::tls_bulk_load_stats; acc != nullptr) { + acc->current_memory_use += leaf_size; + ++acc->node_counts[as_i]; + return; + } increase_memory_use(leaf_size); ++node_counts[as_i]; } @@ -1029,6 +1043,17 @@ class db final { template constexpr void account_shrinking_inode() noexcept; + /// Merge accumulated stats from a parallel bulk_load subtree. + constexpr void merge_bulk_load_stats( + const detail::bulk_load_stats_accumulator& acc) noexcept { + current_memory_use += acc.current_memory_use; + for (std::size_t i = 0; i < node_counts.size(); ++i) + node_counts[i] += acc.node_counts[i]; + for (std::size_t i = 0; i < growing_inode_counts.size(); ++i) + growing_inode_counts[i] += acc.growing_inode_counts[i]; + key_prefix_splits += acc.key_prefix_splits; + } + #endif // UNODB_DETAIL_WITH_STATS /// Root of the tree (nullptr if empty). @@ -1063,8 +1088,8 @@ class db final { db&); /// detail::bulk_load_impl - template - friend void detail::bulk_load_impl(Db2&, Ep2&&, It2, It2); + template + friend void detail::bulk_load_impl(Db2&, Fork2&&, std::size_t, It2, It2); /// detail::basic_db_leaf_deleter template @@ -2687,6 +2712,11 @@ template constexpr void db::increment_inode_count() noexcept { static_assert(inode_defs_type::template is_inode()); + if (auto* const acc = detail::tls_bulk_load_stats; acc != nullptr) { + ++acc->node_counts[as_i]; + acc->current_memory_use += sizeof(INode); + return; + } ++node_counts[as_i]; increase_memory_use(sizeof(INode)); } @@ -2706,6 +2736,10 @@ template constexpr void db::account_growing_inode() noexcept { static_assert(NodeType != node_type::LEAF); + if (auto* const acc = detail::tls_bulk_load_stats; acc != nullptr) { + ++acc->growing_inode_counts[internal_as_i]; + return; + } // NOLINTNEXTLINE(google-readability-casting) ++growing_inode_counts[internal_as_i]; UNODB_DETAIL_ASSERT(growing_inode_counts[internal_as_i] >= @@ -2839,15 +2873,15 @@ bool db::upsert(Key k, value_type v, FN fn) { } template -template -void db::bulk_load(ExecutionPolicy&& policy, RandomIt first, - RandomIt last) { +template +void db::bulk_load(Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last) { if (!empty()) { throw std::invalid_argument("bulk_load requires empty tree"); } if (first == last) return; try { - detail::bulk_load_impl(*this, std::forward(policy), first, + detail::bulk_load_impl(*this, std::forward(fork), max_tasks, first, last); } catch (...) { clear(); diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index a12783b9..94fabefb 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -24,9 +24,11 @@ namespace unodb::detail { /// /// \pre Tree must be empty (caller validates). /// \pre [first, last) is sorted and non-empty (caller validates). -template -void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, - RandomIt last) { +/// \tparam Fork Callable satisfying fork(Callable) -> Future where +/// Future has .get(). Ignored (nullptr_t) for sequential. +template +void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last) { using art_key_type = typename Db::art_key_type; using art_policy = typename Db::art_policy; using tree_depth_type = typename Db::tree_depth_type; @@ -105,6 +107,8 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, #endif pos = depth; } + // LCOV_EXCL_START — chain_consumed is always a multiple of (prefix_cap+1), + // so the while loop always decrements pos to exactly start. if (pos > start) { const auto dispatch = full_key[pos - 1]; auto chain{inode_4::create( @@ -115,6 +119,7 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, self.template account_growing_inode(); #endif } + // LCOV_EXCL_STOP return current; }; @@ -292,11 +297,8 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, build_single_leaf, make_bulk_inode}; - using policy_t = std::remove_cvref_t; constexpr bool is_parallel = - Db::supports_parallel_bulk_load && - (std::is_same_v || - std::is_same_v); + !std::is_same_v, std::nullptr_t>; if constexpr (!is_parallel) { auto result = builder(first, last, tree_depth_type{0}); @@ -319,15 +321,49 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, const auto next_depth = tree_depth_type{static_cast( static_cast(dispatch_depth) + 1)}; - std::vector> futures; - futures.reserve(child_count); - for (std::size_t i = 0; i < child_count; ++i) { + // Fork up to min(child_count, max_tasks) subtrees; inline the rest. + const auto fork_count = std::min(child_count, max_tasks); + +#ifdef UNODB_DETAIL_WITH_STATS + struct parallel_result { + build_result_t build; + bulk_load_stats_accumulator stats; + }; +#else + using parallel_result = build_result_t; +#endif + + auto make_task = [&builder, &parts, child_count, last, + next_depth](std::size_t i) { const auto part_begin = parts[i].begin; const auto part_end = (i + 1 < child_count) ? parts[i + 1].begin : last; - futures.push_back(std::async( - std::launch::async, [&builder, part_begin, part_end, next_depth] { - return builder(part_begin, part_end, next_depth); - })); +#ifdef UNODB_DETAIL_WITH_STATS + bulk_load_stats_accumulator acc; + tls_bulk_load_stats = &acc; + auto result = builder(part_begin, part_end, next_depth); + tls_bulk_load_stats = nullptr; + return parallel_result{result, acc}; +#else + return builder(part_begin, part_end, next_depth); +#endif + }; + + // Submit forked tasks — first fork deduces the future type. + auto first_future = + fork([&make_task] { return make_task(std::size_t{0}); }); + using future_t = decltype(first_future); + std::vector futures; + futures.reserve(fork_count); + futures.push_back(std::move(first_future)); + for (std::size_t i = 1; i < fork_count; ++i) { + futures.push_back(fork([&make_task, i] { return make_task(i); })); + } + + // Run remaining partitions inline on the calling thread. + std::vector inline_results; + inline_results.reserve(child_count - fork_count); + for (std::size_t i = fork_count; i < child_count; ++i) { + inline_results.push_back(make_task(i)); } boost::container::small_vector guards; @@ -335,26 +371,55 @@ void bulk_load_impl(Db& self, ExecutionPolicy&&, RandomIt first, boost::container::small_vector children; children.reserve(child_count); - for (std::size_t i = 0; i < child_count; ++i) { + // Gather forked results first (indices 0..fork_count-1). + for (std::size_t i = 0; i < fork_count; ++i) { try { - auto result = futures[i].get(); + auto future_result = futures[i].get(); +#ifdef UNODB_DETAIL_WITH_STATS + const auto& result = future_result.build; + const auto& acc = future_result.stats; + self.merge_bulk_load_stats(acc); +#else + const auto& result = future_result; +#endif guards.emplace_back(self); guards.back().ptr = result.ptr; guards.back().is_packed_value = result.is_packed_value; children.push_back({parts[i].key_byte, result.ptr}); - } catch (...) { + } catch (...) { // LCOV_EXCL_START — exception drain; no reliable test // Drain remaining futures into guards to prevent leaks. - for (std::size_t j = i + 1; j < child_count; ++j) { + for (std::size_t j = i + 1; j < fork_count; ++j) { try { auto r = futures[j].get(); guards.emplace_back(self); +#ifdef UNODB_DETAIL_WITH_STATS + guards.back().ptr = r.build.ptr; + guards.back().is_packed_value = r.build.is_packed_value; +#else guards.back().ptr = r.ptr; guards.back().is_packed_value = r.is_packed_value; +#endif } catch (...) { - } // LCOV_EXCL_LINE + } } throw; - } + } // LCOV_EXCL_STOP + } + + // Gather inline results (indices fork_count..child_count-1). + for (std::size_t i = 0; i < inline_results.size(); ++i) { + const auto part_idx = fork_count + i; +#ifdef UNODB_DETAIL_WITH_STATS + const auto& result = inline_results[i].build; + const auto& acc = inline_results[i].stats; + self.merge_bulk_load_stats(acc); +#else + const auto& result = inline_results[i]; +#endif + guards.emplace_back(self); + guards.back().ptr = result.ptr; + guards.back().is_packed_value = result.is_packed_value; + children.push_back({parts[part_idx].key_byte, result.ptr}); } const std::size_t chain_consumed = diff --git a/benchmark/micro_benchmark_bulk_load.cpp b/benchmark/micro_benchmark_bulk_load.cpp index 3f09cd48..8bcfbc82 100644 --- a/benchmark/micro_benchmark_bulk_load.cpp +++ b/benchmark/micro_benchmark_bulk_load.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -17,7 +18,6 @@ #include "micro_benchmark_utils.hpp" #include "mutex_art.hpp" #include "olc_art.hpp" -#include "portability_execution.hpp" UNODB_DETAIL_DISABLE_MSVC_WARNING(26445) UNODB_DETAIL_DISABLE_MSVC_WARNING(26496) @@ -179,7 +179,11 @@ void BM_BulkLoad_par_db(benchmark::State& state) { const auto kv = random_keys(state.range(0)); for (const auto _ : state) { db test_db; - test_db.bulk_load(std::execution::par, kv.begin(), kv.end()); + test_db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); benchmark::DoNotOptimize(test_db); state.PauseTiming(); test_db.clear(); @@ -193,7 +197,11 @@ void BM_BulkLoad_par_mutex(benchmark::State& state) { const auto kv = random_keys(state.range(0)); for (const auto _ : state) { mutex_db test_db; - test_db.bulk_load(std::execution::par, kv.begin(), kv.end()); + test_db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); benchmark::DoNotOptimize(test_db); state.PauseTiming(); test_db.clear(); @@ -207,7 +215,11 @@ void BM_BulkLoad_par_olc(benchmark::State& state) { const auto kv = random_keys(state.range(0)); for (const auto _ : state) { olc_db test_db; - test_db.bulk_load(std::execution::par, kv.begin(), kv.end()); + test_db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); benchmark::DoNotOptimize(test_db); state.PauseTiming(); test_db.clear(); diff --git a/examples/example_bulk_load.cpp b/examples/example_bulk_load.cpp index 973996eb..29f75505 100644 --- a/examples/example_bulk_load.cpp +++ b/examples/example_bulk_load.cpp @@ -1,6 +1,6 @@ // Copyright 2026 UnoDB contributors -// Example: bulk_load with sequential and parallel execution policies. +// Example: bulk_load with sequential and parallel execution. #include "global.hpp" @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,7 +16,6 @@ #include "art.hpp" #include "art_common.hpp" #include "mutex_art.hpp" -#include "portability_execution.hpp" namespace { @@ -41,20 +41,23 @@ int main() { // ─── Sequential bulk_load (default) ──────────────────────────────────────── { unodb::db tree; - tree.bulk_load(data.begin(), data.end()); // default: std::execution::seq + tree.bulk_load(data.begin(), data.end()); // sequential (default) std::cerr << "Sequential bulk_load: " << key_count << " keys loaded\n"; std::cerr << " get(42) found: " << tree.get(42).has_value() << '\n'; tree.clear(); } // ─── Parallel bulk_load ──────────────────────────────────────────────────── - // std::execution::par enables concurrent subtree construction. + // The caller provides a fork callable to submit parallel tasks. // The implementation partitions at the root level and builds each - // subtree on a separate thread. Safe for all tree modes because + // subtree via the fork callable. Safe for all tree modes because // bulk_load operates on an unpublished tree (no concurrent readers). + auto async_fork = [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }; { unodb::db tree; - tree.bulk_load(std::execution::par, data.begin(), data.end()); + tree.bulk_load(async_fork, 8, data.begin(), data.end()); std::cerr << "Parallel bulk_load: " << key_count << " keys loaded\n"; std::cerr << " get(99999) found: " << tree.get(99999).has_value() << '\n'; tree.clear(); @@ -63,7 +66,7 @@ int main() { // ─── mutex_db: same API ──────────────────────────────────────────────────── { unodb::mutex_db tree; - tree.bulk_load(std::execution::par, data.begin(), data.end()); + tree.bulk_load(async_fork, 8, data.begin(), data.end()); std::cerr << "mutex_db parallel bulk_load: " << key_count << " keys loaded\n"; tree.clear(); diff --git a/mutex_art.hpp b/mutex_art.hpp index c05c5412..643131ba 100644 --- a/mutex_art.hpp +++ b/mutex_art.hpp @@ -22,10 +22,6 @@ namespace unodb { template class mutex_db final { public: - /// Whether this tree type supports parallel bulk_load execution. - /// mutex_db wraps db which uses non-atomic counters. - static constexpr bool supports_parallel_bulk_load = false; - /// The type of the keys in the index. using key_type = Key; /// The type of the value associated with the keys in the index. @@ -153,20 +149,23 @@ class mutex_db final { /// Bulk-load sorted key-value pairs into an empty tree. /// + /// \tparam Fork Callable: fork(Callable) -> Future with .get() /// \tparam RandomIt Random access iterator over pairs of (key, value) + /// \param fork Callable to submit parallel tasks + /// \param max_tasks Maximum tasks to fork /// \param first Start of sorted range /// \param last End of sorted range - /// \param policy Execution policy (std::execution::seq or par) - template - void bulk_load(ExecutionPolicy&& policy, RandomIt first, RandomIt last) { + template + void bulk_load(Fork&& fork, std::size_t max_tasks, RandomIt first, + RandomIt last) { const std::lock_guard guard{mutex}; - db_.bulk_load(std::forward(policy), first, last); + db_.bulk_load(std::forward(fork), max_tasks, first, last); } /// Convenience overload: sequential execution. template void bulk_load(RandomIt first, RandomIt last) { - bulk_load(std::execution::seq, first, last); + bulk_load(nullptr, 0, first, last); } // diff --git a/node_type.hpp b/node_type.hpp index 91597e07..4699903c 100644 --- a/node_type.hpp +++ b/node_type.hpp @@ -87,6 +87,28 @@ inline constexpr auto internal_as_i{ NodeType) - 1}; +namespace detail { + +/// Thread-local stats accumulator for parallel bulk_load. +/// +/// During parallel subtree construction on non-thread-safe db types, +/// each thread writes stats here instead of directly to the db instance +/// (which has non-atomic counters). After all async tasks complete, +/// accumulators are merged single-threaded into the db. +struct bulk_load_stats_accumulator { + std::size_t current_memory_use{0}; + node_type_counter_array node_counts{}; + inode_type_counter_array growing_inode_counts{}; + std::uint64_t key_prefix_splits{0}; +}; + +/// Per-thread pointer to the active stats accumulator. +/// Null during normal operation; set per-thread during parallel bulk_load. +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +inline thread_local bulk_load_stats_accumulator* tls_bulk_load_stats{nullptr}; + +} // namespace detail + #endif // UNODB_DETAIL_WITH_STATS } // namespace unodb diff --git a/olc_art.hpp b/olc_art.hpp index 7b5a2a2e..6c0d09e5 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -31,7 +30,6 @@ #include "node_type.hpp" #include "optimistic_lock.hpp" #include "portability_arch.hpp" -#include "portability_execution.hpp" #include "qsbr.hpp" #include "sync.hpp" @@ -170,9 +168,9 @@ using olc_leaf_unique_ptr = basic_db_leaf_unique_ptr; /// Forward declaration of shared bulk_load algorithm. -template -void bulk_load_impl(Db& self, ExecutionPolicy&& policy, RandomIt first, - RandomIt last); +template +void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last); } // namespace detail @@ -185,9 +183,6 @@ void bulk_load_impl(Db& self, ExecutionPolicy&& policy, RandomIt first, template class olc_db final { public: - /// Whether this tree type supports parallel bulk_load execution. - static constexpr bool supports_parallel_bulk_load = true; - /// The type of the keys in the index. using key_type = Key; /// The type of the value associated with the key in the index. @@ -309,17 +304,20 @@ class olc_db final { /// Bulk-load sorted key-value pairs into an empty tree. /// /// \note Only legal in single-threaded context (same as clear). + /// \tparam Fork Callable: fork(Callable) -> Future with .get() /// \tparam RandomIt Random access iterator over pairs of (key, value) - /// \param policy Execution policy (parallelism hint; 0=auto, 1=sequential) + /// \param fork Callable to submit parallel tasks + /// \param max_tasks Maximum tasks to fork /// \param first Start of sorted range /// \param last End of sorted range - template - void bulk_load(ExecutionPolicy&& policy, RandomIt first, RandomIt last); + template + void bulk_load(Fork&& fork, std::size_t max_tasks, RandomIt first, + RandomIt last); /// Convenience overload: sequential execution. template void bulk_load(RandomIt first, RandomIt last) { - bulk_load(std::execution::seq, first, last); + bulk_load(nullptr, 0, first, last); } // @@ -906,12 +904,14 @@ class olc_db final { void release() noexcept { ptr = nullptr; } bulk_subtree_guard(const bulk_subtree_guard&) = delete; + // LCOV_EXCL_START — move ctor only called on vector reallocation bulk_subtree_guard(bulk_subtree_guard&& other) noexcept : db_{other.db_}, ptr{other.ptr}, is_packed_value{other.is_packed_value} { other.ptr = nullptr; } + // LCOV_EXCL_STOP auto& operator=(const bulk_subtree_guard&) = delete; auto& operator=(bulk_subtree_guard&&) = delete; }; @@ -1040,6 +1040,11 @@ class olc_db final { template constexpr void account_shrinking_inode() noexcept; + /// Merge accumulated stats from a parallel bulk_load subtree. + /// For olc_db this is a no-op: atomic counters are updated directly. + constexpr void merge_bulk_load_stats( + const detail::bulk_load_stats_accumulator& /*acc*/) noexcept {} + #endif // UNODB_DETAIL_WITH_STATS // optimistic lock guarding the [root]. @@ -1086,8 +1091,8 @@ class olc_db final { value_type, olc_db&); /// detail::bulk_load_impl - template - friend void detail::bulk_load_impl(Db2&, Ep2&&, It2, It2); + template + friend void detail::bulk_load_impl(Db2&, Fork2&&, std::size_t, It2, It2); template friend class detail::basic_db_leaf_deleter; @@ -5039,9 +5044,9 @@ olc_db::try_chain_cut( namespace unodb { template -template -void olc_db::bulk_load(ExecutionPolicy&& policy, RandomIt first, - RandomIt last) { +template +void olc_db::bulk_load(Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last) { UNODB_DETAIL_QSBR_ASSERT( qsbr_state::single_thread_mode(qsbr::instance().get_state())); @@ -5050,7 +5055,7 @@ void olc_db::bulk_load(ExecutionPolicy&& policy, RandomIt first, } if (first == last) return; try { - detail::bulk_load_impl(*this, std::forward(policy), first, + detail::bulk_load_impl(*this, std::forward(fork), max_tasks, first, last); } catch (...) { clear(); diff --git a/portability_execution.hpp b/portability_execution.hpp deleted file mode 100644 index 233cf7aa..00000000 --- a/portability_execution.hpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2026 UnoDB contributors -#ifndef UNODB_DETAIL_PORTABILITY_EXECUTION_HPP -#define UNODB_DETAIL_PORTABILITY_EXECUTION_HPP - -/// \file -/// Portability shim for C++17 execution policies (``). -/// -/// Apple libc++ and some older standard libraries lack execution policy -/// support. This header detects availability and provides a minimal -/// sequential-only fallback when the real header is absent. - -// Should be the first include -#include "global.hpp" // IWYU pragma: keep - -#if __has_include() -#include // IWYU pragma: export -#endif - -// Detect usable execution policies via the library feature-test macro. -// __cpp_lib_execution >= 201603 means parallel_policy is available. -#if defined(__cpp_lib_execution) && __cpp_lib_execution >= 201603L -#define UNODB_DETAIL_HAS_EXECUTION_POLICIES 1 -#else -#define UNODB_DETAIL_HAS_EXECUTION_POLICIES 0 -#endif - -#if !UNODB_DETAIL_HAS_EXECUTION_POLICIES - -/// Minimal fallback: sequential execution tag only. -namespace std::execution { // NOLINT(cert-dcl58-cpp) - -struct sequenced_policy {}; -inline constexpr sequenced_policy seq{}; - -struct parallel_policy {}; -inline constexpr parallel_policy par{}; - -struct parallel_unsequenced_policy {}; -inline constexpr parallel_unsequenced_policy par_unseq{}; - -} // namespace std::execution - -#endif // !UNODB_DETAIL_HAS_EXECUTION_POLICIES - -#endif // UNODB_DETAIL_PORTABILITY_EXECUTION_HPP diff --git a/test/test_art_bulk_load.cpp b/test/test_art_bulk_load.cpp index 388ce8a9..d592b4f3 100644 --- a/test/test_art_bulk_load.cpp +++ b/test/test_art_bulk_load.cpp @@ -168,6 +168,59 @@ UNODB_TEST(BulkLoad, Growth260) { UNODB_ASSERT_EQ(counts[as_i], 260U); } +// T11: Keys sharing a common prefix exercise the prefix chain path. +// For uint64_t, max key length is 8, prefix_cap is 7, so we need a recursive +// scenario where a subtree has all keys sharing a long prefix. +// Here keys share bytes 0-5 (prefix_len=6 at depth 0), which is <= prefix_cap, +// so chain_consumed=0. Instead, verify the prefix is handled correctly. +UNODB_TEST(BulkLoad, SharedPrefix) { + u64_db db; + std::vector> kv; + kv.reserve(20); + // All keys share byte 0 = 0x01, differ at byte 1. + for (std::uint64_t i = 0; i < 20; ++i) { + kv.emplace_back((1ULL << 56U) | (i << 48U), val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +// T12: Deep prefix chain — keys with long shared prefix that exceeds +// prefix_cap (7). This requires key_view (variable-length) keys. +// Keys share 10 prefix bytes, dispatch at byte 10. +UNODB_TEST(BulkLoad, DeepPrefixChain) { + using unodb::test::key_view_db; + key_view_db db; + + // Create keys that share 10 bytes then diverge at byte 10. + // Keys are 13 bytes to ensure sufficient depth beyond dispatch. + std::vector, value_view>> raw_keys; + raw_keys.reserve(10); + for (int i = 0; i < 10; ++i) { + std::vector k(13); + for (int j = 0; j < 10; ++j) { + k[static_cast(j)] = static_cast(j + 1); + } + k[10] = static_cast(i); + k[11] = std::byte{0xAA}; + k[12] = std::byte{0xBB}; + raw_keys.emplace_back(std::move(k), val); + } + // Already sorted since they share a prefix and last byte is 0..9 + std::vector> kv; + kv.reserve(raw_keys.size()); + for (auto& [k, v] : raw_keys) { + kv.emplace_back(unodb::key_view{k.data(), k.size()}, v); + } + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + } // namespace UNODB_DETAIL_RESTORE_MSVC_WARNINGS() diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp index a4bb2655..2c410f14 100644 --- a/test/test_art_bulk_load_ops.cpp +++ b/test/test_art_bulk_load_ops.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -17,7 +18,6 @@ #include "db_test_utils.hpp" #include "gtest_utils.hpp" #include "node_type.hpp" -#include "portability_execution.hpp" #include "qsbr.hpp" #include "qsbr_test_utils.hpp" @@ -59,6 +59,18 @@ UNODB_TEST(BulkLoadError, NonEmpty) { UNODB_ASSERT_TRUE(result.has_value()); } +// T26b: bulk_load on non-empty olc_db throws (covers olc_art.hpp error path). +UNODB_TEST(BulkLoadError, OlcNonEmpty) { + u64_olc_db db; + constexpr std::uint64_t key = 42; + ASSERT_TRUE(db.insert(key, val)); + unodb::this_thread().quiescent(); + std::vector> kv{{100, val}}; + EXPECT_THROW(db.bulk_load(kv.begin(), kv.end()), std::invalid_argument); + const auto result = db.get(key); + ASSERT_TRUE(result.has_value()); +} + // T36: db ignores parallelism parameter UNODB_TEST(BulkLoadError, DbIgnoresParallelism) { u64_db db; @@ -69,8 +81,12 @@ UNODB_TEST(BulkLoadError, DbIgnoresParallelism) { kv.emplace_back(key, val); } std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - // std::execution::par should enable parallel subtree construction - db.bulk_load(std::execution::par, kv.begin(), kv.end()); + // Parallel fork callable enables concurrent subtree construction + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); for (const auto& [k, v] : kv) { const auto result = db.get(k); ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; @@ -209,7 +225,11 @@ UNODB_TEST(BulkLoadOps, OlcDbParallel) { kv.emplace_back(i << 48U, val); } std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - db.bulk_load(std::execution::par, kv.begin(), kv.end()); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); for (const auto& [k, v] : kv) { const auto result = db.get(k); ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; @@ -228,7 +248,11 @@ UNODB_TEST(BulkLoadOps, OlcDbConcurrentReaders) { kv.emplace_back(i << 40U, val); } std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - db.bulk_load(std::execution::par, kv.begin(), kv.end()); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); // Pause main thread QSBR so reader threads can register unodb::this_thread().qsbr_pause(); @@ -252,6 +276,123 @@ UNODB_TEST(BulkLoadOps, OlcDbConcurrentReaders) { unodb::test::expect_idle_qsbr(); } +// ─── Coverage: parallel path with inline remainder +// ──────────────────────────── + +// T42: Parallel bulk_load with max_tasks < child_count exercises the inline +// results gathering path (some partitions forked, remainder runs inline). +UNODB_TEST(BulkLoadOps, ParallelInlineRemainder) { + u64_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + // max_tasks=4 with 100 partitions: 4 forked, 96 inline + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()) << "key " << k << " not found"; + } +} + +// T43: Parallel bulk_load with a single element exercises the n<=1 early +// return. +UNODB_TEST(BulkLoadOps, ParallelSingleElement) { + u64_db db; + std::vector> kv; + kv.emplace_back(42ULL << 56U, val); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + ASSERT_TRUE(db.get(42ULL << 56U).has_value()); +} + +// T44: Parallel bulk_load with keys sharing a long common prefix exercises the +// prefix chain construction (chain_consumed > 0 path). Uses key_view for +// keys longer than prefix_cap (7). +UNODB_TEST(BulkLoadOps, ParallelPrefixChain) { + using unodb::test::key_view_db; + key_view_db db; + // Create keys that share 10 bytes then diverge at byte 10, with 2 more bytes + // to ensure subtree depth doesn't hit key boundary. + std::vector> raw_keys; + raw_keys.reserve(20); + for (int i = 0; i < 20; ++i) { + std::vector k(13); + for (int j = 0; j < 10; ++j) { + k[static_cast(j)] = static_cast(j + 1); + } + k[10] = static_cast(i); + k[11] = std::byte{0xAA}; + k[12] = std::byte{0xBB}; + raw_keys.push_back(std::move(k)); + } + std::vector> kv; + kv.reserve(raw_keys.size()); + for (auto& k : raw_keys) { + kv.emplace_back(unodb::key_view{k.data(), k.size()}, val); + } + // Already sorted (shared prefix, byte 10 is 0..19) + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()); + } +} + +// T45: olc_db parallel with inline remainder (covers olc_art.hpp bulk_load +// entry and bulk_subtree_guard move constructor). +UNODB_TEST(BulkLoadOps, OlcDbParallelInline) { + u64_olc_db db; + std::vector> kv; + kv.reserve(50); + for (std::uint64_t i = 0; i < 50; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + // max_tasks=4 with 50 partitions: 4 forked, 46 inline + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +// T46: olc_db parallel with shared prefix (all keys share byte 0). +UNODB_TEST(BulkLoadOps, OlcDbParallelSharedPrefix) { + u64_olc_db db; + std::vector> kv; + kv.reserve(20); + // All keys share byte 0 = 0x01, differ at byte 1. + for (std::uint64_t i = 0; i < 20; ++i) { + kv.emplace_back((1ULL << 56U) | (i << 48U), val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + } // namespace UNODB_DETAIL_RESTORE_MSVC_WARNINGS() diff --git a/test/test_art_bulk_load_structural.cpp b/test/test_art_bulk_load_structural.cpp index 074c4d6a..3c8388b8 100644 --- a/test/test_art_bulk_load_structural.cpp +++ b/test/test_art_bulk_load_structural.cpp @@ -101,6 +101,7 @@ UNODB_TEST(BulkLoadStructural, BulkLoadPrefix16) { UNODB_TEST(BulkLoadStructural, BulkLoadVIS) { std::vector> s{{std::byte{1}}, {std::byte{2}}}; std::vector> kv; + kv.reserve(s.size()); for (auto& v : s) kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) key_view{v}, 42ULL); @@ -115,6 +116,7 @@ UNODB_TEST(BulkLoadStructural, BulkLoadVISWithChain) { std::vector> s{{std::byte{1}, std::byte{0xAA}}, {std::byte{1}, std::byte{0xBB}}}; std::vector> kv; + kv.reserve(s.size()); for (auto& v : s) kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) key_view{v}, 99ULL); @@ -131,6 +133,7 @@ UNODB_TEST(BulkLoadStructural, BulkLoadVISLongPrefix) { s[0][8] = std::byte{0}; s[1][8] = std::byte{1}; std::vector> kv; + kv.reserve(s.size()); for (auto& v : s) kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) key_view{v}, 77ULL); @@ -291,6 +294,66 @@ UNODB_TEST(BulkLoadStructural, BulkLoadScanOrder) { UNODB_ASSERT_EQ(count, 256U); } +// T27 — VIS with 5 keys → Node16 root (covers vmask loop in dispatch_node). +UNODB_TEST(BulkLoadStructural, BulkLoadVISNode16) { + std::vector> s; + s.reserve(8); + for (int i = 0; i < 8; ++i) { + s.push_back({static_cast(i)}); + } + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +// T28 — VIS with 20 keys → Node48 root. +UNODB_TEST(BulkLoadStructural, BulkLoadVISNode48) { + std::vector> s; + s.reserve(20); + for (int i = 0; i < 20; ++i) { + s.push_back({static_cast(i)}); + } + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +// T29 — VIS with 60 keys → Node256 root. +UNODB_TEST(BulkLoadStructural, BulkLoadVISNode256) { + std::vector> s; + s.reserve(60); + for (int i = 0; i < 60; ++i) { + s.push_back({static_cast(i)}); + } + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + } // namespace UNODB_DETAIL_RESTORE_MSVC_WARNINGS() From d928f95a030d81360b42b8b785112653e47958a6 Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Tue, 7 Jul 2026 17:07:52 +0000 Subject: [PATCH 18/23] fix: address cppcheck warnings on macOS CI - Pass value_mask by const ref in Node48/Node256 create_bulk (cppcheck passedByValue for 6-byte and 32-byte arrays) - Suppress functionStatic on olc_db::merge_bulk_load_stats (intentionally non-static for API consistency) --- art_internal_impl.hpp | 4 ++-- olc_art.hpp | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index 1b05860e..65159378 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -4557,7 +4557,7 @@ class basic_inode_48 db_type& db_instance, detail::key_prefix_size prefix_len, unodb::key_view prefix_key, tree_depth_type depth, std::span> children_span, - std::array value_mask = {}) { + const std::array& value_mask = {}) { const auto n = static_cast(children_span.size()); UNODB_DETAIL_ASSERT(n >= 17 && n <= 48); auto result = @@ -5044,7 +5044,7 @@ class basic_inode_256 db_type& db_instance, detail::key_prefix_size prefix_len, unodb::key_view prefix_key, tree_depth_type depth, std::span> children_span, - std::array value_mask = {}) { + const std::array& value_mask = {}) { const auto n = static_cast(children_span.size()); UNODB_DETAIL_ASSERT(n >= 49 && n <= 256); auto result = diff --git a/olc_art.hpp b/olc_art.hpp index 6c0d09e5..62c8ad93 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -1042,6 +1042,7 @@ class olc_db final { /// Merge accumulated stats from a parallel bulk_load subtree. /// For olc_db this is a no-op: atomic counters are updated directly. + // cppcheck-suppress functionStatic constexpr void merge_bulk_load_stats( const detail::bulk_load_stats_accumulator& /*acc*/) noexcept {} From ed24f28fdc8b73c6616093ae5c23db7929a7845c Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Tue, 7 Jul 2026 17:54:51 +0000 Subject: [PATCH 19/23] fix: inode48 bulk_load_tag ctor and cppcheck warnings - Remove child_indexes{} from inode48 bulk_load_tag constructor init list so the default member initializer (all 0xFF / empty_child) is used. The previous {} zero-initialized, leaving absent key slots as 0 instead of the empty_child sentinel. (CodeRabbit finding) - Pass value_mask by const ref in Node48/Node256 create_bulk (cppcheck passedByValue on macOS) - Suppress cppcheck functionStatic on olc_db::merge_bulk_load_stats (intentionally non-static for API consistency) --- art_internal_impl.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index 65159378..c7acd97a 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -3856,7 +3856,6 @@ class basic_inode_48 tree_depth_type depth, typename parent_class::bulk_load_tag tag) noexcept : parent_class{n_children, prefix_len, k1, depth, tag}, - child_indexes{}, children{} {} /// Initialize by growing from basic_inode_16 and adding a child. From c971952c6b39cf898941dfda2edb8e818067e329 Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Wed, 8 Jul 2026 11:30:34 +0000 Subject: [PATCH 20/23] fix: cppcheck useStlAlgorithm in benchmark, formatting - Replace raw loop with std::transform for random_keys (cppcheck useStlAlgorithm on macOS) - Suppress useStlAlgorithm for key_view_set index loops (no iterator) - Add for std::back_inserter - clang-format fix in inode48 bulk_load_tag constructor --- art_internal_impl.hpp | 3 +-- benchmark/micro_benchmark_bulk_load.cpp | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index c7acd97a..50144499 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -3855,8 +3855,7 @@ class basic_inode_48 // cppcheck-suppress passedByValue tree_depth_type depth, typename parent_class::bulk_load_tag tag) noexcept - : parent_class{n_children, prefix_len, k1, depth, tag}, - children{} {} + : parent_class{n_children, prefix_len, k1, depth, tag}, children{} {} /// Initialize by growing from basic_inode_16 and adding a child. /// diff --git a/benchmark/micro_benchmark_bulk_load.cpp b/benchmark/micro_benchmark_bulk_load.cpp index 8bcfbc82..67949e4c 100644 --- a/benchmark/micro_benchmark_bulk_load.cpp +++ b/benchmark/micro_benchmark_bulk_load.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -57,7 +58,8 @@ const auto value = unodb::value_view{val8}; keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); std::vector> kv; kv.reserve(keys.size()); - for (auto k : keys) kv.emplace_back(k, value); + std::transform(keys.begin(), keys.end(), std::back_inserter(kv), + [](auto k) { return std::make_pair(k, value); }); return kv; } @@ -139,6 +141,7 @@ void BM_BulkLoad_kv(benchmark::State& state) { auto keys = unodb::benchmark::key_view_set::compound(0x01, n); std::vector> kv; kv.reserve(n); + // cppcheck-suppress useStlAlgorithm for (std::size_t i = 0; i < n; ++i) kv.emplace_back(keys[i], value); for (const auto _ : state) { @@ -158,6 +161,7 @@ void BM_Insert_kv(benchmark::State& state) { auto keys = unodb::benchmark::key_view_set::compound(0x01, n); std::vector> kv; kv.reserve(n); + // cppcheck-suppress useStlAlgorithm for (std::size_t i = 0; i < n; ++i) kv.emplace_back(keys[i], value); for (const auto _ : state) { From 361378e27080e8cf9a534670ad044519fcd7fe4f Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Thu, 9 Jul 2026 16:36:13 +0000 Subject: [PATCH 21/23] refactor: shared bulk_build_result, bulk_subtree_guard, bulk_build_chain Move duplicated bulk_load infrastructure from art.hpp and olc_art.hpp into shared ArtPolicy-parameterized templates in art_internal_impl.hpp: - bulk_build_result: tagged return type (ptr + is_packed) - bulk_subtree_guard: RAII guard with policy-aware cleanup - bulk_build_chain(): prefix chain builder (was build_chain) db::build_chain and olc_db::build_chain now delegate to the shared implementation. This eliminates ~80 lines of duplication. --- art.hpp | 112 ++++----------------------------- art_bulk_load_detail.hpp | 2 + art_internal_impl.hpp | 129 +++++++++++++++++++++++++++++++++++++++ olc_art.hpp | 106 ++++---------------------------- 4 files changed, 154 insertions(+), 195 deletions(-) diff --git a/art.hpp b/art.hpp index 0f5b48ac..773438ef 100644 --- a/art.hpp +++ b/art.hpp @@ -206,42 +206,13 @@ class db final { /// Tagged return from build_subtree: pointer + whether it's a VIS packed /// value (not a real heap pointer — must not be passed to delete_subtree). - struct build_result { - detail::node_ptr ptr{nullptr}; - bool is_packed_value{false}; - }; + using build_result = + detail::bulk_build_result>; /// RAII guard for subtrees during bulk construction. /// Skips deallocation for VIS packed values (not heap-allocated). - struct bulk_subtree_guard { - db_type& db_; - detail::node_ptr ptr{nullptr}; - bool is_packed_value{false}; - - constexpr explicit bulk_subtree_guard(db_type& db - UNODB_DETAIL_LIFETIMEBOUND) noexcept - : db_{db} {} - - ~bulk_subtree_guard() noexcept { - if (ptr != nullptr && !is_packed_value) { - art_policy::delete_subtree(ptr, db_); - } - } - - void release() noexcept { ptr = nullptr; } - - bulk_subtree_guard(const bulk_subtree_guard&) = delete; - // LCOV_EXCL_START — move ctor only called on vector reallocation - bulk_subtree_guard(bulk_subtree_guard&& other) noexcept - : db_{other.db_}, - ptr{other.ptr}, - is_packed_value{other.is_packed_value} { - other.ptr = nullptr; - } - // LCOV_EXCL_STOP - auto& operator=(const bulk_subtree_guard&) = delete; - auto& operator=(bulk_subtree_guard&&) = delete; - }; + using bulk_subtree_guard = + detail::bulk_subtree_guard>; /// Extract byte at \a depth from a key. Handles big-endian uint64_t /// (via art_key_type bswap) and key_view (direct byte access). @@ -1091,6 +1062,13 @@ class db final { template friend void detail::bulk_load_impl(Db2&, Fork2&&, std::size_t, It2, It2); + /// detail::bulk_build_chain + template + friend typename ArtPolicy2::node_ptr detail::bulk_build_chain( + typename ArtPolicy2::db_type&, typename ArtPolicy2::art_key_type, + typename ArtPolicy2::node_ptr, + detail::tree_depth); + /// detail::basic_db_leaf_deleter template friend class detail::basic_db_leaf_deleter; @@ -1741,73 +1719,7 @@ template detail::node_ptr db::build_chain(art_key_type k, detail::node_ptr child, tree_depth_type start_depth) { - constexpr std::size_t cap = detail::key_prefix_capacity; - const auto full_key = k.get_key_view(); - const auto key_len = k.size(); - const auto start = static_cast(start_depth); - auto current = child; - bool child_is_value = art_policy::can_eliminate_leaf; - bool owns_current = - false; // set true once we've built at least one chain node - // Build bottom-up: start from end of key, work toward start_depth. - // Each chain I4 consumes up to cap prefix bytes + 1 dispatch byte. - try { - std::size_t pos = key_len; - while (pos > start + cap) { - const auto depth = pos - cap - 1; - const auto dispatch = full_key[pos - 1]; - auto remaining = k; - remaining.shift_right(depth); - auto chain{ - inode_4::create(*this, full_key, remaining, - tree_depth_type{static_cast(depth)}, - dispatch, current)}; - if (child_is_value) { - chain->set_value_bit(0); - child_is_value = false; - } - current = detail::node_ptr{chain.release(), node_type::I4}; - owns_current = true; -#ifdef UNODB_DETAIL_WITH_STATS - account_growing_inode(); -#endif - pos = depth; - } - // Tail: remaining bytes from start_depth to pos. - if (pos > start) { - const auto dispatch = full_key[pos - 1]; - auto chain{inode_4::create( - *this, full_key, tree_depth_type{static_cast(start)}, - static_cast(pos - start - 1), dispatch, - current)}; - if (child_is_value) { - chain->set_value_bit(0); - } - current = detail::node_ptr{chain.release(), node_type::I4}; - owns_current = true; -#ifdef UNODB_DETAIL_WITH_STATS - account_growing_inode(); -#endif - } - } catch (...) { - // On failure, free whatever current points to. If owns_current is - // true, current is a partial chain (I4 tree). If false, current is - // the original child — a leaf node_ptr on the leaf path, or a packed - // value on the VIS path. delete_subtree handles both I4 and LEAF. - // For VIS packed values (!child_is_value is false only after the - // first chain node is built), the caller's packed bits are not a - // real pointer — but we only reach here with a packed value if - // owns_current is false AND child_is_value is true, meaning no - // chain was built and the packed value was never wrapped. In that - // case we must NOT call delete_subtree on packed bits. - if (owns_current) { - art_policy::delete_subtree(current, *this); - } else if (!child_is_value) { - art_policy::delete_subtree(current, *this); - } - throw; - } - return current; + return detail::bulk_build_chain(*this, k, child, start_depth); } UNODB_DETAIL_RESTORE_GCC_WARNINGS() UNODB_DETAIL_RESTORE_GCC_WARNINGS() diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index 94fabefb..16ccc1ee 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -80,9 +80,11 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, ++cur; } } + UNODB_DETAIL_DISABLE_GCC_WARNING("-Wpessimizing-move") UNODB_DETAIL_DISABLE_MSVC_WARNING(26479) return std::move(parts); // NOLINT(performance-move-const-arg) UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_DETAIL_RESTORE_GCC_WARNINGS() }; auto build_prefix_chain = [&self](art_key_type k, node_ptr_t child_inode, diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index 50144499..19be7245 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -158,6 +158,66 @@ struct bulk_child { NodePtr child; }; +/// Tagged return from build_subtree: pointer + whether it's a VIS packed +/// value (not a real heap pointer — must not be passed to delete_subtree). +template +struct bulk_build_result { + typename ArtPolicy::node_ptr ptr{nullptr}; + bool is_packed_value{false}; +}; + +/// RAII guard for subtrees during bulk construction. +/// Skips deallocation for VIS packed values (not heap-allocated). +template +struct bulk_subtree_guard { + using db_type = typename ArtPolicy::db_type; + using node_ptr = typename ArtPolicy::node_ptr; + + db_type& db_; + node_ptr ptr{nullptr}; + bool is_packed_value{false}; + + constexpr explicit bulk_subtree_guard(db_type& db + UNODB_DETAIL_LIFETIMEBOUND) noexcept + : db_{db} {} + + ~bulk_subtree_guard() noexcept { + if (ptr != nullptr && !is_packed_value) { + ArtPolicy::delete_subtree(ptr, db_); + } + } + + void release() noexcept { ptr = nullptr; } + + bulk_subtree_guard(const bulk_subtree_guard&) = delete; + // LCOV_EXCL_START — move ctor only called on vector reallocation + bulk_subtree_guard(bulk_subtree_guard&& other) noexcept + : db_{other.db_}, ptr{other.ptr}, is_packed_value{other.is_packed_value} { + other.ptr = nullptr; + } + // LCOV_EXCL_STOP + auto& operator=(const bulk_subtree_guard&) = delete; + auto& operator=(bulk_subtree_guard&&) = delete; +}; + +/// Build a chain of inode4 nodes for prefix compression during bulk load. +/// +/// Constructs bottom-up from key end toward start_depth. Each chain node +/// consumes up to key_prefix_capacity prefix bytes + 1 dispatch byte. +/// +/// \tparam ArtPolicy Policy type (provides node_ptr, db_type, inode types) +/// \param self Database instance +/// \param k Encoded key for prefix extraction +/// \param child Child pointer to wrap in the chain +/// \param start_depth Depth at which the chain terminates +/// \return The top node pointer of the chain +/// \note Defined at end of art_internal_impl.hpp (after inode types). +template +[[nodiscard]] typename ArtPolicy::node_ptr bulk_build_chain( + typename ArtPolicy::db_type& self, typename ArtPolicy::art_key_type k, + typename ArtPolicy::node_ptr child, + tree_depth start_depth); + #ifdef UNODB_DETAIL_X86_64 /// Compare packed unsigned 8-bit integers for less-than-or-equal. @@ -5066,6 +5126,75 @@ class basic_inode_256 UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() }; // class basic_inode_256 +// ─── Bulk Load: build_chain ───────────────────────────────────────────────── + +UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) +UNODB_DETAIL_DISABLE_MSVC_WARNING(4459) + +template +[[nodiscard]] typename ArtPolicy::node_ptr bulk_build_chain( + typename ArtPolicy::db_type& self, typename ArtPolicy::art_key_type k, + typename ArtPolicy::node_ptr child, + tree_depth start_depth) { + using node_ptr = typename ArtPolicy::node_ptr; + using inode_4 = typename ArtPolicy::inode4_type; + using tree_depth_type = tree_depth; + constexpr std::size_t cap = key_prefix_capacity; + const auto full_key = k.get_key_view(); + const auto key_len = k.size(); + const auto start = static_cast(start_depth); + auto current = child; + bool child_is_value = ArtPolicy::can_eliminate_leaf; + bool owns_current = false; + try { + std::size_t pos = key_len; + while (pos > start + cap) { + const auto depth = pos - cap - 1; + const auto dispatch = full_key[pos - 1]; + auto remaining = k; + remaining.shift_right(depth); + auto chain{ + inode_4::create(self, full_key, remaining, + tree_depth_type{static_cast(depth)}, + dispatch, current)}; + if (child_is_value) { + chain->set_value_bit(0); + child_is_value = false; + } + current = node_ptr{chain.release(), node_type::I4}; + owns_current = true; +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + pos = depth; + } + if (pos > start) { + const auto dispatch = full_key[pos - 1]; + auto chain{inode_4::create( + self, full_key, tree_depth_type{static_cast(start)}, + static_cast(pos - start - 1), dispatch, current)}; + if (child_is_value) { + chain->set_value_bit(0); + } + current = node_ptr{chain.release(), node_type::I4}; +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + } + return current; + } catch (...) { + if (owns_current) { + ArtPolicy::delete_subtree(current, self); + } else if (!child_is_value) { + ArtPolicy::delete_subtree(current, self); + } + throw; + } +} + +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + } // namespace unodb::detail #endif // UNODB_DETAIL_ART_INTERNAL_IMPL_HPP diff --git a/olc_art.hpp b/olc_art.hpp index 62c8ad93..a12d9a39 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -880,41 +880,11 @@ class olc_db final { detail::olc_db_leaf_unique_ptr; /// Result of subtree construction during bulk_load. - struct build_result { - detail::olc_node_ptr ptr{nullptr}; - bool is_packed_value{false}; - }; + /// Tagged return from build_subtree. + using build_result = detail::bulk_build_result; /// RAII guard for subtrees during bulk construction. - struct bulk_subtree_guard { - olc_db& db_; - detail::olc_node_ptr ptr{nullptr}; - bool is_packed_value{false}; - - constexpr explicit bulk_subtree_guard(olc_db& db - UNODB_DETAIL_LIFETIMEBOUND) noexcept - : db_{db} {} - - ~bulk_subtree_guard() noexcept { - if (ptr != nullptr && !is_packed_value) { - art_policy::delete_subtree(ptr, db_); - } - } - - void release() noexcept { ptr = nullptr; } - - bulk_subtree_guard(const bulk_subtree_guard&) = delete; - // LCOV_EXCL_START — move ctor only called on vector reallocation - bulk_subtree_guard(bulk_subtree_guard&& other) noexcept - : db_{other.db_}, - ptr{other.ptr}, - is_packed_value{other.is_packed_value} { - other.ptr = nullptr; - } - // LCOV_EXCL_STOP - auto& operator=(const bulk_subtree_guard&) = delete; - auto& operator=(bulk_subtree_guard&&) = delete; - }; + using bulk_subtree_guard = detail::bulk_subtree_guard; // If get_result is not present, the search was interrupted. Yes, this // resolves to std::optional>, but IMHO both @@ -1095,6 +1065,13 @@ class olc_db final { template friend void detail::bulk_load_impl(Db2&, Fork2&&, std::size_t, It2, It2); + /// detail::bulk_build_chain + template + friend typename ArtPolicy2::node_ptr detail::bulk_build_chain( + typename ArtPolicy2::db_type&, typename ArtPolicy2::art_key_type, + typename ArtPolicy2::node_ptr, + detail::tree_depth); + template friend class detail::basic_db_leaf_deleter; @@ -2382,68 +2359,7 @@ bool olc_db::insert_internal(art_key_type insert_key, template detail::olc_node_ptr olc_db::build_chain( art_key_type k, detail::olc_node_ptr child, tree_depth_type start_depth) { - constexpr std::size_t cap = detail::key_prefix_capacity; - const auto full_key = k.get_key_view(); - const auto key_len = k.size(); - const auto start = static_cast(start_depth); - auto current = child; - bool child_is_value = art_policy::can_eliminate_leaf; - bool owns_current = - false; // set true once we've built at least one chain node - try { - std::size_t pos = key_len; - while (pos > start + cap) { - const auto depth = pos - cap - 1; - const auto dispatch = full_key[pos - 1]; - auto remaining = k; - remaining.shift_right(depth); - auto chain{ - inode_4::create(*this, full_key, remaining, - tree_depth_type{static_cast(depth)}, - dispatch, current)}; - if (child_is_value) { - UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) - chain->set_value_bit(0); - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - child_is_value = false; - } - UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) - current = detail::olc_node_ptr{chain.release(), node_type::I4}; - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - owns_current = true; -#ifdef UNODB_DETAIL_WITH_STATS - account_growing_inode(); -#endif - pos = depth; - } - if (pos > start) { - const auto dispatch = full_key[pos - 1]; - auto chain{inode_4::create( - *this, full_key, tree_depth_type{static_cast(start)}, - static_cast(pos - start - 1), dispatch, - current)}; - if (child_is_value) { - UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) - chain->set_value_bit(0); - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - } - UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) - current = detail::olc_node_ptr{chain.release(), node_type::I4}; - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - owns_current = true; -#ifdef UNODB_DETAIL_WITH_STATS - account_growing_inode(); -#endif - } - } catch (...) { - if (owns_current) { - art_policy::delete_subtree(current, *this); - } else if (!child_is_value) { - art_policy::delete_subtree(current, *this); - } - throw; - } - return current; + return detail::bulk_build_chain(*this, k, child, start_depth); } template From 171f0f4a8604f8075eab2a22fcda3da8bff87ff2 Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Sun, 12 Jul 2026 22:52:49 +0000 Subject: [PATCH 22/23] refactor: address laurynas review feedback - Pull all lambdas to bulk_load_helpers struct with named static methods (common_prefix_length, partition_by_byte, build_prefix_chain, build_single_leaf, make_inode, build_subtree) - partition_entry struct at namespace scope - Use std::exchange in bulk_subtree_guard move ctor - Add ptr_type alias to bulk_build_result for cleaner signatures - Make art_key_type, tree_depth_type, build_result, bulk_subtree_guard public in db and olc_db (needed for struct method signatures) - Pass vector by const& in bulk_load_oom_test - Replace mutex_db with olc_db in example - Clarify LCOV_EXCL with UNODB_DETAIL_ASSERT(pos == start) - Comment on inner catch: future.get() may rethrow task exception - NRVO: return parts directly (explicit return type enables it) --- art.hpp | 21 ++- art_bulk_load_detail.hpp | 269 +++++++++++++++++---------------- art_internal_impl.hpp | 9 +- examples/example_bulk_load.cpp | 9 +- olc_art.hpp | 23 ++- test/test_art_oom.cpp | 3 +- 6 files changed, 183 insertions(+), 151 deletions(-) diff --git a/art.hpp b/art.hpp index 773438ef..03ddedbe 100644 --- a/art.hpp +++ b/art.hpp @@ -113,11 +113,14 @@ using leaf_type = basic_leaf, node_header>; template class inode : public inode_base {}; -/// Forward declaration of shared bulk_load algorithm. +/// Forward declarations of bulk_load algorithm and helpers. template void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, RandomIt first, RandomIt last); +template +struct bulk_load_helpers; + } // namespace detail template @@ -146,10 +149,10 @@ class db final { /// Base class type for internal nodes. using inode_base = detail::inode_base; - private: /// Internal encoded key type used for tree operations. using art_key_type = detail::basic_art_key; + private: /// Leaf node type (keyless when can_eliminate_key_in_leaf). using leaf_type = detail::leaf_type; @@ -202,8 +205,9 @@ class db final { /// tree and the associated index entry was removed). [[nodiscard]] bool remove_internal(art_key_type remove_key); - // ─── Bulk Load Infrastructure ───────────────────────────────────────── + // ─── Bulk Load Infrastructure (public for detail:: free functions) ───── + public: /// Tagged return from build_subtree: pointer + whether it's a VIS packed /// value (not a real heap pointer — must not be passed to delete_subtree). using build_result = @@ -214,6 +218,10 @@ class db final { using bulk_subtree_guard = detail::bulk_subtree_guard>; + /// Tree depth tracking type. + using tree_depth_type = detail::tree_depth; + + private: /// Extract byte at \a depth from a key. Handles big-endian uint64_t /// (via art_key_type bswap) and key_view (direct byte access). [[nodiscard]] static constexpr std::byte key_byte_at( @@ -934,9 +942,6 @@ class db final { /// Node type with 256 children. using inode_256 = detail::inode_256; - /// Tree depth tracking type. - using tree_depth_type = detail::tree_depth; - /// Visitor type for scan operations. using visitor_type = visitor; @@ -1062,6 +1067,10 @@ class db final { template friend void detail::bulk_load_impl(Db2&, Fork2&&, std::size_t, It2, It2); + /// detail::bulk_load_helpers + template + friend struct detail::bulk_load_helpers; + /// detail::bulk_build_chain template friend typename ArtPolicy2::node_ptr detail::bulk_build_chain( diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp index 16ccc1ee..2254e284 100644 --- a/art_bulk_load_detail.hpp +++ b/art_bulk_load_detail.hpp @@ -20,15 +20,21 @@ UNODB_DETAIL_DISABLE_MSVC_WARNING(26496) /// \cond UNODB_DETAIL_INTERNAL namespace unodb::detail { -/// Core bulk_load algorithm parameterized on database type. -/// -/// \pre Tree must be empty (caller validates). -/// \pre [first, last) is sorted and non-empty (caller validates). -/// \tparam Fork Callable satisfying fork(Callable) -> Future where -/// Future has .get(). Ignored (nullptr_t) for sequential. -template -void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, - RandomIt first, RandomIt last) { +// ─── Helper types ────────────────────────────────────────────────────────── + +/// Entry produced by partition_by_byte: the start iterator and dispatch byte. +template +struct partition_entry { + RandomIt begin; + std::byte key_byte; +}; + +// ─── Bulk load helper struct ─────────────────────────────────────────────── + +/// Named helper functions for bulk_load, collected in a struct to simplify +/// friend declarations and avoid forward-declaration complexity. +template +struct bulk_load_helpers { using art_key_type = typename Db::art_key_type; using art_policy = typename Db::art_policy; using tree_depth_type = typename Db::tree_depth_type; @@ -38,12 +44,12 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, using inode_256 = typename Db::inode_256; using build_result_t = typename Db::build_result; using guard_t = typename Db::bulk_subtree_guard; - using node_ptr_t = decltype(build_result_t{}.ptr); + using node_ptr_t = typename build_result_t::ptr_type; using bulk_child_t = bulk_child; - constexpr std::size_t prefix_cap = key_prefix_capacity; - auto common_prefix_length = [](RandomIt f, RandomIt l, - tree_depth_type depth) -> std::size_t { + /// Compute the length of the common prefix between first and last keys. + [[nodiscard]] static std::size_t common_prefix_length(RandomIt f, RandomIt l, + tree_depth_type depth) { const art_key_type first_ak{f->first}; const art_key_type last_ak{std::prev(l)->first}; const auto fk = first_ak.get_key_view(); @@ -53,16 +59,13 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, std::size_t len = 0; while (d + len < max_len && fk[d + len] == lk[d + len]) ++len; return len; - }; - - struct partition_entry { - RandomIt begin; - std::byte key_byte; - }; + } - auto partition_by_byte = [](RandomIt f, RandomIt l, - tree_depth_type dispatch_depth) { - boost::container::small_vector parts; + /// Partition sorted range [f, l) by the key byte at dispatch_depth. + [[nodiscard]] static boost::container::small_vector, + 16> + partition_by_byte(RandomIt f, RandomIt l, tree_depth_type dispatch_depth) { + boost::container::small_vector, 16> parts; const auto dd = static_cast(dispatch_depth); auto cur = f; while (cur != l) { @@ -80,16 +83,15 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, ++cur; } } - UNODB_DETAIL_DISABLE_GCC_WARNING("-Wpessimizing-move") - UNODB_DETAIL_DISABLE_MSVC_WARNING(26479) - return std::move(parts); // NOLINT(performance-move-const-arg) - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - UNODB_DETAIL_RESTORE_GCC_WARNINGS() - }; - - auto build_prefix_chain = [&self](art_key_type k, node_ptr_t child_inode, - tree_depth_type start_depth, - std::size_t end_depth) -> node_ptr_t { + return parts; + } + + /// Build a chain of I4 nodes encoding the prefix from start_depth to + /// end_depth, with child_inode at the bottom. + [[nodiscard]] static node_ptr_t build_prefix_chain( + Db& self, art_key_type k, node_ptr_t child_inode, + tree_depth_type start_depth, std::size_t end_depth) { + constexpr std::size_t prefix_cap = key_prefix_capacity; const auto full_key = k.get_key_view(); const auto start = static_cast(start_depth); auto current = child_inode; @@ -109,8 +111,9 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, #endif pos = depth; } - // LCOV_EXCL_START — chain_consumed is always a multiple of (prefix_cap+1), - // so the while loop always decrements pos to exactly start. + // LCOV_EXCL_START — always-false: chain_consumed is always a multiple of + // (prefix_cap+1), so the while loop always decrements pos to exactly start. + UNODB_DETAIL_ASSERT(pos == start); if (pos > start) { const auto dispatch = full_key[pos - 1]; auto chain{inode_4::create( @@ -123,10 +126,11 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, } // LCOV_EXCL_STOP return current; - }; + } - auto build_single_leaf = [&self](RandomIt it, - tree_depth_type depth) -> build_result_t { + /// Build a single leaf (or packed value) from the iterator at given depth. + [[nodiscard]] static build_result_t build_single_leaf(Db& self, RandomIt it, + tree_depth_type depth) { const art_key_type ak{it->first}; if constexpr (art_policy::can_eliminate_leaf) { const auto packed = art_policy::pack_value(it->second); @@ -147,16 +151,15 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, const auto leaf_ptr = node_ptr_t{leaf.release(), node_type::LEAF}; return {leaf_ptr, false}; } - }; - - // ─── Shared inode factory ──────────────────────────────────────────── + } - auto make_bulk_inode = - [&self](std::span cs, - const boost::container::small_vector& guards, - const boost::container::small_vector& children, - key_prefix_size inode_prefix_len, key_view prefix_kv, - tree_depth_type inode_depth) -> node_ptr_t { + /// Assemble an internal node of the appropriate fan-out. + [[nodiscard]] static node_ptr_t make_inode( + Db& self, std::span cs, + const boost::container::small_vector& guards, + const boost::container::small_vector& children, + key_prefix_size inode_prefix_len, key_view prefix_kv, + tree_depth_type inode_depth) { if constexpr (!art_policy::can_eliminate_leaf) { static_cast(guards); static_cast(children); @@ -223,101 +226,109 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, self.template account_growing_inode(); #endif return node_ptr_t{ptr.release(), node_type::I256}; - }; - - // ─── Recursive subtree builder ───────────────────────────────────────── - - struct subtree_builder { - Db& self; - decltype(common_prefix_length)& cpl; - decltype(partition_by_byte)& pbb; - decltype(build_prefix_chain)& bpc; - decltype(build_single_leaf)& bsl; - decltype(make_bulk_inode)& mbi; - - build_result_t operator()(RandomIt f, RandomIt l, - tree_depth_type depth) const { - const auto n = std::distance(f, l); - if (n == 0) return {node_ptr_t{nullptr}, false}; - if (n == 1) return bsl(f, depth); - - const auto prefix_len = cpl(f, l, depth); - const auto dispatch_depth = tree_depth_type{static_cast( - static_cast(depth) + prefix_len)}; - auto parts = pbb(f, l, dispatch_depth); - const auto child_count = parts.size(); - - boost::container::small_vector guards; - guards.reserve(child_count); - boost::container::small_vector children; - children.reserve(child_count); - - const auto next_depth = tree_depth_type{static_cast( - static_cast(dispatch_depth) + 1)}; + } - for (std::size_t i = 0; i < child_count; ++i) { - const auto part_begin = parts[i].begin; - const auto part_end = (i + 1 < child_count) ? parts[i + 1].begin : l; - auto result = (*this)(part_begin, part_end, next_depth); - guards.emplace_back(self); - guards.back().ptr = result.ptr; - guards.back().is_packed_value = result.is_packed_value; - children.push_back({parts[i].key_byte, result.ptr}); - } + /// Recursively build a subtree from a sorted range of key-value pairs. + [[nodiscard]] static build_result_t build_subtree(Db& self, RandomIt f, + RandomIt l, + tree_depth_type depth) { + constexpr std::size_t prefix_cap = key_prefix_capacity; - const std::size_t chain_consumed = - (prefix_len > prefix_cap) - ? (prefix_len / (prefix_cap + 1)) * (prefix_cap + 1) - : 0; - const auto inode_prefix_len = - static_cast(prefix_len - chain_consumed); - const auto inode_depth = tree_depth_type{static_cast( - static_cast(depth) + chain_consumed)}; - - const art_key_type prefix_key{f->first}; - const auto prefix_kv = prefix_key.get_key_view(); - - const auto cs = - std::span{children.data(), children.size()}; - auto inode_ptr = - mbi(cs, guards, children, inode_prefix_len, prefix_kv, inode_depth); - for (auto& g : guards) g.release(); - - if (chain_consumed > 0) { - return {bpc(prefix_key, inode_ptr, depth, - chain_consumed + static_cast(depth)), - false}; - } - return {inode_ptr, false}; + const auto n = std::distance(f, l); + if (n == 0) return {node_ptr_t{nullptr}, false}; + if (n == 1) return build_single_leaf(self, f, depth); + + const auto prefix_len = common_prefix_length(f, l, depth); + const auto dispatch_depth = tree_depth_type{static_cast( + static_cast(depth) + prefix_len)}; + auto parts = partition_by_byte(f, l, dispatch_depth); + const auto child_count = parts.size(); + + boost::container::small_vector guards; + guards.reserve(child_count); + boost::container::small_vector children; + children.reserve(child_count); + + const auto next_depth = tree_depth_type{static_cast( + static_cast(dispatch_depth) + 1)}; + + for (std::size_t i = 0; i < child_count; ++i) { + const auto part_begin = parts[i].begin; + const auto part_end = (i + 1 < child_count) ? parts[i + 1].begin : l; + auto result = build_subtree(self, part_begin, part_end, next_depth); + guards.emplace_back(self); + guards.back().ptr = result.ptr; + guards.back().is_packed_value = result.is_packed_value; + children.push_back({parts[i].key_byte, result.ptr}); + } + + const std::size_t chain_consumed = + (prefix_len > prefix_cap) + ? (prefix_len / (prefix_cap + 1)) * (prefix_cap + 1) + : 0; + const auto inode_prefix_len = + static_cast(prefix_len - chain_consumed); + const auto inode_depth = tree_depth_type{static_cast( + static_cast(depth) + chain_consumed)}; + + const art_key_type prefix_key{f->first}; + const auto prefix_kv = prefix_key.get_key_view(); + + const auto cs = + std::span{children.data(), children.size()}; + auto inode_ptr = make_inode(self, cs, guards, children, inode_prefix_len, + prefix_kv, inode_depth); + for (auto& g : guards) g.release(); + + if (chain_consumed > 0) { + return { + build_prefix_chain(self, prefix_key, inode_ptr, depth, + chain_consumed + static_cast(depth)), + false}; } - }; + return {inode_ptr, false}; + } +}; - subtree_builder builder{self, - common_prefix_length, - partition_by_byte, - build_prefix_chain, - build_single_leaf, - make_bulk_inode}; +// ─── Top-level bulk_load_impl ──────────────────────────────────────────────── + +/// Core bulk_load algorithm parameterized on database type. +/// +/// \pre Tree must be empty (caller validates). +/// \pre [first, last) is sorted and non-empty (caller validates). +/// \tparam Fork Callable satisfying fork(Callable) -> Future where +/// Future has .get(). Ignored (nullptr_t) for sequential. +template +void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last) { + using H = bulk_load_helpers; + using build_result_t = typename Db::build_result; + using guard_t = typename Db::bulk_subtree_guard; + using node_ptr_t = typename build_result_t::ptr_type; + using bulk_child_t = bulk_child; + using tree_depth_type = typename Db::tree_depth_type; + using art_key_type = typename Db::art_key_type; + constexpr std::size_t prefix_cap = key_prefix_capacity; constexpr bool is_parallel = !std::is_same_v, std::nullptr_t>; if constexpr (!is_parallel) { - auto result = builder(first, last, tree_depth_type{0}); + auto result = H::build_subtree(self, first, last, tree_depth_type{0}); self.root = result.ptr; } else { const auto n = std::distance(first, last); if (n <= 1) { - auto result = builder(first, last, tree_depth_type{0}); + auto result = H::build_subtree(self, first, last, tree_depth_type{0}); self.root = result.ptr; return; } const auto prefix_len = - common_prefix_length(first, last, tree_depth_type{0}); + H::common_prefix_length(first, last, tree_depth_type{0}); const auto dispatch_depth = tree_depth_type{static_cast(prefix_len)}; - auto parts = partition_by_byte(first, last, dispatch_depth); + auto parts = H::partition_by_byte(first, last, dispatch_depth); const auto child_count = parts.size(); const auto next_depth = tree_depth_type{static_cast( @@ -335,18 +346,18 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, using parallel_result = build_result_t; #endif - auto make_task = [&builder, &parts, child_count, last, + auto make_task = [&self, &parts, child_count, last, next_depth](std::size_t i) { const auto part_begin = parts[i].begin; const auto part_end = (i + 1 < child_count) ? parts[i + 1].begin : last; #ifdef UNODB_DETAIL_WITH_STATS bulk_load_stats_accumulator acc; tls_bulk_load_stats = &acc; - auto result = builder(part_begin, part_end, next_depth); + auto result = H::build_subtree(self, part_begin, part_end, next_depth); tls_bulk_load_stats = nullptr; return parallel_result{result, acc}; #else - return builder(part_begin, part_end, next_depth); + return H::build_subtree(self, part_begin, part_end, next_depth); #endif }; @@ -401,7 +412,7 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, guards.back().ptr = r.ptr; guards.back().is_packed_value = r.is_packed_value; #endif - } catch (...) { + } catch (...) { // future.get() may rethrow task exception } } throw; @@ -438,13 +449,13 @@ void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, const auto cs = std::span{children.data(), children.size()}; - auto inode_ptr = make_bulk_inode(cs, guards, children, inode_prefix_len, - prefix_kv, inode_depth); + auto inode_ptr = H::make_inode(self, cs, guards, children, inode_prefix_len, + prefix_kv, inode_depth); for (auto& g : guards) g.release(); if (chain_consumed > 0) { - self.root = build_prefix_chain(prefix_key, inode_ptr, tree_depth_type{0}, - chain_consumed); + self.root = H::build_prefix_chain(self, prefix_key, inode_ptr, + tree_depth_type{0}, chain_consumed); } else { self.root = inode_ptr; } diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index 19be7245..30c3b61e 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -162,7 +162,8 @@ struct bulk_child { /// value (not a real heap pointer — must not be passed to delete_subtree). template struct bulk_build_result { - typename ArtPolicy::node_ptr ptr{nullptr}; + using ptr_type = typename ArtPolicy::node_ptr; + ptr_type ptr{nullptr}; bool is_packed_value{false}; }; @@ -192,9 +193,9 @@ struct bulk_subtree_guard { bulk_subtree_guard(const bulk_subtree_guard&) = delete; // LCOV_EXCL_START — move ctor only called on vector reallocation bulk_subtree_guard(bulk_subtree_guard&& other) noexcept - : db_{other.db_}, ptr{other.ptr}, is_packed_value{other.is_packed_value} { - other.ptr = nullptr; - } + : db_{other.db_}, + ptr{std::exchange(other.ptr, nullptr)}, + is_packed_value{other.is_packed_value} {} // LCOV_EXCL_STOP auto& operator=(const bulk_subtree_guard&) = delete; auto& operator=(bulk_subtree_guard&&) = delete; diff --git a/examples/example_bulk_load.cpp b/examples/example_bulk_load.cpp index 29f75505..babd5aca 100644 --- a/examples/example_bulk_load.cpp +++ b/examples/example_bulk_load.cpp @@ -15,7 +15,7 @@ #include "art.hpp" #include "art_common.hpp" -#include "mutex_art.hpp" +#include "olc_art.hpp" namespace { @@ -63,12 +63,11 @@ int main() { tree.clear(); } - // ─── mutex_db: same API ──────────────────────────────────────────────────── + // ─── olc_db: same API ────────────────────────────────────────────────────── { - unodb::mutex_db tree; + unodb::olc_db tree; tree.bulk_load(async_fork, 8, data.begin(), data.end()); - std::cerr << "mutex_db parallel bulk_load: " << key_count - << " keys loaded\n"; + std::cerr << "olc_db parallel bulk_load: " << key_count << " keys loaded\n"; tree.clear(); } diff --git a/olc_art.hpp b/olc_art.hpp index a12d9a39..b5ff1a92 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -167,11 +167,14 @@ template using olc_leaf_unique_ptr = basic_db_leaf_unique_ptr; -/// Forward declaration of shared bulk_load algorithm. +/// Forward declarations of bulk_load algorithm and helpers. template void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, RandomIt first, RandomIt last); +template +struct bulk_load_helpers; + } // namespace detail /// A thread-safe Adaptive Radix Tree that is synchronized using optimistic lock @@ -192,9 +195,10 @@ class olc_db final { using leaf_type = detail::olc_leaf_type; using db_type = olc_db; - private: + /// Internal encoded key type used for tree operations. using art_key_type = detail::basic_art_key; + private: /// Query for a value associated with an encoded key. [[nodiscard, gnu::pure]] get_result get_internal( art_key_type search_key) const noexcept; @@ -874,10 +878,10 @@ class olc_db final { using inode_16 = detail::olc_inode_16; using inode_48 = detail::olc_inode_48; using inode_256 = detail::olc_inode_256; + + public: + /// Tree depth tracking type. using tree_depth_type = detail::tree_depth; - using visitor_type = visitor; - using olc_db_leaf_unique_ptr_type = - detail::olc_db_leaf_unique_ptr; /// Result of subtree construction during bulk_load. /// Tagged return from build_subtree. @@ -886,6 +890,11 @@ class olc_db final { /// RAII guard for subtrees during bulk construction. using bulk_subtree_guard = detail::bulk_subtree_guard; + private: + using visitor_type = visitor; + using olc_db_leaf_unique_ptr_type = + detail::olc_db_leaf_unique_ptr; + // If get_result is not present, the search was interrupted. Yes, this // resolves to std::optional>, but IMHO both // levels of std::optional are clear here @@ -1065,6 +1074,10 @@ class olc_db final { template friend void detail::bulk_load_impl(Db2&, Fork2&&, std::size_t, It2, It2); + /// detail::bulk_load_helpers + template + friend struct detail::bulk_load_helpers; + /// detail::bulk_build_chain template friend typename ArtPolicy2::node_ptr detail::bulk_build_chain( diff --git a/test/test_art_oom.cpp b/test/test_art_oom.cpp index efb75cdb..b2bc1636 100644 --- a/test/test_art_oom.cpp +++ b/test/test_art_oom.cpp @@ -653,8 +653,7 @@ UNODB_TYPED_TEST(ARTKeyViewOOMTest, BuildChainMultiNode) { template void bulk_load_oom_test( unsigned fail_limit, - std::vector> - kv) { // NOLINT(performance-unnecessary-value-param) + const std::vector>& kv) { unsigned fail_n; for (fail_n = 1; fail_n <= fail_limit; ++fail_n) { TypeParam test_db; From 2405da90690d13af552e1e07ed59c64d27133c4a Mon Sep 17 00:00:00 2001 From: Bryan Thompson Date: Mon, 20 Jul 2026 22:46:14 +0000 Subject: [PATCH 23/23] test: merge 3 bulk load test files into one Combines test_art_bulk_load.cpp, test_art_bulk_load_ops.cpp, and test_art_bulk_load_structural.cpp into a single test_art_bulk_load.cpp (45 tests). Stats-dependent tests are inside #ifdef UNODB_DETAIL_WITH_STATS; error, operational, and parallel tests are unconditional. --- test/CMakeLists.txt | 12 +- test/test_art_bulk_load.cpp | 678 ++++++++++++++++++++++++- test/test_art_bulk_load_ops.cpp | 400 --------------- test/test_art_bulk_load_structural.cpp | 361 ------------- 4 files changed, 656 insertions(+), 795 deletions(-) delete mode 100644 test/test_art_bulk_load_ops.cpp delete mode 100644 test/test_art_bulk_load_structural.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d8118d28..cf2b25c1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -60,9 +60,7 @@ add_db_test_target(test_art) target_compile_options(test_art PRIVATE "$<${is_msvc}:/bigobj>") add_db_test_target(test_art_bulk_load) -add_db_test_target(test_art_bulk_load_structural) -add_db_test_target(test_art_bulk_load_ops) -target_link_libraries(test_art_bulk_load_ops PRIVATE qsbr_test_utils) +target_link_libraries(test_art_bulk_load PRIVATE qsbr_test_utils) add_db_test_target(test_art_key_view) add_db_test_target(test_art_key_view_full_chain) @@ -104,7 +102,7 @@ add_test(NAME test_olc_no_qsbr COMMAND test_olc_no_qsbr) if(COVERAGE) add_custom_target(tests_for_coverage ctest DEPENDS test_key_encode_decode test_art test_art_key_view test_art_key_view_full_chain test_art_iter test_art_scan test_art_concurrency test_art_allocator test_art_upsert test_olc_no_qsbr test_qsbr_ptr test_qsbr test_art_oom - test_qsbr_oom test_art_bulk_load test_art_bulk_load_structural test_art_bulk_load_ops) + test_qsbr_oom test_art_bulk_load) add_coverage_target(TARGET coverage DEPENDENCY tests_for_coverage) endif() @@ -121,7 +119,5 @@ add_custom_target(valgrind_tests COMMAND ${VALGRIND_COMMAND} ./test_art_scan; COMMAND ${VALGRIND_COMMAND} ./test_art_concurrency; COMMAND ${VALGRIND_COMMAND} ./test_art_upsert; - COMMAND ${VALGRIND_COMMAND} ./test_art_bulk_load; - COMMAND ${VALGRIND_COMMAND} ./test_art_bulk_load_structural; - COMMAND ${VALGRIND_COMMAND} ./test_art_bulk_load_ops - DEPENDS test_qsbr_ptr test_qsbr test_key_encode_decode test_art test_art_key_view test_art_key_view_full_chain test_art_iter test_art_scan test_art_concurrency test_art_upsert test_art_bulk_load test_art_bulk_load_structural test_art_bulk_load_ops) + COMMAND ${VALGRIND_COMMAND} ./test_art_bulk_load + DEPENDS test_qsbr_ptr test_qsbr test_key_encode_decode test_art test_art_key_view test_art_key_view_full_chain test_art_iter test_art_scan test_art_concurrency test_art_upsert test_art_bulk_load) diff --git a/test/test_art_bulk_load.cpp b/test/test_art_bulk_load.cpp index d592b4f3..61a59293 100644 --- a/test/test_art_bulk_load.cpp +++ b/test/test_art_bulk_load.cpp @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include #include @@ -16,23 +19,35 @@ #include "db_test_utils.hpp" #include "gtest_utils.hpp" #include "node_type.hpp" +#include "qsbr.hpp" +#include "qsbr_test_utils.hpp" -#ifdef UNODB_DETAIL_WITH_STATS - +UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) +UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) UNODB_DETAIL_DISABLE_MSVC_WARNING(26445) namespace { +#ifdef UNODB_DETAIL_WITH_STATS using unodb::as_i; using unodb::node_type; +#endif +using unodb::key_view; using unodb::value_view; using unodb::test::u64_db; +using unodb::test::u64_mutex_db; +using unodb::test::u64_olc_db; constexpr auto val_bytes = std::array{std::byte{0x68}, std::byte{0x65}, std::byte{0x6C}, std::byte{0x6C}, std::byte{0x6F}}; constexpr auto val = value_view{val_bytes}; +#ifdef UNODB_DETAIL_WITH_STATS + +constexpr auto val16 = std::array{}; +constexpr auto large_val = value_view{val16}; + /// Generate N keys that differ at byte 0 (big-endian uint64_t). /// For N <= 256, all keys differ only at byte 0 (single root node). /// For N > 256, extra keys share byte 0 with key 0 but differ at byte 1. @@ -53,7 +68,8 @@ constexpr auto val = value_view{val_bytes}; return kv; } -// T01 +// ─── Node Count Tests ──────────────────────────────────────────────────────── + UNODB_TEST(BulkLoad, Empty) { u64_db db; std::vector> kv; @@ -65,7 +81,6 @@ UNODB_TEST(BulkLoad, Empty) { } } -// T02 UNODB_TEST(BulkLoad, Single) { u64_db db; constexpr std::uint64_t key = 1; @@ -80,7 +95,6 @@ UNODB_TEST(BulkLoad, Single) { UNODB_ASSERT_EQ(counts[as_i], 0U); } -// T03 UNODB_TEST(BulkLoad, Small4Keys) { u64_db db; auto kv = make_keys(4); @@ -93,7 +107,6 @@ UNODB_TEST(BulkLoad, Small4Keys) { UNODB_ASSERT_EQ(counts[as_i], 1U); } -// T04 UNODB_TEST(BulkLoad, Boundary4) { u64_db db; auto kv = make_keys(4); @@ -105,7 +118,6 @@ UNODB_TEST(BulkLoad, Boundary4) { UNODB_ASSERT_EQ(counts[as_i], 0U); } -// T05 UNODB_TEST(BulkLoad, Boundary5) { u64_db db; auto kv = make_keys(5); @@ -115,7 +127,6 @@ UNODB_TEST(BulkLoad, Boundary5) { UNODB_ASSERT_EQ(counts[as_i], 0U); } -// T06 UNODB_TEST(BulkLoad, Boundary16) { u64_db db; auto kv = make_keys(16); @@ -125,7 +136,6 @@ UNODB_TEST(BulkLoad, Boundary16) { UNODB_ASSERT_EQ(counts[as_i], 0U); } -// T07 UNODB_TEST(BulkLoad, Boundary17) { u64_db db; auto kv = make_keys(17); @@ -135,7 +145,6 @@ UNODB_TEST(BulkLoad, Boundary17) { UNODB_ASSERT_EQ(counts[as_i], 0U); } -// T08 UNODB_TEST(BulkLoad, Boundary48) { u64_db db; auto kv = make_keys(48); @@ -145,7 +154,6 @@ UNODB_TEST(BulkLoad, Boundary48) { UNODB_ASSERT_EQ(counts[as_i], 0U); } -// T09 UNODB_TEST(BulkLoad, Boundary49) { u64_db db; auto kv = make_keys(49); @@ -155,7 +163,6 @@ UNODB_TEST(BulkLoad, Boundary49) { UNODB_ASSERT_EQ(counts[as_i], 0U); } -// T10 UNODB_TEST(BulkLoad, Growth260) { u64_db db; auto kv = make_keys(260); @@ -168,11 +175,6 @@ UNODB_TEST(BulkLoad, Growth260) { UNODB_ASSERT_EQ(counts[as_i], 260U); } -// T11: Keys sharing a common prefix exercise the prefix chain path. -// For uint64_t, max key length is 8, prefix_cap is 7, so we need a recursive -// scenario where a subtree has all keys sharing a long prefix. -// Here keys share bytes 0-5 (prefix_len=6 at depth 0), which is <= prefix_cap, -// so chain_consumed=0. Instead, verify the prefix is handled correctly. UNODB_TEST(BulkLoad, SharedPrefix) { u64_db db; std::vector> kv; @@ -188,15 +190,9 @@ UNODB_TEST(BulkLoad, SharedPrefix) { } } -// T12: Deep prefix chain — keys with long shared prefix that exceeds -// prefix_cap (7). This requires key_view (variable-length) keys. -// Keys share 10 prefix bytes, dispatch at byte 10. UNODB_TEST(BulkLoad, DeepPrefixChain) { using unodb::test::key_view_db; key_view_db db; - - // Create keys that share 10 bytes then diverge at byte 10. - // Keys are 13 bytes to ensure sufficient depth beyond dispatch. std::vector, value_view>> raw_keys; raw_keys.reserve(10); for (int i = 0; i < 10; ++i) { @@ -209,7 +205,6 @@ UNODB_TEST(BulkLoad, DeepPrefixChain) { k[12] = std::byte{0xBB}; raw_keys.emplace_back(std::move(k), val); } - // Already sorted since they share a prefix and last byte is 0..9 std::vector> kv; kv.reserve(raw_keys.size()); for (auto& [k, v] : raw_keys) { @@ -221,8 +216,639 @@ UNODB_TEST(BulkLoad, DeepPrefixChain) { } } -} // namespace +// ─── Structural Tests ──────────────────────────────────────────────────────── -UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix7) { + u64_db db; + std::vector> kv{ + {0x0102030405060700ULL, val}, {0x0102030405060701ULL, val}}; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix8) { + using unodb::test::key_view_db; + std::vector> s{ + {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}, std::byte{5}, + std::byte{6}, std::byte{7}, std::byte{8}, std::byte{0}}, + {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}, std::byte{5}, + std::byte{6}, std::byte{7}, std::byte{8}, std::byte{1}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 2U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix15) { + using unodb::test::key_view_db; + std::vector> s( + 2, std::vector(16, std::byte{0xAA})); + s[0][15] = std::byte{0}; + s[1][15] = std::byte{1}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 2U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix16) { + using unodb::test::key_view_db; + std::vector> s( + 2, std::vector(17, std::byte{0xBB})); + s[0][16] = std::byte{0}; + s[1][16] = std::byte{1}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 3U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVIS) { + using unodb::test::key_view_u64val_db; + std::vector> s{{std::byte{1}}, {std::byte{2}}}; + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISWithChain) { + using unodb::test::key_view_u64val_db; + std::vector> s{{std::byte{1}, std::byte{0xAA}}, + {std::byte{1}, std::byte{0xBB}}}; + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, 99ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISLongPrefix) { + using unodb::test::key_view_u64val_db; + std::vector> s( + 2, std::vector(9, std::byte{0xCC})); + s[0][8] = std::byte{0}; + s[1][8] = std::byte{1}; + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, 77ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 2U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeaf) { + using unodb::test::key_view_db; + std::vector> s{ + {std::byte{1}, std::byte{2}, std::byte{3}}, + {std::byte{4}, std::byte{5}, std::byte{6}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 3U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeafWithChain) { + using unodb::test::key_view_db; + std::vector> s{ + {std::byte{1}, std::byte{0xAA}, std::byte{0xFF}}, + {std::byte{1}, std::byte{0xBB}, std::byte{0xFF}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 3U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadLarge) { + std::mt19937 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) + std::vector> kv; + kv.reserve(100000); + UNODB_DETAIL_DISABLE_GCC_WARNING("-Wuseless-cast") + for (int i = 0; i < 100000; ++i) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + (static_cast(rng()) << 32U) | rng(), val); + UNODB_DETAIL_RESTORE_GCC_WARNINGS() + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + kv.erase(std::unique( // NOLINT(modernize-use-ranges) + kv.begin(), kv.end(), + [](const auto& a, const auto& b) { return a.first == b.first; }), + kv.end()); + u64_db db; + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) UNODB_ASSERT_TRUE(db.get(k).has_value()); + std::uint64_t prev = 0; + bool first = true; + UNODB_DETAIL_DISABLE_MSVC_WARNING(26440) + db.scan([&](auto& visitor) { + unodb::key_decoder dec{visitor.get_key()}; + std::uint64_t k{}; + dec.decode(k); + if (!first) { + UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) + UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) + EXPECT_GE(k, prev); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + } + prev = k; + first = false; + return false; + }); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +} + +UNODB_TEST(BulkLoadStructural, BulkLoadKeyView) { + using unodb::test::key_view_db; + constexpr std::size_t key_len = 10; + std::mt19937 rng(123); // NOLINT(cert-msc32-c,cert-msc51-cpp) + std::vector> storage; + for (int i = 0; i < 1000; ++i) { + std::vector k(key_len); + for (auto& b : k) b = static_cast(rng() & 0xFFU); + storage.push_back(std::move(k)); + } + std::ranges::sort(storage); + // NOLINTNEXTLINE(modernize-use-ranges) + storage.erase(std::unique(storage.begin(), storage.end()), storage.end()); + std::vector> kv; + kv.reserve(storage.size()); + for (auto& v : storage) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) UNODB_ASSERT_TRUE(db.get(k).has_value()); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISRootSingle) { + using unodb::test::key_view_u64val_db; + std::vector> s{{std::byte{1}, std::byte{2}}}; + std::vector> kv; + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{s[0]}, 55ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + UNODB_ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeafRootSingle) { + using unodb::test::key_view_db; + std::vector> s{ + {std::byte{1}, std::byte{2}, std::byte{3}}}; + std::vector> kv; + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{s[0]}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + UNODB_ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadFullLeafRootSingle) { + u64_db db; + std::vector> kv{{0xDEADBEEFULL, val}}; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadScanOrder) { + std::vector> kv; + for (std::uint64_t i = 0; i < 256; ++i) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + i << 56U, val); + u64_db db; + db.bulk_load(kv.begin(), kv.end()); + std::uint64_t prev = 0; + std::size_t count = 0; + UNODB_DETAIL_DISABLE_MSVC_WARNING(26440) + db.scan([&](auto& visitor) { + unodb::key_decoder dec{visitor.get_key()}; + std::uint64_t k{}; + dec.decode(k); + if (count > 0) { + UNODB_EXPECT_GT(k, prev); + } + prev = k; + ++count; + return false; + }); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_ASSERT_EQ(count, 256U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISNode16) { + using unodb::test::key_view_u64val_db; + std::vector> s; + s.reserve(8); + for (int i = 0; i < 8; ++i) { + s.push_back({static_cast(i)}); + } + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISNode48) { + using unodb::test::key_view_u64val_db; + std::vector> s; + s.reserve(20); + for (int i = 0; i < 20; ++i) { + s.push_back({static_cast(i)}); + } + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISNode256) { + using unodb::test::key_view_u64val_db; + std::vector> s; + s.reserve(60); + for (int i = 0; i < 60; ++i) { + s.push_back({static_cast(i)}); + } + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +// ─── Stats-dependent Operational Tests ─────────────────────────────────────── + +UNODB_TEST(BulkLoadOps, Stats) { + u64_db db; + std::vector> kv; + kv.reserve(17); + for (std::uint64_t i = 0; i < 17; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 17U); + UNODB_ASSERT_EQ(counts[as_i], 0U); + UNODB_ASSERT_EQ(counts[as_i], 0U); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} #endif // UNODB_DETAIL_WITH_STATS + +// ─── Error Tests (no stats dependency) ─────────────────────────────────────── + +UNODB_TEST(BulkLoadError, NonEmpty) { + u64_db db; + constexpr std::uint64_t key = 42; + UNODB_ASSERT_TRUE(db.insert(key, val)); + std::vector> kv{{100, val}}; + UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) + UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) + EXPECT_THROW(db.bulk_load(kv.begin(), kv.end()), std::invalid_argument); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + const auto result = db.get(key); + UNODB_ASSERT_TRUE(result.has_value()); +} + +UNODB_TEST(BulkLoadError, OlcNonEmpty) { + u64_olc_db db; + constexpr std::uint64_t key = 42; + ASSERT_TRUE(db.insert(key, val)); + unodb::this_thread().quiescent(); + std::vector> kv{{100, val}}; + EXPECT_THROW(db.bulk_load(kv.begin(), kv.end()), std::invalid_argument); + const auto result = db.get(key); + ASSERT_TRUE(result.has_value()); +} + +UNODB_TEST(BulkLoadError, DbIgnoresParallelism) { + u64_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + const auto key = i << 56U; + kv.emplace_back(key, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +// ─── Operational Tests (no stats dependency) ───────────────────────────────── + +UNODB_TEST(BulkLoadOps, ThenOperations) { + u64_db db; + std::vector> kv; + kv.reserve(10); + for (std::uint64_t i = 0; i < 10; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } + constexpr std::uint64_t new_key = 0xFFULL << 56U; + UNODB_ASSERT_TRUE(db.insert(new_key, val)); + UNODB_ASSERT_TRUE(db.get(new_key).has_value()); + UNODB_ASSERT_FALSE(db.insert(kv[0].first, val)); + UNODB_ASSERT_TRUE(db.remove(kv[0].first)); + UNODB_ASSERT_FALSE(db.get(kv[0].first).has_value()); + + std::size_t count = 0; + db.scan([&count](auto&) { + ++count; + return false; + }); + UNODB_ASSERT_EQ(count, 10U); // 10 - 1 removed + 1 new = 10 +} + +UNODB_TEST(BulkLoadOps, NoGrowthEvents) { + u64_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()) << "key " << k << " not found"; + } +} + +UNODB_TEST(BulkLoadOps, MutexDb) { + u64_mutex_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.first.has_value()) << "key " << k << " not found"; + } +} + +UNODB_TEST(BulkLoadOps, ClearAndReload) { + u64_db db; + std::vector> kv; + kv.reserve(10); + for (std::uint64_t i = 0; i < 10; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + db.clear(); + UNODB_ASSERT_TRUE(db.empty()); + std::vector> kv2; + kv2.reserve(10); + for (std::uint64_t i = 100; i < 110; ++i) { + kv2.emplace_back(i << 56U, val); + } + std::ranges::sort(kv2, {}, &decltype(kv2)::value_type::first); + db.bulk_load(kv2.begin(), kv2.end()); + for (const auto& [k, v] : kv2) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +UNODB_TEST(BulkLoadOps, OlcDbParallel) { + u64_olc_db db; + std::vector> kv; + kv.reserve(1000); + for (std::uint64_t i = 0; i < 1000; ++i) { + kv.emplace_back(i << 48U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +UNODB_TEST(BulkLoadOps, OlcDbConcurrentReaders) { + u64_olc_db db; + constexpr std::size_t n_keys = 10000; + std::vector> kv; + kv.reserve(n_keys); + for (std::uint64_t i = 0; i < n_keys; ++i) { + kv.emplace_back(i << 40U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); + + unodb::this_thread().qsbr_pause(); + + constexpr std::size_t n_threads = 4; + std::array threads; + for (std::size_t t = 0; t < n_threads; ++t) { + threads[t] = unodb::qsbr_thread([&kv, &db] { + for (const auto& [k, v] : kv) { + const unodb::quiescent_state_on_scope_exit qsbr{}; + const auto result = db.get(k); + EXPECT_TRUE(result.has_value()) << "key " << k << " not found"; + } + unodb::this_thread().quiescent(); + }); + } + for (auto& t : threads) t.join(); + + unodb::this_thread().qsbr_resume(); + unodb::this_thread().quiescent(); + unodb::test::expect_idle_qsbr(); +} + +// ─── Parallel Path Coverage ────────────────────────────────────────────────── + +UNODB_TEST(BulkLoadOps, ParallelInlineRemainder) { + u64_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()) << "key " << k << " not found"; + } +} + +UNODB_TEST(BulkLoadOps, ParallelSingleElement) { + u64_db db; + std::vector> kv; + kv.emplace_back(42ULL << 56U, val); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + ASSERT_TRUE(db.get(42ULL << 56U).has_value()); +} + +UNODB_TEST(BulkLoadOps, ParallelPrefixChain) { + using unodb::test::key_view_db; + key_view_db db; + std::vector> raw_keys; + raw_keys.reserve(20); + for (int i = 0; i < 20; ++i) { + std::vector k(13); + for (int j = 0; j < 10; ++j) { + k[static_cast(j)] = static_cast(j + 1); + } + k[10] = static_cast(i); + k[11] = std::byte{0xAA}; + k[12] = std::byte{0xBB}; + raw_keys.push_back(std::move(k)); + } + std::vector> kv; + kv.reserve(raw_keys.size()); + for (auto& k : raw_keys) { + kv.emplace_back(unodb::key_view{k.data(), k.size()}, val); + } + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()); + } +} + +UNODB_TEST(BulkLoadOps, OlcDbParallelInline) { + u64_olc_db db; + std::vector> kv; + kv.reserve(50); + for (std::uint64_t i = 0; i < 50; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +UNODB_TEST(BulkLoadOps, OlcDbParallelSharedPrefix) { + u64_olc_db db; + std::vector> kv; + kv.reserve(20); + for (std::uint64_t i = 0; i < 20; ++i) { + kv.emplace_back((1ULL << 56U) | (i << 48U), val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +} // namespace + +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() diff --git a/test/test_art_bulk_load_ops.cpp b/test/test_art_bulk_load_ops.cpp deleted file mode 100644 index 2c410f14..00000000 --- a/test/test_art_bulk_load_ops.cpp +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2026 UnoDB contributors - -// Should be the first include -#include "global.hpp" // IWYU pragma: keep - -#include -#include -#include -#include -#include -#include -#include -#include - -#include // NOLINT(misc-include-cleaner) - -#include "art_common.hpp" -#include "db_test_utils.hpp" -#include "gtest_utils.hpp" -#include "node_type.hpp" -#include "qsbr.hpp" -#include "qsbr_test_utils.hpp" - -UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) -UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) -UNODB_DETAIL_DISABLE_MSVC_WARNING(26445) - -namespace { - -#ifdef UNODB_DETAIL_WITH_STATS -using unodb::as_i; -using unodb::node_type; -#endif -using unodb::value_view; -using unodb::test::u64_db; -using unodb::test::u64_mutex_db; -using unodb::test::u64_olc_db; - -constexpr auto val_bytes = - std::array{std::byte{0x68}, std::byte{0x65}, std::byte{0x6C}, - std::byte{0x6C}, std::byte{0x6F}}; -constexpr auto val = value_view{val_bytes}; - -// ─── Error Tests ───────────────────────────────────────────────────────────── - -// T26: bulk_load on non-empty tree throws -UNODB_TEST(BulkLoadError, NonEmpty) { - u64_db db; - constexpr std::uint64_t key = 42; - UNODB_ASSERT_TRUE(db.insert(key, val)); - std::vector> kv{{100, val}}; - UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) - UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) - EXPECT_THROW(db.bulk_load(kv.begin(), kv.end()), std::invalid_argument); - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - // Original key still present - const auto result = db.get(key); - UNODB_ASSERT_TRUE(result.has_value()); -} - -// T26b: bulk_load on non-empty olc_db throws (covers olc_art.hpp error path). -UNODB_TEST(BulkLoadError, OlcNonEmpty) { - u64_olc_db db; - constexpr std::uint64_t key = 42; - ASSERT_TRUE(db.insert(key, val)); - unodb::this_thread().quiescent(); - std::vector> kv{{100, val}}; - EXPECT_THROW(db.bulk_load(kv.begin(), kv.end()), std::invalid_argument); - const auto result = db.get(key); - ASSERT_TRUE(result.has_value()); -} - -// T36: db ignores parallelism parameter -UNODB_TEST(BulkLoadError, DbIgnoresParallelism) { - u64_db db; - std::vector> kv; - kv.reserve(100); - for (std::uint64_t i = 0; i < 100; ++i) { - const auto key = i << 56U; - kv.emplace_back(key, val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - // Parallel fork callable enables concurrent subtree construction - db.bulk_load( - [](auto&& f) { - return std::async(std::launch::async, std::forward(f)); - }, - 256, kv.begin(), kv.end()); - for (const auto& [k, v] : kv) { - const auto result = db.get(k); - ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; - } -} - -// ─── Operational Tests ─────────────────────────────────────────────────────── - -// T38: Operations work correctly after bulk_load -UNODB_TEST(BulkLoadOps, ThenOperations) { - u64_db db; - std::vector> kv; - kv.reserve(10); - for (std::uint64_t i = 0; i < 10; ++i) { - kv.emplace_back(i << 56U, val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - db.bulk_load(kv.begin(), kv.end()); - - // get works for all keys - for (const auto& [k, v] : kv) { - UNODB_ASSERT_TRUE(db.get(k).has_value()); - } - - // insert new key works - constexpr std::uint64_t new_key = 0xFFULL << 56U; - UNODB_ASSERT_TRUE(db.insert(new_key, val)); - UNODB_ASSERT_TRUE(db.get(new_key).has_value()); - - // insert duplicate fails - UNODB_ASSERT_FALSE(db.insert(kv[0].first, val)); - - // remove works - UNODB_ASSERT_TRUE(db.remove(kv[0].first)); - UNODB_ASSERT_FALSE(db.get(kv[0].first).has_value()); - - // scan works - std::size_t count = 0; - db.scan([&count](auto&) { - ++count; - return false; // continue - }); - UNODB_ASSERT_EQ(count, 10U); // 10 original - 1 removed + 1 new = 10 -} - -// T39: Stats are correct after bulk_load -#ifdef UNODB_DETAIL_WITH_STATS -UNODB_TEST(BulkLoadOps, Stats) { - u64_db db; - // 17 keys differing at byte 0 → should produce 1 inode48 - std::vector> kv; - kv.reserve(17); - for (std::uint64_t i = 0; i < 17; ++i) { - kv.emplace_back(i << 56U, val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - db.bulk_load(kv.begin(), kv.end()); - - const auto counts = db.get_node_counts(); - UNODB_ASSERT_EQ(counts[as_i], 17U); - UNODB_ASSERT_EQ(counts[as_i], 0U); - UNODB_ASSERT_EQ(counts[as_i], 0U); - UNODB_ASSERT_EQ(counts[as_i], 1U); - UNODB_ASSERT_EQ(counts[as_i], 0U); -} -#endif // UNODB_DETAIL_WITH_STATS - -// T40: No growth events during bulk_load (all right-sized at allocation) -UNODB_TEST(BulkLoadOps, NoGrowthEvents) { - u64_db db; - std::vector> kv; - kv.reserve(100); - for (std::uint64_t i = 0; i < 100; ++i) { - kv.emplace_back(i << 56U, val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - db.bulk_load(kv.begin(), kv.end()); - - // In bulk_load, no inode should ever grow (they're right-sized at creation). - // The growing_inode_counts track create events, not grow events — but the - // key insight is that no shrink events should exist. - // Just verify all keys are present: - for (const auto& [k, v] : kv) { - ASSERT_TRUE(db.get(k).has_value()) << "key " << k << " not found"; - } -} - -// T42: mutex_db bulk_load works -UNODB_TEST(BulkLoadOps, MutexDb) { - u64_mutex_db db; - std::vector> kv; - kv.reserve(100); - for (std::uint64_t i = 0; i < 100; ++i) { - kv.emplace_back(i << 56U, val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - db.bulk_load(kv.begin(), kv.end()); - - for (const auto& [k, v] : kv) { - const auto result = db.get(k); - ASSERT_TRUE(result.first.has_value()) << "key " << k << " not found"; - } -} - -// T38 extended: clear after bulk_load then re-bulk_load -UNODB_TEST(BulkLoadOps, ClearAndReload) { - u64_db db; - std::vector> kv; - kv.reserve(10); - for (std::uint64_t i = 0; i < 10; ++i) { - kv.emplace_back(i << 56U, val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - db.bulk_load(kv.begin(), kv.end()); - db.clear(); - UNODB_ASSERT_TRUE(db.empty()); - // Re-load different data - std::vector> kv2; - kv2.reserve(10); - for (std::uint64_t i = 100; i < 110; ++i) { - kv2.emplace_back(i << 56U, val); - } - std::ranges::sort(kv2, {}, &decltype(kv2)::value_type::first); - db.bulk_load(kv2.begin(), kv2.end()); - for (const auto& [k, v] : kv2) { - UNODB_ASSERT_TRUE(db.get(k).has_value()); - } -} - -// T43: olc_db parallel bulk_load -UNODB_TEST(BulkLoadOps, OlcDbParallel) { - u64_olc_db db; - std::vector> kv; - kv.reserve(1000); - for (std::uint64_t i = 0; i < 1000; ++i) { - kv.emplace_back(i << 48U, val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - db.bulk_load( - [](auto&& f) { - return std::async(std::launch::async, std::forward(f)); - }, - 256, kv.begin(), kv.end()); - for (const auto& [k, v] : kv) { - const auto result = db.get(k); - ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; - } -} - -// T44: olc_db concurrent readers after parallel bulk_load -// Verifies that optimistic locks are correctly initialized after bulk_load so -// concurrent readers can traverse the tree without deadlock or data corruption. -UNODB_TEST(BulkLoadOps, OlcDbConcurrentReaders) { - u64_olc_db db; - constexpr std::size_t n_keys = 10000; - std::vector> kv; - kv.reserve(n_keys); - for (std::uint64_t i = 0; i < n_keys; ++i) { - kv.emplace_back(i << 40U, val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - db.bulk_load( - [](auto&& f) { - return std::async(std::launch::async, std::forward(f)); - }, - 256, kv.begin(), kv.end()); - - // Pause main thread QSBR so reader threads can register - unodb::this_thread().qsbr_pause(); - - constexpr std::size_t n_threads = 4; - std::array threads; - for (std::size_t t = 0; t < n_threads; ++t) { - threads[t] = unodb::qsbr_thread([&kv, &db] { - for (const auto& [k, v] : kv) { - const unodb::quiescent_state_on_scope_exit qsbr{}; - const auto result = db.get(k); - EXPECT_TRUE(result.has_value()) << "key " << k << " not found"; - } - unodb::this_thread().quiescent(); - }); - } - for (auto& t : threads) t.join(); - - unodb::this_thread().qsbr_resume(); - unodb::this_thread().quiescent(); - unodb::test::expect_idle_qsbr(); -} - -// ─── Coverage: parallel path with inline remainder -// ──────────────────────────── - -// T42: Parallel bulk_load with max_tasks < child_count exercises the inline -// results gathering path (some partitions forked, remainder runs inline). -UNODB_TEST(BulkLoadOps, ParallelInlineRemainder) { - u64_db db; - std::vector> kv; - kv.reserve(100); - for (std::uint64_t i = 0; i < 100; ++i) { - kv.emplace_back(i << 56U, val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - // max_tasks=4 with 100 partitions: 4 forked, 96 inline - db.bulk_load( - [](auto&& f) { - return std::async(std::launch::async, std::forward(f)); - }, - 4, kv.begin(), kv.end()); - for (const auto& [k, v] : kv) { - ASSERT_TRUE(db.get(k).has_value()) << "key " << k << " not found"; - } -} - -// T43: Parallel bulk_load with a single element exercises the n<=1 early -// return. -UNODB_TEST(BulkLoadOps, ParallelSingleElement) { - u64_db db; - std::vector> kv; - kv.emplace_back(42ULL << 56U, val); - db.bulk_load( - [](auto&& f) { - return std::async(std::launch::async, std::forward(f)); - }, - 4, kv.begin(), kv.end()); - ASSERT_TRUE(db.get(42ULL << 56U).has_value()); -} - -// T44: Parallel bulk_load with keys sharing a long common prefix exercises the -// prefix chain construction (chain_consumed > 0 path). Uses key_view for -// keys longer than prefix_cap (7). -UNODB_TEST(BulkLoadOps, ParallelPrefixChain) { - using unodb::test::key_view_db; - key_view_db db; - // Create keys that share 10 bytes then diverge at byte 10, with 2 more bytes - // to ensure subtree depth doesn't hit key boundary. - std::vector> raw_keys; - raw_keys.reserve(20); - for (int i = 0; i < 20; ++i) { - std::vector k(13); - for (int j = 0; j < 10; ++j) { - k[static_cast(j)] = static_cast(j + 1); - } - k[10] = static_cast(i); - k[11] = std::byte{0xAA}; - k[12] = std::byte{0xBB}; - raw_keys.push_back(std::move(k)); - } - std::vector> kv; - kv.reserve(raw_keys.size()); - for (auto& k : raw_keys) { - kv.emplace_back(unodb::key_view{k.data(), k.size()}, val); - } - // Already sorted (shared prefix, byte 10 is 0..19) - db.bulk_load( - [](auto&& f) { - return std::async(std::launch::async, std::forward(f)); - }, - 4, kv.begin(), kv.end()); - for (const auto& [k, v] : kv) { - ASSERT_TRUE(db.get(k).has_value()); - } -} - -// T45: olc_db parallel with inline remainder (covers olc_art.hpp bulk_load -// entry and bulk_subtree_guard move constructor). -UNODB_TEST(BulkLoadOps, OlcDbParallelInline) { - u64_olc_db db; - std::vector> kv; - kv.reserve(50); - for (std::uint64_t i = 0; i < 50; ++i) { - kv.emplace_back(i << 56U, val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - // max_tasks=4 with 50 partitions: 4 forked, 46 inline - db.bulk_load( - [](auto&& f) { - return std::async(std::launch::async, std::forward(f)); - }, - 4, kv.begin(), kv.end()); - for (const auto& [k, v] : kv) { - const auto result = db.get(k); - ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; - } -} - -// T46: olc_db parallel with shared prefix (all keys share byte 0). -UNODB_TEST(BulkLoadOps, OlcDbParallelSharedPrefix) { - u64_olc_db db; - std::vector> kv; - kv.reserve(20); - // All keys share byte 0 = 0x01, differ at byte 1. - for (std::uint64_t i = 0; i < 20; ++i) { - kv.emplace_back((1ULL << 56U) | (i << 48U), val); - } - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - db.bulk_load( - [](auto&& f) { - return std::async(std::launch::async, std::forward(f)); - }, - 4, kv.begin(), kv.end()); - for (const auto& [k, v] : kv) { - const auto result = db.get(k); - ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; - } -} - -} // namespace - -UNODB_DETAIL_RESTORE_MSVC_WARNINGS() -UNODB_DETAIL_RESTORE_MSVC_WARNINGS() -UNODB_DETAIL_RESTORE_MSVC_WARNINGS() diff --git a/test/test_art_bulk_load_structural.cpp b/test/test_art_bulk_load_structural.cpp deleted file mode 100644 index 3c8388b8..00000000 --- a/test/test_art_bulk_load_structural.cpp +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2026 UnoDB contributors - -#include "global.hpp" // IWYU pragma: keep - -#include -#include -#include -#include -#include -#include -#include - -#include // NOLINT(misc-include-cleaner) - -#include "art_common.hpp" -#include "db_test_utils.hpp" -#include "gtest_utils.hpp" -#include "node_type.hpp" - -#ifdef UNODB_DETAIL_WITH_STATS - -UNODB_DETAIL_DISABLE_MSVC_WARNING(26445) - -namespace { - -using unodb::as_i; -using unodb::key_view; -using unodb::node_type; -using unodb::value_view; -using unodb::test::key_view_db; -using unodb::test::key_view_u64val_db; -using unodb::test::u64_db; - -constexpr auto val16 = std::array{}; -constexpr auto large_val = value_view{val16}; -constexpr auto sval_bytes = - std::array{std::byte{0x68}, std::byte{0x65}, std::byte{0x6C}, - std::byte{0x6C}, std::byte{0x6F}}; -constexpr auto sval = value_view{sval_bytes}; - -// T11 -UNODB_TEST(BulkLoadStructural, BulkLoadPrefix7) { - u64_db db; - std::vector> kv{ - {0x0102030405060700ULL, sval}, {0x0102030405060701ULL, sval}}; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 1U); - UNODB_ASSERT_EQ(c[as_i], 2U); -} -// T12 -UNODB_TEST(BulkLoadStructural, BulkLoadPrefix8) { - std::vector> s{ - {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}, std::byte{5}, - std::byte{6}, std::byte{7}, std::byte{8}, std::byte{0}}, - {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}, std::byte{5}, - std::byte{6}, std::byte{7}, std::byte{8}, std::byte{1}}}; - std::vector> kv; - for (auto& v : s) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{v}, large_val); - key_view_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 2U); - UNODB_ASSERT_EQ(c[as_i], 2U); -} -// T13 -UNODB_TEST(BulkLoadStructural, BulkLoadPrefix15) { - std::vector> s( - 2, std::vector(16, std::byte{0xAA})); - s[0][15] = std::byte{0}; - s[1][15] = std::byte{1}; - std::vector> kv; - for (auto& v : s) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{v}, large_val); - key_view_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 2U); - UNODB_ASSERT_EQ(c[as_i], 2U); -} -// T14 -UNODB_TEST(BulkLoadStructural, BulkLoadPrefix16) { - std::vector> s( - 2, std::vector(17, std::byte{0xBB})); - s[0][16] = std::byte{0}; - s[1][16] = std::byte{1}; - std::vector> kv; - for (auto& v : s) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{v}, large_val); - key_view_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 3U); - UNODB_ASSERT_EQ(c[as_i], 2U); -} -// T15 -UNODB_TEST(BulkLoadStructural, BulkLoadVIS) { - std::vector> s{{std::byte{1}}, {std::byte{2}}}; - std::vector> kv; - kv.reserve(s.size()); - for (auto& v : s) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{v}, 42ULL); - key_view_u64val_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 1U); - UNODB_ASSERT_EQ(c[as_i], 0U); -} -// T16 -UNODB_TEST(BulkLoadStructural, BulkLoadVISWithChain) { - std::vector> s{{std::byte{1}, std::byte{0xAA}}, - {std::byte{1}, std::byte{0xBB}}}; - std::vector> kv; - kv.reserve(s.size()); - for (auto& v : s) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{v}, 99ULL); - key_view_u64val_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 1U); - UNODB_ASSERT_EQ(c[as_i], 0U); -} -// T17 -UNODB_TEST(BulkLoadStructural, BulkLoadVISLongPrefix) { - std::vector> s( - 2, std::vector(9, std::byte{0xCC})); - s[0][8] = std::byte{0}; - s[1][8] = std::byte{1}; - std::vector> kv; - kv.reserve(s.size()); - for (auto& v : s) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{v}, 77ULL); - key_view_u64val_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 2U); - UNODB_ASSERT_EQ(c[as_i], 0U); -} -// T18 -UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeaf) { - std::vector> s{ - {std::byte{1}, std::byte{2}, std::byte{3}}, - {std::byte{4}, std::byte{5}, std::byte{6}}}; - std::vector> kv; - for (auto& v : s) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{v}, large_val); - key_view_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - // Root I4 (dispatch byte 0) + 2 chain I4s wrapping keyless leaves - UNODB_ASSERT_EQ(c[as_i], 3U); - UNODB_ASSERT_EQ(c[as_i], 2U); -} -// T19 -UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeafWithChain) { - std::vector> s{ - {std::byte{1}, std::byte{0xAA}, std::byte{0xFF}}, - {std::byte{1}, std::byte{0xBB}, std::byte{0xFF}}}; - std::vector> kv; - for (auto& v : s) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{v}, large_val); - key_view_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 3U); - UNODB_ASSERT_EQ(c[as_i], 2U); -} -// T20 -UNODB_TEST(BulkLoadStructural, BulkLoadLarge) { - std::mt19937 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) - std::vector> kv; - kv.reserve(100000); - UNODB_DETAIL_DISABLE_GCC_WARNING("-Wuseless-cast") - for (int i = 0; i < 100000; ++i) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - (static_cast(rng()) << 32U) | rng(), sval); - UNODB_DETAIL_RESTORE_GCC_WARNINGS() - std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); - kv.erase(std::unique( // NOLINT(modernize-use-ranges) - kv.begin(), kv.end(), - [](const auto& a, const auto& b) { return a.first == b.first; }), - kv.end()); - u64_db db; - db.bulk_load(kv.begin(), kv.end()); - for (const auto& [k, v] : kv) UNODB_ASSERT_TRUE(db.get(k).has_value()); - std::uint64_t prev = 0; - bool first = true; - UNODB_DETAIL_DISABLE_MSVC_WARNING(26440) - db.scan([&](auto& visitor) { - unodb::key_decoder dec{visitor.get_key()}; - std::uint64_t k{}; - dec.decode(k); - if (!first) { - UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) - UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) - EXPECT_GE(k, prev); - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - } - prev = k; - first = false; - return false; - }); - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() -} -// T21 — key_view bulk_load with random fixed-length keys (prefix-free). -UNODB_TEST(BulkLoadStructural, BulkLoadKeyView) { - constexpr std::size_t key_len = - 10; // Fixed length → no key is prefix of another - std::mt19937 rng(123); // NOLINT(cert-msc32-c,cert-msc51-cpp) - std::vector> storage; - for (int i = 0; i < 1000; ++i) { - std::vector k(key_len); - for (auto& b : k) b = static_cast(rng() & 0xFFU); - storage.push_back(std::move(k)); - } - std::ranges::sort(storage); - // NOLINTNEXTLINE(modernize-use-ranges) - storage.erase(std::unique(storage.begin(), storage.end()), storage.end()); - std::vector> kv; - kv.reserve(storage.size()); - for (auto& v : storage) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{v}, large_val); - key_view_db db; - db.bulk_load(kv.begin(), kv.end()); - for (const auto& [k, v] : kv) UNODB_ASSERT_TRUE(db.get(k).has_value()); -} -// T22 -UNODB_TEST(BulkLoadStructural, BulkLoadVISRootSingle) { - std::vector> s{{std::byte{1}, std::byte{2}}}; - std::vector> kv; - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{s[0]}, 55ULL); - key_view_u64val_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 1U); - UNODB_ASSERT_EQ(c[as_i], 0U); - UNODB_ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); -} -// T23 -UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeafRootSingle) { - std::vector> s{ - {std::byte{1}, std::byte{2}, std::byte{3}}}; - std::vector> kv; - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - key_view{s[0]}, large_val); - key_view_db db; - db.bulk_load(kv.begin(), kv.end()); - UNODB_ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); -} -// T24 -UNODB_TEST(BulkLoadStructural, BulkLoadFullLeafRootSingle) { - u64_db db; - std::vector> kv{{0xDEADBEEFULL, sval}}; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 1U); - UNODB_ASSERT_EQ(c[as_i], 0U); -} -// T25 -UNODB_TEST(BulkLoadStructural, BulkLoadScanOrder) { - std::vector> kv; - for (std::uint64_t i = 0; i < 256; ++i) - kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) - i << 56U, sval); - u64_db db; - db.bulk_load(kv.begin(), kv.end()); - std::uint64_t prev = 0; - std::size_t count = 0; - UNODB_DETAIL_DISABLE_MSVC_WARNING(26440) - db.scan([&](auto& visitor) { - unodb::key_decoder dec{visitor.get_key()}; - std::uint64_t k{}; - dec.decode(k); - if (count > 0) { - UNODB_EXPECT_GT(k, prev); - } - prev = k; - ++count; - return false; - }); - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - UNODB_ASSERT_EQ(count, 256U); -} - -// T27 — VIS with 5 keys → Node16 root (covers vmask loop in dispatch_node). -UNODB_TEST(BulkLoadStructural, BulkLoadVISNode16) { - std::vector> s; - s.reserve(8); - for (int i = 0; i < 8; ++i) { - s.push_back({static_cast(i)}); - } - std::vector> kv; - kv.reserve(s.size()); - for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); - key_view_u64val_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 1U); - UNODB_ASSERT_EQ(c[as_i], 0U); - for (const auto& [k, v] : kv) { - UNODB_ASSERT_TRUE(db.get(k).has_value()); - } -} - -// T28 — VIS with 20 keys → Node48 root. -UNODB_TEST(BulkLoadStructural, BulkLoadVISNode48) { - std::vector> s; - s.reserve(20); - for (int i = 0; i < 20; ++i) { - s.push_back({static_cast(i)}); - } - std::vector> kv; - kv.reserve(s.size()); - for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); - key_view_u64val_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 1U); - UNODB_ASSERT_EQ(c[as_i], 0U); - for (const auto& [k, v] : kv) { - UNODB_ASSERT_TRUE(db.get(k).has_value()); - } -} - -// T29 — VIS with 60 keys → Node256 root. -UNODB_TEST(BulkLoadStructural, BulkLoadVISNode256) { - std::vector> s; - s.reserve(60); - for (int i = 0; i < 60; ++i) { - s.push_back({static_cast(i)}); - } - std::vector> kv; - kv.reserve(s.size()); - for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); - key_view_u64val_db db; - db.bulk_load(kv.begin(), kv.end()); - const auto c = db.get_node_counts(); - UNODB_ASSERT_EQ(c[as_i], 1U); - UNODB_ASSERT_EQ(c[as_i], 0U); - for (const auto& [k, v] : kv) { - UNODB_ASSERT_TRUE(db.get(k).has_value()); - } -} - -} // namespace - -UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - -#endif // UNODB_DETAIL_WITH_STATS