diff --git a/art.hpp b/art.hpp index f6af4d77..03ddedbe 100644 --- a/art.hpp +++ b/art.hpp @@ -18,8 +18,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -111,6 +113,14 @@ using leaf_type = basic_leaf, node_header>; template class inode : public inode_base {}; +/// Forward declarations of bulk_load algorithm and helpers. +template +void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last); + +template +struct bulk_load_helpers; + } // namespace detail template @@ -139,10 +149,10 @@ class db final { /// Base class type for internal nodes. using inode_base = detail::inode_base; - private: /// Internal encoded key type used for tree operations. using art_key_type = detail::basic_art_key; + private: /// Leaf node type (keyless when can_eliminate_key_in_leaf). using leaf_type = detail::leaf_type; @@ -195,6 +205,31 @@ class db final { /// tree and the associated index entry was removed). [[nodiscard]] bool remove_internal(art_key_type remove_key); + // ─── Bulk Load Infrastructure (public for detail:: free functions) ───── + + public: + /// Tagged return from build_subtree: pointer + whether it's a VIS packed + /// value (not a real heap pointer — must not be passed to delete_subtree). + using build_result = + detail::bulk_build_result>; + + /// RAII guard for subtrees during bulk construction. + /// Skips deallocation for VIS packed values (not heap-allocated). + using bulk_subtree_guard = + detail::bulk_subtree_guard>; + + /// Tree depth tracking type. + using tree_depth_type = detail::tree_depth; + + private: + /// Extract byte at \a depth from a key. Handles big-endian uint64_t + /// (via art_key_type bswap) and key_view (direct byte access). + [[nodiscard]] static constexpr std::byte key_byte_at( + const key_type& key, detail::tree_depth depth) noexcept { + const art_key_type ak{key}; + return ak[depth]; + } + /// Two-pass removal with chain cleanup for variable-length keys. [[nodiscard]] bool remove_internal_key_view(art_key_type remove_key); @@ -304,6 +339,34 @@ class db final { /// nodes is freed. Node growth/shrink counters are preserved. void clear() noexcept; + /// Bulk-load an ART tree from a pre-sorted range of key-value pairs. + /// + /// Bulk-load the tree from a sorted range, with caller-provided parallelism. + /// + /// \tparam Fork Callable satisfying: fork(Callable) -> Future where + /// Future has .get(). The callable is invoked to submit + /// subtree-building tasks; the library never spawns + /// threads. + /// \tparam RandomIt Random-access iterator to std::pair + /// \param fork Callable to submit parallel tasks. + /// \param max_tasks Maximum number of tasks to fork (remaining partitions + /// run inline on the calling thread). + /// \param first Start of sorted range (ART byte order, no duplicates). + /// \param last End of sorted range. + /// \pre empty() == true + /// \throws std::invalid_argument if tree is non-empty. + /// \throws std::bad_alloc (strong guarantee — tree left empty). + template + void bulk_load(Fork&& fork, std::size_t max_tasks, RandomIt first, + RandomIt last); + + /// Convenience overload: sequential execution (no parallelism). + template + void bulk_load(RandomIt first, RandomIt last) { + bulk_load(nullptr, 0, first, last); + } + /// Internal iterator for tree traversal. /// /// Iterator is an internal API. Use scan() for the public API. @@ -870,8 +933,14 @@ class db final { /// Node type with 4 children. using inode_4 = detail::inode_4; - /// Tree depth tracking type. - using tree_depth_type = detail::tree_depth; + /// Node type with 16 children. + using inode_16 = detail::inode_16; + + /// Node type with 48 children. + using inode_48 = detail::inode_48; + + /// Node type with 256 children. + using inode_256 = detail::inode_256; /// Visitor type for scan operations. using visitor_type = visitor; @@ -887,6 +956,11 @@ class db final { /// Increase tracked memory usage by \a delta bytes. constexpr void increase_memory_use(std::size_t delta) noexcept { UNODB_DETAIL_ASSERT(delta > 0); + + if (auto* const acc = detail::tls_bulk_load_stats; acc != nullptr) { + acc->current_memory_use += delta; // LCOV_EXCL_LINE — exercised by + return; // LCOV_EXCL_LINE — parallel tests + } UNODB_DETAIL_ASSERT( std::numeric_limits::max() - delta >= current_memory_use); @@ -904,6 +978,11 @@ class db final { /// Increment leaf node count and bump memory usage by \a leaf_size bytes. constexpr void increment_leaf_count(std::size_t leaf_size) noexcept { + if (auto* const acc = detail::tls_bulk_load_stats; acc != nullptr) { + acc->current_memory_use += leaf_size; + ++acc->node_counts[as_i]; + return; + } increase_memory_use(leaf_size); ++node_counts[as_i]; } @@ -940,6 +1019,17 @@ class db final { template constexpr void account_shrinking_inode() noexcept; + /// Merge accumulated stats from a parallel bulk_load subtree. + constexpr void merge_bulk_load_stats( + const detail::bulk_load_stats_accumulator& acc) noexcept { + current_memory_use += acc.current_memory_use; + for (std::size_t i = 0; i < node_counts.size(); ++i) + node_counts[i] += acc.node_counts[i]; + for (std::size_t i = 0; i < growing_inode_counts.size(); ++i) + growing_inode_counts[i] += acc.growing_inode_counts[i]; + key_prefix_splits += acc.key_prefix_splits; + } + #endif // UNODB_DETAIL_WITH_STATS /// Root of the tree (nullptr if empty). @@ -973,6 +1063,21 @@ class db final { friend auto detail::make_db_leaf_ptr(art_key_type, value_type, db&); + /// detail::bulk_load_impl + template + friend void detail::bulk_load_impl(Db2&, Fork2&&, std::size_t, It2, It2); + + /// detail::bulk_load_helpers + template + friend struct detail::bulk_load_helpers; + + /// detail::bulk_build_chain + template + friend typename ArtPolicy2::node_ptr detail::bulk_build_chain( + typename ArtPolicy2::db_type&, typename ArtPolicy2::art_key_type, + typename ArtPolicy2::node_ptr, + detail::tree_depth); + /// detail::basic_db_leaf_deleter template friend class detail::basic_db_leaf_deleter; @@ -1623,73 +1728,7 @@ template detail::node_ptr db::build_chain(art_key_type k, detail::node_ptr child, tree_depth_type start_depth) { - constexpr std::size_t cap = detail::key_prefix_capacity; - const auto full_key = k.get_key_view(); - const auto key_len = k.size(); - const auto start = static_cast(start_depth); - auto current = child; - bool child_is_value = art_policy::can_eliminate_leaf; - bool owns_current = - false; // set true once we've built at least one chain node - // Build bottom-up: start from end of key, work toward start_depth. - // Each chain I4 consumes up to cap prefix bytes + 1 dispatch byte. - try { - std::size_t pos = key_len; - while (pos > start + cap) { - const auto depth = pos - cap - 1; - const auto dispatch = full_key[pos - 1]; - auto remaining = k; - remaining.shift_right(depth); - auto chain{ - inode_4::create(*this, full_key, remaining, - tree_depth_type{static_cast(depth)}, - dispatch, current)}; - if (child_is_value) { - chain->set_value_bit(0); - child_is_value = false; - } - current = detail::node_ptr{chain.release(), node_type::I4}; - owns_current = true; -#ifdef UNODB_DETAIL_WITH_STATS - account_growing_inode(); -#endif - pos = depth; - } - // Tail: remaining bytes from start_depth to pos. - if (pos > start) { - const auto dispatch = full_key[pos - 1]; - auto chain{inode_4::create( - *this, full_key, tree_depth_type{static_cast(start)}, - static_cast(pos - start - 1), dispatch, - current)}; - if (child_is_value) { - chain->set_value_bit(0); - } - current = detail::node_ptr{chain.release(), node_type::I4}; - owns_current = true; -#ifdef UNODB_DETAIL_WITH_STATS - account_growing_inode(); -#endif - } - } catch (...) { - // On failure, free whatever current points to. If owns_current is - // true, current is a partial chain (I4 tree). If false, current is - // the original child — a leaf node_ptr on the leaf path, or a packed - // value on the VIS path. delete_subtree handles both I4 and LEAF. - // For VIS packed values (!child_is_value is false only after the - // first chain node is built), the caller's packed bits are not a - // real pointer — but we only reach here with a packed value if - // owns_current is false AND child_is_value is true, meaning no - // chain was built and the packed value was never wrapped. In that - // case we must NOT call delete_subtree on packed bits. - if (owns_current) { - art_policy::delete_subtree(current, *this); - } else if (!child_is_value) { - art_policy::delete_subtree(current, *this); - } - throw; - } - return current; + return detail::bulk_build_chain(*this, k, child, start_depth); } UNODB_DETAIL_RESTORE_GCC_WARNINGS() UNODB_DETAIL_RESTORE_GCC_WARNINGS() @@ -2594,6 +2633,11 @@ template constexpr void db::increment_inode_count() noexcept { static_assert(inode_defs_type::template is_inode()); + if (auto* const acc = detail::tls_bulk_load_stats; acc != nullptr) { + ++acc->node_counts[as_i]; + acc->current_memory_use += sizeof(INode); + return; + } ++node_counts[as_i]; increase_memory_use(sizeof(INode)); } @@ -2613,6 +2657,10 @@ template constexpr void db::account_growing_inode() noexcept { static_assert(NodeType != node_type::LEAF); + if (auto* const acc = detail::tls_bulk_load_stats; acc != nullptr) { + ++acc->growing_inode_counts[internal_as_i]; + return; + } // NOLINTNEXTLINE(google-readability-casting) ++growing_inode_counts[internal_as_i]; UNODB_DETAIL_ASSERT(growing_inode_counts[internal_as_i] >= @@ -2745,6 +2793,25 @@ bool db::upsert(Key k, value_type v, FN fn) { } } +template +template +void db::bulk_load(Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last) { + if (!empty()) { + throw std::invalid_argument("bulk_load requires empty tree"); + } + if (first == last) return; + try { + detail::bulk_load_impl(*this, std::forward(fork), max_tasks, first, + last); + } catch (...) { + clear(); + throw; + } +} + } // namespace unodb +#include "art_bulk_load_detail.hpp" + #endif // UNODB_DETAIL_ART_HPP diff --git a/art_bulk_load_detail.hpp b/art_bulk_load_detail.hpp new file mode 100644 index 00000000..2254e284 --- /dev/null +++ b/art_bulk_load_detail.hpp @@ -0,0 +1,471 @@ +// Copyright 2026 UnoDB contributors +// +// Shared implementation of the bulk_load algorithm for all ART variants. +// Included at the bottom of art.hpp and olc_art.hpp after class definitions. +// +// Template parameter Db: must expose art_key_type, art_policy, tree_depth_type, +// inode_4, inode_16, inode_48, inode_256, build_result, bulk_subtree_guard, +// build_chain(), account_growing_inode<>(), and root member. + +#ifndef UNODB_DETAIL_ART_BULK_LOAD_DETAIL_HPP +#define UNODB_DETAIL_ART_BULK_LOAD_DETAIL_HPP + +// MSVC static analysis false positive: claims pointers dangle after +// smart-pointer release() + reassignment. Suppressed file-wide. +UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) +// C26496: vmask variables are mutated in if-constexpr branches that +// MSVC SA doesn't track; cannot be const in the general case. +UNODB_DETAIL_DISABLE_MSVC_WARNING(26496) + +/// \cond UNODB_DETAIL_INTERNAL +namespace unodb::detail { + +// ─── Helper types ────────────────────────────────────────────────────────── + +/// Entry produced by partition_by_byte: the start iterator and dispatch byte. +template +struct partition_entry { + RandomIt begin; + std::byte key_byte; +}; + +// ─── Bulk load helper struct ─────────────────────────────────────────────── + +/// Named helper functions for bulk_load, collected in a struct to simplify +/// friend declarations and avoid forward-declaration complexity. +template +struct bulk_load_helpers { + using art_key_type = typename Db::art_key_type; + using art_policy = typename Db::art_policy; + using tree_depth_type = typename Db::tree_depth_type; + using inode_4 = typename Db::inode_4; + using inode_16 = typename Db::inode_16; + using inode_48 = typename Db::inode_48; + using inode_256 = typename Db::inode_256; + using build_result_t = typename Db::build_result; + using guard_t = typename Db::bulk_subtree_guard; + using node_ptr_t = typename build_result_t::ptr_type; + using bulk_child_t = bulk_child; + + /// Compute the length of the common prefix between first and last keys. + [[nodiscard]] static std::size_t common_prefix_length(RandomIt f, RandomIt l, + tree_depth_type depth) { + const art_key_type first_ak{f->first}; + const art_key_type last_ak{std::prev(l)->first}; + const auto fk = first_ak.get_key_view(); + const auto lk = last_ak.get_key_view(); + const auto d = static_cast(depth); + const auto max_len = std::min(fk.size(), lk.size()); + std::size_t len = 0; + while (d + len < max_len && fk[d + len] == lk[d + len]) ++len; + return len; + } + + /// Partition sorted range [f, l) by the key byte at dispatch_depth. + [[nodiscard]] static boost::container::small_vector, + 16> + partition_by_byte(RandomIt f, RandomIt l, tree_depth_type dispatch_depth) { + boost::container::small_vector, 16> parts; + const auto dd = static_cast(dispatch_depth); + auto cur = f; + while (cur != l) { + const art_key_type ak{cur->first}; + const auto kv = ak.get_key_view(); + UNODB_DETAIL_ASSERT(dd < kv.size()); + const auto byte = kv[dd]; + parts.push_back({cur, byte}); + ++cur; + while (cur != l) { + const art_key_type ak2{cur->first}; + const auto kv2 = ak2.get_key_view(); + UNODB_DETAIL_ASSERT(dd < kv2.size()); + if (kv2[dd] != byte) break; + ++cur; + } + } + return parts; + } + + /// Build a chain of I4 nodes encoding the prefix from start_depth to + /// end_depth, with child_inode at the bottom. + [[nodiscard]] static node_ptr_t build_prefix_chain( + Db& self, art_key_type k, node_ptr_t child_inode, + tree_depth_type start_depth, std::size_t end_depth) { + constexpr std::size_t prefix_cap = key_prefix_capacity; + const auto full_key = k.get_key_view(); + const auto start = static_cast(start_depth); + auto current = child_inode; + std::size_t pos = end_depth; + while (pos > start + prefix_cap) { + const auto depth = pos - prefix_cap - 1; + const auto dispatch = full_key[pos - 1]; + auto remaining = k; + remaining.shift_right(depth); + auto chain{ + inode_4::create(self, full_key, remaining, + tree_depth_type{static_cast(depth)}, + dispatch, current)}; + current = node_ptr_t{chain.release(), node_type::I4}; +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + pos = depth; + } + // LCOV_EXCL_START — always-false: chain_consumed is always a multiple of + // (prefix_cap+1), so the while loop always decrements pos to exactly start. + UNODB_DETAIL_ASSERT(pos == start); + if (pos > start) { + const auto dispatch = full_key[pos - 1]; + auto chain{inode_4::create( + self, full_key, tree_depth_type{static_cast(start)}, + static_cast(pos - start - 1), dispatch, current)}; + current = node_ptr_t{chain.release(), node_type::I4}; +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + } + // LCOV_EXCL_STOP + return current; + } + + /// Build a single leaf (or packed value) from the iterator at given depth. + [[nodiscard]] static build_result_t build_single_leaf(Db& self, RandomIt it, + tree_depth_type depth) { + const art_key_type ak{it->first}; + if constexpr (art_policy::can_eliminate_leaf) { + const auto packed = art_policy::pack_value(it->second); + if (static_cast(depth) >= ak.size()) { + return {packed, true}; + } + return {self.build_chain(ak, packed, depth), false}; + } else if constexpr (art_policy::can_eliminate_key_in_leaf) { + auto leaf = art_policy::make_db_leaf_ptr(ak, it->second, self); + auto leaf_ptr = node_ptr_t{leaf.release(), node_type::LEAF}; + if (static_cast(depth) >= ak.size()) { + return {leaf_ptr, false}; + } + return {self.build_chain(ak, leaf_ptr, depth), false}; + } else { + static_cast(depth); + auto leaf = art_policy::make_db_leaf_ptr(ak, it->second, self); + const auto leaf_ptr = node_ptr_t{leaf.release(), node_type::LEAF}; + return {leaf_ptr, false}; + } + } + + /// Assemble an internal node of the appropriate fan-out. + [[nodiscard]] static node_ptr_t make_inode( + Db& self, std::span cs, + const boost::container::small_vector& guards, + const boost::container::small_vector& children, + key_prefix_size inode_prefix_len, key_view prefix_kv, + tree_depth_type inode_depth) { + if constexpr (!art_policy::can_eliminate_leaf) { + static_cast(guards); + static_cast(children); + } + const auto child_count = cs.size(); + if (child_count <= 4) { + std::uint8_t vmask = 0; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask |= static_cast(1U << i); + } + } + auto ptr = inode_4::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I4}; + } + if (child_count <= 16) { + std::uint16_t vmask = 0; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask |= static_cast(1U << i); + } + } + auto ptr = inode_16::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I16}; + } + if (child_count <= 48) { + std::array vmask{}; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) + vmask[i / 8] |= static_cast(1U << (i % 8)); + } + } + auto ptr = inode_48::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I48}; + } + std::array vmask{}; + if constexpr (art_policy::can_eliminate_leaf) { + for (std::size_t i = 0; i < child_count; ++i) { + if (guards[i].is_packed_value) { + const auto kb = static_cast(children[i].key_byte); + vmask[static_cast(kb / 8)] |= + static_cast(1U << (kb % 8)); + } + } + } + auto ptr = inode_256::create_bulk(self, inode_prefix_len, prefix_kv, + inode_depth, cs, vmask); +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + return node_ptr_t{ptr.release(), node_type::I256}; + } + + /// Recursively build a subtree from a sorted range of key-value pairs. + [[nodiscard]] static build_result_t build_subtree(Db& self, RandomIt f, + RandomIt l, + tree_depth_type depth) { + constexpr std::size_t prefix_cap = key_prefix_capacity; + + const auto n = std::distance(f, l); + if (n == 0) return {node_ptr_t{nullptr}, false}; + if (n == 1) return build_single_leaf(self, f, depth); + + const auto prefix_len = common_prefix_length(f, l, depth); + const auto dispatch_depth = tree_depth_type{static_cast( + static_cast(depth) + prefix_len)}; + auto parts = partition_by_byte(f, l, dispatch_depth); + const auto child_count = parts.size(); + + boost::container::small_vector guards; + guards.reserve(child_count); + boost::container::small_vector children; + children.reserve(child_count); + + const auto next_depth = tree_depth_type{static_cast( + static_cast(dispatch_depth) + 1)}; + + for (std::size_t i = 0; i < child_count; ++i) { + const auto part_begin = parts[i].begin; + const auto part_end = (i + 1 < child_count) ? parts[i + 1].begin : l; + auto result = build_subtree(self, part_begin, part_end, next_depth); + guards.emplace_back(self); + guards.back().ptr = result.ptr; + guards.back().is_packed_value = result.is_packed_value; + children.push_back({parts[i].key_byte, result.ptr}); + } + + const std::size_t chain_consumed = + (prefix_len > prefix_cap) + ? (prefix_len / (prefix_cap + 1)) * (prefix_cap + 1) + : 0; + const auto inode_prefix_len = + static_cast(prefix_len - chain_consumed); + const auto inode_depth = tree_depth_type{static_cast( + static_cast(depth) + chain_consumed)}; + + const art_key_type prefix_key{f->first}; + const auto prefix_kv = prefix_key.get_key_view(); + + const auto cs = + std::span{children.data(), children.size()}; + auto inode_ptr = make_inode(self, cs, guards, children, inode_prefix_len, + prefix_kv, inode_depth); + for (auto& g : guards) g.release(); + + if (chain_consumed > 0) { + return { + build_prefix_chain(self, prefix_key, inode_ptr, depth, + chain_consumed + static_cast(depth)), + false}; + } + return {inode_ptr, false}; + } +}; + +// ─── Top-level bulk_load_impl ──────────────────────────────────────────────── + +/// Core bulk_load algorithm parameterized on database type. +/// +/// \pre Tree must be empty (caller validates). +/// \pre [first, last) is sorted and non-empty (caller validates). +/// \tparam Fork Callable satisfying fork(Callable) -> Future where +/// Future has .get(). Ignored (nullptr_t) for sequential. +template +void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last) { + using H = bulk_load_helpers; + using build_result_t = typename Db::build_result; + using guard_t = typename Db::bulk_subtree_guard; + using node_ptr_t = typename build_result_t::ptr_type; + using bulk_child_t = bulk_child; + using tree_depth_type = typename Db::tree_depth_type; + using art_key_type = typename Db::art_key_type; + constexpr std::size_t prefix_cap = key_prefix_capacity; + + constexpr bool is_parallel = + !std::is_same_v, std::nullptr_t>; + + if constexpr (!is_parallel) { + auto result = H::build_subtree(self, first, last, tree_depth_type{0}); + self.root = result.ptr; + } else { + const auto n = std::distance(first, last); + if (n <= 1) { + auto result = H::build_subtree(self, first, last, tree_depth_type{0}); + self.root = result.ptr; + return; + } + + const auto prefix_len = + H::common_prefix_length(first, last, tree_depth_type{0}); + const auto dispatch_depth = + tree_depth_type{static_cast(prefix_len)}; + auto parts = H::partition_by_byte(first, last, dispatch_depth); + const auto child_count = parts.size(); + + const auto next_depth = tree_depth_type{static_cast( + static_cast(dispatch_depth) + 1)}; + + // Fork up to min(child_count, max_tasks) subtrees; inline the rest. + const auto fork_count = std::min(child_count, max_tasks); + +#ifdef UNODB_DETAIL_WITH_STATS + struct parallel_result { + build_result_t build; + bulk_load_stats_accumulator stats; + }; +#else + using parallel_result = build_result_t; +#endif + + auto make_task = [&self, &parts, child_count, last, + next_depth](std::size_t i) { + const auto part_begin = parts[i].begin; + const auto part_end = (i + 1 < child_count) ? parts[i + 1].begin : last; +#ifdef UNODB_DETAIL_WITH_STATS + bulk_load_stats_accumulator acc; + tls_bulk_load_stats = &acc; + auto result = H::build_subtree(self, part_begin, part_end, next_depth); + tls_bulk_load_stats = nullptr; + return parallel_result{result, acc}; +#else + return H::build_subtree(self, part_begin, part_end, next_depth); +#endif + }; + + // Submit forked tasks — first fork deduces the future type. + auto first_future = + fork([&make_task] { return make_task(std::size_t{0}); }); + using future_t = decltype(first_future); + std::vector futures; + futures.reserve(fork_count); + futures.push_back(std::move(first_future)); + for (std::size_t i = 1; i < fork_count; ++i) { + futures.push_back(fork([&make_task, i] { return make_task(i); })); + } + + // Run remaining partitions inline on the calling thread. + std::vector inline_results; + inline_results.reserve(child_count - fork_count); + for (std::size_t i = fork_count; i < child_count; ++i) { + inline_results.push_back(make_task(i)); + } + + boost::container::small_vector guards; + guards.reserve(child_count); + boost::container::small_vector children; + children.reserve(child_count); + + // Gather forked results first (indices 0..fork_count-1). + for (std::size_t i = 0; i < fork_count; ++i) { + try { + auto future_result = futures[i].get(); +#ifdef UNODB_DETAIL_WITH_STATS + const auto& result = future_result.build; + const auto& acc = future_result.stats; + self.merge_bulk_load_stats(acc); +#else + const auto& result = future_result; +#endif + guards.emplace_back(self); + guards.back().ptr = result.ptr; + guards.back().is_packed_value = result.is_packed_value; + children.push_back({parts[i].key_byte, result.ptr}); + } catch (...) { // LCOV_EXCL_START — exception drain; no reliable test + // Drain remaining futures into guards to prevent leaks. + for (std::size_t j = i + 1; j < fork_count; ++j) { + try { + auto r = futures[j].get(); + guards.emplace_back(self); +#ifdef UNODB_DETAIL_WITH_STATS + guards.back().ptr = r.build.ptr; + guards.back().is_packed_value = r.build.is_packed_value; +#else + guards.back().ptr = r.ptr; + guards.back().is_packed_value = r.is_packed_value; +#endif + } catch (...) { // future.get() may rethrow task exception + } + } + throw; + } // LCOV_EXCL_STOP + } + + // Gather inline results (indices fork_count..child_count-1). + for (std::size_t i = 0; i < inline_results.size(); ++i) { + const auto part_idx = fork_count + i; +#ifdef UNODB_DETAIL_WITH_STATS + const auto& result = inline_results[i].build; + const auto& acc = inline_results[i].stats; + self.merge_bulk_load_stats(acc); +#else + const auto& result = inline_results[i]; +#endif + guards.emplace_back(self); + guards.back().ptr = result.ptr; + guards.back().is_packed_value = result.is_packed_value; + children.push_back({parts[part_idx].key_byte, result.ptr}); + } + + const std::size_t chain_consumed = + (prefix_len > prefix_cap) + ? (prefix_len / (prefix_cap + 1)) * (prefix_cap + 1) + : 0; + const auto inode_prefix_len = + static_cast(prefix_len - chain_consumed); + const auto inode_depth = + tree_depth_type{static_cast(chain_consumed)}; + + const art_key_type prefix_key{first->first}; + const auto prefix_kv = prefix_key.get_key_view(); + + const auto cs = + std::span{children.data(), children.size()}; + auto inode_ptr = H::make_inode(self, cs, guards, children, inode_prefix_len, + prefix_kv, inode_depth); + for (auto& g : guards) g.release(); + + if (chain_consumed > 0) { + self.root = H::build_prefix_chain(self, prefix_key, inode_ptr, + tree_depth_type{0}, chain_consumed); + } else { + self.root = inode_ptr; + } + } +} + +} // namespace unodb::detail +/// \endcond + +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + +#endif // UNODB_DETAIL_ART_BULK_LOAD_DETAIL_HPP diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index 40755b64..30c3b61e 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -149,6 +150,75 @@ struct value_bitmask_field, CritSec> { static constexpr void insert_at(std::uint8_t) noexcept {} }; +/// A child entry for bulk inode construction. +/// Pre-sorted by key_byte; used by create_bulk factories. +template +struct bulk_child { + std::byte key_byte; + NodePtr child; +}; + +/// Tagged return from build_subtree: pointer + whether it's a VIS packed +/// value (not a real heap pointer — must not be passed to delete_subtree). +template +struct bulk_build_result { + using ptr_type = typename ArtPolicy::node_ptr; + ptr_type ptr{nullptr}; + bool is_packed_value{false}; +}; + +/// RAII guard for subtrees during bulk construction. +/// Skips deallocation for VIS packed values (not heap-allocated). +template +struct bulk_subtree_guard { + using db_type = typename ArtPolicy::db_type; + using node_ptr = typename ArtPolicy::node_ptr; + + db_type& db_; + node_ptr ptr{nullptr}; + bool is_packed_value{false}; + + constexpr explicit bulk_subtree_guard(db_type& db + UNODB_DETAIL_LIFETIMEBOUND) noexcept + : db_{db} {} + + ~bulk_subtree_guard() noexcept { + if (ptr != nullptr && !is_packed_value) { + ArtPolicy::delete_subtree(ptr, db_); + } + } + + void release() noexcept { ptr = nullptr; } + + bulk_subtree_guard(const bulk_subtree_guard&) = delete; + // LCOV_EXCL_START — move ctor only called on vector reallocation + bulk_subtree_guard(bulk_subtree_guard&& other) noexcept + : db_{other.db_}, + ptr{std::exchange(other.ptr, nullptr)}, + is_packed_value{other.is_packed_value} {} + // LCOV_EXCL_STOP + auto& operator=(const bulk_subtree_guard&) = delete; + auto& operator=(bulk_subtree_guard&&) = delete; +}; + +/// Build a chain of inode4 nodes for prefix compression during bulk load. +/// +/// Constructs bottom-up from key end toward start_depth. Each chain node +/// consumes up to key_prefix_capacity prefix bytes + 1 dispatch byte. +/// +/// \tparam ArtPolicy Policy type (provides node_ptr, db_type, inode types) +/// \param self Database instance +/// \param k Encoded key for prefix extraction +/// \param child Child pointer to wrap in the chain +/// \param start_depth Depth at which the chain terminates +/// \return The top node pointer of the chain +/// \note Defined at end of art_internal_impl.hpp (after inode types). +template +[[nodiscard]] typename ArtPolicy::node_ptr bulk_build_chain( + typename ArtPolicy::db_type& self, typename ArtPolicy::art_key_type k, + typename ArtPolicy::node_ptr child, + tree_depth start_depth); + #ifdef UNODB_DETAIL_X86_64 /// Compare packed unsigned 8-bit integers for less-than-or-equal. @@ -2193,6 +2263,16 @@ class [[nodiscard]] basic_inode : public basic_inode_impl { tree_depth depth, single_child_tag) noexcept : parent{1, prefix_len, k1, depth} {} + + /// Tag type to select the bulk-load constructor. + struct bulk_load_tag {}; + + /// Construct with arbitrary children count and explicit prefix. + /// Used by create_bulk for bulk-loading sorted data. + constexpr basic_inode(unsigned n_children, detail::key_prefix_size prefix_len, + unodb::key_view k1, tree_depth depth, + bulk_load_tag) noexcept + : parent{n_children, prefix_len, k1, depth} {} }; /// Type alias for basic_inode_4 parent class. @@ -2380,6 +2460,14 @@ class basic_inode_4 init(key_byte, child); } + /// Construct for bulk load with explicit children count and prefix. + constexpr basic_inode_4(db_type&, unsigned n_children, + detail::key_prefix_size prefix_len, key_view k1, + // cppcheck-suppress passedByValue + tree_depth_type depth, + typename parent_class::bulk_load_tag tag) noexcept + : parent_class{n_children, prefix_len, k1, depth, tag}, children{} {} + /// \} /// \name Initialization methods @@ -3037,6 +3125,43 @@ class basic_inode_4 template friend class basic_inode_16; friend class basic_inode_impl; + + public: + /// Bulk-construct an inode4 with pre-sorted children. + /// + /// \pre children.size() in [2, 4] + /// \pre children sorted by key_byte + /// \param db_instance Database for allocation + /// \param prefix_len Prefix bytes to store + /// \param prefix_key Key view for prefix extraction + /// \param depth Depth at which prefix starts + /// \param children_span Span of sorted (key_byte, child_ptr) pairs + /// \param value_mask Precomputed VIS bitmask + UNODB_DETAIL_DISABLE_CLANG_21_WARNING("-Wnrvo") + [[nodiscard]] static auto create_bulk( + db_type& db_instance, detail::key_prefix_size prefix_len, + unodb::key_view prefix_key, tree_depth_type depth, + std::span> children_span, + std::uint8_t value_mask = 0) { + const auto n = static_cast(children_span.size()); + UNODB_DETAIL_ASSERT(n >= 2 && n <= 4); + auto result = + inode4_type::create(db_instance, n, prefix_len, prefix_key, depth, + typename parent_class::bulk_load_tag{}); + auto* node = result.get(); + for (unsigned i = 0; i < n; ++i) { + node->keys.byte_array[i] = children_span[i].key_byte; + node->children[i] = children_span[i].child; + } +#ifndef UNODB_DETAIL_X86_64 + for (unsigned i = n; i < 4; ++i) node->keys.byte_array[i] = unused_key_byte; +#endif + if constexpr (ArtPolicy::can_eliminate_leaf) { + node->bitmask_base::bits = value_mask; + } + return result; + } + UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() }; // class basic_inode_4 /// Type alias for basic_inode_16 parent class. @@ -3123,6 +3248,14 @@ class basic_inode_16 init(db_instance, source_node, child_to_delete); } + /// Construct for bulk load with explicit children count and prefix. + constexpr basic_inode_16(db_type&, unsigned n_children, + detail::key_prefix_size prefix_len, key_view k1, + // cppcheck-suppress passedByValue + tree_depth_type depth, + typename parent_class::bulk_load_tag tag) noexcept + : parent_class{n_children, prefix_len, k1, depth, tag}, children{} {} + /// Initialize by growing from basic_inode_4 and adding a child. /// /// \param db_instance Database instance @@ -3659,6 +3792,34 @@ class basic_inode_16 friend class basic_inode_impl; template friend class basic_inode_48; + + public: + /// Bulk-construct an inode16 with pre-sorted children. + /// + /// \pre children_span.size() in [5, 16] + /// \pre children sorted by key_byte + UNODB_DETAIL_DISABLE_CLANG_21_WARNING("-Wnrvo") + [[nodiscard]] static auto create_bulk( + db_type& db_instance, detail::key_prefix_size prefix_len, + unodb::key_view prefix_key, tree_depth_type depth, + std::span> children_span, + std::uint16_t value_mask = 0) { + const auto n = static_cast(children_span.size()); + UNODB_DETAIL_ASSERT(n >= 5 && n <= 16); + auto result = + inode16_type::create(db_instance, n, prefix_len, prefix_key, depth, + typename parent_class::bulk_load_tag{}); + auto* node = result.get(); + for (unsigned i = 0; i < n; ++i) { + node->keys.byte_array[i] = children_span[i].key_byte; + node->children[i] = children_span[i].child; + } + if constexpr (ArtPolicy::can_eliminate_leaf) { + node->bitmask_base::bits = value_mask; + } + return result; + } + UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() }; // class basic_inode_16 /// Type alias for basic_inode_48 parent class. @@ -3749,6 +3910,14 @@ class basic_inode_48 init(db_instance, source_node, child_to_delete); } + /// Construct for bulk load with explicit children count and prefix. + constexpr basic_inode_48(db_type&, unsigned n_children, + detail::key_prefix_size prefix_len, key_view k1, + // cppcheck-suppress passedByValue + tree_depth_type depth, + typename parent_class::bulk_load_tag tag) noexcept + : parent_class{n_children, prefix_len, k1, depth, tag}, children{} {} + /// Initialize by growing from basic_inode_16 and adding a child. /// /// \param db_instance Database instance @@ -4436,6 +4605,40 @@ class basic_inode_48 friend class basic_inode_impl; template friend class basic_inode_256; + + public: + /// Bulk-construct an inode48 with pre-sorted children. + /// + /// \pre children_span.size() in [17, 48] + /// \pre children sorted by key_byte + UNODB_DETAIL_DISABLE_CLANG_21_WARNING("-Wnrvo") + [[nodiscard]] static auto create_bulk( + db_type& db_instance, detail::key_prefix_size prefix_len, + unodb::key_view prefix_key, tree_depth_type depth, + std::span> children_span, + const std::array& value_mask = {}) { + const auto n = static_cast(children_span.size()); + UNODB_DETAIL_ASSERT(n >= 17 && n <= 48); + auto result = + inode48_type::create(db_instance, n, prefix_len, prefix_key, depth, + typename parent_class::bulk_load_tag{}); + auto* node = result.get(); + for (unsigned i = 0; i < n; ++i) { + const auto kb = static_cast(children_span[i].key_byte); + node->child_indexes[kb] = static_cast(i); + node->children.pointer_array[i] = children_span[i].child; + } + for (unsigned i = n; i < 48; ++i) { + node->children.pointer_array[i] = node_ptr{nullptr}; + } + if constexpr (ArtPolicy::can_eliminate_leaf) { + for (std::size_t i = 0; i < value_mask.size(); ++i) { + node->bitmask_base::bits[i] = value_mask[i]; + } + } + return result; + } + UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() }; // class basic_inode_48 /// Type alias for basic_inode_256 parent class. @@ -4508,6 +4711,14 @@ class basic_inode_256 init(db_instance, source_node, packed_value, depth, key_byte); } + /// Construct for bulk load with explicit children count and prefix. + constexpr basic_inode_256(db_type&, unsigned n_children, + detail::key_prefix_size prefix_len, key_view k1, + // cppcheck-suppress passedByValue + tree_depth_type depth, + typename parent_class::bulk_load_tag tag) noexcept + : parent_class{n_children, prefix_len, k1, depth, tag}, children{} {} + /// Initialize by growing from basic_inode_48 and adding a child. /// /// \param db_instance Database for memory tracking @@ -4881,8 +5092,110 @@ class basic_inode_256 template friend class basic_inode_48; friend class basic_inode_impl; + + public: + /// Bulk-construct an inode256 with pre-sorted children. + /// + /// \pre children_span.size() in [49, 256] + /// \pre children sorted by key_byte + UNODB_DETAIL_DISABLE_CLANG_21_WARNING("-Wnrvo") + [[nodiscard]] static auto create_bulk( + db_type& db_instance, detail::key_prefix_size prefix_len, + unodb::key_view prefix_key, tree_depth_type depth, + std::span> children_span, + const std::array& value_mask = {}) { + const auto n = static_cast(children_span.size()); + UNODB_DETAIL_ASSERT(n >= 49 && n <= 256); + auto result = + parent_class::create(db_instance, n, prefix_len, prefix_key, depth, + typename parent_class::bulk_load_tag{}); + auto* node = result.get(); + for (unsigned i = 0; i < 256; ++i) { + node->children[i] = node_ptr{nullptr}; + } + for (unsigned i = 0; i < n; ++i) { + const auto kb = static_cast(children_span[i].key_byte); + node->children[kb] = children_span[i].child; + } + if constexpr (ArtPolicy::can_eliminate_leaf) { + for (std::size_t i = 0; i < value_mask.size(); ++i) { + node->bitmask_base::bits[i] = value_mask[i]; + } + } + return result; + } + UNODB_DETAIL_RESTORE_CLANG_21_WARNINGS() }; // class basic_inode_256 +// ─── Bulk Load: build_chain ───────────────────────────────────────────────── + +UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) +UNODB_DETAIL_DISABLE_MSVC_WARNING(4459) + +template +[[nodiscard]] typename ArtPolicy::node_ptr bulk_build_chain( + typename ArtPolicy::db_type& self, typename ArtPolicy::art_key_type k, + typename ArtPolicy::node_ptr child, + tree_depth start_depth) { + using node_ptr = typename ArtPolicy::node_ptr; + using inode_4 = typename ArtPolicy::inode4_type; + using tree_depth_type = tree_depth; + constexpr std::size_t cap = key_prefix_capacity; + const auto full_key = k.get_key_view(); + const auto key_len = k.size(); + const auto start = static_cast(start_depth); + auto current = child; + bool child_is_value = ArtPolicy::can_eliminate_leaf; + bool owns_current = false; + try { + std::size_t pos = key_len; + while (pos > start + cap) { + const auto depth = pos - cap - 1; + const auto dispatch = full_key[pos - 1]; + auto remaining = k; + remaining.shift_right(depth); + auto chain{ + inode_4::create(self, full_key, remaining, + tree_depth_type{static_cast(depth)}, + dispatch, current)}; + if (child_is_value) { + chain->set_value_bit(0); + child_is_value = false; + } + current = node_ptr{chain.release(), node_type::I4}; + owns_current = true; +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + pos = depth; + } + if (pos > start) { + const auto dispatch = full_key[pos - 1]; + auto chain{inode_4::create( + self, full_key, tree_depth_type{static_cast(start)}, + static_cast(pos - start - 1), dispatch, current)}; + if (child_is_value) { + chain->set_value_bit(0); + } + current = node_ptr{chain.release(), node_type::I4}; +#ifdef UNODB_DETAIL_WITH_STATS + self.template account_growing_inode(); +#endif + } + return current; + } catch (...) { + if (owns_current) { + ArtPolicy::delete_subtree(current, self); + } else if (!child_is_value) { + ArtPolicy::delete_subtree(current, self); + } + throw; + } +} + +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + } // namespace unodb::detail #endif // UNODB_DETAIL_ART_INTERNAL_IMPL_HPP diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index a6b12656..fa4e7fb3 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -10,6 +10,7 @@ set(micro_benchmark_quick_arg "--benchmark_filter=\".*/100$$|.*/1000/.*:800$$|.*/100/.*:0$$\"") set(micro_benchmark_mutex_quick_arg "--benchmark_filter=\"/4/70000/\"") set(micro_benchmark_olc_quick_arg "--benchmark_filter=\"/4/70000/\"") +set(micro_benchmark_bulk_load_quick_arg "--benchmark_filter=\"/1024\"") add_custom_target(benchmarks env ${SANITIZER_ENV} ./micro_benchmark_key_prefix @@ -20,7 +21,8 @@ add_custom_target(benchmarks COMMAND env ${SANITIZER_ENV} ./micro_benchmark_n256 COMMAND env ${SANITIZER_ENV} ./micro_benchmark COMMAND env ${SANITIZER_ENV} ./micro_benchmark_mutex - COMMAND env ${SANITIZER_ENV} ./micro_benchmark_olc) + COMMAND env ${SANITIZER_ENV} ./micro_benchmark_olc + COMMAND env ${SANITIZER_ENV} ./micro_benchmark_bulk_load) add_custom_target(quick_benchmarks env ${SANITIZER_ENV} @@ -40,7 +42,9 @@ add_custom_target(quick_benchmarks COMMAND env ${SANITIZER_ENV} ./micro_benchmark_mutex ${micro_benchmark_mutex_quick_arg} COMMAND env ${SANITIZER_ENV} - ./micro_benchmark_olc ${micro_benchmark_olc_quick_arg}) + ./micro_benchmark_olc ${micro_benchmark_olc_quick_arg} + COMMAND env ${SANITIZER_ENV} + ./micro_benchmark_bulk_load ${micro_benchmark_bulk_load_quick_arg}) add_custom_target(valgrind_benchmarks COMMAND ${VALGRIND_COMMAND} ./micro_benchmark_key_prefix @@ -59,7 +63,9 @@ add_custom_target(valgrind_benchmarks COMMAND ${VALGRIND_COMMAND} ./micro_benchmark_mutex ${micro_benchmark_mutex_quick_arg} COMMAND ${VALGRIND_COMMAND} ./micro_benchmark_olc - ${micro_benchmark_olc_quick_arg}) + ${micro_benchmark_olc_quick_arg} + COMMAND ${VALGRIND_COMMAND} ./micro_benchmark_bulk_load + ${micro_benchmark_bulk_load_quick_arg}) add_library(micro_benchmark_utils STATIC micro_benchmark_utils.cpp micro_benchmark_utils.hpp) @@ -102,3 +108,4 @@ add_node_benchmark_target(micro_benchmark_n256) add_node_benchmark_target(micro_benchmark) add_concurrent_benchmark_target(micro_benchmark_mutex) add_concurrent_benchmark_target(micro_benchmark_olc) +add_benchmark_target(micro_benchmark_bulk_load) diff --git a/benchmark/micro_benchmark_bulk_load.cpp b/benchmark/micro_benchmark_bulk_load.cpp new file mode 100644 index 00000000..67949e4c --- /dev/null +++ b/benchmark/micro_benchmark_bulk_load.cpp @@ -0,0 +1,477 @@ +// Copyright 2026 UnoDB contributors + +// Should be the first include +#include "global.hpp" // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "art.hpp" +#include "art_common.hpp" +#include "micro_benchmark_utils.hpp" +#include "mutex_art.hpp" +#include "olc_art.hpp" + +UNODB_DETAIL_DISABLE_MSVC_WARNING(26445) +UNODB_DETAIL_DISABLE_MSVC_WARNING(26496) + +namespace { + +using db = unodb::benchmark::db; +using mutex_db = unodb::benchmark::mutex_db; +using olc_db = unodb::benchmark::olc_db; + +using kv_db = unodb::benchmark::kv_db; +using kv_mutex_db = unodb::mutex_db; +using kv_olc_db = unodb::benchmark::kv_olc_db; + +constexpr auto val8 = std::array{}; +const auto value = unodb::value_view{val8}; + +// =================================================================== +// Helpers +// =================================================================== + +/// Generate sorted sequential keys [0..n). +[[nodiscard]] auto sequential_keys(std::int64_t n) { + std::vector> kv; + kv.reserve(static_cast(n)); + for (std::int64_t i = 0; i < n; ++i) + kv.emplace_back(static_cast(i), value); + return kv; +} + +/// Generate sorted random unique keys. +[[nodiscard]] auto random_keys(std::int64_t n) { + std::mt19937_64 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) + std::vector keys(static_cast(n)); + std::generate(keys.begin(), keys.end(), rng); + std::sort(keys.begin(), keys.end()); + keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); + std::vector> kv; + kv.reserve(keys.size()); + std::transform(keys.begin(), keys.end(), std::back_inserter(kv), + [](auto k) { return std::make_pair(k, value); }); + return kv; +} + +// =================================================================== +// BM01: bulk_load sequential uint64 keys (all tree modes) +// =================================================================== + +template +void BM_BulkLoad_seq(benchmark::State& state) { + const auto kv = sequential_keys(state.range(0)); + for (const auto _ : state) { + Db test_db; + test_db.bulk_load(kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +template +void BM_Insert_seq(benchmark::State& state) { + const auto kv = sequential_keys(state.range(0)); + for (const auto _ : state) { + Db test_db; + for (const auto& [k, v] : kv) static_cast(test_db.insert(k, v)); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +// =================================================================== +// BM02: bulk_load random uint64 keys (all tree modes) +// =================================================================== + +template +void BM_BulkLoad_rand(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + Db test_db; + test_db.bulk_load(kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +template +void BM_Insert_rand(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + Db test_db; + for (const auto& [k, v] : kv) static_cast(test_db.insert(k, v)); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +// =================================================================== +// BM03: bulk_load key_view compound keys (all key_view tree modes) +// =================================================================== + +template +void BM_BulkLoad_kv(benchmark::State& state) { + const auto n = static_cast(state.range(0)); + auto keys = unodb::benchmark::key_view_set::compound(0x01, n); + std::vector> kv; + kv.reserve(n); + // cppcheck-suppress useStlAlgorithm + for (std::size_t i = 0; i < n; ++i) kv.emplace_back(keys[i], value); + + for (const auto _ : state) { + Db test_db; + test_db.bulk_load(kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} + +template +void BM_Insert_kv(benchmark::State& state) { + const auto n = static_cast(state.range(0)); + auto keys = unodb::benchmark::key_view_set::compound(0x01, n); + std::vector> kv; + kv.reserve(n); + // cppcheck-suppress useStlAlgorithm + for (std::size_t i = 0; i < n; ++i) kv.emplace_back(keys[i], value); + + for (const auto _ : state) { + Db test_db; + for (const auto& [k, v] : kv) static_cast(test_db.insert(k, v)); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} + +// =================================================================== +// BM04: bulk_load parallel scaling (olc_db) +// =================================================================== + +void BM_BulkLoad_par_db(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + db test_db; + test_db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +void BM_BulkLoad_par_mutex(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + mutex_db test_db; + test_db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +void BM_BulkLoad_par_olc(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + olc_db test_db; + test_db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +// =================================================================== +// BM05: bulk_load scaling (N varies, all tree modes) +// =================================================================== + +template +void BM_BulkLoad_scaling(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + Db test_db; + test_db.bulk_load(kv.begin(), kv.end()); + benchmark::DoNotOptimize(test_db); + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +// =================================================================== +// BM07: memory comparison (requires STATS build) +// =================================================================== + +#ifdef UNODB_DETAIL_WITH_STATS + +template +void BM_BulkLoad_memory(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + std::size_t bulk_mem = 0; + std::size_t insert_mem = 0; + + for (const auto _ : state) { + { + Db bulk_db; + bulk_db.bulk_load(kv.begin(), kv.end()); + bulk_mem = bulk_db.get_current_memory_use(); + benchmark::DoNotOptimize(bulk_mem); + state.PauseTiming(); + bulk_db.clear(); + state.ResumeTiming(); + } + { + state.PauseTiming(); + Db insert_db; + for (const auto& [k, v] : kv) static_cast(insert_db.insert(k, v)); + insert_mem = insert_db.get_current_memory_use(); + insert_db.clear(); + state.ResumeTiming(); + } + } + state.counters["bulk_bytes"] = benchmark::Counter( + static_cast(bulk_mem), benchmark::Counter::kDefaults, + benchmark::Counter::OneK::kIs1024); + state.counters["insert_bytes"] = benchmark::Counter( + static_cast(insert_mem), benchmark::Counter::kDefaults, + benchmark::Counter::OneK::kIs1024); +} + +#endif // UNODB_DETAIL_WITH_STATS + +// =================================================================== +// BM08: first-scan penalty after bulk_load (all tree modes) +// =================================================================== + +template +void BM_BulkLoad_first_scan(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + state.PauseTiming(); + Db test_db; + test_db.bulk_load(kv.begin(), kv.end()); + state.ResumeTiming(); + + std::size_t count = 0; + test_db.scan([&count](const auto& /*visitor*/) { + ++count; + return false; // continue + }); + benchmark::DoNotOptimize(count); + + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +template +void BM_Insert_first_scan(benchmark::State& state) { + const auto kv = random_keys(state.range(0)); + for (const auto _ : state) { + state.PauseTiming(); + Db test_db; + for (const auto& [k, v] : kv) static_cast(test_db.insert(k, v)); + state.ResumeTiming(); + + std::size_t count = 0; + test_db.scan([&count](const auto& /*visitor*/) { + ++count; + return false; // continue + }); + benchmark::DoNotOptimize(count); + + state.PauseTiming(); + test_db.clear(); + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * + static_cast(kv.size())); +} + +} // namespace + +// =================================================================== +// Registration — all tree modes +// =================================================================== + +UNODB_START_BENCHMARKS() + +// BM01: Sequential bulk_load vs insert (db, mutex_db, olc_db) +BENCHMARK_TEMPLATE(BM_BulkLoad_seq, db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_seq, mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_seq, olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_seq, db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_seq, mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_seq, olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); + +// BM02: Random bulk_load vs insert (db, mutex_db, olc_db) +BENCHMARK_TEMPLATE(BM_BulkLoad_rand, db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_rand, mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_rand, olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_rand, db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_rand, mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_rand, olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); + +// BM03: key_view bulk_load vs insert (kv_db, kv_mutex_db, kv_olc_db) +BENCHMARK_TEMPLATE(BM_BulkLoad_kv, kv_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_kv, kv_mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_kv, kv_olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_kv, kv_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_kv, kv_mutex_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_kv, kv_olc_db) + ->Range(1 << 10, 1 << 20) + ->Unit(benchmark::kMillisecond); + +// BM04: Parallel execution (all tree modes, par vs seq) +BENCHMARK(BM_BulkLoad_par_db)->Arg(1 << 20)->Unit(benchmark::kMillisecond); +BENCHMARK(BM_BulkLoad_par_mutex)->Arg(1 << 20)->Unit(benchmark::kMillisecond); +BENCHMARK(BM_BulkLoad_par_olc)->Arg(1 << 20)->Unit(benchmark::kMillisecond); + +// BM05: Size scaling (db, mutex_db, olc_db) +BENCHMARK_TEMPLATE(BM_BulkLoad_scaling, db) + ->Arg(1 << 10) + ->Arg(1 << 14) + ->Arg(1 << 17) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_scaling, mutex_db) + ->Arg(1 << 10) + ->Arg(1 << 14) + ->Arg(1 << 17) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_scaling, olc_db) + ->Arg(1 << 10) + ->Arg(1 << 14) + ->Arg(1 << 17) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); + +// BM07: Memory comparison (stats build only; db, mutex_db, olc_db) +#ifdef UNODB_DETAIL_WITH_STATS +BENCHMARK_TEMPLATE(BM_BulkLoad_memory, db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_memory, mutex_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_memory, olc_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +#endif + +// BM08: First-scan penalty (db, mutex_db, olc_db) +BENCHMARK_TEMPLATE(BM_BulkLoad_first_scan, db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_first_scan, mutex_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_BulkLoad_first_scan, olc_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_first_scan, db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_first_scan, mutex_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(BM_Insert_first_scan, olc_db) + ->Arg(1 << 20) + ->Unit(benchmark::kMillisecond); + +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + +UNODB_BENCHMARK_MAIN(); diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index cc5fd84b..85767e4c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -39,12 +39,16 @@ target_link_libraries(example_olc_art PRIVATE unodb) add_executable(example_upsert example_upsert.cpp) target_link_libraries(example_upsert PRIVATE unodb) +add_executable(example_bulk_load example_bulk_load.cpp) +target_link_libraries(example_bulk_load PRIVATE unodb) + add_custom_target(examples ./example_art COMMAND ./example_art_stats COMMAND ./example_olc_art COMMAND ./example_upsert - DEPENDS example_art example_art_stats example_olc_art example_upsert) + COMMAND ./example_bulk_load + DEPENDS example_art example_art_stats example_olc_art example_upsert example_bulk_load) set(VALGRIND_COMMAND "valgrind" "--error-exitcode=1" "--leak-check=full" "--trace-children=yes" "-v") @@ -54,4 +58,5 @@ add_custom_target(valgrind_examples COMMAND ${VALGRIND_COMMAND} ./example_art_stats COMMAND ${VALGRIND_COMMAND} ./example_olc_art COMMAND ${VALGRIND_COMMAND} ./example_upsert - DEPENDS example_art example_art_stats example_olc_art example_upsert) + COMMAND ${VALGRIND_COMMAND} ./example_bulk_load + DEPENDS example_art example_art_stats example_olc_art example_upsert example_bulk_load) diff --git a/examples/example_bulk_load.cpp b/examples/example_bulk_load.cpp new file mode 100644 index 00000000..babd5aca --- /dev/null +++ b/examples/example_bulk_load.cpp @@ -0,0 +1,75 @@ +// Copyright 2026 UnoDB contributors + +// Example: bulk_load with sequential and parallel execution. + +#include "global.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "art.hpp" +#include "art_common.hpp" +#include "olc_art.hpp" + +namespace { + +constexpr std::size_t key_count = 100'000; + +/// Build sorted key-value pairs. The caller is responsible for +/// pre-encoding and pre-sorting keys before calling bulk_load. +[[nodiscard]] auto make_sorted_data() { + constexpr std::array val{}; + const auto value = unodb::value_view{val}; + std::vector> kv; + kv.reserve(key_count); + for (std::size_t i = 0; i < key_count; ++i) + kv.emplace_back(static_cast(i), value); + return kv; +} + +} // namespace + +int main() { + const auto data = make_sorted_data(); + + // ─── Sequential bulk_load (default) ──────────────────────────────────────── + { + unodb::db tree; + tree.bulk_load(data.begin(), data.end()); // sequential (default) + std::cerr << "Sequential bulk_load: " << key_count << " keys loaded\n"; + std::cerr << " get(42) found: " << tree.get(42).has_value() << '\n'; + tree.clear(); + } + + // ─── Parallel bulk_load ──────────────────────────────────────────────────── + // The caller provides a fork callable to submit parallel tasks. + // The implementation partitions at the root level and builds each + // subtree via the fork callable. Safe for all tree modes because + // bulk_load operates on an unpublished tree (no concurrent readers). + auto async_fork = [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }; + { + unodb::db tree; + tree.bulk_load(async_fork, 8, data.begin(), data.end()); + std::cerr << "Parallel bulk_load: " << key_count << " keys loaded\n"; + std::cerr << " get(99999) found: " << tree.get(99999).has_value() << '\n'; + tree.clear(); + } + + // ─── olc_db: same API ────────────────────────────────────────────────────── + { + unodb::olc_db tree; + tree.bulk_load(async_fork, 8, data.begin(), data.end()); + std::cerr << "olc_db parallel bulk_load: " << key_count << " keys loaded\n"; + tree.clear(); + } + + std::cerr << "Done.\n"; +} diff --git a/mutex_art.hpp b/mutex_art.hpp index 8aaaf223..643131ba 100644 --- a/mutex_art.hpp +++ b/mutex_art.hpp @@ -147,6 +147,27 @@ class mutex_db final { db_.clear(); } + /// Bulk-load sorted key-value pairs into an empty tree. + /// + /// \tparam Fork Callable: fork(Callable) -> Future with .get() + /// \tparam RandomIt Random access iterator over pairs of (key, value) + /// \param fork Callable to submit parallel tasks + /// \param max_tasks Maximum tasks to fork + /// \param first Start of sorted range + /// \param last End of sorted range + template + void bulk_load(Fork&& fork, std::size_t max_tasks, RandomIt first, + RandomIt last) { + const std::lock_guard guard{mutex}; + db_.bulk_load(std::forward(fork), max_tasks, first, last); + } + + /// Convenience overload: sequential execution. + template + void bulk_load(RandomIt first, RandomIt last) { + bulk_load(nullptr, 0, first, last); + } + // // scan API. // diff --git a/node_type.hpp b/node_type.hpp index 91597e07..4699903c 100644 --- a/node_type.hpp +++ b/node_type.hpp @@ -87,6 +87,28 @@ inline constexpr auto internal_as_i{ NodeType) - 1}; +namespace detail { + +/// Thread-local stats accumulator for parallel bulk_load. +/// +/// During parallel subtree construction on non-thread-safe db types, +/// each thread writes stats here instead of directly to the db instance +/// (which has non-atomic counters). After all async tasks complete, +/// accumulators are merged single-threaded into the db. +struct bulk_load_stats_accumulator { + std::size_t current_memory_use{0}; + node_type_counter_array node_counts{}; + inode_type_counter_array growing_inode_counts{}; + std::uint64_t key_prefix_splits{0}; +}; + +/// Per-thread pointer to the active stats accumulator. +/// Null during normal operation; set per-thread during parallel bulk_load. +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +inline thread_local bulk_load_stats_accumulator* tls_bulk_load_stats{nullptr}; + +} // namespace detail + #endif // UNODB_DETAIL_WITH_STATS } // namespace unodb diff --git a/olc_art.hpp b/olc_art.hpp index 0e207ef0..b5ff1a92 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -167,6 +167,14 @@ template using olc_leaf_unique_ptr = basic_db_leaf_unique_ptr; +/// Forward declarations of bulk_load algorithm and helpers. +template +void bulk_load_impl(Db& self, Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last); + +template +struct bulk_load_helpers; + } // namespace detail /// A thread-safe Adaptive Radix Tree that is synchronized using optimistic lock @@ -187,9 +195,10 @@ class olc_db final { using leaf_type = detail::olc_leaf_type; using db_type = olc_db; - private: + /// Internal encoded key type used for tree operations. using art_key_type = detail::basic_art_key; + private: /// Query for a value associated with an encoded key. [[nodiscard, gnu::pure]] get_result get_internal( art_key_type search_key) const noexcept; @@ -296,6 +305,25 @@ class olc_db final { /// \note Only legal in single-threaded context, as destructor void clear() noexcept; + /// Bulk-load sorted key-value pairs into an empty tree. + /// + /// \note Only legal in single-threaded context (same as clear). + /// \tparam Fork Callable: fork(Callable) -> Future with .get() + /// \tparam RandomIt Random access iterator over pairs of (key, value) + /// \param fork Callable to submit parallel tasks + /// \param max_tasks Maximum tasks to fork + /// \param first Start of sorted range + /// \param last End of sorted range + template + void bulk_load(Fork&& fork, std::size_t max_tasks, RandomIt first, + RandomIt last); + + /// Convenience overload: sequential execution. + template + void bulk_load(RandomIt first, RandomIt last) { + bulk_load(nullptr, 0, first, last); + } + // // iterator (the iterator is an internal API, the public API is scan()). // @@ -850,10 +878,23 @@ class olc_db final { using inode_16 = detail::olc_inode_16; using inode_48 = detail::olc_inode_48; using inode_256 = detail::olc_inode_256; + + public: + /// Tree depth tracking type. using tree_depth_type = detail::tree_depth; + + /// Result of subtree construction during bulk_load. + /// Tagged return from build_subtree. + using build_result = detail::bulk_build_result; + + /// RAII guard for subtrees during bulk construction. + using bulk_subtree_guard = detail::bulk_subtree_guard; + + private: using visitor_type = visitor; using olc_db_leaf_unique_ptr_type = detail::olc_db_leaf_unique_ptr; + // If get_result is not present, the search was interrupted. Yes, this // resolves to std::optional>, but IMHO both // levels of std::optional are clear here @@ -978,6 +1019,12 @@ class olc_db final { template constexpr void account_shrinking_inode() noexcept; + /// Merge accumulated stats from a parallel bulk_load subtree. + /// For olc_db this is a no-op: atomic counters are updated directly. + // cppcheck-suppress functionStatic + constexpr void merge_bulk_load_stats( + const detail::bulk_load_stats_accumulator& /*acc*/) noexcept {} + #endif // UNODB_DETAIL_WITH_STATS // optimistic lock guarding the [root]. @@ -1023,6 +1070,21 @@ class olc_db final { friend auto detail::make_db_leaf_ptr(art_key_type, value_type, olc_db&); + /// detail::bulk_load_impl + template + friend void detail::bulk_load_impl(Db2&, Fork2&&, std::size_t, It2, It2); + + /// detail::bulk_load_helpers + template + friend struct detail::bulk_load_helpers; + + /// detail::bulk_build_chain + template + friend typename ArtPolicy2::node_ptr detail::bulk_build_chain( + typename ArtPolicy2::db_type&, typename ArtPolicy2::art_key_type, + typename ArtPolicy2::node_ptr, + detail::tree_depth); + template friend class detail::basic_db_leaf_deleter; @@ -2310,68 +2372,7 @@ bool olc_db::insert_internal(art_key_type insert_key, template detail::olc_node_ptr olc_db::build_chain( art_key_type k, detail::olc_node_ptr child, tree_depth_type start_depth) { - constexpr std::size_t cap = detail::key_prefix_capacity; - const auto full_key = k.get_key_view(); - const auto key_len = k.size(); - const auto start = static_cast(start_depth); - auto current = child; - bool child_is_value = art_policy::can_eliminate_leaf; - bool owns_current = - false; // set true once we've built at least one chain node - try { - std::size_t pos = key_len; - while (pos > start + cap) { - const auto depth = pos - cap - 1; - const auto dispatch = full_key[pos - 1]; - auto remaining = k; - remaining.shift_right(depth); - auto chain{ - inode_4::create(*this, full_key, remaining, - tree_depth_type{static_cast(depth)}, - dispatch, current)}; - if (child_is_value) { - UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) - chain->set_value_bit(0); - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - child_is_value = false; - } - UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) - current = detail::olc_node_ptr{chain.release(), node_type::I4}; - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - owns_current = true; -#ifdef UNODB_DETAIL_WITH_STATS - account_growing_inode(); -#endif - pos = depth; - } - if (pos > start) { - const auto dispatch = full_key[pos - 1]; - auto chain{inode_4::create( - *this, full_key, tree_depth_type{static_cast(start)}, - static_cast(pos - start - 1), dispatch, - current)}; - if (child_is_value) { - UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) - chain->set_value_bit(0); - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - } - UNODB_DETAIL_DISABLE_MSVC_WARNING(26815) - current = detail::olc_node_ptr{chain.release(), node_type::I4}; - UNODB_DETAIL_RESTORE_MSVC_WARNINGS() - owns_current = true; -#ifdef UNODB_DETAIL_WITH_STATS - account_growing_inode(); -#endif - } - } catch (...) { - if (owns_current) { - art_policy::delete_subtree(current, *this); - } else if (!child_is_value) { - art_policy::delete_subtree(current, *this); - } - throw; - } - return current; + return detail::bulk_build_chain(*this, k, child, start_depth); } template @@ -4968,4 +4969,32 @@ olc_db::try_chain_cut( } } // namespace unodb +// ─── olc_db::bulk_load ─────────────────────────────────────────────────────── + +namespace unodb { + +template +template +void olc_db::bulk_load(Fork&& fork, std::size_t max_tasks, + RandomIt first, RandomIt last) { + UNODB_DETAIL_QSBR_ASSERT( + qsbr_state::single_thread_mode(qsbr::instance().get_state())); + + if (!empty()) { + throw std::invalid_argument("bulk_load requires empty tree"); + } + if (first == last) return; + try { + detail::bulk_load_impl(*this, std::forward(fork), max_tasks, first, + last); + } catch (...) { + clear(); + throw; + } +} + +} // namespace unodb + +#include "art_bulk_load_detail.hpp" + #endif // UNODB_DETAIL_OLC_ART_HPP diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e58704a4..cf2b25c1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,9 @@ add_db_test_target(test_key_encode_decode) add_db_test_target(test_art) target_compile_options(test_art PRIVATE "$<${is_msvc}:/bigobj>") +add_db_test_target(test_art_bulk_load) +target_link_libraries(test_art_bulk_load PRIVATE qsbr_test_utils) + add_db_test_target(test_art_key_view) add_db_test_target(test_art_key_view_full_chain) add_db_test_target(test_art_iter) @@ -99,7 +102,7 @@ add_test(NAME test_olc_no_qsbr COMMAND test_olc_no_qsbr) if(COVERAGE) add_custom_target(tests_for_coverage ctest DEPENDS test_key_encode_decode test_art test_art_key_view test_art_key_view_full_chain test_art_iter test_art_scan test_art_concurrency test_art_allocator test_art_upsert test_olc_no_qsbr test_qsbr_ptr test_qsbr test_art_oom - test_qsbr_oom) + test_qsbr_oom test_art_bulk_load) add_coverage_target(TARGET coverage DEPENDENCY tests_for_coverage) endif() @@ -115,5 +118,6 @@ add_custom_target(valgrind_tests COMMAND ${VALGRIND_COMMAND} ./test_art_iter; COMMAND ${VALGRIND_COMMAND} ./test_art_scan; COMMAND ${VALGRIND_COMMAND} ./test_art_concurrency; - COMMAND ${VALGRIND_COMMAND} ./test_art_upsert - DEPENDS test_qsbr_ptr test_qsbr test_key_encode_decode test_art test_art_key_view test_art_key_view_full_chain test_art_iter test_art_scan test_art_concurrency test_art_upsert) + COMMAND ${VALGRIND_COMMAND} ./test_art_upsert; + COMMAND ${VALGRIND_COMMAND} ./test_art_bulk_load + DEPENDS test_qsbr_ptr test_qsbr test_key_encode_decode test_art test_art_key_view test_art_key_view_full_chain test_art_iter test_art_scan test_art_concurrency test_art_upsert test_art_bulk_load) diff --git a/test/test_art_bulk_load.cpp b/test/test_art_bulk_load.cpp new file mode 100644 index 00000000..61a59293 --- /dev/null +++ b/test/test_art_bulk_load.cpp @@ -0,0 +1,854 @@ +// Copyright 2026 UnoDB contributors + +// Should be the first include +#include "global.hpp" // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include // NOLINT(misc-include-cleaner) + +#include "art_common.hpp" +#include "db_test_utils.hpp" +#include "gtest_utils.hpp" +#include "node_type.hpp" +#include "qsbr.hpp" +#include "qsbr_test_utils.hpp" + +UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) +UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) +UNODB_DETAIL_DISABLE_MSVC_WARNING(26445) + +namespace { + +#ifdef UNODB_DETAIL_WITH_STATS +using unodb::as_i; +using unodb::node_type; +#endif +using unodb::key_view; +using unodb::value_view; +using unodb::test::u64_db; +using unodb::test::u64_mutex_db; +using unodb::test::u64_olc_db; + +constexpr auto val_bytes = + std::array{std::byte{0x68}, std::byte{0x65}, std::byte{0x6C}, + std::byte{0x6C}, std::byte{0x6F}}; +constexpr auto val = value_view{val_bytes}; + +#ifdef UNODB_DETAIL_WITH_STATS + +constexpr auto val16 = std::array{}; +constexpr auto large_val = value_view{val16}; + +/// Generate N keys that differ at byte 0 (big-endian uint64_t). +/// For N <= 256, all keys differ only at byte 0 (single root node). +/// For N > 256, extra keys share byte 0 with key 0 but differ at byte 1. +[[nodiscard]] std::vector> make_keys( + std::size_t n) { + std::vector> kv; + kv.reserve(n); + for (std::uint64_t i = 0; i < n && i < 256; ++i) { + const auto key = i << 56U; + kv.emplace_back(key, val); + } + // Overflow keys: share byte 0 == 0x00, differ at byte 1 + for (std::uint64_t i = 256; i < n; ++i) { + const auto key = (i - 255) << 48U; + kv.emplace_back(key, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + return kv; +} + +// ─── Node Count Tests ──────────────────────────────────────────────────────── + +UNODB_TEST(BulkLoad, Empty) { + u64_db db; + std::vector> kv; + db.bulk_load(kv.begin(), kv.end()); + UNODB_ASSERT_TRUE(db.empty()); + const auto counts = db.get_node_counts(); + for (const auto c : counts) { + UNODB_ASSERT_EQ(c, 0U); + } +} + +UNODB_TEST(BulkLoad, Single) { + u64_db db; + constexpr std::uint64_t key = 1; + std::vector> kv{{key, val}}; + db.bulk_load(kv.begin(), kv.end()); + const auto result = db.get(key); + UNODB_ASSERT_TRUE(result.has_value()); + UNODB_ASSERT_EQ( // NOLINT(bugprone-unchecked-optional-access) + result.value().size(), val.size()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +UNODB_TEST(BulkLoad, Small4Keys) { + u64_db db; + auto kv = make_keys(4); + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 4U); + UNODB_ASSERT_EQ(counts[as_i], 1U); +} + +UNODB_TEST(BulkLoad, Boundary4) { + u64_db db; + auto kv = make_keys(4); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); + UNODB_ASSERT_EQ(counts[as_i], 0U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +UNODB_TEST(BulkLoad, Boundary5) { + u64_db db; + auto kv = make_keys(5); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +UNODB_TEST(BulkLoad, Boundary16) { + u64_db db; + auto kv = make_keys(16); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +UNODB_TEST(BulkLoad, Boundary17) { + u64_db db; + auto kv = make_keys(17); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +UNODB_TEST(BulkLoad, Boundary48) { + u64_db db; + auto kv = make_keys(48); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +UNODB_TEST(BulkLoad, Boundary49) { + u64_db db; + auto kv = make_keys(49); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +UNODB_TEST(BulkLoad, Growth260) { + u64_db db; + auto kv = make_keys(260); + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 260U); +} + +UNODB_TEST(BulkLoad, SharedPrefix) { + u64_db db; + std::vector> kv; + kv.reserve(20); + // All keys share byte 0 = 0x01, differ at byte 1. + for (std::uint64_t i = 0; i < 20; ++i) { + kv.emplace_back((1ULL << 56U) | (i << 48U), val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +UNODB_TEST(BulkLoad, DeepPrefixChain) { + using unodb::test::key_view_db; + key_view_db db; + std::vector, value_view>> raw_keys; + raw_keys.reserve(10); + for (int i = 0; i < 10; ++i) { + std::vector k(13); + for (int j = 0; j < 10; ++j) { + k[static_cast(j)] = static_cast(j + 1); + } + k[10] = static_cast(i); + k[11] = std::byte{0xAA}; + k[12] = std::byte{0xBB}; + raw_keys.emplace_back(std::move(k), val); + } + std::vector> kv; + kv.reserve(raw_keys.size()); + for (auto& [k, v] : raw_keys) { + kv.emplace_back(unodb::key_view{k.data(), k.size()}, v); + } + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +// ─── Structural Tests ──────────────────────────────────────────────────────── + +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix7) { + u64_db db; + std::vector> kv{ + {0x0102030405060700ULL, val}, {0x0102030405060701ULL, val}}; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix8) { + using unodb::test::key_view_db; + std::vector> s{ + {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}, std::byte{5}, + std::byte{6}, std::byte{7}, std::byte{8}, std::byte{0}}, + {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}, std::byte{5}, + std::byte{6}, std::byte{7}, std::byte{8}, std::byte{1}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 2U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix15) { + using unodb::test::key_view_db; + std::vector> s( + 2, std::vector(16, std::byte{0xAA})); + s[0][15] = std::byte{0}; + s[1][15] = std::byte{1}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 2U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadPrefix16) { + using unodb::test::key_view_db; + std::vector> s( + 2, std::vector(17, std::byte{0xBB})); + s[0][16] = std::byte{0}; + s[1][16] = std::byte{1}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 3U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVIS) { + using unodb::test::key_view_u64val_db; + std::vector> s{{std::byte{1}}, {std::byte{2}}}; + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISWithChain) { + using unodb::test::key_view_u64val_db; + std::vector> s{{std::byte{1}, std::byte{0xAA}}, + {std::byte{1}, std::byte{0xBB}}}; + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, 99ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISLongPrefix) { + using unodb::test::key_view_u64val_db; + std::vector> s( + 2, std::vector(9, std::byte{0xCC})); + s[0][8] = std::byte{0}; + s[1][8] = std::byte{1}; + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, 77ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 2U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeaf) { + using unodb::test::key_view_db; + std::vector> s{ + {std::byte{1}, std::byte{2}, std::byte{3}}, + {std::byte{4}, std::byte{5}, std::byte{6}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 3U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeafWithChain) { + using unodb::test::key_view_db; + std::vector> s{ + {std::byte{1}, std::byte{0xAA}, std::byte{0xFF}}, + {std::byte{1}, std::byte{0xBB}, std::byte{0xFF}}}; + std::vector> kv; + for (auto& v : s) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 3U); + UNODB_ASSERT_EQ(c[as_i], 2U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadLarge) { + std::mt19937 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp) + std::vector> kv; + kv.reserve(100000); + UNODB_DETAIL_DISABLE_GCC_WARNING("-Wuseless-cast") + for (int i = 0; i < 100000; ++i) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + (static_cast(rng()) << 32U) | rng(), val); + UNODB_DETAIL_RESTORE_GCC_WARNINGS() + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + kv.erase(std::unique( // NOLINT(modernize-use-ranges) + kv.begin(), kv.end(), + [](const auto& a, const auto& b) { return a.first == b.first; }), + kv.end()); + u64_db db; + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) UNODB_ASSERT_TRUE(db.get(k).has_value()); + std::uint64_t prev = 0; + bool first = true; + UNODB_DETAIL_DISABLE_MSVC_WARNING(26440) + db.scan([&](auto& visitor) { + unodb::key_decoder dec{visitor.get_key()}; + std::uint64_t k{}; + dec.decode(k); + if (!first) { + UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) + UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) + EXPECT_GE(k, prev); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + } + prev = k; + first = false; + return false; + }); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +} + +UNODB_TEST(BulkLoadStructural, BulkLoadKeyView) { + using unodb::test::key_view_db; + constexpr std::size_t key_len = 10; + std::mt19937 rng(123); // NOLINT(cert-msc32-c,cert-msc51-cpp) + std::vector> storage; + for (int i = 0; i < 1000; ++i) { + std::vector k(key_len); + for (auto& b : k) b = static_cast(rng() & 0xFFU); + storage.push_back(std::move(k)); + } + std::ranges::sort(storage); + // NOLINTNEXTLINE(modernize-use-ranges) + storage.erase(std::unique(storage.begin(), storage.end()), storage.end()); + std::vector> kv; + kv.reserve(storage.size()); + for (auto& v : storage) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{v}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) UNODB_ASSERT_TRUE(db.get(k).has_value()); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISRootSingle) { + using unodb::test::key_view_u64val_db; + std::vector> s{{std::byte{1}, std::byte{2}}}; + std::vector> kv; + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{s[0]}, 55ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + UNODB_ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadKeylessLeafRootSingle) { + using unodb::test::key_view_db; + std::vector> s{ + {std::byte{1}, std::byte{2}, std::byte{3}}}; + std::vector> kv; + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + key_view{s[0]}, large_val); + key_view_db db; + db.bulk_load(kv.begin(), kv.end()); + UNODB_ASSERT_TRUE(db.get(key_view{s[0]}).has_value()); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadFullLeafRootSingle) { + u64_db db; + std::vector> kv{{0xDEADBEEFULL, val}}; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadScanOrder) { + std::vector> kv; + for (std::uint64_t i = 0; i < 256; ++i) + kv.emplace_back( // NOLINT(performance-inefficient-vector-operation) + i << 56U, val); + u64_db db; + db.bulk_load(kv.begin(), kv.end()); + std::uint64_t prev = 0; + std::size_t count = 0; + UNODB_DETAIL_DISABLE_MSVC_WARNING(26440) + db.scan([&](auto& visitor) { + unodb::key_decoder dec{visitor.get_key()}; + std::uint64_t k{}; + dec.decode(k); + if (count > 0) { + UNODB_EXPECT_GT(k, prev); + } + prev = k; + ++count; + return false; + }); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_ASSERT_EQ(count, 256U); +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISNode16) { + using unodb::test::key_view_u64val_db; + std::vector> s; + s.reserve(8); + for (int i = 0; i < 8; ++i) { + s.push_back({static_cast(i)}); + } + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISNode48) { + using unodb::test::key_view_u64val_db; + std::vector> s; + s.reserve(20); + for (int i = 0; i < 20; ++i) { + s.push_back({static_cast(i)}); + } + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +UNODB_TEST(BulkLoadStructural, BulkLoadVISNode256) { + using unodb::test::key_view_u64val_db; + std::vector> s; + s.reserve(60); + for (int i = 0; i < 60; ++i) { + s.push_back({static_cast(i)}); + } + std::vector> kv; + kv.reserve(s.size()); + for (auto& v : s) kv.emplace_back(key_view{v}, 42ULL); + key_view_u64val_db db; + db.bulk_load(kv.begin(), kv.end()); + const auto c = db.get_node_counts(); + UNODB_ASSERT_EQ(c[as_i], 1U); + UNODB_ASSERT_EQ(c[as_i], 0U); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +// ─── Stats-dependent Operational Tests ─────────────────────────────────────── + +UNODB_TEST(BulkLoadOps, Stats) { + u64_db db; + std::vector> kv; + kv.reserve(17); + for (std::uint64_t i = 0; i < 17; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + const auto counts = db.get_node_counts(); + UNODB_ASSERT_EQ(counts[as_i], 17U); + UNODB_ASSERT_EQ(counts[as_i], 0U); + UNODB_ASSERT_EQ(counts[as_i], 0U); + UNODB_ASSERT_EQ(counts[as_i], 1U); + UNODB_ASSERT_EQ(counts[as_i], 0U); +} + +#endif // UNODB_DETAIL_WITH_STATS + +// ─── Error Tests (no stats dependency) ─────────────────────────────────────── + +UNODB_TEST(BulkLoadError, NonEmpty) { + u64_db db; + constexpr std::uint64_t key = 42; + UNODB_ASSERT_TRUE(db.insert(key, val)); + std::vector> kv{{100, val}}; + UNODB_DETAIL_DISABLE_MSVC_WARNING(6326) + UNODB_DETAIL_DISABLE_MSVC_WARNING(26818) + EXPECT_THROW(db.bulk_load(kv.begin(), kv.end()), std::invalid_argument); + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + UNODB_DETAIL_RESTORE_MSVC_WARNINGS() + const auto result = db.get(key); + UNODB_ASSERT_TRUE(result.has_value()); +} + +UNODB_TEST(BulkLoadError, OlcNonEmpty) { + u64_olc_db db; + constexpr std::uint64_t key = 42; + ASSERT_TRUE(db.insert(key, val)); + unodb::this_thread().quiescent(); + std::vector> kv{{100, val}}; + EXPECT_THROW(db.bulk_load(kv.begin(), kv.end()), std::invalid_argument); + const auto result = db.get(key); + ASSERT_TRUE(result.has_value()); +} + +UNODB_TEST(BulkLoadError, DbIgnoresParallelism) { + u64_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + const auto key = i << 56U; + kv.emplace_back(key, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +// ─── Operational Tests (no stats dependency) ───────────────────────────────── + +UNODB_TEST(BulkLoadOps, ThenOperations) { + u64_db db; + std::vector> kv; + kv.reserve(10); + for (std::uint64_t i = 0; i < 10; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } + constexpr std::uint64_t new_key = 0xFFULL << 56U; + UNODB_ASSERT_TRUE(db.insert(new_key, val)); + UNODB_ASSERT_TRUE(db.get(new_key).has_value()); + UNODB_ASSERT_FALSE(db.insert(kv[0].first, val)); + UNODB_ASSERT_TRUE(db.remove(kv[0].first)); + UNODB_ASSERT_FALSE(db.get(kv[0].first).has_value()); + + std::size_t count = 0; + db.scan([&count](auto&) { + ++count; + return false; + }); + UNODB_ASSERT_EQ(count, 10U); // 10 - 1 removed + 1 new = 10 +} + +UNODB_TEST(BulkLoadOps, NoGrowthEvents) { + u64_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()) << "key " << k << " not found"; + } +} + +UNODB_TEST(BulkLoadOps, MutexDb) { + u64_mutex_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.first.has_value()) << "key " << k << " not found"; + } +} + +UNODB_TEST(BulkLoadOps, ClearAndReload) { + u64_db db; + std::vector> kv; + kv.reserve(10); + for (std::uint64_t i = 0; i < 10; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load(kv.begin(), kv.end()); + db.clear(); + UNODB_ASSERT_TRUE(db.empty()); + std::vector> kv2; + kv2.reserve(10); + for (std::uint64_t i = 100; i < 110; ++i) { + kv2.emplace_back(i << 56U, val); + } + std::ranges::sort(kv2, {}, &decltype(kv2)::value_type::first); + db.bulk_load(kv2.begin(), kv2.end()); + for (const auto& [k, v] : kv2) { + UNODB_ASSERT_TRUE(db.get(k).has_value()); + } +} + +UNODB_TEST(BulkLoadOps, OlcDbParallel) { + u64_olc_db db; + std::vector> kv; + kv.reserve(1000); + for (std::uint64_t i = 0; i < 1000; ++i) { + kv.emplace_back(i << 48U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +UNODB_TEST(BulkLoadOps, OlcDbConcurrentReaders) { + u64_olc_db db; + constexpr std::size_t n_keys = 10000; + std::vector> kv; + kv.reserve(n_keys); + for (std::uint64_t i = 0; i < n_keys; ++i) { + kv.emplace_back(i << 40U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 256, kv.begin(), kv.end()); + + unodb::this_thread().qsbr_pause(); + + constexpr std::size_t n_threads = 4; + std::array threads; + for (std::size_t t = 0; t < n_threads; ++t) { + threads[t] = unodb::qsbr_thread([&kv, &db] { + for (const auto& [k, v] : kv) { + const unodb::quiescent_state_on_scope_exit qsbr{}; + const auto result = db.get(k); + EXPECT_TRUE(result.has_value()) << "key " << k << " not found"; + } + unodb::this_thread().quiescent(); + }); + } + for (auto& t : threads) t.join(); + + unodb::this_thread().qsbr_resume(); + unodb::this_thread().quiescent(); + unodb::test::expect_idle_qsbr(); +} + +// ─── Parallel Path Coverage ────────────────────────────────────────────────── + +UNODB_TEST(BulkLoadOps, ParallelInlineRemainder) { + u64_db db; + std::vector> kv; + kv.reserve(100); + for (std::uint64_t i = 0; i < 100; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()) << "key " << k << " not found"; + } +} + +UNODB_TEST(BulkLoadOps, ParallelSingleElement) { + u64_db db; + std::vector> kv; + kv.emplace_back(42ULL << 56U, val); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + ASSERT_TRUE(db.get(42ULL << 56U).has_value()); +} + +UNODB_TEST(BulkLoadOps, ParallelPrefixChain) { + using unodb::test::key_view_db; + key_view_db db; + std::vector> raw_keys; + raw_keys.reserve(20); + for (int i = 0; i < 20; ++i) { + std::vector k(13); + for (int j = 0; j < 10; ++j) { + k[static_cast(j)] = static_cast(j + 1); + } + k[10] = static_cast(i); + k[11] = std::byte{0xAA}; + k[12] = std::byte{0xBB}; + raw_keys.push_back(std::move(k)); + } + std::vector> kv; + kv.reserve(raw_keys.size()); + for (auto& k : raw_keys) { + kv.emplace_back(unodb::key_view{k.data(), k.size()}, val); + } + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + ASSERT_TRUE(db.get(k).has_value()); + } +} + +UNODB_TEST(BulkLoadOps, OlcDbParallelInline) { + u64_olc_db db; + std::vector> kv; + kv.reserve(50); + for (std::uint64_t i = 0; i < 50; ++i) { + kv.emplace_back(i << 56U, val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +UNODB_TEST(BulkLoadOps, OlcDbParallelSharedPrefix) { + u64_olc_db db; + std::vector> kv; + kv.reserve(20); + for (std::uint64_t i = 0; i < 20; ++i) { + kv.emplace_back((1ULL << 56U) | (i << 48U), val); + } + std::ranges::sort(kv, {}, &decltype(kv)::value_type::first); + db.bulk_load( + [](auto&& f) { + return std::async(std::launch::async, std::forward(f)); + }, + 4, kv.begin(), kv.end()); + for (const auto& [k, v] : kv) { + const auto result = db.get(k); + ASSERT_TRUE(result.has_value()) << "key " << k << " not found"; + } +} + +} // namespace + +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() +UNODB_DETAIL_RESTORE_MSVC_WARNINGS() diff --git a/test/test_art_oom.cpp b/test/test_art_oom.cpp index cda64e41..b2bc1636 100644 --- a/test/test_art_oom.cpp +++ b/test/test_art_oom.cpp @@ -1,4 +1,4 @@ -// Copyright 2022-2025 UnoDB contributors +// Copyright 2022-2026 UnoDB contributors #ifndef NDEBUG @@ -11,6 +11,8 @@ #include #include // IWYU pragma: keep +#include +#include #include @@ -644,6 +646,86 @@ UNODB_TYPED_TEST(ARTKeyViewOOMTest, BuildChainMultiNode) { [](unodb::test::tree_verifier&) {}); } +// =================================================================== +// bulk_load OOM tests — strong exception guarantee: tree stays empty +// =================================================================== + +template +void bulk_load_oom_test( + unsigned fail_limit, + const std::vector>& kv) { + unsigned fail_n; + for (fail_n = 1; fail_n <= fail_limit; ++fail_n) { + TypeParam test_db; + + unodb::test::allocation_failure_injector::fail_on_nth_allocation(fail_n); + try { + test_db.bulk_load(kv.begin(), kv.end()); + // Success: we've found the limit + unodb::test::allocation_failure_injector::reset(); + UNODB_ASSERT_FALSE(test_db.empty()); + for (const auto& [k, v] : kv) { + UNODB_ASSERT_TRUE(TypeParam::key_found(test_db.get(k))); + } + return; + } catch (const std::bad_alloc&) { + unodb::test::allocation_failure_injector::reset(); + // Strong guarantee: tree must be empty after failed bulk_load + UNODB_ASSERT_TRUE(test_db.empty()); +#ifdef UNODB_DETAIL_WITH_STATS + UNODB_ASSERT_EQ(test_db.get_current_memory_use(), 0); +#endif + } + } + FAIL() << "bulk_load did not succeed within " << fail_limit << " allocations"; +} + +// Small tree (single leaf → one allocation) +UNODB_TYPED_TEST(ARTOOMTest, BulkLoadSingleKey) { + constexpr auto val = unodb::test::test_values[0]; + const std::vector> kv{{42, val}}; + bulk_load_oom_test(5, kv); +} + +// Tree that creates one inode4 (4 leaves + 1 inode4) +UNODB_TYPED_TEST(ARTOOMTest, BulkLoadInode4) { + constexpr auto val = unodb::test::test_values[0]; + std::vector> kv; + kv.reserve(4); + for (std::uint64_t i = 0; i < 4; ++i) kv.emplace_back(i << 56U, val); + bulk_load_oom_test(10, kv); +} + +// Tree that creates an inode16 (10 leaves + 1 inode16) +UNODB_TYPED_TEST(ARTOOMTest, BulkLoadInode16) { + constexpr auto val = unodb::test::test_values[0]; + std::vector> kv; + kv.reserve(10); + for (std::uint64_t i = 0; i < 10; ++i) kv.emplace_back(i << 56U, val); + bulk_load_oom_test(15, kv); +} + +// Tree that creates an inode48 (20 leaves + 1 inode48) +UNODB_TYPED_TEST(ARTOOMTest, BulkLoadInode48) { + constexpr auto val = unodb::test::test_values[0]; + std::vector> kv; + kv.reserve(20); + for (std::uint64_t i = 0; i < 20; ++i) kv.emplace_back(i << 56U, val); + bulk_load_oom_test(30, kv); +} + +// Tree with nested inodes (two inode4s under one root inode4) +UNODB_TYPED_TEST(ARTOOMTest, BulkLoadNested) { + constexpr auto val = unodb::test::test_values[0]; + std::vector> kv; + kv.reserve(8); + for (std::uint64_t i = 0; i < 4; ++i) + kv.emplace_back((1ULL << 56U) | (i << 48U), val); + for (std::uint64_t i = 0; i < 4; ++i) + kv.emplace_back((2ULL << 56U) | (i << 48U), val); + bulk_load_oom_test(20, kv); +} + } // namespace #endif // #ifndef NDEBUG