diff --git a/CPPLINT.cfg b/CPPLINT.cfg index 1106f9db..83e30e13 100644 --- a/CPPLINT.cfg +++ b/CPPLINT.cfg @@ -7,6 +7,8 @@ filter=-build/include_order, filter=-build/include_subdir, # Nice but reports a false positive in qsbr.hpp on std::max filter=-build/include_what_you_use, +# False positives with [[likely]] and [[unlikely]] +filter=-readability/braces, # clang-tidy owns NOLINT filter=-readability/nolint, filter=-runtime/references, diff --git a/art.cpp b/art.cpp index 1034b6b9..84696ad7 100644 --- a/art.cpp +++ b/art.cpp @@ -181,7 +181,7 @@ detail::node_ptr *impl_helpers::add_or_choose_subtree( const auto children_count = inode.get_children_count(); if constexpr (!std::is_same_v) { - if (UNODB_DETAIL_UNLIKELY(children_count == INode::capacity)) { + if (children_count == INode::capacity) [[unlikely]] { auto larger_node{INode::larger_derived_type::create( db_instance, inode, std::move(aleaf), depth)}; *node_in_parent = @@ -212,7 +212,7 @@ std::optional impl_helpers::remove_or_choose_subtree( const auto *const aleaf{child_ptr_val.template ptr<::leaf *>()}; if (!aleaf->matches(k)) return {}; - if (UNODB_DETAIL_UNLIKELY(inode.is_min_size())) { + if (inode.is_min_size()) [[unlikely]] { if constexpr (std::is_same_v) { auto current_node{ art_policy::make_db_inode_unique_ptr(&inode, db_instance)}; @@ -280,7 +280,8 @@ constexpr void db::account_shrinking_inode() noexcept { #endif // UNODB_DETAIL_WITH_STATS db::get_result db::get(key search_key) const noexcept { - if (UNODB_DETAIL_UNLIKELY(root == nullptr)) return {}; + if (root == nullptr) [[unlikely]] + return {}; auto node{root}; const detail::art_key k{search_key}; @@ -315,7 +316,7 @@ UNODB_DETAIL_DISABLE_MSVC_WARNING(26430) bool db::insert(key insert_key, value_view v) { const auto k = detail::art_key{insert_key}; - if (UNODB_DETAIL_UNLIKELY(root == nullptr)) { + if (root == nullptr) [[unlikely]] { auto leaf = unodb::detail::art_policy::make_db_leaf_ptr(k, v, *this); root = detail::node_ptr{leaf.release(), node_type::LEAF}; return true; @@ -330,7 +331,8 @@ bool db::insert(key insert_key, value_view v) { if (node_type == node_type::LEAF) { auto *const leaf{node->ptr<::leaf *>()}; const auto existing_key{leaf->get_key()}; - if (UNODB_DETAIL_UNLIKELY(k == existing_key)) return false; + if (k == existing_key) [[unlikely]] + return false; auto new_leaf = unodb::detail::art_policy::make_db_leaf_ptr(k, v, *this); auto new_node{detail::inode_4::create(*this, existing_key, remaining_key, @@ -381,7 +383,8 @@ UNODB_DETAIL_RESTORE_MSVC_WARNINGS() bool db::remove(key remove_key) { const auto k = detail::art_key{remove_key}; - if (UNODB_DETAIL_UNLIKELY(root == nullptr)) return false; + if (root == nullptr) [[unlikely]] + return false; if (root.type() == node_type::LEAF) { auto *const root_leaf{root.ptr()}; @@ -416,7 +419,8 @@ bool db::remove(key remove_key) { const auto remove_result{ inode->remove_or_choose_subtree>( node_type, remaining_key[0], k, *this, node)}; - if (UNODB_DETAIL_UNLIKELY(!remove_result)) return false; + if (!remove_result) [[unlikely]] + return false; auto *const child_ptr{*remove_result}; if (child_ptr == nullptr) return true; @@ -501,7 +505,8 @@ void db::iterator::dump() const { dump(std::cerr); } // If the tree is empty, then the result is the same as end(). db::iterator &db::iterator::first() { invalidate(); // clear the stack - if (UNODB_DETAIL_UNLIKELY(db_.root == nullptr)) return *this; // empty tree. + if (db_.root == nullptr) [[unlikely]] + return *this; // empty tree. auto node{db_.root}; return left_most_traversal(node); } @@ -511,7 +516,8 @@ db::iterator &db::iterator::first() { // If the tree is empty, then the result is the same as end(). db::iterator &db::iterator::last() { invalidate(); // clear the stack - if (UNODB_DETAIL_UNLIKELY(db_.root == nullptr)) return *this; // empty tree. + if (db_.root == nullptr) [[unlikely]] + return *this; // empty tree. auto node{db_.root}; return right_most_traversal(node); } @@ -625,7 +631,8 @@ db::iterator &db::iterator::seek(detail::art_key search_key, bool &match, bool fwd) { invalidate(); // invalidate the iterator (clear the stack). match = false; // unless we wind up with an exact match. - if (UNODB_DETAIL_UNLIKELY(db_.root == nullptr)) return *this; // aka end() + if (db_.root == nullptr) [[unlikely]] + return *this; // aka end() auto node{db_.root}; const detail::art_key k = search_key; diff --git a/art.hpp b/art.hpp index 8ea1bc67..d0c24dd0 100644 --- a/art.hpp +++ b/art.hpp @@ -573,7 +573,8 @@ inline void db::scan(FN fn, bool fwd) { it.first(); visitor v{it}; while (it.valid()) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.next(); } } else { @@ -581,7 +582,8 @@ inline void db::scan(FN fn, bool fwd) { it.last(); visitor v{it}; while (it.valid()) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.prior(); } } @@ -596,7 +598,8 @@ inline void db::scan_from(key from_key, FN fn, bool fwd) { it.seek(from_key_, match, true /*fwd*/); visitor v{it}; while (it.valid()) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.next(); } } else { @@ -604,7 +607,8 @@ inline void db::scan_from(key from_key, FN fn, bool fwd) { it.seek(from_key_, match, false /*fwd*/); visitor v{it}; while (it.valid()) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.prior(); } } @@ -629,7 +633,8 @@ inline void db::scan_range(key from_key, const key to_key, FN fn) { } visitor v{it}; while (it.valid() && it.cmp(to_key_) < 0) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.next(); if constexpr (debug) { std::cerr << "scan_range:: next()\n"; @@ -646,7 +651,8 @@ inline void db::scan_range(key from_key, const key to_key, FN fn) { } visitor v{it}; while (it.valid() && it.cmp(to_key_) > 0) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.prior(); if constexpr (debug) { std::cerr << "scan_range:: prior()\n"; diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index 28b54f6b..d9b564d3 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -160,7 +160,7 @@ template Db &db UNODB_DETAIL_LIFETIMEBOUND) { using leaf_type = basic_leaf
; - if (UNODB_DETAIL_UNLIKELY(v.size() > leaf_type::max_value_size)) { + if (v.size() > leaf_type::max_value_size) [[unlikely]] { throw std::length_error("Value length must fit in std::uint32_t"); } diff --git a/global.hpp b/global.hpp index 7abd615a..0deba785 100644 --- a/global.hpp +++ b/global.hpp @@ -1,4 +1,4 @@ -// Copyright 2019-2022 Laurynas Biveinis +// Copyright 2019-2025 Laurynas Biveinis #ifndef UNODB_DETAIL_GLOBAL_HPP #define UNODB_DETAIL_GLOBAL_HPP @@ -102,8 +102,11 @@ #endif +// Use only in the contexts where C++20 [[likely]] and [[unlikely]] are illegal, +// i.e. in subexpressions, return statements, ternary operator. #define UNODB_DETAIL_LIKELY(x) __builtin_expect(x, 1) #define UNODB_DETAIL_UNLIKELY(x) __builtin_expect(x, 0) + #define UNODB_DETAIL_UNUSED [[gnu::unused]] #define UNODB_DETAIL_FORCE_INLINE __attribute__((always_inline)) #define UNODB_DETAIL_NOINLINE __attribute__((noinline)) @@ -113,8 +116,12 @@ #else // #ifndef UNODB_DETAIL_MSVC #define UNODB_DETAIL_BUILTIN_ASSUME(x) __assume(x) + +// Use only in the contexts where C++20 [[likely]] and [[unlikely]] are illegal, +// i.e. in subexpressions, return statements, ternary operator. #define UNODB_DETAIL_LIKELY(x) (!!(x)) #define UNODB_DETAIL_UNLIKELY(x) (!!(x)) + #define UNODB_DETAIL_UNUSED [[maybe_unused]] #define UNODB_DETAIL_FORCE_INLINE __forceinline #define UNODB_DETAIL_NOINLINE __declspec(noinline) diff --git a/heap.hpp b/heap.hpp index 0f81ba28..139e2653 100644 --- a/heap.hpp +++ b/heap.hpp @@ -45,7 +45,8 @@ template #ifndef _MSC_VER const auto err = posix_memalign(&result, alignment, size); - if (UNODB_DETAIL_UNLIKELY(err != 0)) result = nullptr; + if (err != 0) [[unlikely]] + result = nullptr; // LCOV_EXCL_LINE #else result = _aligned_malloc(size, alignment); #ifndef NDEBUG @@ -57,8 +58,8 @@ template // NOLINTNEXTLINE(readability-simplify-boolean-expr) UNODB_DETAIL_ASSERT(result != nullptr || err == ENOMEM); - if (UNODB_DETAIL_UNLIKELY(result == nullptr)) { - throw std::bad_alloc{}; + if (result == nullptr) [[unlikely]] { + throw std::bad_alloc{}; // LCOV_EXCL_LINE } return result; diff --git a/olc_art.cpp b/olc_art.cpp index 09eda278..c3e2135e 100644 --- a/olc_art.cpp +++ b/olc_art.cpp @@ -601,7 +601,7 @@ UNODB_DETAIL_RESTORE_MSVC_WARNINGS() void create_leaf_if_needed(olc_db_leaf_unique_ptr &cached_leaf, unodb::detail::art_key k, unodb::value_view v, unodb::olc_db &db_instance) { - if (UNODB_DETAIL_LIKELY(cached_leaf == nullptr)) { + if (cached_leaf == nullptr) [[likely]] { UNODB_DETAIL_ASSERT(&cached_leaf.get_deleter().get_db() == &db_instance); // Do not assign because we do not need to assign the deleter // NOLINTNEXTLINE(misc-uniqueptr-reset-release) @@ -628,18 +628,19 @@ olc_impl_helpers::add_or_choose_subtree( const auto children_count = inode.get_children_count(); if constexpr (!std::is_same_v) { - if (UNODB_DETAIL_UNLIKELY(children_count == INode::capacity)) { + if (children_count == INode::capacity) [[unlikely]] { auto larger_node{ INode::larger_derived_type::create(db_instance, inode)}; { const optimistic_lock::write_guard write_unlock_on_exit{ std::move(parent_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(write_unlock_on_exit.must_restart())) + if (write_unlock_on_exit.must_restart()) [[unlikely]] return {}; // LCOV_EXCL_LINE optimistic_lock::write_guard node_write_guard{ std::move(node_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(node_write_guard.must_restart())) return {}; + if (node_write_guard.must_restart()) [[unlikely]] + return {}; larger_node->init(db_instance, inode, node_write_guard, std::move(cached_leaf), depth); @@ -659,9 +660,10 @@ olc_impl_helpers::add_or_choose_subtree( const optimistic_lock::write_guard write_unlock_on_exit{ std::move(node_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(write_unlock_on_exit.must_restart())) return {}; + if (write_unlock_on_exit.must_restart()) [[unlikely]] + return {}; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE inode.add_to_nonfull(std::move(cached_leaf), depth, children_count); @@ -683,9 +685,9 @@ template const auto [child_i, found_child]{inode.find_child(key_byte)}; if (found_child == nullptr) { - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE return false; @@ -693,28 +695,30 @@ template *child = found_child->load(); - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.check())) return {}; + if (!node_critical_section.check()) [[unlikely]] + return {}; // LCOV_EXCL_LINE auto &child_lock{node_ptr_lock(*child)}; *child_critical_section = child_lock.try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(child_critical_section->must_restart())) return {}; + if (child_critical_section->must_restart()) [[unlikely]] + return {}; // LCOV_EXCL_LINE *child_type = child->type(); if (*child_type != node_type::LEAF) { *child_in_parent = found_child; - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE return true; } const auto *const aleaf{child->ptr()}; if (!aleaf->matches(k)) { - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY(!child_critical_section->try_read_unlock())) + if (!child_critical_section->try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE return false; @@ -722,17 +726,19 @@ template const auto is_node_min_size{inode.is_min_size()}; - if (UNODB_DETAIL_LIKELY(!is_node_min_size)) { - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (!is_node_min_size) [[likely]] { + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE const optimistic_lock::write_guard node_guard{ std::move(node_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(node_guard.must_restart())) return {}; + if (node_guard.must_restart()) [[unlikely]] + return {}; optimistic_lock::write_guard child_guard{ std::move(*child_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(child_guard.must_restart())) return {}; + if (child_guard.must_restart()) [[unlikely]] + return {}; child_guard.unlock_and_obsolete(); @@ -747,14 +753,17 @@ template if constexpr (std::is_same_v) { const optimistic_lock::write_guard parent_guard{ std::move(parent_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(parent_guard.must_restart())) return {}; + if (parent_guard.must_restart()) [[unlikely]] + return {}; optimistic_lock::write_guard node_guard{std::move(node_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(node_guard.must_restart())) return {}; + if (node_guard.must_restart()) [[unlikely]] + return {}; optimistic_lock::write_guard child_guard{ std::move(*child_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(child_guard.must_restart())) return {}; + if (child_guard.must_restart()) [[unlikely]] + return {}; auto current_node{ olc_art_policy::make_db_inode_reclaimable_ptr(&inode, db_instance)}; @@ -771,14 +780,17 @@ template const optimistic_lock::write_guard parent_guard{ std::move(parent_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(parent_guard.must_restart())) return {}; + if (parent_guard.must_restart()) [[unlikely]] + return {}; optimistic_lock::write_guard node_guard{std::move(node_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(node_guard.must_restart())) return {}; + if (node_guard.must_restart()) [[unlikely]] + return {}; optimistic_lock::write_guard child_guard{ std::move(*child_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(child_guard.must_restart())) return {}; + if (child_guard.must_restart()) [[unlikely]] + return {}; smaller_node->init(db_instance, inode, node_guard, child_i, child_guard); *node_in_parent = detail::olc_node_ptr{smaller_node.release(), @@ -862,7 +874,7 @@ olc_db::get_result olc_db::get(key search_key) const noexcept { olc_db::try_get_result_type olc_db::try_get(detail::art_key k) const noexcept { auto parent_critical_section = root_pointer_lock.try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(parent_critical_section.must_restart())) { + if (parent_critical_section.must_restart()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return {}; @@ -871,8 +883,8 @@ olc_db::try_get_result_type olc_db::try_get(detail::art_key k) const noexcept { detail::olc_node_ptr node{root.load()}; // load root into [node]. - if (UNODB_DETAIL_UNLIKELY(node == nullptr)) { // special path if empty tree. - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) { + if (node == nullptr) [[unlikely]] { // special path if empty tree. + if (!parent_critical_section.try_read_unlock()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return {}; @@ -883,7 +895,7 @@ olc_db::try_get_result_type olc_db::try_get(detail::art_key k) const noexcept { } // A check() is required before acting on [node] by taking the lock. - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.check())) { + if (!parent_critical_section.check()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return {}; @@ -895,8 +907,9 @@ olc_db::try_get_result_type olc_db::try_get(detail::art_key k) const noexcept { while (true) { // Lock version chaining (node and parent) auto node_critical_section = node_ptr_lock(node).try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(node_critical_section.must_restart())) return {}; - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (node_critical_section.must_restart()) [[unlikely]] + return {}; // LCOV_EXCL_LINE + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE const auto node_type = node.type(); @@ -905,11 +918,11 @@ olc_db::try_get_result_type olc_db::try_get(detail::art_key k) const noexcept { const auto *const aleaf{node.ptr()}; if (aleaf->matches(k)) { const auto val_view{aleaf->get_value_view()}; - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE return qsbr_ptr_span{val_view}; } - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE return std::make_optional(std::nullopt); } @@ -921,7 +934,7 @@ olc_db::try_get_result_type olc_db::try_get(detail::art_key k) const noexcept { key_prefix.get_shared_length(remaining_key)}; if (shared_key_prefix_length < key_prefix_length) { - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE return std::make_optional(std::nullopt); } @@ -934,7 +947,7 @@ olc_db::try_get_result_type olc_db::try_get(detail::art_key k) const noexcept { inode->find_child(node_type, remaining_key[0]).second}; if (child_in_parent == nullptr) { - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE return std::make_optional(std::nullopt); } @@ -945,7 +958,8 @@ olc_db::try_get_result_type olc_db::try_get(detail::art_key k) const noexcept { node = child; remaining_key.shift_right(1); - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.check())) return {}; + if (!parent_critical_section.check()) [[unlikely]] + return {}; // LCOV_EXCL_LINE } } @@ -969,7 +983,7 @@ olc_db::try_update_result_type olc_db::try_insert( detail::art_key k, unodb::value_view v, unodb::detail::olc_db_leaf_unique_ptr &cached_leaf) { auto parent_critical_section = root_pointer_lock.try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(parent_critical_section.must_restart())) { + if (parent_critical_section.must_restart()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return {}; @@ -978,12 +992,12 @@ olc_db::try_update_result_type olc_db::try_insert( auto node{root.load()}; - if (UNODB_DETAIL_UNLIKELY(node == nullptr)) { + if (node == nullptr) [[unlikely]] { create_leaf_if_needed(cached_leaf, k, v, *this); const optimistic_lock::write_guard write_unlock_on_exit{ std::move(parent_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(write_unlock_on_exit.must_restart())) { + if (write_unlock_on_exit.must_restart()) [[unlikely]] { // Do not call spin_wait_loop_body here - creating the leaf took some time return {}; // LCOV_EXCL_LINE } @@ -996,7 +1010,7 @@ olc_db::try_update_result_type olc_db::try_insert( detail::tree_depth depth{}; auto remaining_key{k}; - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.check())) { + if (!parent_critical_section.check()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return {}; @@ -1005,20 +1019,21 @@ olc_db::try_update_result_type olc_db::try_insert( while (true) { auto node_critical_section = node_ptr_lock(node).try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(node_critical_section.must_restart())) return {}; + if (node_critical_section.must_restart()) [[unlikely]] + return {}; // LCOV_EXCL_LINE const auto node_type = node.type(); if (node_type == node_type::LEAF) { auto *const aleaf{node.ptr()}; const auto existing_key{aleaf->get_key()}; - if (UNODB_DETAIL_UNLIKELY(k == existing_key)) { - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (k == existing_key) [[unlikely]] { + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY(cached_leaf != nullptr)) { + if (cached_leaf != nullptr) [[unlikely]] { cached_leaf.reset(); // LCOV_EXCL_LINE } return false; @@ -1031,11 +1046,13 @@ olc_db::try_update_result_type olc_db::try_insert( { const optimistic_lock::write_guard parent_guard{ std::move(parent_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(parent_guard.must_restart())) return {}; + if (parent_guard.must_restart()) [[unlikely]] + return {}; // LCOV_EXCL_LINE const optimistic_lock::write_guard node_guard{ std::move(node_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(node_guard.must_restart())) return {}; + if (node_guard.must_restart()) [[unlikely]] + return {}; // LCOV_EXCL_LINE new_node->init(existing_key, remaining_key, depth, aleaf, std::move(cached_leaf)); @@ -1065,11 +1082,13 @@ olc_db::try_update_result_type olc_db::try_insert( { const optimistic_lock::write_guard parent_guard{ std::move(parent_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(parent_guard.must_restart())) return {}; + if (parent_guard.must_restart()) [[unlikely]] + return {}; // LCOV_EXCL_LINE const optimistic_lock::write_guard node_guard{ std::move(node_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(node_guard.must_restart())) return {}; + if (node_guard.must_restart()) [[unlikely]] + return {}; // LCOV_EXCL_LINE new_node->init(node, shared_prefix_length, depth, std::move(cached_leaf)); @@ -1095,12 +1114,13 @@ olc_db::try_update_result_type olc_db::try_insert( node_type, remaining_key[0], k, v, *this, depth, node_critical_section, node_in_parent, parent_critical_section, cached_leaf)}; - if (UNODB_DETAIL_UNLIKELY(!add_result)) return {}; + if (!add_result) [[unlikely]] + return {}; // LCOV_EXCL_LINE auto *const child_in_parent = *add_result; if (child_in_parent == nullptr) return true; - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE const auto child = child_in_parent->load(); @@ -1111,7 +1131,8 @@ olc_db::try_update_result_type olc_db::try_insert( ++depth; remaining_key.shift_right(1); - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.check())) return {}; + if (!parent_critical_section.check()) [[unlikely]] + return {}; // LCOV_EXCL_LINE } } @@ -1129,7 +1150,7 @@ bool olc_db::remove(key remove_key) { olc_db::try_update_result_type olc_db::try_remove(detail::art_key k) { auto parent_critical_section = root_pointer_lock.try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(parent_critical_section.must_restart())) { + if (parent_critical_section.must_restart()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return {}; @@ -1138,17 +1159,18 @@ olc_db::try_update_result_type olc_db::try_remove(detail::art_key k) { auto node{root.load()}; - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.check())) { + if (!parent_critical_section.check()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return {}; // LCOV_EXCL_STOP } - if (UNODB_DETAIL_UNLIKELY(node == nullptr)) return false; + if (node == nullptr) [[unlikely]] + return false; auto node_critical_section = node_ptr_lock(node).try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(node_critical_section.must_restart())) { + if (node_critical_section.must_restart()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return {}; @@ -1164,10 +1186,12 @@ olc_db::try_update_result_type olc_db::try_remove(detail::art_key k) { std::move(parent_critical_section)}; // Do not call spin_wait_loop_body from this point on - assume // the above took enough time - if (UNODB_DETAIL_UNLIKELY(parent_guard.must_restart())) return {}; + if (parent_guard.must_restart()) [[unlikely]] + return {}; // LCOV_EXCL_LINE optimistic_lock::write_guard node_guard{std::move(node_critical_section)}; - if (UNODB_DETAIL_UNLIKELY(node_guard.must_restart())) return {}; + if (node_guard.must_restart()) [[unlikely]] + return {}; // LCOV_EXCL_LINE node_guard.unlock_and_obsolete(); @@ -1177,7 +1201,7 @@ olc_db::try_update_result_type olc_db::try_remove(detail::art_key k) { return true; } - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE return false; @@ -1198,9 +1222,9 @@ olc_db::try_update_result_type olc_db::try_remove(detail::art_key k) { key_prefix.get_shared_length(remaining_key)}; if (shared_prefix_length < key_prefix_length) { - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return {}; // LCOV_EXCL_LINE return false; @@ -1221,7 +1245,8 @@ olc_db::try_update_result_type olc_db::try_remove(detail::art_key k) { node_critical_section, node_in_parent, &child_in_parent, &child_critical_section, &child_type, &child)}; - if (UNODB_DETAIL_UNLIKELY(!opt_remove_result)) return {}; + if (!opt_remove_result) [[unlikely]] + return {}; // LCOV_EXCL_LINE if (const auto remove_result{*opt_remove_result}; !remove_result) return false; @@ -1367,7 +1392,8 @@ olc_db::iterator &olc_db::iterator::next() { // reconstruct the key. So maybe we have two internal buffers on // the iterator to support this? const auto &akey = aleaf->get_key(); // access the key on the leaf. - if (UNODB_DETAIL_LIKELY(try_next())) return *this; + if (try_next()) [[likely]] + return *this; while (true) { bool match{}; // seek to the current key (or its successor). @@ -1394,7 +1420,8 @@ olc_db::iterator &olc_db::iterator::prior() { // reconstruct the key. So maybe we have two internal buffers on // the iterator to support this? const auto &akey = aleaf->get_key(); // access the key on the leaf. - if (UNODB_DETAIL_LIKELY(try_prior())) return *this; + if (try_prior()) [[likely]] + return *this; while (true) { bool match{}; // seek to the current key (or its predecessor) @@ -1425,10 +1452,10 @@ olc_db::iterator &olc_db::iterator::seek(detail::art_key search_key, bool olc_db::iterator::try_first() { invalidate(); // clear the stack auto parent_critical_section = db_.root_pointer_lock.try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(parent_critical_section.must_restart())) + if (parent_critical_section.must_restart()) [[unlikely]] return false; // LCOV_EXCL_LINE auto node{db_.root.load()}; - if (UNODB_DETAIL_UNLIKELY(node == nullptr)) { + if (node == nullptr) [[unlikely]] { return UNODB_DETAIL_LIKELY(parent_critical_section.try_read_unlock()); } return try_left_most_traversal(node, parent_critical_section); @@ -1440,10 +1467,10 @@ bool olc_db::iterator::try_first() { bool olc_db::iterator::try_last() { invalidate(); // clear the stack auto parent_critical_section = db_.root_pointer_lock.try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(parent_critical_section.must_restart())) + if (parent_critical_section.must_restart()) [[unlikely]] return false; // LCOV_EXCL_LINE auto node{db_.root.load()}; - if (UNODB_DETAIL_UNLIKELY(node == nullptr)) { + if (node == nullptr) [[unlikely]] { return UNODB_DETAIL_LIKELY(parent_critical_section.try_read_unlock()); } return try_right_most_traversal(node, parent_critical_section); @@ -1458,11 +1485,12 @@ bool olc_db::iterator::try_next() { node_ptr_lock(node).rehydrate_read_lock(e.version)); // Restart check (fails if node was modified after it was pushed // onto the stack). - if (!UNODB_DETAIL_UNLIKELY(node_critical_section.check())) return false; + if (!node_critical_section.check()) [[unlikely]] + return false; // LCOV_EXCL_LINE auto node_type = node.type(); if (node_type == node_type::LEAF) { pop(); // pop off the leaf - if (!UNODB_DETAIL_UNLIKELY(node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return false; // LCOV_EXCL_LINE continue; // falls through loop if just a root leaf since stack now // empty. @@ -1470,13 +1498,12 @@ bool olc_db::iterator::try_next() { auto *inode{node.ptr()}; auto nxt = inode->next(node_type, e.child_index); // next child of that parent. - if (!UNODB_DETAIL_UNLIKELY(node_critical_section.check())) { - // restart check + if (!node_critical_section.check()) [[unlikely]] { return false; // LCOV_EXCL_LINE } if (!nxt) { pop(); // Nothing more for that inode. - if (!UNODB_DETAIL_UNLIKELY(node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return false; // LCOV_EXCL_LINE continue; // We will look for the right sibling of the parent inode. } @@ -1486,9 +1513,8 @@ bool olc_db::iterator::try_next() { pop(); push(e2, node_critical_section); auto child = inode->get_child(node_type, e2.child_index); // descend - if (UNODB_DETAIL_UNLIKELY( - !node_critical_section.check())) // before using [child] - return false; // LCOV_EXCL_LINE + if (!node_critical_section.check()) [[unlikely]] // before using [child] + return false; // LCOV_EXCL_LINE return try_left_most_traversal(child, node_critical_section); } return true; // stack is empty, so iterator == end(). @@ -1502,23 +1528,23 @@ bool olc_db::iterator::try_prior() { UNODB_DETAIL_ASSERT(node != nullptr); auto node_critical_section( node_ptr_lock(node).rehydrate_read_lock(e.version)); - if (!UNODB_DETAIL_UNLIKELY(node_critical_section.check())) + if (!node_critical_section.check()) [[unlikely]] return false; // LCOV_EXCL_LINE auto node_type = node.type(); if (node_type == node_type::LEAF) { pop(); // pop off the leaf - if (!UNODB_DETAIL_UNLIKELY(node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return false; // LCOV_EXCL_LINE continue; // falls through loop if just a root leaf since stack now empty } auto *inode{node.ptr()}; auto nxt = inode->prior(node_type, e.child_index); // previous child of that parent. - if (!UNODB_DETAIL_UNLIKELY(node_critical_section.check())) + if (!node_critical_section.check()) [[unlikely]] return false; // LCOV_EXCL_LINE if (!nxt) { pop(); // Nothing more for that inode. - if (!UNODB_DETAIL_UNLIKELY(node_critical_section.try_read_unlock())) + if (!node_critical_section.try_read_unlock()) [[unlikely]] return false; // LCOV_EXCL_LINE continue; // We will look for the left sibling of the parent inode. } @@ -1528,9 +1554,8 @@ bool olc_db::iterator::try_prior() { pop(); push(e2, node_critical_section); auto child = inode->get_child(node_type, e2.child_index); // get child - if (UNODB_DETAIL_UNLIKELY( - !node_critical_section.check())) // before using [child] - return false; // LCOV_EXCL_LINE + if (!node_critical_section.check()) [[unlikely]] // before using [child] + return false; // LCOV_EXCL_LINE return try_right_most_traversal(child, node_critical_section); } return true; // stack is empty, so iterator == end(). @@ -1545,7 +1570,7 @@ inline bool olc_db::iterator::try_left_most_traversal( detail::olc_node_ptr node, optimistic_lock::read_critical_section &parent_critical_section) { // A check() is required before acting on [node] by taking the lock. - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.check())) { + if (!parent_critical_section.check()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return {}; @@ -1555,9 +1580,9 @@ inline bool olc_db::iterator::try_left_most_traversal( UNODB_DETAIL_ASSERT(node != nullptr); // Lock version chaining (node and parent) auto node_critical_section = node_ptr_lock(node).try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(node_critical_section.must_restart())) - return false; - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (node_critical_section.must_restart()) [[unlikely]] + return false; // LCOV_EXCL_LINE + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return false; // LCOV_EXCL_LINE const auto node_type = node.type(); if (node_type == node_type::LEAF) { @@ -1568,13 +1593,12 @@ inline bool olc_db::iterator::try_left_most_traversal( auto *const inode{node.ptr()}; // first child index of the current internal node. auto t = inode->begin(node_type); - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.check())) + if (!node_critical_section.check()) [[unlikely]] return false; // LCOV_EXCL_LINE push(t, node_critical_section); node = inode->get_child(node_type, t.child_index); // get child - if (UNODB_DETAIL_UNLIKELY( - !node_critical_section.check())) // before using [child] - return false; // LCOV_EXCL_LINE + if (!node_critical_section.check()) [[unlikely]] // before using [child] + return false; // LCOV_EXCL_LINE // Move RCS (will check invariant at top of loop) parent_critical_section = std::move(node_critical_section); } @@ -1590,7 +1614,7 @@ inline bool olc_db::iterator::try_right_most_traversal( detail::olc_node_ptr node, optimistic_lock::read_critical_section &parent_critical_section) { // A check() is required before acting on [node] by taking the lock. - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.check())) { + if (!parent_critical_section.check()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return {}; @@ -1600,9 +1624,9 @@ inline bool olc_db::iterator::try_right_most_traversal( UNODB_DETAIL_ASSERT(node != nullptr); // Lock version chaining (node and parent) auto node_critical_section = node_ptr_lock(node).try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(node_critical_section.must_restart())) + if (node_critical_section.must_restart()) [[unlikely]] return false; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.try_read_unlock())) + if (!parent_critical_section.try_read_unlock()) [[unlikely]] return false; // LCOV_EXCL_LINE const auto node_type = node.type(); if (node_type == node_type::LEAF) { @@ -1612,13 +1636,12 @@ inline bool olc_db::iterator::try_right_most_traversal( // recursive descent. auto *const inode{node.ptr()}; auto t = inode->last(node_type); // first child of current internal node - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.check())) + if (!node_critical_section.check()) [[unlikely]] return false; // LCOV_EXCL_LINE push(t, node_critical_section); node = inode->get_child(node_type, t.child_index); // get the child - if (UNODB_DETAIL_UNLIKELY( - !node_critical_section.check())) // before using [child] - return false; // LCOV_EXCL_LINE + if (!node_critical_section.check()) [[unlikely]] // before using [child] + return false; // LCOV_EXCL_LINE // Move RCS (will check invariant at top of loop) parent_critical_section = std::move(node_critical_section); } @@ -1643,21 +1666,21 @@ bool olc_db::iterator::try_seek(detail::art_key search_key, bool &match, invalidate(); // invalidate the iterator (clear the stack). match = false; // unless we wind up with an exact match. auto parent_critical_section = db_.root_pointer_lock.try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(parent_critical_section.must_restart())) { + if (parent_critical_section.must_restart()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); return false; // LCOV_EXCL_STOP } auto node{db_.root.load()}; - if (UNODB_DETAIL_UNLIKELY(node == nullptr)) { + if (node == nullptr) [[unlikely]] { return UNODB_DETAIL_LIKELY(parent_critical_section.try_read_unlock()); } // A check() is required before acting on [node] by taking the lock. - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.check())) { + if (!parent_critical_section.check()) [[unlikely]] { // LCOV_EXCL_START spin_wait_loop_body(); - return {}; + return false; // LCOV_EXCL_STOP } const detail::art_key k = search_key; @@ -1666,24 +1689,23 @@ bool olc_db::iterator::try_seek(detail::art_key search_key, bool &match, UNODB_DETAIL_ASSERT(node != nullptr); // Lock version chaining (node and parent) auto node_critical_section = node_ptr_lock(node).try_read_lock(); - if (UNODB_DETAIL_UNLIKELY(node_critical_section.must_restart())) - return false; + if (node_critical_section.must_restart()) [[unlikely]] + return false; // LCOV_EXCL_LINE // TODO(thompsonbry) Should be redundant. Checked before entering // the while() loop and at the bottom of the while() loop. - if (UNODB_DETAIL_UNLIKELY(!parent_critical_section.check())) return false; + if (!parent_critical_section.check()) [[unlikely]] + return false; // LCOV_EXCL_LINE // Note: We DO NOT unlock the parent_critical_section here. It is // done below along all code paths. const auto node_type = node.type(); if (node_type == node_type::LEAF) { - if (UNODB_DETAIL_UNLIKELY( - !parent_critical_section.try_read_unlock())) // unlock parent - return false; // LCOV_EXCL_LINE + if (!parent_critical_section.try_read_unlock()) [[unlikely]] + return false; // LCOV_EXCL_LINE const auto *const leaf{node.ptr()}; push_leaf(node, node_critical_section); const auto cmp_ = leaf->cmp(k); - if (UNODB_DETAIL_UNLIKELY( - !node_critical_section.try_read_unlock())) // unlock leaf - return false; // LCOV_EXCL_LINE + if (!node_critical_section.try_read_unlock()) [[unlikely]] + return false; // LCOV_EXCL_LINE if (cmp_ == 0) { match = true; return true; // done @@ -1769,34 +1791,32 @@ bool olc_db::iterator::try_seek(detail::art_key search_key, bool &match, // do a left-most descent under that right-sibling. If there // is no such parent, we will wind up with an empty stack // (aka the end() iterator) and return that state. - if (UNODB_DETAIL_UNLIKELY( - !parent_critical_section.try_read_unlock())) // unlock parent + if (!parent_critical_section.try_read_unlock()) [[unlikely]] + return false; // LCOV_EXCL_LINE + if (!node_critical_section.try_read_unlock()) [[unlikely]] return false; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY( - !node_critical_section.try_read_unlock())) // unlock node - return false; // LCOV_EXCL_LINE if (!empty()) pop(); while (!empty()) { const auto centry = top(); const auto cnode{centry.node}; // a possible parent from the stack. auto c_critical_section( node_ptr_lock(cnode).rehydrate_read_lock(centry.version)); - if (!UNODB_DETAIL_UNLIKELY(c_critical_section.check())) - return false; // restart check + if (!c_critical_section.check()) [[unlikely]] + return false; // LCOV_EXCL_LINE auto *const icnode{cnode.ptr()}; const auto cnxt = icnode->next( cnode.type(), centry.child_index); // right-sibling. if (cnxt) { auto nchild = icnode->get_child( cnode.type(), centry.child_index); // get the child - if (UNODB_DETAIL_UNLIKELY( - !c_critical_section.check())) // before using [nchild] - return false; // LCOV_EXCL_LINE + if (!c_critical_section.check()) + [[unlikely]] // before using [nchild] + return false; // LCOV_EXCL_LINE return try_left_most_traversal(nchild, c_critical_section); } pop(); - if (!UNODB_DETAIL_UNLIKELY(c_critical_section.try_read_unlock())) - return false; + if (!c_critical_section.try_read_unlock()) [[unlikely]] + return false; // LCOV_EXCL_LINE } return true; // stack is empty (aka end()). } @@ -1804,12 +1824,11 @@ bool olc_db::iterator::try_seek(detail::art_key search_key, bool &match, const auto child_index = tmp.child_index; const auto child = inode->get_child(node_type, child_index); // get child - if (UNODB_DETAIL_UNLIKELY( - !node_critical_section.check())) // before using [child] - return false; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY( - !parent_critical_section.try_read_unlock())) // unlock parent - return false; // LCOV_EXCL_LINE + if (!node_critical_section.check()) + [[unlikely]] // before using [child] + return false; // LCOV_EXCL_LINE + if (!parent_critical_section.try_read_unlock()) [[unlikely]] + return false; // LCOV_EXCL_LINE push(node, tmp.key_byte, child_index, // push the path we took node_critical_section); return try_left_most_traversal(child, node_critical_section); @@ -1823,47 +1842,43 @@ bool olc_db::iterator::try_seek(detail::art_key search_key, bool &match, // left-sibling and then do a right-most descent under that // left-sibling. In the extreme case there is no such // previous entry and we will wind up with an empty stack. - if (UNODB_DETAIL_UNLIKELY( - !parent_critical_section.try_read_unlock())) // unlock parent - return false; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY( - !node_critical_section.try_read_unlock())) // unlock node - return false; // LCOV_EXCL_LINE + if (!parent_critical_section.try_read_unlock()) [[unlikely]] + return false; // LCOV_EXCL_LINE + if (!node_critical_section.try_read_unlock()) [[unlikely]] + return false; // LCOV_EXCL_LINE if (!empty()) pop(); while (!empty()) { const auto centry = top(); const auto cnode{centry.node}; // a possible parent from stack auto c_critical_section( node_ptr_lock(cnode).rehydrate_read_lock(centry.version)); - if (!UNODB_DETAIL_UNLIKELY(c_critical_section.check())) - return false; // restart check + if (!c_critical_section.check()) [[unlikely]] + return false; // LCOV_EXCL_LINE auto *const icnode{cnode.ptr()}; const auto cnxt = icnode->prior(cnode.type(), centry.child_index); // left-sibling. if (cnxt) { auto nchild = icnode->get_child( cnode.type(), centry.child_index); // get the child - if (UNODB_DETAIL_UNLIKELY( - !c_critical_section.check())) // before using [nchild] - return false; // LCOV_EXCL_LINE + if (!c_critical_section.check()) + [[unlikely]] // before using [nchild] + return false; // LCOV_EXCL_LINE return try_right_most_traversal(nchild, c_critical_section); } pop(); - if (!UNODB_DETAIL_UNLIKELY(c_critical_section.try_read_unlock())) - return false; + if (!c_critical_section.try_read_unlock()) [[unlikely]] + return false; // LCOV_EXCL_LINE } return true; // stack is empty (aka end()). } auto tmp = nxt.value(); // unwrap. const auto child_index = tmp.child_index; const auto child = - inode->get_child(node_type, child_index); // get the child - if (UNODB_DETAIL_UNLIKELY( - !node_critical_section.check())) // before using [child] - return false; // LCOV_EXCL_LINE - if (UNODB_DETAIL_UNLIKELY( - !parent_critical_section.try_read_unlock())) // unlock parent - return false; // LCOV_EXCL_LINE + inode->get_child(node_type, child_index); // get the child + if (!node_critical_section.check()) [[unlikely]] // before using [child] + return false; // LCOV_EXCL_LINE + if (!parent_critical_section.try_read_unlock()) [[unlikely]] + return false; // LCOV_EXCL_LINE push(node, tmp.key_byte, child_index, // push the path we took node_critical_section); return try_right_most_traversal(child, node_critical_section); @@ -1875,7 +1890,7 @@ bool olc_db::iterator::try_seek(detail::art_key search_key, bool &match, node = *child; remaining_key.shift_right(1); // check node before using [child] and before we std::move() the RCS. - if (UNODB_DETAIL_UNLIKELY(!node_critical_section.check())) + if (!node_critical_section.check()) [[unlikely]] return false; // LCOV_EXCL_LINE // Move RCS (will check invariant at top of loop) parent_critical_section = std::move(node_critical_section); diff --git a/olc_art.hpp b/olc_art.hpp index 09dadfc7..694d03f5 100644 --- a/olc_art.hpp +++ b/olc_art.hpp @@ -486,7 +486,8 @@ class olc_db final { it.first(); visitor v{it}; while (it.valid()) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.next(); } } else { @@ -494,7 +495,8 @@ class olc_db final { it.last(); visitor v{it}; while (it.valid()) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.prior(); } } @@ -521,7 +523,8 @@ class olc_db final { it.seek(from_key_, match, true /*fwd*/); visitor v{it}; while (it.valid()) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.next(); } } else { @@ -529,7 +532,8 @@ class olc_db final { it.seek(from_key_, match, false /*fwd*/); visitor v{it}; while (it.valid()) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.prior(); } } @@ -574,7 +578,8 @@ class olc_db final { } visitor v{it}; while (it.valid() && it.cmp(to_key_) < 0) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.next(); if constexpr (debug) { std::cerr << "scan: next()\n"; @@ -591,7 +596,8 @@ class olc_db final { } visitor v{it}; while (it.valid() && it.cmp(to_key_) > 0) { - if (UNODB_DETAIL_UNLIKELY(fn(v))) break; + if (fn(v)) [[unlikely]] + break; it.prior(); if constexpr (debug) { std::cerr << "scan: prior()\n"; diff --git a/optimistic_lock.hpp b/optimistic_lock.hpp index 7e3a9ca7..fc99f5d5 100644 --- a/optimistic_lock.hpp +++ b/optimistic_lock.hpp @@ -264,7 +264,8 @@ class [[nodiscard]] optimistic_lock final { [[nodiscard]] bool check() const noexcept { const auto result = lock->check(version); #ifndef NDEBUG - if (UNODB_DETAIL_UNLIKELY(!result)) lock = nullptr; // LCOV_EXCL_LINE + if (!result) [[unlikely]] + lock = nullptr; // LCOV_EXCL_LINE #endif return UNODB_DETAIL_LIKELY(result); } @@ -318,7 +319,8 @@ class [[nodiscard]] optimistic_lock final { #endif const auto result = lock->try_upgrade_to_write_lock(critical_section.version); - if (UNODB_DETAIL_UNLIKELY(!result)) lock = nullptr; // LCOV_EXCL_LINE + if (!result) [[unlikely]] + lock = nullptr; // LCOV_EXCL_LINE } ~write_guard() noexcept { @@ -381,12 +383,12 @@ class [[nodiscard]] optimistic_lock final { [[nodiscard]] read_critical_section try_read_lock() noexcept { while (true) { const auto current_version = version.load(); - if (UNODB_DETAIL_LIKELY(current_version.is_free())) { + if (current_version.is_free()) [[likely]] { inc_read_lock_count(); return read_critical_section{*this, current_version}; } // LCOV_EXCL_START - if (UNODB_DETAIL_UNLIKELY(current_version.is_obsolete())) + if (current_version.is_obsolete()) [[unlikely]] return read_critical_section{}; UNODB_DETAIL_ASSERT(current_version.is_write_locked()); spin_wait_loop_body(); @@ -448,7 +450,8 @@ class [[nodiscard]] optimistic_lock final { #endif const auto result{locked_version == version.load_relaxed()}; #ifndef NDEBUG - if (UNODB_DETAIL_UNLIKELY(!result)) dec_read_lock_count(); + if (!result) [[unlikely]] + dec_read_lock_count(); // LCOV_EXCL_LINE #endif return UNODB_DETAIL_LIKELY(result); } @@ -457,7 +460,8 @@ class [[nodiscard]] optimistic_lock final { version_type locked_version) const noexcept { const auto result{check(locked_version)}; #ifndef NDEBUG - if (UNODB_DETAIL_LIKELY(result)) dec_read_lock_count(); + if (result) [[likely]] + dec_read_lock_count(); // LCOV_EXCL_LINE #endif return UNODB_DETAIL_LIKELY(result); } diff --git a/qsbr.cpp b/qsbr.cpp index 325a0ea1..c9ba672c 100644 --- a/qsbr.cpp +++ b/qsbr.cpp @@ -152,9 +152,9 @@ void add_to_orphan_list( list_node_ptr->next = orphan_list.load(std::memory_order_acquire); while (true) { - if (UNODB_DETAIL_LIKELY(orphan_list.compare_exchange_weak( - list_node_ptr->next, list_node_ptr, std::memory_order_acq_rel, - std::memory_order_acquire))) + if (orphan_list.compare_exchange_weak(list_node_ptr->next, list_node_ptr, + std::memory_order_acq_rel, + std::memory_order_acquire)) [[likely]] return; } } @@ -211,14 +211,13 @@ qsbr_epoch qsbr::register_thread() noexcept { const auto old_threads_in_previous_epoch = qsbr_state::get_threads_in_previous_epoch(old_state); - if (UNODB_DETAIL_LIKELY(old_threads_in_previous_epoch > 0 || - old_thread_count == 0)) { + if (old_threads_in_previous_epoch > 0 || old_thread_count == 0) [[likely]] { const auto new_state = qsbr_state::inc_thread_count_and_threads_in_previous_epoch(old_state); - if (UNODB_DETAIL_LIKELY(state.compare_exchange_weak( - old_state, new_state, std::memory_order_acq_rel, - std::memory_order_acquire))) + if (state.compare_exchange_weak(old_state, new_state, + std::memory_order_acq_rel, + std::memory_order_acquire)) [[likely]] return old_epoch; // LCOV_EXCL_START @@ -234,9 +233,9 @@ qsbr_epoch qsbr::register_thread() noexcept { // Epoch change in progress - try to bump the thread count only const auto new_state = qsbr_state::inc_thread_count(old_state); - if (UNODB_DETAIL_LIKELY(state.compare_exchange_weak( - old_state, new_state, std::memory_order_acq_rel, - std::memory_order_acquire))) { + if (state.compare_exchange_weak(old_state, new_state, + std::memory_order_acq_rel, + std::memory_order_acquire)) [[likely]] { // Spin until the epoch change completes. An alternative would be to // return the new epoch early, and then handle seeing it in quiescent // state as a no-op, but that trades spinning here for more work in a @@ -265,15 +264,15 @@ void qsbr::unregister_thread(std::uint64_t quiescent_states_since_epoch_change, const auto old_threads_in_previous_epoch = qsbr_state::get_threads_in_previous_epoch(old_state); - if (UNODB_DETAIL_UNLIKELY(old_threads_in_previous_epoch == 0)) { + if (old_threads_in_previous_epoch == 0) [[unlikely]] { // LCOV_EXCL_START UNODB_DETAIL_ASSERT(thread_epoch == qsbr_state::get_epoch(old_state)); // Epoch change in progress - try to decrement the thread count only const auto new_state = qsbr_state::dec_thread_count(old_state); - if (UNODB_DETAIL_LIKELY(state.compare_exchange_weak( - old_state, new_state, std::memory_order_acq_rel, - std::memory_order_acquire))) { + if (state.compare_exchange_weak(old_state, new_state, + std::memory_order_acq_rel, + std::memory_order_acquire)) [[likely]] { qsbr_thread.orphan_deferred_requests(); return; } @@ -299,7 +298,7 @@ void qsbr::unregister_thread(std::uint64_t quiescent_states_since_epoch_change, old_state, advance_epoch) : qsbr_state::dec_thread_count(old_state); - if (UNODB_DETAIL_UNLIKELY(remove_thread_from_old_epoch)) { + if (remove_thread_from_old_epoch) [[unlikely]] { thread_epoch_change_barrier(); if (UNODB_DETAIL_UNLIKELY(advance_epoch) && @@ -312,13 +311,13 @@ void qsbr::unregister_thread(std::uint64_t quiescent_states_since_epoch_change, } } - if (UNODB_DETAIL_LIKELY(state.compare_exchange_weak( - old_state, new_state, std::memory_order_acq_rel, - std::memory_order_acquire))) { + if (state.compare_exchange_weak(old_state, new_state, + std::memory_order_acq_rel, + std::memory_order_acquire)) [[likely]] { // Might be the first time the quitting thread saw the old epoch too, if a // second-to-last thread quit before, advancing the epoch. qsbr_thread.advance_last_seen_epoch(old_single_thread_mode, old_epoch); - if (UNODB_DETAIL_UNLIKELY(advance_epoch)) { + if (advance_epoch) [[unlikely]] { #ifdef UNODB_DETAIL_WITH_STATS bump_epoch_change_count(); #endif // UNODB_DETAIL_WITH_STATS @@ -328,7 +327,7 @@ void qsbr::unregister_thread(std::uint64_t quiescent_states_since_epoch_change, qsbr_thread.orphan_deferred_requests(); #ifdef UNODB_DETAIL_WITH_STATS - if (UNODB_DETAIL_UNLIKELY(thread_epoch != old_epoch)) { + if (thread_epoch != old_epoch) [[unlikely]] { register_quiescent_states_per_thread_between_epoch_changes( quiescent_states_since_epoch_change); } @@ -446,13 +445,12 @@ void qsbr::epoch_change_barrier_and_handle_orphans( free_orphan_list(orphaned_previous_requests); - if (UNODB_DETAIL_LIKELY(!single_thread_mode)) { + if (!single_thread_mode) [[likely]] { detail::dealloc_vector_list_node *new_previous_requests = nullptr; - if (UNODB_DETAIL_UNLIKELY( - !orphaned_previous_interval_dealloc_requests - .compare_exchange_strong( - new_previous_requests, orphaned_current_requests, - std::memory_order_acq_rel, std::memory_order_acquire))) { + if (!orphaned_previous_interval_dealloc_requests.compare_exchange_strong( + new_previous_requests, orphaned_current_requests, + std::memory_order_acq_rel, std::memory_order_acquire)) + [[unlikely]] { // Someone added new previous requests since we took the previous batch // above. Append ours at the tail then, only one thread can do this, as // everybody else add at the list head. The list should be short in @@ -477,9 +475,9 @@ qsbr_epoch qsbr::change_epoch(qsbr_epoch current_global_epoch, qsbr_state::get_epoch(old_state)); const auto new_state = qsbr_state::inc_epoch_reset_previous(old_state); - if (UNODB_DETAIL_LIKELY(state.compare_exchange_weak( - old_state, new_state, std::memory_order_acq_rel, - std::memory_order_acquire))) { + if (state.compare_exchange_weak(old_state, new_state, + std::memory_order_acq_rel, + std::memory_order_acquire)) [[likely]] { UNODB_DETAIL_ASSERT(current_global_epoch.advance() == qsbr_state::get_epoch(new_state)); diff --git a/qsbr.hpp b/qsbr.hpp index dfe39c39..9d5d6a02 100644 --- a/qsbr.hpp +++ b/qsbr.hpp @@ -13,6 +13,12 @@ #include "global.hpp" // IWYU pragma: keep // IWYU pragma: no_include <__fwd/ostream.h> +// IWYU pragma: no_include <__ostream/basic_ostream.h> +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include // IWYU pragma: no_include #include @@ -943,7 +949,7 @@ inline void qsbr_per_thread::on_next_epoch_deallocate( const auto single_thread_mode = qsbr_state::single_thread_mode(current_qsbr_state); - if (UNODB_DETAIL_UNLIKELY(single_thread_mode)) { + if (single_thread_mode) [[unlikely]] { advance_last_seen_epoch(single_thread_mode, current_global_epoch); qsbr::deallocate(pointer #ifndef NDEBUG @@ -1029,7 +1035,7 @@ inline void qsbr_per_thread::update_requests( current_interval_total_dealloc_size = 0; #endif // UNODB_DETAIL_WITH_STATS - if (UNODB_DETAIL_LIKELY(!single_thread_mode)) { + if (!single_thread_mode) [[likely]] { previous_interval_dealloc_requests = std::move(current_interval_dealloc_requests); } else { diff --git a/test/qsbr_test_utils.cpp b/test/qsbr_test_utils.cpp index 1ceeeaa5..2e95a662 100644 --- a/test/qsbr_test_utils.cpp +++ b/test/qsbr_test_utils.cpp @@ -1,7 +1,4 @@ -// Copyright 2021-2024 Laurynas Biveinis - -// IWYU pragma: no_include -// IWYU pragma: no_include "gtest/gtest.h" +// Copyright 2021-2025 Laurynas Biveinis // // CAUTION: [global.hpp] MUST BE THE FIRST INCLUDE IN ALL SOURCE AND @@ -13,6 +10,10 @@ // container internal structure layouts and that is Not Good. #include "global.hpp" // IWYU pragma: keep +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include "gtest/gtest.h" + #include "gtest_utils.hpp" #include "qsbr.hpp" #include "qsbr_test_utils.hpp" diff --git a/test_heap.cpp b/test_heap.cpp index 8634ebf0..03f6db6b 100644 --- a/test_heap.cpp +++ b/test_heap.cpp @@ -29,7 +29,8 @@ void* operator new(std::size_t count) { while (true) { // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory,hicpp-no-malloc) void* const result = malloc(count); - if (UNODB_DETAIL_LIKELY(result != nullptr)) return result; + if (result != nullptr) [[likely]] + return result; // LCOV_EXCL_START auto* new_handler = std::get_new_handler(); if (new_handler == nullptr) throw std::bad_alloc{};