diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e1cd462..473d9c41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ jobs: - run: name: Installing dependencies (common) command: | - sudo apt-get install -y libboost-dev + sudo apt-get install -y libboost-dev libjemalloc-dev - when: condition: equal: [ "gcc", << parameters.compiler >> ] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d91e3c12..4865dadb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -225,7 +225,7 @@ jobs: - name: Setup common dependencies for Linux run: | - sudo apt-get install -y libboost-dev libc6-dev-i386 + sudo apt-get install -y libboost-dev libc6-dev-i386 libjemalloc-dev sudo apt-get remove clang-tidy-11 if: runner.os == 'Linux' @@ -268,7 +268,7 @@ jobs: - name: Set up dependencies for macOS (common) run: | - brew install boost + brew install boost jemalloc if: runner.os == 'macOS' - name: Set up dependencies for macOS (coverage) diff --git a/.github/workflows/experimental-build.yml b/.github/workflows/experimental-build.yml index 78746c5d..4e6f5679 100644 --- a/.github/workflows/experimental-build.yml +++ b/.github/workflows/experimental-build.yml @@ -28,7 +28,7 @@ jobs: - name: Set up dependencies for macOS run: | brew update - brew install boost cppcheck iwyu + brew install boost cppcheck iwyu jemalloc - name: Create build environment run: mkdir ${{github.workspace}}/build diff --git a/.github/workflows/msvc-build.yml b/.github/workflows/msvc-build.yml index 35c5dac2..4d484de0 100644 --- a/.github/workflows/msvc-build.yml +++ b/.github/workflows/msvc-build.yml @@ -53,11 +53,17 @@ jobs: platform_version: 2019 toolset: msvc + - name: Install jemalloc + run: | + vcpkg install jemalloc + - name: Setup command line tools uses: ilammy/msvc-dev-cmd@v1 - name: Configure CMake - run: cmake --preset "${{ matrix.preset }}" + run: | + $env:PKG_CONFIG_PATH += ";C:\vcpkg\packages\jemalloc_x86-windows\lib\pkgconfig" + cmake --preset "${{ matrix.preset }}" env: BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} diff --git a/.github/workflows/old-compilers.yml b/.github/workflows/old-compilers.yml index 361bfc6c..9fc8dad7 100644 --- a/.github/workflows/old-compilers.yml +++ b/.github/workflows/old-compilers.yml @@ -355,7 +355,7 @@ jobs: - name: Setup common dependencies for Linux run: | - sudo apt-get install -y libboost-dev libc6-dev-i386 + sudo apt-get install -y libboost-dev libc6-dev-i386 libjemalloc-dev if: runner.os == 'Linux' - name: Setup dependencies for Linux LLVM 11 & 12 diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 532af081..073cc560 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -37,7 +37,7 @@ jobs: - name: Setup dependencies run: | sudo add-apt-repository -y 'ppa:mhier/libboost-latest' - sudo apt-get install -y boost1.74 + sudo apt-get install -y boost1.74 libjemalloc-dev - name: Produce Compilation Database shell: bash diff --git a/CMakeLists.txt b/CMakeLists.txt index daec2b8c..46170d6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -256,6 +256,20 @@ find_package(Threads REQUIRED) find_package(Boost REQUIRED) +if(NOT("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" + AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "arm64")) + set(USE_JEMALLOC ON) +else() + set(USE_JEMALLOC OFF) +endif() + +if(USE_JEMALLOC) + find_package(PkgConfig REQUIRED) + + pkg_check_modules(JEMALLOC jemalloc) + pkg_search_module(JEMALLOC REQUIRED jemalloc) +endif() + string(REPLACE ";" " " CXX_FLAGS_FOR_SUBDIR_STR "${SANITIZER_CXX_FLAGS}") if(MSVC) string(REPLACE ";" " " MSVC_WARNING_FLAGS_FOR_SUBDIR_STR_TMP "${MSVC_CXX_WARNING_FLAGS}") @@ -395,6 +409,7 @@ set(is_clang_ge_13_not_windows "$") set(is_clang_ge_14_not_windows "$") set(is_gxx_ge_11 "$") set(is_gxx_ge_12 "$") +set(has_jemalloc "$") set(has_avx2 "$") set(fatal_warnings_on "$") set(coverage_on "$") @@ -496,6 +511,9 @@ function(COMMON_TARGET_PROPERTIES TARGET) set_target_properties(${TARGET} PROPERTIES CXX_EXTENSIONS OFF) target_compile_definitions(${TARGET} PRIVATE "$<${is_standalone}:UNODB_DETAIL_STANDALONE>") + target_compile_definitions(${TARGET} PRIVATE + # FIXME(laurynas): UNODB_DETAIL_USE_SIZED_FREE + "$<${has_jemalloc}:UNODB_DETAIL_USE_JEMALLOC>") target_compile_options(${TARGET} PRIVATE "${CXX_FLAGS}" "${SANITIZER_CXX_FLAGS}" "$<${is_msvc}:${MSVC_CXX_FLAGS}>" @@ -520,6 +538,8 @@ function(COMMON_TARGET_PROPERTIES TARGET) "$<$:$>" # Misc "$<${coverage_on}:--coverage>") + target_link_libraries(${TARGET} PRIVATE + "$<${has_jemalloc}:${JEMALLOC_LDFLAGS}>") # Change to target_link_options on 3.13 minimum CMake version target_link_libraries(${TARGET} INTERFACE "$<${coverage_on}:--coverage>") target_link_libraries(${TARGET} PRIVATE diff --git a/art_internal_impl.hpp b/art_internal_impl.hpp index 61db1fe0..4f5c1c15 100644 --- a/art_internal_impl.hpp +++ b/art_internal_impl.hpp @@ -167,7 +167,11 @@ inline void basic_db_leaf_deleter::operator()( leaf_type *to_delete) const noexcept { const auto leaf_size = to_delete->get_size(); +#ifdef UNODB_DETAIL_USE_JEMALLOC + free_sized_aligned(to_delete, leaf_size); +#else free_aligned(to_delete); +#endif db.decrement_leaf_count(leaf_size); } @@ -177,7 +181,11 @@ inline void basic_db_inode_deleter::operator()( INode *inode_ptr) noexcept { static_assert(std::is_trivially_destructible_v); +#ifdef UNODB_DETAIL_USE_JEMALLOC + free_sized_aligned(inode_ptr, sizeof(INode)); +#else free_aligned(inode_ptr); +#endif db.template decrement_inode_count(); } diff --git a/heap.hpp b/heap.hpp index 32000144..21c971f4 100644 --- a/heap.hpp +++ b/heap.hpp @@ -17,6 +17,10 @@ #include #endif +#ifdef UNODB_DETAIL_USE_JEMALLOC +#include +#endif + #include "assert.hpp" namespace unodb::test { @@ -81,7 +85,7 @@ template void* result; -#ifndef _MSC_VER +#if !defined(_MSC_VER) || defined(UNODB_DETAIL_USE_JEMALLOC) const auto err = posix_memalign(&result, alignment, size); if (UNODB_DETAIL_UNLIKELY(err != 0)) result = nullptr; #else @@ -102,6 +106,7 @@ template return result; } +#ifndef UNODB_DETAIL_USE_JEMALLOC inline void free_aligned(void* ptr) noexcept { #ifndef _MSC_VER // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory,hicpp-no-malloc) @@ -110,6 +115,29 @@ inline void free_aligned(void* ptr) noexcept { _aligned_free(ptr); #endif } +#endif // !UNODB_DETAIL_USE_JEMALLOC + +inline void free_sized(void* ptr, std::size_t size) noexcept { +#ifdef UNODB_DETAIL_USE_JEMALLOC + sdallocx(ptr, size, 0); +#else + (void)size; + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory,hicpp-no-malloc) + free(ptr); +#endif +} + +inline void free_sized_aligned(void* ptr, std::size_t size) noexcept { +#ifdef UNODB_DETAIL_USE_JEMALLOC + sdallocx(ptr, size, 0); +#elif defined(_MSC_VER) + _aligned_free(ptr); +#else + (void)size; + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory,hicpp-no-malloc) + free(ptr); +#endif +} } // namespace unodb::detail diff --git a/qsbr.hpp b/qsbr.hpp index 2ca1f373..409a9ec5 100644 --- a/qsbr.hpp +++ b/qsbr.hpp @@ -350,8 +350,12 @@ namespace detail { struct [[nodiscard]] deallocation_request final { void *const pointer; +#ifdef UNODB_DETAIL_USE_JEMALLOC + std::size_t size; +#endif #ifndef NDEBUG + // FIXME(laurynas): pass size too? using debug_callback = std::function; // Non-const to support move @@ -360,19 +364,25 @@ struct [[nodiscard]] deallocation_request final { #endif explicit deallocation_request(void *pointer_ +#ifdef UNODB_DETAIL_USE_JEMALLOC + , + std::size_t size_ +#endif #ifndef NDEBUG , qsbr_epoch request_epoch_, debug_callback dealloc_callback_ #endif ) noexcept - : pointer { - pointer_ - } + : pointer{pointer_} +#ifdef UNODB_DETAIL_USE_JEMALLOC + , + size{size_} +#endif #ifndef NDEBUG - , dealloc_callback{std::move(dealloc_callback_)}, request_epoch { - request_epoch_ - } + , + dealloc_callback{std::move(dealloc_callback_)}, + request_epoch{request_epoch_} #endif { #ifndef NDEBUG @@ -410,16 +420,15 @@ class [[nodiscard]] deferred_requests final { std::optional dealloc_epoch_single_thread_mode_ #endif ) noexcept - : requests { - std::move(requests_) - } + : requests{std::move(requests_)} #ifndef NDEBUG - , orphaned_requests{orphaned_requests_}, dealloc_epoch{request_epoch_}, - dealloc_epoch_single_thread_mode { - dealloc_epoch_single_thread_mode_ - } + , + orphaned_requests{orphaned_requests_}, + dealloc_epoch{request_epoch_}, + dealloc_epoch_single_thread_mode{dealloc_epoch_single_thread_mode_} #endif - {} + { + } deferred_requests(const deferred_requests &) noexcept = delete; deferred_requests(deferred_requests &&) noexcept = delete; @@ -640,6 +649,10 @@ class qsbr final { UNODB_DETAIL_DISABLE_MSVC_WARNING(26447) static void deallocate( void *pointer +#ifdef UNODB_DETAIL_USE_JEMALLOC + , + std::size_t size +#endif #ifndef NDEBUG , const detail::deallocation_request::debug_callback &debug_callback @@ -648,7 +661,11 @@ class qsbr final { #ifndef NDEBUG if (debug_callback != nullptr) debug_callback(pointer); #endif +#ifdef UNODB_DETAIL_USE_JEMALLOC + detail::free_sized_aligned(pointer, size); +#else detail::free_aligned(pointer); +#endif } UNODB_DETAIL_RESTORE_MSVC_WARNINGS() @@ -856,6 +873,10 @@ inline void qsbr_per_thread::on_next_epoch_deallocate( if (UNODB_DETAIL_UNLIKELY(single_thread_mode)) { advance_last_seen_epoch(single_thread_mode, current_global_epoch); qsbr::deallocate(pointer +#ifdef UNODB_DETAIL_USE_JEMALLOC + , + size +#endif #ifndef NDEBUG , dealloc_callback @@ -867,6 +888,10 @@ inline void qsbr_per_thread::on_next_epoch_deallocate( if (last_seen_epoch != current_global_epoch) { detail::dealloc_request_vector new_current_requests; new_current_requests.emplace_back(pointer +#ifdef UNODB_DETAIL_USE_JEMALLOC + , + size +#endif #ifndef NDEBUG , current_global_epoch, @@ -882,6 +907,10 @@ inline void qsbr_per_thread::on_next_epoch_deallocate( } current_interval_dealloc_requests.emplace_back(pointer +#ifdef UNODB_DETAIL_USE_JEMALLOC + , + size +#endif #ifndef NDEBUG , last_seen_epoch, @@ -1008,6 +1037,10 @@ inline void deallocation_request::deallocate( *dealloc_epoch == request_epoch.advance(2)))); qsbr::deallocate(pointer +#ifdef UNODB_DETAIL_USE_JEMALLOC + , + size +#endif #ifndef NDEBUG , dealloc_callback diff --git a/test_heap.cpp b/test_heap.cpp index db4e4f94..815cc0d2 100644 --- a/test_heap.cpp +++ b/test_heap.cpp @@ -35,9 +35,13 @@ void operator delete(void* ptr) noexcept { } UNODB_DETAIL_DISABLE_CLANG_WARNING("-Wmissing-prototypes") -void operator delete(void* ptr, std::size_t) noexcept { - // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory,hicpp-no-malloc) +void operator delete(void* ptr, std::size_t size) noexcept { +#ifdef UNODB_DETAIL_USE_JEMALLOC + unodb::detail::free_sized(ptr, size); +#else + (void)size; free(ptr); +#endif } UNODB_DETAIL_RESTORE_CLANG_WARNINGS()