[v26.2.x] Port chunked_vector, chunked_hash_map, and use it in prometheus-impl.hh

CMakeLists.txt

@@ -417,6 +417,35 @@ set (Seastar_GEN_BINARY_DIR ${Seastar_BINARY_DIR}/gen)
 include (SeastarDependencies)
 seastar_find_dependencies ()
 
+# unordered_dense is header-only and not reliably packaged by distributions.
+# When seastar_find_dependencies() did not locate an installed copy (e.g. in
+# CI), fetch the pinned version so the build is self-contained. This lives in
+# the Seastar build only, not in SeastarDependencies, so consumers that use
+# find_package(Seastar) never trigger a download.
+if (NOT TARGET unordered_dense::unordered_dense)
+  include (FetchContent)
+  FetchContent_Declare (unordered_dense
+    URL https://github.com/martinus/unordered_dense/archive/f30ed41b58af8c79788e8581fe57a6faf856258e.tar.gz
+    URL_HASH MD5=0370e4a35c1e573aa6639fde97f0c93f)
+  FetchContent_GetProperties (unordered_dense)
+  if (NOT unordered_dense_POPULATED)
+    # Populate (download) only; we deliberately do not add it as a subproject
+    # (see below). CMP0169 OLD keeps the bare Populate() form available.
+    if (POLICY CMP0169)
+      cmake_policy (SET CMP0169 OLD)
+    endif ()
+    FetchContent_Populate (unordered_dense)
+  endif ()
+  # Expose the fetched headers as an IMPORTED target rather than building it as
+  # a subproject, so that install(EXPORT) for Seastar (which PUBLIC-links it)
+  # does not require exporting a fetched target. Downstream consumers resolve
+  # unordered_dense themselves via SeastarDependencies.
+  add_library (unordered_dense_fetched INTERFACE IMPORTED GLOBAL)
+  set_target_properties (unordered_dense_fetched PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${unordered_dense_SOURCE_DIR}/include")
+  add_library (unordered_dense::unordered_dense ALIAS unordered_dense_fetched)
+endif ()
+
 # Private build dependencies not visible to consumers
 find_package (ragel 6.10 REQUIRED)
 find_package (Threads REQUIRED)
@@ -550,6 +579,8 @@ add_library (seastar
   include/seastar/core/cacheline.hh
   include/seastar/core/checked_ptr.hh
   include/seastar/core/chunked_fifo.hh
+  include/seastar/core/chunked_hash_map.hh
+  include/seastar/core/chunked_vector.hh
   include/seastar/core/circular_buffer.hh
   include/seastar/core/circular_buffer_fixed_capacity.hh
   include/seastar/core/condition-variable.hh
@@ -914,6 +945,8 @@ target_link_libraries (seastar
     c-ares::cares
     fmt::fmt
     lz4::lz4
+    unordered_dense::unordered_dense
+    absl::hash
   PRIVATE
     ${CMAKE_DL_LIBS}
     StdAtomic::atomic

cmake/SeastarDependencies.cmake

@@ -88,6 +88,11 @@ macro (seastar_find_dependencies)
   endif()
   seastar_find_dep (fmt 8.1.1 REQUIRED)
   seastar_find_dep (lz4 1.7.3 REQUIRED)
+  # Not REQUIRED: unordered_dense is header-only and not reliably packaged by
+  # distributions. When it is not installed, Seastar's own build fetches it
+  # (see CMakeLists.txt); consumers that lack it must provide it themselves.
+  seastar_find_dep (unordered_dense)
+  seastar_find_dep (absl CONFIG REQUIRED)
   seastar_find_dep (GnuTLS 3.7.4)
   seastar_find_dep (OpenSSL 3.0)
   if (Seastar_IO_URING)

cooking_recipe.cmake

@@ -336,3 +336,21 @@ cooking_ingredient (lz4
     CONFIGURE_COMMAND <DISABLE>
     BUILD_COMMAND <DISABLE>
     INSTALL_COMMAND ${make_command} PREFIX=<INSTALL_DIR> install)
+
+# Header-only library used by chunked_hash_map. Pinned to the same commit
+# (v4.4.0) consumed by Redpanda.
+cooking_ingredient (unordered_dense
+  EXTERNAL_PROJECT_ARGS
+    URL https://github.com/martinus/unordered_dense/archive/f30ed41b58af8c79788e8581fe57a6faf856258e.tar.gz
+    URL_MD5 0370e4a35c1e573aa6639fde97f0c93f)
+
+# Used by chunked_hash_map for absl::Hash support. Pinned to the same LTS
+# release (20250814.1) consumed by Redpanda.
+cooking_ingredient (absl
+  EXTERNAL_PROJECT_ARGS
+    URL https://github.com/abseil/abseil-cpp/releases/download/20250814.1/abseil-cpp-20250814.1.tar.gz
+    URL_MD5 d4d3c25f78e28d61ad83e54cd1116933
+  CMAKE_ARGS
+    -DABSL_PROPAGATE_CXX_STD=ON
+    -DABSL_ENABLE_INSTALL=ON
+    -DBUILD_TESTING=OFF)

include/seastar/core/chunked_hash_map.hh

@@ -0,0 +1,262 @@
+/*
+ * This file is open source software, licensed to you under the terms
+ * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
+ * distributed with this work for additional information regarding copyright
+ * ownership.  You may not use this file except in compliance with the License.
+ *
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Copyright 2024 Redpanda Data, Inc.
+ */
+#pragma once
+
+#include <seastar/core/chunked_vector.hh>
+
+#include <absl/hash/hash.h>
+#include <ankerl/unordered_dense.h>
+#include <fmt/format.h>
+#include <fmt/ranges.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <ranges>
+#include <type_traits>
+#include <utility>
+
+namespace seastar {
+
+namespace internal {
+
+template<typename T>
+concept has_absl_hash = requires(T val) {
+    { AbslHashValue(std::declval<absl::HashState>(), val) };
+};
+
+/// Wrapper around absl::Hash that disables the extra hash mixing in
+/// unordered_dense
+template<typename T>
+struct avalanching_absl_hash {
+    // absl always hash mixes itself so no need to do it again
+    using is_avalanching = void;
+
+    auto operator()(const T& x) const noexcept -> uint64_t {
+        return absl::Hash<T>()(x);
+    }
+};
+
+} // namespace internal
+
+/**
+ * @brief A hash map that uses a chunked vector as the underlying storage.
+ *
+ * Use when the hash map is expected to have a large number of elements (e.g.:
+ * scales with partitions or topics). Performance wise it's equal to the abseil
+ * hashmaps.
+ *
+ * NB: References and iterators are not stable across insertions and deletions.
+ *
+ * Both std::hash and abseil's AbslHashValue are supported. We dispatch to the
+ * latter if available. Given AbslHashValue also supports std::hash we could
+ * also unconditionally dispatch to it. However, absl's hash mixing seems more
+ * extensive (and hence less performant) so we only do that when needed.
+ *
+ * For more info please see
+ * https://github.com/martinus/unordered_dense/?tab=readme-ov-file#1-overview
+ */
+template<
+  typename Key,
+  typename Value,
+  typename Hash = std::conditional_t<
+    internal::has_absl_hash<Key>,
+    internal::avalanching_absl_hash<Key>,
+    ankerl::unordered_dense::hash<Key>>,
+  typename EqualTo = std::equal_to<Key>>
+using chunked_hash_map = ankerl::unordered_dense::segmented_map<
+  Key,
+  Value,
+  Hash,
+  EqualTo,
+  chunked_vector<std::pair<Key, Value>>,
+  ankerl::unordered_dense::bucket_type::standard,
+  chunked_vector<ankerl::unordered_dense::bucket_type::standard>>;
+
+namespace internal {
+template<typename Range>
+struct chunked_hash_map_from_range_impl {
+    using value_t = std::ranges::range_value_t<std::decay_t<Range>>;
+    using first_t = typename value_t::first_type;
+    using second_t = typename value_t::second_type;
+    using ret_t = chunked_hash_map<first_t, second_t>;
+};
+} // namespace internal
+
+// reserves if range size is known
+template<typename Range>
+typename internal::chunked_hash_map_from_range_impl<Range>::ret_t
+chunked_hash_map_from_range(Range&& range) {
+    size_t size = 0;
+    if constexpr (std::ranges::sized_range<Range>) {
+        size = std::ranges::size(range);
+    }
+    return {std::ranges::begin(range), std::ranges::end(range), size};
+};
+
+/**
+ * @brief A set counterpart of chunked_hash_map (uses a chunked vector as the
+ * underlying storage).
+ */
+template<
+  typename Key,
+  typename Hash = std::conditional_t<
+    internal::has_absl_hash<Key>,
+    internal::avalanching_absl_hash<Key>,
+    ankerl::unordered_dense::hash<Key>>,
+  typename EqualTo = std::equal_to<Key>>
+using chunked_hash_set = ankerl::unordered_dense::segmented_set<
+  Key,
+  Hash,
+  EqualTo,
+  chunked_vector<Key>,
+  ankerl::unordered_dense::bucket_type::standard,
+  chunked_vector<ankerl::unordered_dense::bucket_type::standard>>;
+
+/// Returns a lower bound on the memory currently being held by `m`.
+template<
+  typename K,
+  typename V,
+  typename Hash = std::conditional_t<
+    internal::has_absl_hash<K>,
+    internal::avalanching_absl_hash<K>,
+    ankerl::unordered_dense::hash<K>>,
+  typename EqualTo = std::equal_to<K>>
+size_t
+memory_usage_lower_bound(const chunked_hash_map<K, V, Hash, EqualTo>& m) {
+    return m.bucket_count()
+             * sizeof(typename chunked_hash_map<K, V>::bucket_type)
+           + m.values().capacity() * sizeof(m.values()[0]);
+}
+
+} // namespace seastar
+
+// Disable fmt's range formatter for chunked_hash_map/set to use our custom
+// formatters instead.
+template<
+  typename K,
+  typename V,
+  typename H,
+  typename E,
+  typename AllocatorOrContainer,
+  typename Bucket,
+  typename BucketContainer,
+  bool IsSegmented,
+  typename Char>
+struct fmt::range_format_kind<
+  ankerl::unordered_dense::detail::table<
+    K,
+    V,
+    H,
+    E,
+    AllocatorOrContainer,
+    Bucket,
+    BucketContainer,
+    IsSegmented>,
+  Char>
+  : std::integral_constant<fmt::range_format, fmt::range_format::disabled> {};
+
+template<
+  typename K,
+  typename V,
+  typename H,
+  typename E,
+  typename AllocatorOrContainer,
+  typename Bucket,
+  typename BucketContainer,
+  bool IsSegmented>
+struct fmt::formatter<ankerl::unordered_dense::detail::table<
+  K,
+  V,
+  H,
+  E,
+  AllocatorOrContainer,
+  Bucket,
+  BucketContainer,
+  IsSegmented>> {
+    using type = ankerl::unordered_dense::detail::table<
+      K,
+      V,
+      H,
+      E,
+      AllocatorOrContainer,
+      Bucket,
+      BucketContainer,
+      IsSegmented>;
+
+    constexpr auto parse(format_parse_context& ctx) const {
+        return ctx.begin();
+    }
+
+    template<typename FormatContext>
+    typename FormatContext::iterator
+    format(const type& map, FormatContext& ctx) const {
+        auto out = ctx.out();
+        out = fmt::format_to(out, "[");
+        auto it = map.begin();
+        if (it != map.end()) {
+            out = fmt::format_to(out, "{{{} -> {}}}", it->first, it->second);
+            for (++it; it != map.end(); ++it) {
+                out = fmt::format_to(
+                  out, ", {{{} -> {}}}", it->first, it->second);
+            }
+        }
+        return fmt::format_to(out, "]");
+    }
+};
+
+template<
+  typename K,
+  typename H,
+  typename E,
+  typename AllocatorOrContainer,
+  typename Bucket,
+  typename BucketContainer,
+  bool IsSegmented>
+struct fmt::formatter<ankerl::unordered_dense::detail::table<
+  K,
+  void,
+  H,
+  E,
+  AllocatorOrContainer,
+  Bucket,
+  BucketContainer,
+  IsSegmented>> {
+    using type = ankerl::unordered_dense::detail::table<
+      K,
+      void,
+      H,
+      E,
+      AllocatorOrContainer,
+      Bucket,
+      BucketContainer,
+      IsSegmented>;
+
+    constexpr auto parse(format_parse_context& ctx) const {
+        return ctx.begin();
+    }
+
+    template<typename FormatContext>
+    typename FormatContext::iterator
+    format(const type& set, FormatContext& ctx) const {
+        return fmt::format_to(ctx.out(), "[{}]", fmt::join(set, ","));
+    }
+};