container: use ss::chunked_vector

bazel/repositories.bzl

@@ -164,9 +164,9 @@ def data_dependency():
     http_archive(
         name = "seastar",
         build_file = "//bazel/thirdparty:seastar.BUILD",
-        sha256 = "e800bfbfeaf514ad90cb480aa5e317c01c514f99ed97ee0ac2ef3c9c55dc43a5",
-        strip_prefix = "seastar-5d474c884fb54f1bbef9fe9bfada0eb48feb47b5",
-        url = "https://github.com/redpanda-data/seastar/archive/5d474c884fb54f1bbef9fe9bfada0eb48feb47b5.tar.gz",
+        sha256 = "7f395262aa42d3ffec092a112971c36e5ce7513c8729be6a1a0c606cc5d4fc12",
+        strip_prefix = "seastar-4027b6ed4ad267594fcfaaada877b1c30fc05e0c",
+        url = "https://github.com/redpanda-data/seastar/archive/4027b6ed4ad267594fcfaaada877b1c30fc05e0c.tar.gz",
     )
 
     http_archive(

bazel/thirdparty/seastar.BUILD

@@ -335,6 +335,9 @@ cc_library(
         "include/seastar/core/cacheline.hh",
         "include/seastar/core/checked_ptr.hh",
         "include/seastar/core/chunked_fifo.hh",
+        "include/seastar/core/chunked_hash_map.hh",
+        "include/seastar/core/chunked_vector.hh",
+        "include/seastar/core/chunked_vector_async.hh",
         "include/seastar/core/circular_buffer.hh",
         "include/seastar/core/circular_buffer_fixed_capacity.hh",
         "include/seastar/core/condition-variable.hh",
@@ -660,6 +663,7 @@ cc_library(
     ],
     deps = [
         ":metrics_cc_proto",
+        "@abseil-cpp//absl/hash",
         "@boost//:algorithm",
         "@boost//:asio",
         "@boost//:endian",
@@ -671,6 +675,7 @@ cc_library(
         "@lksctp",
         "@lz4",
         "@protobuf",
+        "@unordered_dense",
         "@yaml-cpp",
     ] + select({
         ":use_io_uring": ["@liburing"],

src/v/cloud_topics/level_one/domain/BUILD

@@ -141,6 +141,7 @@ redpanda_cc_library(
         "//src/v/cloud_topics/level_one/metastore/lsm:replicated_db",
         "//src/v/cloud_topics/level_one/metastore/lsm:stm",
         "//src/v/container:chunked_hash_map",
+        "//src/v/container:chunked_vector",
         "//src/v/model",
         "//src/v/ssx:checkpoint_mutex",
         "@abseil-cpp//absl/container:btree",

src/v/cloud_topics/level_one/domain/entity_lock_map.h

@@ -12,6 +12,7 @@
 #include "absl/container/btree_set.h"
 #include "base/seastarx.h"
 #include "container/chunked_hash_map.h"
+#include "container/chunked_vector.h"
 #include "ssx/checkpoint_mutex.h"
 
 #include <seastar/core/future.hh>

src/v/cluster_link/model/BUILD

@@ -19,6 +19,7 @@ redpanda_cc_library(
         "//src/v/base",
         "//src/v/cluster_link:errc",
         "//src/v/container:chunked_hash_map",
+        "//src/v/container:chunked_vector",
         "//src/v/kafka/protocol",
         "//src/v/model",
         "//src/v/serde",

src/v/cluster_link/model/types.h

@@ -15,6 +15,7 @@
 #include "base/format_to.h"
 #include "cluster_link/errc.h"
 #include "container/chunked_hash_map.h"
+#include "container/chunked_vector.h"
 #include "kafka/protocol/topic_properties.h"
 #include "model/fundamental.h"
 #include "model/metadata.h"

src/v/compaction/key_offset_map.cc

@@ -10,7 +10,7 @@
  */
 #include "compaction/key_offset_map.h"
 
-#include "container/chunked_vector_async.h"
+#include <seastar/core/chunked_vector_async.hh>
 
 namespace compaction {
 
@@ -189,7 +189,7 @@ size_t hash_key_offset_map::size() const { return size_; }
 size_t hash_key_offset_map::capacity() const { return capacity_; }
 
 seastar::future<> hash_key_offset_map::initialize(size_t size_bytes) {
-    co_await chunked_vector_clear_async(entries_);
+    co_await ss::chunked_vector_clear_async(entries_);
     while (entries_.memory_size() < size_bytes) {
         const auto required_size = size_bytes - entries_.memory_size();
         const auto required_entries = std::max(
@@ -218,7 +218,7 @@ seastar::future<> hash_key_offset_map::initialize(size_t size_bytes) {
 }
 
 seastar::future<> hash_key_offset_map::reset() {
-    co_await chunked_vector_fill_async(entries_, entry{});
+    co_await ss::chunked_vector_fill_async(entries_, entry{});
     size_ = 0;
     max_offset_ = model::offset{};
     search_count_ = 0;

src/v/container/BUILD

@@ -15,12 +15,11 @@ redpanda_cc_library(
     name = "chunked_vector",
     hdrs = [
         "chunked_vector.h",
-        "chunked_vector_async.h",
     ],
     visibility = ["//visibility:public"],
     deps = [
         "//src/v/base",
-        "@fmt",
+        "@seastar",
     ],
 )
 

src/v/container/chunked_hash_map.h

@@ -10,234 +10,9 @@
  */
 #pragma once
 
-#include "absl/hash/hash.h"
-#include "container/chunked_vector.h"
+#include "base/seastarx.h"
 
-#include <ankerl/unordered_dense.h>
-#include <fmt/format.h>
-#include <fmt/ranges.h>
+#include <seastar/core/chunked_hash_map.hh>
 
-#include <type_traits>
-
-namespace detail {
-
-template<typename T>
-concept has_absl_hash = requires(T val) {
-    { AbslHashValue(std::declval<absl::HashState>(), val) };
-};
-
-/// Wrapper around absl::Hash that disables the extra hash mixing in
-/// unordered_dense
-template<typename T>
-struct avalanching_absl_hash {
-    // absl always hash mixes itself so no need to do it again
-    using is_avalanching = void;
-
-    auto operator()(const T& x) const noexcept -> uint64_t {
-        return absl::Hash<T>()(x);
-    }
-};
-
-} // namespace detail
-
-/**
- * @brief A hash map that uses a chunked vector as the underlying storage.
- *
- * Use when the hash map is expected to have a large number of elements (e.g.:
- * scales with partitions or topics). Performance wise it's equal to the abseil
- * hashmaps.
- *
- * NB: References and iterators are not stable across insertions and deletions.
- *
- * Both std::hash and abseil's AbslHashValue are supported. We dispatch to the
- * latter if available. Given AbslHashValue also supports std::hash we could
- * also unconditionally dispatch to it. However, absl's hash mixing seems more
- * extensive (and hence less performant) so we only do that when needed.
- *
- * For more info please see
- * https://github.com/martinus/unordered_dense/?tab=readme-ov-file#1-overview
- */
-template<
-  typename Key,
-  typename Value,
-  typename Hash = std::conditional_t<
-    detail::has_absl_hash<Key>,
-    detail::avalanching_absl_hash<Key>,
-    ankerl::unordered_dense::hash<Key>>,
-  typename EqualTo = std::equal_to<Key>>
-using chunked_hash_map = ankerl::unordered_dense::segmented_map<
-  Key,
-  Value,
-  Hash,
-  EqualTo,
-  chunked_vector<std::pair<Key, Value>>,
-  ankerl::unordered_dense::bucket_type::standard,
-  chunked_vector<ankerl::unordered_dense::bucket_type::standard>>;
-
-namespace detail {
-template<typename Range>
-struct chunked_hash_map_from_range_impl {
-    using value_t = std::ranges::range_value_t<std::decay_t<Range>>;
-    using first_t = typename value_t::first_type;
-    using second_t = typename value_t::second_type;
-    using ret_t = chunked_hash_map<first_t, second_t>;
-};
-} // namespace detail
-
-// reserves if range size is known
-template<typename Range>
-typename detail::chunked_hash_map_from_range_impl<Range>::ret_t
-chunked_hash_map_from_range(Range&& range) {
-    size_t size = 0;
-    if constexpr (std::ranges::sized_range<Range>) {
-        size = std::ranges::size(range);
-    }
-    return {std::ranges::begin(range), std::ranges::end(range), size};
-};
-
-/**
- * @brief A set counterpart of chunked_hash_map (uses a chunked vector as the
- * underlying storage).
- */
-template<
-  typename Key,
-  typename Hash = std::conditional_t<
-    detail::has_absl_hash<Key>,
-    detail::avalanching_absl_hash<Key>,
-    ankerl::unordered_dense::hash<Key>>,
-  typename EqualTo = std::equal_to<Key>>
-using chunked_hash_set = ankerl::unordered_dense::segmented_set<
-  Key,
-  Hash,
-  EqualTo,
-  chunked_vector<Key>,
-  ankerl::unordered_dense::bucket_type::standard,
-  chunked_vector<ankerl::unordered_dense::bucket_type::standard>>;
-
-// Disable fmt's range formatter for chunked_hash_map/set to use our custom
-// formatters instead.
-template<
-  typename K,
-  typename V,
-  typename H,
-  typename E,
-  typename AllocatorOrContainer,
-  typename Bucket,
-  typename BucketContainer,
-  bool IsSegmented,
-  typename Char>
-struct fmt::range_format_kind<
-  ankerl::unordered_dense::detail::table<
-    K,
-    V,
-    H,
-    E,
-    AllocatorOrContainer,
-    Bucket,
-    BucketContainer,
-    IsSegmented>,
-  Char>
-  : std::integral_constant<fmt::range_format, fmt::range_format::disabled> {};
-
-template<
-  typename K,
-  typename V,
-  typename H,
-  typename E,
-  typename AllocatorOrContainer,
-  typename Bucket,
-  typename BucketContainer,
-  bool IsSegmented>
-struct fmt::formatter<ankerl::unordered_dense::detail::table<
-  K,
-  V,
-  H,
-  E,
-  AllocatorOrContainer,
-  Bucket,
-  BucketContainer,
-  IsSegmented>> {
-    using type = ankerl::unordered_dense::detail::table<
-      K,
-      V,
-      H,
-      E,
-      AllocatorOrContainer,
-      Bucket,
-      BucketContainer,
-      IsSegmented>;
-
-    constexpr auto parse(format_parse_context& ctx) const {
-        return ctx.begin();
-    }
-
-    template<typename FormatContext>
-    typename FormatContext::iterator
-    format(const type& map, FormatContext& ctx) const {
-        auto out = ctx.out();
-        out = fmt::format_to(out, "[");
-        auto it = map.begin();
-        if (it != map.end()) {
-            out = fmt::format_to(out, "{{{} -> {}}}", it->first, it->second);
-            for (++it; it != map.end(); ++it) {
-                out = fmt::format_to(
-                  out, ", {{{} -> {}}}", it->first, it->second);
-            }
-        }
-        return fmt::format_to(out, "]");
-    }
-};
-
-/// Returns a lower bound on the memory currently being held by `m`.
-template<
-  typename K,
-  typename V,
-  typename Hash = std::conditional_t<
-    detail::has_absl_hash<K>,
-    detail::avalanching_absl_hash<K>,
-    ankerl::unordered_dense::hash<K>>,
-  typename EqualTo = std::equal_to<K>>
-size_t
-memory_usage_lower_bound(const chunked_hash_map<K, V, Hash, EqualTo>& m) {
-    return m.bucket_count()
-             * sizeof(typename chunked_hash_map<K, V>::bucket_type)
-           + m.values().capacity() * sizeof(m.values()[0]);
-}
-
-template<
-  typename K,
-  typename H,
-  typename E,
-  typename AllocatorOrContainer,
-  typename Bucket,
-  typename BucketContainer,
-  bool IsSegmented>
-struct fmt::formatter<ankerl::unordered_dense::detail::table<
-  K,
-  void,
-  H,
-  E,
-  AllocatorOrContainer,
-  Bucket,
-  BucketContainer,
-  IsSegmented>> {
-    using type = ankerl::unordered_dense::detail::table<
-      K,
-      void,
-      H,
-      E,
-      AllocatorOrContainer,
-      Bucket,
-      BucketContainer,
-      IsSegmented>;
-
-    constexpr auto parse(format_parse_context& ctx) const {
-        return ctx.begin();
-    }
-
-    template<typename FormatContext>
-    typename FormatContext::iterator
-    format(const type& set, FormatContext& ctx) const {
-        return fmt::format_to(ctx.out(), "[{}]", fmt::join(set, ","));
-    }
-};
+using ss::chunked_hash_map;
+using ss::chunked_hash_set;