Keep writing until the repl throttler activates in throttle tests

.config/typos.toml

@@ -18,11 +18,17 @@ Collet = "Collet" # LZ4 author Yann Collet
 nd = "nd"
 Ba = "Ba"
 Addd = "Addd"
+forkless = "forkless"
+
+[default.extend-identifiers]
+dbe = "dbe"
 
 [default]
 extend-ignore-re = [
     "SELECTed",
+    "SELECTs",
     "WATCHed",
+    "AIMD",
 ]
 
 [type.c]
@@ -39,6 +45,11 @@ extend-ignore-re = [
     "DUMPed",
 ]
 
+[type.sh]
+extend-ignore-re = [
+    "passin", # gen-test-certs.sh
+]
+
 [type.c.extend-identifiers]
 advices = "advices"
 clen = "clen"
@@ -59,6 +70,7 @@ seeked = "seeked"
 
 [type.c.extend-words]
 arange = "arange"
+Forkless = "Forkless"
 fo = "fo"
 frst = "frst"
 limite = "limite"
@@ -67,13 +79,13 @@ pn = "pn"
 seeked = "seeked"
 tre = "tre"
 
+[type.cpp.extend-words]
+fo = "fo"
+
 [type.systemd.extend-words]
 # systemd = .conf
 ake = "ake"
 
 [type.tcl.extend-words]
 fo = "fo"
 tre = "tre"
-
-[type.cpp.extend-words]
-fo = "fo"

.github/workflows/ci.yml

@@ -176,6 +176,7 @@ jobs:
       - name: test
         run: sudo ./runtest-rdma --install-rxe
       - name: show-kernel-log
+        if: always()
         run: sudo dmesg -c
 
   test-tls-only:
@@ -199,7 +200,7 @@ jobs:
 
   build-debian-old:
     runs-on: ubuntu-latest
-    container: debian:bullseye
+    container: debian:bookworm
     steps:
       - name: Install libbacktrace
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

.gitignore

@@ -58,3 +58,5 @@ cmake-build-debug/
 cmake-build-release/
 __pycache__
 src/unit/.flags
+valkey-unit-tests
+src/.idea/*

cmake/Modules/SourceFiles.cmake

@@ -5,6 +5,7 @@
 # valkey-server source files
 set(VALKEY_SERVER_SRCS
     ${CMAKE_SOURCE_DIR}/src/threads_mngr.c
+    ${CMAKE_SOURCE_DIR}/src/forkless.c
     ${CMAKE_SOURCE_DIR}/src/adlist.c
     ${CMAKE_SOURCE_DIR}/src/vector.c
     ${CMAKE_SOURCE_DIR}/src/quicklist.c
@@ -38,6 +39,7 @@ set(VALKEY_SERVER_SRCS
     ${CMAKE_SOURCE_DIR}/src/t_hash.c
     ${CMAKE_SOURCE_DIR}/src/config.c
     ${CMAKE_SOURCE_DIR}/src/aof.c
+    ${CMAKE_SOURCE_DIR}/src/bgiteration.c
     ${CMAKE_SOURCE_DIR}/src/pubsub.c
     ${CMAKE_SOURCE_DIR}/src/multi.c
     ${CMAKE_SOURCE_DIR}/src/debug.c
@@ -126,7 +128,13 @@ set(VALKEY_SERVER_SRCS
     ${CMAKE_SOURCE_DIR}/src/queues.c
     ${CMAKE_SOURCE_DIR}/src/compression.c
     ${CMAKE_SOURCE_DIR}/src/compression_lz4.c
-    ${CMAKE_SOURCE_DIR}/src/compression_stream.c)
+    ${CMAKE_SOURCE_DIR}/src/compression_stream.c
+    ${CMAKE_SOURCE_DIR}/src/hotkeys.c
+    ${CMAKE_SOURCE_DIR}/src/space_saving.c
+    ${CMAKE_SOURCE_DIR}/src/throttle_token_bucket.c
+    ${CMAKE_SOURCE_DIR}/src/stat_calc.c
+    ${CMAKE_SOURCE_DIR}/src/throttle_repl.c
+    ${CMAKE_SOURCE_DIR}/src/throttle.c)
 
 
 # valkey-cli

design-docs/io-threads.md

@@ -196,10 +196,67 @@ policy: ignite when main-thread active time crosses
 non-empty, scale down after `IO_COOLDOWN_MS` of idle. `io-threads-always-active`
 disables the policy and keeps all configured workers awake.
 
+## Cluster Bus I/O
+
+Cluster bus reads, writes, and the inbound TLS handshake run on the same worker
+pool as client I/O. The main thread alone applies cluster packets and mutates
+cluster state; workers do transport work only.
+
+Scope: the **outbound** handshake is not offloaded. `connTLSAccept` has an
+offload hook, `connTLSConnect` does not, so `SSL_connect` still runs on the main
+thread from `tlsHandleEvent`. Links are bidirectional and the bus uses mutual
+TLS, so roughly half the handshake cost stays on the main thread; offloading
+connect is planned follow-up work.
+
+Key design decisions:
+
+- **Read framing boundary.** Workers read into `clusterLink->rcvbuf`, scan
+  the prefix of complete packets, and publish `io_complete_bytes` /
+  `io_complete_packets`. The main thread drains exactly that prefix on
+  completion, then shrinks `rcvbuf` around any leftover partial packet.
+- **Write snapshot boundary.** A single canonical `send_msg_queue` is shared
+  with the worker via `io_last_send_block` + `io_head_offset`. New messages
+  enqueued while a write is in flight are picked up by the next dispatch.
+- **Bounded jobs.** A read job stops at `RCVBUF_MAX_PREALLOC`, a write job at
+  `NET_MAX_WRITES_PER_EVENT`, so one link cannot hold a worker or balloon its
+  buffer. The remainder goes out on the next event.
+- **Read and write are mutually exclusive per link**, because both directions
+  share the connection: TLS forbids concurrent use of one `SSL` object, and both
+  workers classify errors from `conn->state`. A one-shot yield
+  (`io_read_deferred`) keeps a permanently backlogged send queue from starving
+  reads, since `CONN_FLAG_WRITE_BARRIER` fires writable first.
+- **Deferred teardown.** `freeClusterLink()` defers final free via
+  `io_refs > 0` + `async_close = 1`; the last completion drops the ref and
+  frees the link. For links, `io_refs` alone guards connection lifetime.
+- **Accept serialization.** `CONN_FLAG_ACCEPT_OFFLOAD_PENDING` ensures only
+  one accept job is in flight per connection across TLS retries, and
+  `clusterConnAcceptHandler` is installed as `conn_handler` before dispatch so
+  every completion path finishes the accept. The generic accept path uses
+  `ConnectionOwnerKind` to route cluster-owned connections back to the cluster
+  dispatcher. Only TLS accepts are offloaded; a plain TCP accept does no real
+  work.
+- **Read/write dispatch is skipped until the connection is established.** A link
+  mid-connect or mid-handshake is left to the connection layer, which drives the
+  read/write handler once connected. This guards the data path; it says nothing
+  about offloading the handshake itself.
+- **Fallback.** If dispatch returns `C_ERR`, the caller runs the I/O on
+  the main thread and increments `cluster_io_main_thread_fallbacks`. Dispatch
+  needs an already-active pool, so on a lightly loaded node most cluster bus I/O
+  takes this path.
+
+`CLUSTER INFO` reports `cluster_io_threaded_reads_processed`,
+`cluster_io_threaded_writes_processed`,
+`cluster_io_threaded_accepts_processed` (all counted when a worker job
+completes) and `cluster_io_main_thread_fallbacks`.
+
 ## Relevant Code
 
 - `src/io_threads.{c,h}` — main thread dispatch helpers, worker loop,
   scaling policy.
 - `src/queues.{c,h}` — SPMC, MPSC, and SPSC queue primitives.
 - `src/networking.c` — client read/write handlers invoked from worker job
   dispatch (`ioThreadReadQueryFromClient`, `ioThreadWriteToClient`).
+- `src/cluster_legacy.c` — cluster bus worker jobs (`clusterReadJob`,
+  `clusterWriteJob`, `clusterAcceptJob`) and their main-thread completions
+  (`clusterHandleReadCompletion`, `clusterHandleWriteCompletion`,
+  `clusterHandleAcceptCompletion`).

src/Makefile

@@ -477,6 +477,7 @@ ENGINE_SERVER_OBJ = \
     allocator_defrag.o \
     anet.o \
     aof.o \
+	bgiteration.o \
     bio.o \
     bitops.o \
     blocked.o \
@@ -507,11 +508,14 @@ ENGINE_SERVER_OBJ = \
     expire.o \
     fbtree.o \
     fifo.o \
+	forkless.o \
     functions.o \
     geo.o \
     geohash.o \
     geohash_helper.o \
     hashtable.o \
+    hotkeys.o \
+    space_saving.o \
     hyperloglog.o \
     intset.o \
     io_threads.o \
@@ -586,7 +590,11 @@ ENGINE_SERVER_OBJ = \
     ziplist.o \
     zipmap.o \
     zmalloc.o \
-    queues.o
+    queues.o \
+    throttle_token_bucket.o \
+    stat_calc.o \
+    throttle_repl.o \
+    throttle.o
 ENGINE_SERVER_OBJ+=$(ENGINE_TRACE_OBJ)
 ENGINE_CLI_NAME=$(ENGINE_NAME)-cli$(PROG_SUFFIX)
 ENGINE_CLI_OBJ = \