diff --git a/.config/typos.toml b/.config/typos.toml index d4bc2684c..13a9e7b40 100644 --- a/.config/typos.toml +++ b/.config/typos.toml @@ -18,11 +18,17 @@ Collet = "Collet" # LZ4 author Yann Collet nd = "nd" Ba = "Ba" Addd = "Addd" +forkless = "forkless" + +[default.extend-identifiers] +dbe = "dbe" [default] extend-ignore-re = [ "SELECTed", + "SELECTs", "WATCHed", + "AIMD", ] [type.c] @@ -39,6 +45,11 @@ extend-ignore-re = [ "DUMPed", ] +[type.sh] +extend-ignore-re = [ + "passin", # gen-test-certs.sh +] + [type.c.extend-identifiers] advices = "advices" clen = "clen" @@ -59,6 +70,7 @@ seeked = "seeked" [type.c.extend-words] arange = "arange" +Forkless = "Forkless" fo = "fo" frst = "frst" limite = "limite" @@ -67,6 +79,9 @@ pn = "pn" seeked = "seeked" tre = "tre" +[type.cpp.extend-words] +fo = "fo" + [type.systemd.extend-words] # systemd = .conf ake = "ake" @@ -74,6 +89,3 @@ ake = "ake" [type.tcl.extend-words] fo = "fo" tre = "tre" - -[type.cpp.extend-words] -fo = "fo" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 350bfaf57..1291bc19c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,6 +176,7 @@ jobs: - name: test run: sudo ./runtest-rdma --install-rxe - name: show-kernel-log + if: always() run: sudo dmesg -c test-tls-only: @@ -199,7 +200,7 @@ jobs: build-debian-old: runs-on: ubuntu-latest - container: debian:bullseye + container: debian:bookworm steps: - name: Install libbacktrace uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.gitignore b/.gitignore index 636e29b86..a86ff7632 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,5 @@ cmake-build-debug/ cmake-build-release/ __pycache__ src/unit/.flags +valkey-unit-tests +src/.idea/* diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake index 9aeebe034..a8e29599c 100644 --- a/cmake/Modules/SourceFiles.cmake +++ b/cmake/Modules/SourceFiles.cmake @@ -5,6 +5,7 @@ # valkey-server source files set(VALKEY_SERVER_SRCS ${CMAKE_SOURCE_DIR}/src/threads_mngr.c + ${CMAKE_SOURCE_DIR}/src/forkless.c ${CMAKE_SOURCE_DIR}/src/adlist.c ${CMAKE_SOURCE_DIR}/src/vector.c ${CMAKE_SOURCE_DIR}/src/quicklist.c @@ -38,6 +39,7 @@ set(VALKEY_SERVER_SRCS ${CMAKE_SOURCE_DIR}/src/t_hash.c ${CMAKE_SOURCE_DIR}/src/config.c ${CMAKE_SOURCE_DIR}/src/aof.c + ${CMAKE_SOURCE_DIR}/src/bgiteration.c ${CMAKE_SOURCE_DIR}/src/pubsub.c ${CMAKE_SOURCE_DIR}/src/multi.c ${CMAKE_SOURCE_DIR}/src/debug.c @@ -126,7 +128,13 @@ set(VALKEY_SERVER_SRCS ${CMAKE_SOURCE_DIR}/src/queues.c ${CMAKE_SOURCE_DIR}/src/compression.c ${CMAKE_SOURCE_DIR}/src/compression_lz4.c - ${CMAKE_SOURCE_DIR}/src/compression_stream.c) + ${CMAKE_SOURCE_DIR}/src/compression_stream.c + ${CMAKE_SOURCE_DIR}/src/hotkeys.c + ${CMAKE_SOURCE_DIR}/src/space_saving.c + ${CMAKE_SOURCE_DIR}/src/throttle_token_bucket.c + ${CMAKE_SOURCE_DIR}/src/stat_calc.c + ${CMAKE_SOURCE_DIR}/src/throttle_repl.c + ${CMAKE_SOURCE_DIR}/src/throttle.c) # valkey-cli diff --git a/design-docs/io-threads.md b/design-docs/io-threads.md index 2241526e5..fca81017d 100644 --- a/design-docs/io-threads.md +++ b/design-docs/io-threads.md @@ -196,6 +196,59 @@ policy: ignite when main-thread active time crosses non-empty, scale down after `IO_COOLDOWN_MS` of idle. `io-threads-always-active` disables the policy and keeps all configured workers awake. +## Cluster Bus I/O + +Cluster bus reads, writes, and the inbound TLS handshake run on the same worker +pool as client I/O. The main thread alone applies cluster packets and mutates +cluster state; workers do transport work only. + +Scope: the **outbound** handshake is not offloaded. `connTLSAccept` has an +offload hook, `connTLSConnect` does not, so `SSL_connect` still runs on the main +thread from `tlsHandleEvent`. Links are bidirectional and the bus uses mutual +TLS, so roughly half the handshake cost stays on the main thread; offloading +connect is planned follow-up work. + +Key design decisions: + +- **Read framing boundary.** Workers read into `clusterLink->rcvbuf`, scan + the prefix of complete packets, and publish `io_complete_bytes` / + `io_complete_packets`. The main thread drains exactly that prefix on + completion, then shrinks `rcvbuf` around any leftover partial packet. +- **Write snapshot boundary.** A single canonical `send_msg_queue` is shared + with the worker via `io_last_send_block` + `io_head_offset`. New messages + enqueued while a write is in flight are picked up by the next dispatch. +- **Bounded jobs.** A read job stops at `RCVBUF_MAX_PREALLOC`, a write job at + `NET_MAX_WRITES_PER_EVENT`, so one link cannot hold a worker or balloon its + buffer. The remainder goes out on the next event. +- **Read and write are mutually exclusive per link**, because both directions + share the connection: TLS forbids concurrent use of one `SSL` object, and both + workers classify errors from `conn->state`. A one-shot yield + (`io_read_deferred`) keeps a permanently backlogged send queue from starving + reads, since `CONN_FLAG_WRITE_BARRIER` fires writable first. +- **Deferred teardown.** `freeClusterLink()` defers final free via + `io_refs > 0` + `async_close = 1`; the last completion drops the ref and + frees the link. For links, `io_refs` alone guards connection lifetime. +- **Accept serialization.** `CONN_FLAG_ACCEPT_OFFLOAD_PENDING` ensures only + one accept job is in flight per connection across TLS retries, and + `clusterConnAcceptHandler` is installed as `conn_handler` before dispatch so + every completion path finishes the accept. The generic accept path uses + `ConnectionOwnerKind` to route cluster-owned connections back to the cluster + dispatcher. Only TLS accepts are offloaded; a plain TCP accept does no real + work. +- **Read/write dispatch is skipped until the connection is established.** A link + mid-connect or mid-handshake is left to the connection layer, which drives the + read/write handler once connected. This guards the data path; it says nothing + about offloading the handshake itself. +- **Fallback.** If dispatch returns `C_ERR`, the caller runs the I/O on + the main thread and increments `cluster_io_main_thread_fallbacks`. Dispatch + needs an already-active pool, so on a lightly loaded node most cluster bus I/O + takes this path. + +`CLUSTER INFO` reports `cluster_io_threaded_reads_processed`, +`cluster_io_threaded_writes_processed`, +`cluster_io_threaded_accepts_processed` (all counted when a worker job +completes) and `cluster_io_main_thread_fallbacks`. + ## Relevant Code - `src/io_threads.{c,h}` — main thread dispatch helpers, worker loop, @@ -203,3 +256,7 @@ disables the policy and keeps all configured workers awake. - `src/queues.{c,h}` — SPMC, MPSC, and SPSC queue primitives. - `src/networking.c` — client read/write handlers invoked from worker job dispatch (`ioThreadReadQueryFromClient`, `ioThreadWriteToClient`). +- `src/cluster_legacy.c` — cluster bus worker jobs (`clusterReadJob`, + `clusterWriteJob`, `clusterAcceptJob`) and their main-thread completions + (`clusterHandleReadCompletion`, `clusterHandleWriteCompletion`, + `clusterHandleAcceptCompletion`). diff --git a/src/Makefile b/src/Makefile index f08d7c480..5976dae38 100644 --- a/src/Makefile +++ b/src/Makefile @@ -477,6 +477,7 @@ ENGINE_SERVER_OBJ = \ allocator_defrag.o \ anet.o \ aof.o \ + bgiteration.o \ bio.o \ bitops.o \ blocked.o \ @@ -507,11 +508,14 @@ ENGINE_SERVER_OBJ = \ expire.o \ fbtree.o \ fifo.o \ + forkless.o \ functions.o \ geo.o \ geohash.o \ geohash_helper.o \ hashtable.o \ + hotkeys.o \ + space_saving.o \ hyperloglog.o \ intset.o \ io_threads.o \ @@ -586,7 +590,11 @@ ENGINE_SERVER_OBJ = \ ziplist.o \ zipmap.o \ zmalloc.o \ - queues.o + queues.o \ + throttle_token_bucket.o \ + stat_calc.o \ + throttle_repl.o \ + throttle.o ENGINE_SERVER_OBJ+=$(ENGINE_TRACE_OBJ) ENGINE_CLI_NAME=$(ENGINE_NAME)-cli$(PROG_SUFFIX) ENGINE_CLI_OBJ = \ diff --git a/src/acl.c b/src/acl.c index 252e77d9e..d43b4f2e2 100644 --- a/src/acl.c +++ b/src/acl.c @@ -41,6 +41,7 @@ * ==========================================================================*/ rax *Users; /* Table mapping usernames to user structures. */ +rax *Roles; /* Table mapping role names to user structures (with USER_FLAG_ROLE). */ user *DefaultUser; /* Global reference to the default user. Every new connection is associated to it, if no @@ -54,11 +55,21 @@ list *UsersToLoad; /* This is a list of users found in the configuration file array of SDS pointers: the first is the user name, all the remaining pointers are ACL rules in the same format as ACLSetUser(). */ +list *RolesToLoad; /* Similar to UsersToLoad, but for ACL roles. Every list + element is a NULL terminated array of SDS pointers: + the first is the role name, all the remaining pointers + are ACL rules. Unlike a user, a role carries no + password and cannot be turned on or off. */ list *ACLLog; /* Our security log, the user is able to inspect that using the ACL LOG command .*/ long long ACLLogEntryCount = 0; /* Number of ACL log entries created */ +static dictType aclMembershipDictType = { + .entryGetKey = dictEntryGetKey, + .entryDestructor = zfree, +}; + static rax *commandId = NULL; /* Command name to id mapping */ static unsigned long nextid = 0; /* Next command id that has not been assigned */ @@ -191,6 +202,13 @@ static void ACLAddAllowedFirstArg(aclSelector *selector, unsigned long id, const static void ACLFreeLogEntry(void *le); static int ACLSetSelector(aclSelector *selector, const char *op, size_t oplen); static struct serverCommand *ACLLookupCommand(const char *name); +static sds ACLDescribeSelector(aclSelector *selector); +static aclSelector *aclCreateSelectorFromOpSet(const char *opset, size_t opsetlen); +static sds *ACLMergeSelectorArguments(sds *argv, int argc, int *merged_argc, int *invalid_idx); +static int ACLStringHasSpaces(const char *s, size_t len); +static int ACLUserHasAllChannels(user *u); +static list *ACLUserGetChannels(user *u); +static int ACLShouldKillPubsubClient(client *c, list *upcoming); /* The length of the string representation of a hashed password. */ #define HASH_PASSWORD_LEN (SHA256_BLOCK_SIZE * 2) @@ -273,6 +291,29 @@ static int ACLStringHasSpaces(const char *s, size_t len) { return 0; } +/* Return an error string if the role name is not valid, or NULL if it is fine. + * + * A role name is any run of printable ASCII characters, except for the four + * that the name could not be read back through: + * + * ',' separates the names in the `role=` list of a user. + * '"', '\'' and '\\' are special to sdssplitargs(), which parses the ACL + * file and valkey.conf, and the writers emit the name + * unquoted. + * + * Space and the other control characters are excluded by the printable range, + * since they end a token in the same parser. */ +static const char *ACLRoleNameError(const char *name, size_t len) { + if (len == 0) return "Role names can't be empty"; + for (size_t i = 0; i < len; i++) { + unsigned char c = name[i]; + if (c <= ' ' || c >= 0x7f) return "Role names can only contain printable ASCII characters"; + if (c == ',') return "Role names can't contain commas"; + if (c == '"' || c == '\'' || c == '\\') return "Role names can't contain quotes or backslashes"; + } + return NULL; +} + /* Given the category name the command returns the corresponding flag, or * zero if there is no match. */ uint64_t ACLGetCommandCategoryFlagByName(const char *name) { @@ -455,6 +496,9 @@ static user *ACLCreateUser(const char *name, size_t namelen) { aclSelector *s = ACLCreateSelector(SELECTOR_FLAG_ROOT); listAddNodeHead(u->selectors, s); + u->roles = listCreate(); + u->members = NULL; + raxInsert(Users, (unsigned char *)name, namelen, u, NULL); return u; } @@ -475,16 +519,49 @@ user *ACLCreateUnlinkedUser(void) { } } +/* Remove user from all roles' member lists and release the roles list. */ +static void ACLUserClearRoles(user *u) { + if (!u->roles) return; + listIter li; + listNode *ln; + listRewind(u->roles, &li); + while ((ln = listNext(&li))) { + user *r = listNodeValue(ln); + dictDelete(r->members, u); + } + listRelease(u->roles); + u->roles = NULL; +} + +/* Copy role assignments from src to dst, registering dst as a member of each + * role. The order src holds the roles in is preserved. */ +static void ACLCopyRoles(user *dst, user *src) { + if (!src->roles) return; + listIter li; + listNode *ln; + listRewind(src->roles, &li); + while ((ln = listNext(&li))) { + user *r = listNodeValue(ln); + listAddNodeTail(dst->roles, r); + serverAssert(dictAdd(r->members, dst, dst) == DICT_OK); + } +} + /* Release the memory used by the user structure. Note that this function * will not remove the user from the Users global radix tree. */ void ACLFreeUser(user *u) { + ACLUserClearRoles(u); sdsfree(u->name); if (u->acl_string) { decrRefCount(u->acl_string); u->acl_string = NULL; } - listRelease(u->passwords); + if (u->passwords) listRelease(u->passwords); listRelease(u->selectors); + if (u->members) { + serverAssert(dictSize(u->members) == 0); + dictRelease(u->members); + } zfree(u); } @@ -522,9 +599,9 @@ void ACLFreeUserAndKillClients(user *u) { * user 'dst' so that at the end of the process they'll have exactly the * same rules (but the names will continue to be the original ones). */ static void ACLCopyUser(user *dst, user *src) { - listRelease(dst->passwords); + if (dst->passwords) listRelease(dst->passwords); listRelease(dst->selectors); - dst->passwords = listDup(src->passwords); + dst->passwords = src->passwords ? listDup(src->passwords) : NULL; dst->selectors = listDup(src->selectors); dst->flags = src->flags; if (dst->acl_string) { @@ -535,6 +612,207 @@ static void ACLCopyUser(user *dst, user *src) { /* if src is NULL, we set it to NULL, if not, need to increment reference count */ incrRefCount(dst->acl_string); } + /* Clean up dst's existing role memberships, then copy from src. */ + ACLUserClearRoles(dst); + if (src->roles) { + dst->roles = listCreate(); + ACLCopyRoles(dst, src); + } else { + dst->roles = NULL; + } +} + +/* ============================================================================= + * ACL Role functions + * ==========================================================================*/ + +/* Create a new ACL role with the given name and register it in the Roles + * radix tree. A role is a user with USER_FLAG_ROLE set, no passwords, and + * a members list. Returns NULL if a role with the same name already exists. */ +static user *ACLCreateRole(const char *name, size_t namelen) { + if (raxFind(Roles, (unsigned char *)name, namelen, NULL)) return NULL; + + user *r = zmalloc(sizeof(*r)); + r->name = sdsnewlen(name, namelen); + r->flags = USER_FLAG_ROLE; + r->passwords = NULL; + r->members = dictCreate(&aclMembershipDictType); + r->roles = NULL; + r->acl_string = NULL; + r->selectors = listCreate(); + + listSetFreeMethod(r->selectors, ACLListFreeSelector); + listSetDupMethod(r->selectors, ACLListDuplicateSelector); + aclSelector *s = ACLCreateSelector(SELECTOR_FLAG_ROOT); + listAddNodeHead(r->selectors, s); + + raxInsert(Roles, (unsigned char *)name, namelen, r, NULL); + return r; +} + +/* Lookup a role by name. Returns NULL if not found. */ +user *ACLGetRoleByName(const char *name, size_t namelen) { + void *myrole = NULL; + raxFind(Roles, (unsigned char *)name, namelen, &myrole); + return myrole; +} + +/* Replace the set of roles held by the user with the comma separated list of + * role names in `spec`, which is the part of the `role=` rule following the + * equal sign. The list has to name at least one role; use `resetroles` to + * leave the user with no role at all. + * + * Every name is resolved before the user is touched, so on error the user + * keeps the roles it had. Returns C_OK, or C_ERR with errno set to ESRCH if a + * role does not exist and EINVAL if the list is malformed. */ +static int ACLSetUserRoles(user *u, const char *spec, size_t speclen) { + /* An empty list, or a trailing comma leaving an empty last name, which the + * loop below cannot see. Leading and repeated commas are caught by the + * zero length check. */ + if (speclen == 0 || spec[speclen - 1] == ',') { + errno = EINVAL; + return C_ERR; + } + + list *resolved = listCreate(); + const char *end = spec + speclen; + for (const char *p = spec; p < end;) { + const char *comma = memchr(p, ',', end - p); + size_t namelen = comma ? (size_t)(comma - p) : (size_t)(end - p); + user *r = namelen ? ACLGetRoleByName(p, namelen) : NULL; + if (!r) { + errno = namelen ? ESRCH : EINVAL; + listRelease(resolved); + return C_ERR; + } + listAddNodeTail(resolved, r); + p = comma ? comma + 1 : end; + } + + ACLUserClearRoles(u); + u->roles = listCreate(); + + listIter li; + listNode *ln; + listRewind(resolved, &li); + while ((ln = listNext(&li))) { + user *r = listNodeValue(ln); + /* The same role may be named twice in the list, keep the first. */ + if (listSearchKey(u->roles, r)) continue; + listAddNodeTail(u->roles, r); + serverAssert(dictAdd(r->members, u, u) == DICT_OK); + } + listRelease(resolved); + return C_OK; +} + +/* High-level function to set multiple ACL rules on a role atomically. + * Uses a temporary role-flagged user + ACLSetUser() for validation. + * Returns NULL on success, or an SDS error string on failure. */ +static sds ACLStringSetRole(user *r, sds rolename, sds *argv, int argc) { + sds error = NULL; + + /* Create a temporary role-flagged user to validate all changes */ + user *tempr = zmalloc(sizeof(*tempr)); + tempr->name = sdsdup(rolename); + tempr->flags = USER_FLAG_ROLE; + tempr->passwords = NULL; + tempr->roles = NULL; + tempr->members = NULL; + tempr->acl_string = NULL; + tempr->selectors = listCreate(); + listSetFreeMethod(tempr->selectors, ACLListFreeSelector); + listSetDupMethod(tempr->selectors, ACLListDuplicateSelector); + + /* If role already exists, copy its selectors */ + if (r) { + listRelease(tempr->selectors); + tempr->selectors = listDup(r->selectors); + } else { + aclSelector *s = ACLCreateSelector(SELECTOR_FLAG_ROOT); + listAddNodeHead(tempr->selectors, s); + } + + int merged_argc = 0, invalid_idx = 0; + sds *acl_args = ACLMergeSelectorArguments(argv, argc, &merged_argc, &invalid_idx); + if (!acl_args) { + error = sdscatfmt(sdsempty(), "Unmatched parenthesis in selector definition starting at '%s'.", + (char *)argv[invalid_idx]); + ACLFreeUser(tempr); + return error; + } + + for (int j = 0; j < merged_argc; j++) { + if (ACLSetUser(tempr, acl_args[j], (ssize_t)sdslen(acl_args[j])) != C_OK) { + const char *errmsg = ACLSetStringError(); + error = sdscatfmt(sdsempty(), "Error in ACL SETROLE modifier '%s': %s", (char *)acl_args[j], errmsg); + goto cleanup; + } + } + + /* Apply changes: if role doesn't exist, create it; otherwise update it */ + if (!r) { + r = ACLCreateRole(rolename, sdslen(rolename)); + serverAssert(r != NULL); + } + + /* Save old selectors before updating. */ + list *old_selectors = r->selectors; + r->selectors = listDup(tempr->selectors); + + /* Kill pubsub clients of member users whose channel access was revoked. + * Since the role's selectors are already updated, the member's effective + * permissions reflect the new state. We build the new channel list from + * the member's effective permissions and check each client against it. */ + if (pubsubTotalSubscriptions() > 0 && dictSize(r->members) > 0) { + dictIterator *mdi = dictGetIterator(r->members); + dictEntry *mde; + while ((mde = dictNext(mdi))) { + user *member = dictGetVal(mde); + if (ACLUserHasAllChannels(member)) continue; + + list *upcoming = ACLUserGetChannels(member); + listIter cli; + listNode *cln; + listRewind(server.clients, &cli); + while ((cln = listNext(&cli))) { + client *c = listNodeValue(cln); + if (c->user != member) continue; + if (ACLShouldKillPubsubClient(c, upcoming)) { + freeClientOrCloseLater(c, 0); + } + } + listRelease(upcoming); + } + dictReleaseIterator(mdi); + } + + listRelease(old_selectors); + + if (r->acl_string) { + decrRefCount(r->acl_string); + r->acl_string = NULL; + } + + /* Invalidate acl_string cache for all member users */ + { + dictIterator *di = dictGetIterator(r->members); + dictEntry *de; + while ((de = dictNext(di))) { + user *u = dictGetVal(de); + if (u->acl_string) { + decrRefCount(u->acl_string); + u->acl_string = NULL; + } + } + dictReleaseIterator(di); + } + +cleanup: + for (int i = 0; i < merged_argc; i++) sdsfree(acl_args[i]); + zfree(acl_args); + ACLFreeUser(tempr); + return error; } /* Given a command ID, this function set by reference 'word' and 'bit' @@ -685,12 +963,11 @@ static void ACLSetSelectorCommandBitsForCategory(hashtable *commands, aclSelecto hashtableCleanupIterator(&iter); } -/* This function is responsible for recomputing the command bits for all selectors of the existing users. - * It uses the 'command_rules', a string representation of the ordered categories and commands, - * to recompute the command bits. */ -void ACLRecomputeCommandBitsFromCommandRulesAllUsers(void) { +/* Recompute the command bits for all selectors of every entry in table, which + * is either the Users or the Roles radix tree. */ +static void ACLRecomputeCommandBitsFromCommandRulesInTable(rax *table) { raxIterator ri; - raxStart(&ri, Users); + raxStart(&ri, table); raxSeek(&ri, "^", NULL, 0); while (raxNext(&ri)) { user *u = ri.data; @@ -722,6 +999,14 @@ void ACLRecomputeCommandBitsFromCommandRulesAllUsers(void) { raxStop(&ri); } +/* This function is responsible for recomputing the command bits for all selectors of the existing users. + * It uses the 'command_rules', a string representation of the ordered categories and commands, + * to recompute the command bits. */ +void ACLRecomputeCommandBitsFromCommandRulesAllUsers(void) { + ACLRecomputeCommandBitsFromCommandRulesInTable(Users); + ACLRecomputeCommandBitsFromCommandRulesInTable(Roles); +} + static int ACLSetSelectorCategory(aclSelector *selector, const char *category, int allow) { uint64_t cflag = ACLGetCommandCategoryFlagByName(category + 1); if (!cflag) return C_ERR; @@ -733,13 +1018,11 @@ static int ACLSetSelectorCategory(aclSelector *selector, const char *category, i return C_OK; } -/* Check if any ACL user has command rules referencing the specified module. - * If rule_out is not NULL, it will be set to a duplicate of the first matching - * rule. - * Returns 1 if any rules are found, 0 otherwise. */ -int ACLModuleHasCommandRules(const struct ValkeyModule *module, sds *rule_out) { +/* Check if any entry in table, which is either the Users or the Roles radix + * tree, has command rules referencing the specified module. */ +static int ACLTableHasModuleCommandRules(rax *table, const struct ValkeyModule *module, sds *rule_out) { raxIterator ri; - raxStart(&ri, Users); + raxStart(&ri, table); raxSeek(&ri, "^", NULL, 0); while (raxNext(&ri)) { user *u = ri.data; @@ -784,6 +1067,15 @@ int ACLModuleHasCommandRules(const struct ValkeyModule *module, sds *rule_out) { return 0; } +/* Check if any ACL user has command rules referencing the specified module. + * If rule_out is not NULL, it will be set to a duplicate of the first matching + * rule. + * Returns 1 if any rules are found, 0 otherwise. */ +int ACLModuleHasCommandRules(const struct ValkeyModule *module, sds *rule_out) { + if (ACLTableHasModuleCommandRules(Users, module, rule_out)) return 1; + return ACLTableHasModuleCommandRules(Roles, module, rule_out); +} + /* This function returns an SDS string representing the specified selector ACL * rules related to command execution, in the same format you could set them * back using ACL SETUSER. The function will return just the set of rules needed @@ -911,26 +1203,31 @@ robj *ACLDescribeUser(user *u) { sds res = sdsempty(); - /* Flags. */ - for (int j = 0; ACLUserFlags[j].flag; j++) { - if (u->flags & ACLUserFlags[j].flag) { - res = sdscat(res, ACLUserFlags[j].name); + /* For roles, skip flags and passwords. */ + if (!(u->flags & USER_FLAG_ROLE)) { + /* Flags. */ + for (int j = 0; ACLUserFlags[j].flag; j++) { + if (u->flags & ACLUserFlags[j].flag) { + res = sdscat(res, ACLUserFlags[j].name); + res = sdscatlen(res, " ", 1); + } + } + + /* Passwords. */ + listIter li; + listNode *ln; + listRewind(u->passwords, &li); + while ((ln = listNext(&li))) { + sds thispass = listNodeValue(ln); + res = sdscatlen(res, "#", 1); + res = sdscatsds(res, thispass); res = sdscatlen(res, " ", 1); } } - /* Passwords. */ + /* Selectors (Commands and keys) */ listIter li; listNode *ln; - listRewind(u->passwords, &li); - while ((ln = listNext(&li))) { - sds thispass = listNodeValue(ln); - res = sdscatlen(res, "#", 1); - res = sdscatsds(res, thispass); - res = sdscatlen(res, " ", 1); - } - - /* Selectors (Commands and keys) */ listRewind(u->selectors, &li); while ((ln = listNext(&li))) { aclSelector *selector = (aclSelector *)listNodeValue(ln); @@ -943,6 +1240,19 @@ robj *ACLDescribeUser(user *u) { sdsfree(default_perm); } + /* Role memberships (only for users, not roles), in assignment order. */ + if (u->roles && listLength(u->roles) > 0) { + listIter li; + listNode *ln; + listRewind(u->roles, &li); + const char *sep = " role="; + while ((ln = listNext(&li))) { + user *r = listNodeValue(ln); + res = sdscatfmt(res, "%s%S", sep, r->name); + sep = ","; + } + } + u->acl_string = createObject(OBJ_STRING, res); /* because we are returning it, have to increase count */ incrRefCount(u->acl_string); @@ -1397,8 +1707,12 @@ static int ACLSetSelector(aclSelector *selector, const char *op, size_t oplen) { * some password (or setting it as "nopass" later). * reset Performs the following actions: resetpass, resetkeys, resetchannels, * allchannels (if acl-pubsub-default is set), alldbs (for backwards compatibility), - * off, sanitize-payload, clearselectors, -@all. + * off, sanitize-payload, clearselectors, -@all, resetroles. * The user returns to the same state it has immediately after its creation. + * role= Replace the set of roles held by the user with the named ones. + * May be used with `,` for naming several roles (e.g "role=a,b"). + * At least one role has to be named. + * resetroles Remove every role from the user. * () Create a new selector with the options specified within the * parentheses and attach it to the user. Each option should be * space separated. The first character must be ( and the last @@ -1445,6 +1759,25 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { if (oplen == -1) oplen = strlen(op); if (oplen == 0) return C_OK; /* Empty string is a no-operation. */ + + /* Roles cannot use user-only operations. */ + if (u->flags & USER_FLAG_ROLE) { + if (!strcasecmp(op, "on") || !strcasecmp(op, "off") || + !strcasecmp(op, "nopass") || !strcasecmp(op, "resetpass") || + !strcasecmp(op, "reset") || + !strcasecmp(op, "skip-sanitize-payload") || + !strcasecmp(op, "sanitize-payload") || + op[0] == '>' || op[0] == '#' || op[0] == '<' || op[0] == '!') { + errno = EINVAL; + return C_ERR; + } + /* Roles cannot have roles */ + if (!strcasecmp(op, "resetroles") || (oplen >= 5 && !strncasecmp(op, "role=", 5))) { + errno = EINVAL; + return C_ERR; + } + } + if (!strcasecmp(op, "on")) { u->flags |= USER_FLAG_ENABLED; u->flags &= ~USER_FLAG_DISABLED; @@ -1527,6 +1860,12 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { serverAssert(ACLSetUser(u, "off", -1) == C_OK); serverAssert(ACLSetUser(u, "clearselectors", -1) == C_OK); serverAssert(ACLSetUser(u, "-@all", -1) == C_OK); + serverAssert(ACLSetUser(u, "resetroles", -1) == C_OK); + } else if (!strcasecmp(op, "resetroles")) { + ACLUserClearRoles(u); + u->roles = listCreate(); + } else if (oplen >= 5 && !strncasecmp(op, "role=", 5)) { + if (ACLSetUserRoles(u, op + 5, oplen - 5) == C_ERR) return C_ERR; } else { aclSelector *selector = ACLUserGetRootSelector(u); if (ACLSetSelector(selector, op, oplen) == C_ERR) { @@ -1536,9 +1875,9 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { return C_OK; } -/* Return a description of the error that occurred in ACLSetUser() according to - * the errno value set by the function on error. */ -const char *ACLSetUserStringError(void) { +/* Return a description of the error that occurred in ACLSetUser() + * according to the errno value set on error. */ +const char *ACLSetStringError(void) { const char *errmsg = "Wrong format"; if (errno == ENOENT) errmsg = "Unknown command or category name in ACL"; @@ -1563,10 +1902,18 @@ const char *ACLSetUserStringError(void) { else if (errno == EALREADY) errmsg = "Duplicate user found. A user can only be defined once in " "config files"; + else if (errno == EBUSY) + errmsg = "Duplicate role found. A role can only be defined once in " + "config files"; + else if (errno == EILSEQ) + errmsg = "Role names can't be empty and can only contain printable " + "ASCII characters, excluding commas, quotes and backslashes"; else if (errno == ECHILD) errmsg = "Allowing first-arg of a subcommand is not supported"; else if (errno == ERANGE) errmsg = "The provided database ID is out of range"; + else if (errno == ESRCH) + errmsg = "The specified ACL role does not exist"; return errmsg; } @@ -1585,9 +1932,12 @@ static user *ACLCreateDefaultUser(void) { /* Initialization of the ACL subsystem. */ void ACLInit(void) { Users = raxNew(); + Roles = raxNew(); UsersToLoad = listCreate(); + RolesToLoad = listCreate(); ACLInitCommandCategories(); listSetMatchMethod(UsersToLoad, ACLListMatchLoadedUser); + listSetMatchMethod(RolesToLoad, ACLListMatchLoadedUser); ACLLog = listCreate(); DefaultUser = ACLCreateDefaultUser(); } @@ -2005,6 +2355,47 @@ static int ACLSelectorCheckCmd(aclSelector *selector, return ACL_OK; } +/* Iterates the selectors that apply to a user: its own first, then those of every + * role it belongs to, in the order the roles were assigned. Roles carry no roles + * of their own, so iterating one just walks its selectors. The iterator holds no + * resources, so it can be abandoned at any point. */ +typedef struct { + user *u; + listIter li; /* Position within the selector list being walked. */ + listIter rli; /* Position within u->roles, valid once in_roles is set. */ + int in_roles; /* True once the walk moved on to the user's roles. */ + int done; +} aclSelectorIterator; + +static void ACLSelectorIteratorInit(aclSelectorIterator *it, user *u) { + it->u = u; + it->in_roles = 0; + it->done = 0; + listRewind(u->selectors, &it->li); +} + +static aclSelector *ACLSelectorIteratorNext(aclSelectorIterator *it) { + if (it->done) return NULL; + while (1) { + listNode *ln = listNext(&it->li); + if (ln) return (aclSelector *)listNodeValue(ln); + if (!it->u->roles) { + it->done = 1; + return NULL; + } + if (!it->in_roles) { + listRewind(it->u->roles, &it->rli); + it->in_roles = 1; + } + listNode *rln = listNext(&it->rli); + if (!rln) { + it->done = 1; + return NULL; + } + listRewind(((user *)listNodeValue(rln))->selectors, &it->li); + } +} + /* Checks whether the given user has permission to access a specified key. * * This function verifies the access control list (ACL) permissions for a user on a key. @@ -2029,16 +2420,13 @@ static int ACLSelectorCheckCmd(aclSelector *selector, * it only evaluates read-only selectors. */ int ACLUserCheckKeyPerm(user *u, const char *key, int keylen, int flags, bool is_prefix) { - listIter li; - listNode *ln; - /* If there is no associated user, the connection can run anything. */ if (u == NULL) return ACL_OK; - /* Check all of the selectors */ - listRewind(u->selectors, &li); - while ((ln = listNext(&li))) { - aclSelector *s = (aclSelector *)listNodeValue(ln); + aclSelectorIterator it; + aclSelector *s; + ACLSelectorIteratorInit(&it, u); + while ((s = ACLSelectorIteratorNext(&it))) { if (ACLSelectorCheckKey(s, key, keylen, flags, is_prefix) == ACL_OK) { return ACL_OK; } @@ -2053,8 +2441,6 @@ int ACLUserCheckKeyPerm(user *u, const char *key, int keylen, int flags, bool is * if the user has access or 0 otherwise. */ int ACLUserCheckCmdWithUnrestrictedKeyAccess(user *u, struct serverCommand *cmd, robj **argv, int argc, int dbid, int flags) { - listIter li; - listNode *ln; int local_idxptr; /* If there is no associated user, the connection can run anything. */ @@ -2065,10 +2451,10 @@ int ACLUserCheckCmdWithUnrestrictedKeyAccess(user *u, struct serverCommand *cmd, aclKeyResultCache cache; initACLKeyResultCache(&cache); - /* Check each selector sequentially */ - listRewind(u->selectors, &li); - while ((ln = listNext(&li))) { - aclSelector *s = (aclSelector *)listNodeValue(ln); + aclSelectorIterator it; + aclSelector *s; + ACLSelectorIteratorInit(&it, u); + while ((s = ACLSelectorIteratorNext(&it))) { int acl_retval = ACLSelectorCheckCmd(s, cmd, argv, argc, &local_idxptr, &cache, dbid); if (acl_retval == ACL_OK && ACLSelectorHasUnrestrictedKeyAccess(s, flags)) { cleanupACLKeyResultCache(&cache); @@ -2085,21 +2471,16 @@ int ACLUserCheckCmdWithUnrestrictedKeyAccess(user *u, struct serverCommand *cmd, * If the user can access the key, ACL_OK is returned, otherwise * ACL_DENIED_CHANNEL is returned. */ int ACLUserCheckChannelPerm(user *u, sds channel, int is_pattern) { - listIter li; - listNode *ln; - /* If there is no associated user, the connection can run anything. */ if (u == NULL) return ACL_OK; - /* Check all of the selectors */ - listRewind(u->selectors, &li); - while ((ln = listNext(&li))) { - aclSelector *s = (aclSelector *)listNodeValue(ln); - /* The selector can run any keys */ - if (s->flags & SELECTOR_FLAG_ALLCHANNELS) return ACL_OK; - - /* Otherwise, loop over the selectors list and check each channel */ - if (ACLCheckChannelAgainstList(s->channels, channel, sdslen(channel), is_pattern) == ACL_OK) { + aclSelectorIterator it; + aclSelector *s; + ACLSelectorIteratorInit(&it, u); + while ((s = ACLSelectorIteratorNext(&it))) { + /* The selector can run any channel, or names this one. */ + if ((s->flags & SELECTOR_FLAG_ALLCHANNELS) || + ACLCheckChannelAgainstList(s->channels, channel, sdslen(channel), is_pattern) == ACL_OK) { return ACL_OK; } } @@ -2112,9 +2493,6 @@ int ACLUserCheckChannelPerm(user *u, sds channel, int is_pattern) { * causes the failure, either 0 if the command itself fails or the idx of the key/channel * that causes the failure */ int ACLCheckAllUserCommandPerm(user *u, struct serverCommand *cmd, robj **argv, int argc, int dbid, int *idxptr) { - listIter li; - listNode *ln; - /* If there is no associated user, the connection can run anything. */ if (u == NULL) return ACL_OK; @@ -2130,9 +2508,10 @@ int ACLCheckAllUserCommandPerm(user *u, struct serverCommand *cmd, robj **argv, initACLKeyResultCache(&cache); /* Check each selector sequentially */ - listRewind(u->selectors, &li); - while ((ln = listNext(&li))) { - aclSelector *s = (aclSelector *)listNodeValue(ln); + aclSelectorIterator it; + aclSelector *s; + ACLSelectorIteratorInit(&it, u); + while ((s = ACLSelectorIteratorNext(&it))) { int acl_retval = ACLSelectorCheckCmd(s, cmd, argv, argc, &local_idxptr, &cache, dbid); if (acl_retval == ACL_OK) { cleanupACLKeyResultCache(&cache); @@ -2151,42 +2530,59 @@ int ACLCheckAllUserCommandPerm(user *u, struct serverCommand *cmd, robj **argv, /* High level API for checking if a client can execute the queued up command */ int ACLCheckAllPerm(client *c, int *idxptr) { - int dbid = (c->flag.multi) ? c->mstate->transaction_db_id : c->db->id; + int dbid = (c->flag.multi && c->cmd->proc != execCommand) ? c->mstate->transaction_db_id : c->db->id; return ACLCheckAllUserCommandPerm(c->user, c->cmd, c->argv, c->argc, dbid, idxptr); } -/* If 'new' can access all channels 'original' could then return NULL; - Otherwise, return a list of channels that the new user can access */ -static list *getUpcomingChannelList(user *new, user *original) { - listIter li, lpi; - listNode *ln, *lpn; - - /* Optimization: we check if any selector has all channel permissions. */ - listRewind(new->selectors, &li); - while ((ln = listNext(&li))) { - aclSelector *s = (aclSelector *)listNodeValue(ln); - if (s->flags & SELECTOR_FLAG_ALLCHANNELS) return NULL; +/* Check if user has allchannels permission from own or role selectors. */ +static int ACLUserHasAllChannels(user *u) { + aclSelectorIterator it; + aclSelector *s; + ACLSelectorIteratorInit(&it, u); + while ((s = ACLSelectorIteratorNext(&it))) { + if (s->flags & SELECTOR_FLAG_ALLCHANNELS) { + return 1; + } } + return 0; +} - /* Next, check if the new list of channels - * is a strict superset of the original. This is done by - * created an "upcoming" list of all channels that are in - * the new user and checking each of the existing channels - * against it. */ - list *upcoming = listCreate(); - listRewind(new->selectors, &li); - while ((ln = listNext(&li))) { - aclSelector *s = (aclSelector *)listNodeValue(ln); +/* Build a list of all channel patterns accessible by user (own + role selectors). + * Caller must listRelease() the returned list. */ +static list *ACLUserGetChannels(user *u) { + list *channels = listCreate(); + listIter lpi; + listNode *lpn; + + aclSelectorIterator it; + aclSelector *s; + ACLSelectorIteratorInit(&it, u); + while ((s = ACLSelectorIteratorNext(&it))) { listRewind(s->channels, &lpi); while ((lpn = listNext(&lpi))) { - listAddNodeTail(upcoming, listNodeValue(lpn)); + listAddNodeTail(channels, listNodeValue(lpn)); } } + return channels; +} + +static list *getUpcomingChannelList(user *new, user *original) { + listIter lpi; + listNode *lpn; + + /* Optimization: if new user has allchannels, no kill needed. */ + if (ACLUserHasAllChannels(new)) return NULL; + /* Build the list of channels the new user can access. */ + list *upcoming = ACLUserGetChannels(new); + + /* Walk the original user's own selectors and then those of each of its + * roles, since both grant channels to the user. */ int match = 1; - listRewind(original->selectors, &li); - while ((ln = listNext(&li)) && match) { - aclSelector *s = (aclSelector *)listNodeValue(ln); + aclSelectorIterator it; + aclSelector *s; + ACLSelectorIteratorInit(&it, original); + while (match && (s = ACLSelectorIteratorNext(&it))) { /* If any of the original selectors has the all-channels permission, but * the new ones don't (this is checked earlier in this function), then the * new list is not a strict superset of the original. */ @@ -2195,7 +2591,7 @@ static list *getUpcomingChannelList(user *new, user *original) { break; } listRewind(s->channels, &lpi); - while ((lpn = listNext(&lpi)) && match) { + while ((lpn = listNext(&lpi))) { if (!listSearchKey(upcoming, listNodeValue(lpn))) { match = 0; break; @@ -2378,7 +2774,7 @@ sds ACLStringSetUser(user *u, sds username, sds *argv, int argc) { for (int j = 0; j < merged_argc; j++) { if (ACLSetUser(tempu, acl_args[j], (ssize_t)sdslen(acl_args[j])) != C_OK) { - const char *errmsg = ACLSetUserStringError(); + const char *errmsg = ACLSetStringError(); error = sdscatfmt(sdsempty(), "Error in ACL SETUSER modifier '%s': %s", (char *)acl_args[j], errmsg); goto cleanup; } @@ -2452,7 +2848,7 @@ int ACLAppendUserForLoading(sds *argv, int argc, int *argc_err) { for (int j = 0; j < merged_argc; j++) { if (ACLSetUser(fakeuser, acl_args[j], sdslen(acl_args[j])) == C_ERR) { - if (errno != ENOENT) { + if (errno != ENOENT && errno != ESRCH) { ACLFreeUser(fakeuser); if (argc_err) *argc_err = j; for (int i = 0; i < merged_argc; i++) sdsfree(acl_args[i]); @@ -2501,7 +2897,7 @@ static int ACLLoadConfiguredUsers(void) { /* Load every rule defined for this user. */ for (int j = 1; aclrules[j]; j++) { if (ACLSetUser(u, aclrules[j], sdslen(aclrules[j])) != C_OK) { - const char *errmsg = ACLSetUserStringError(); + const char *errmsg = ACLSetStringError(); serverLog(LL_WARNING, "Error loading ACL rule '%s' for " "the user named '%s': %s", @@ -2523,6 +2919,144 @@ static int ACLLoadConfiguredUsers(void) { return C_OK; } +/* Append a role definition for deferred loading (from valkey.conf). */ +int ACLAppendRoleForLoading(sds *argv, int argc, int *argc_err) { + if (argc < 2 || strcasecmp(argv[0], "role")) { + if (argc_err) *argc_err = 0; + return C_ERR; + } + + if (listSearchKey(RolesToLoad, argv[1])) { + if (argc_err) *argc_err = 1; + errno = EBUSY; + return C_ERR; + } + + if (ACLRoleNameError(argv[1], sdslen(argv[1]))) { + if (argc_err) *argc_err = 1; + errno = EILSEQ; + return C_ERR; + } + + /* Store the role definition for later loading. */ + int merged_argc = 0, invalid_idx = 0; + sds *acl_args = ACLMergeSelectorArguments(argv + 2, argc - 2, &merged_argc, &invalid_idx); + if (!acl_args) { + if (argc_err) *argc_err = invalid_idx + 2; + return C_ERR; + } + + /* Try to apply the role rules in a fake role-flagged user to validate them. */ + user fakeRole = {0}; + fakeRole.flags = USER_FLAG_ROLE; + fakeRole.passwords = NULL; + fakeRole.roles = NULL; + fakeRole.members = NULL; + fakeRole.selectors = listCreate(); + listSetFreeMethod(fakeRole.selectors, ACLListFreeSelector); + aclSelector *s = ACLCreateSelector(SELECTOR_FLAG_ROOT); + listAddNodeHead(fakeRole.selectors, s); + + for (int j = 0; j < merged_argc; j++) { + if (ACLSetUser(&fakeRole, acl_args[j], sdslen(acl_args[j])) == C_ERR) { + if (errno != ENOENT && errno != ESRCH) { + listRelease(fakeRole.selectors); + if (argc_err) *argc_err = j + 2; + for (int i = 0; i < merged_argc; i++) sdsfree(acl_args[i]); + zfree(acl_args); + return C_ERR; + } + } + } + listRelease(fakeRole.selectors); + + /* Rules look valid, store for deferred loading. */ + sds *copy = zmalloc(sizeof(sds) * (merged_argc + 2)); + copy[0] = sdsdup(argv[1]); + for (int j = 0; j < merged_argc; j++) copy[j + 1] = sdsdup(acl_args[j]); + copy[merged_argc + 1] = NULL; + listAddNodeTail(RolesToLoad, copy); + for (int i = 0; i < merged_argc; i++) sdsfree(acl_args[i]); + zfree(acl_args); + return C_OK; +} + +/* Load configured roles from RolesToLoad. Must be called before + * ACLLoadConfiguredUsers so that users can reference roles. */ +static int ACLLoadConfiguredRoles(void) { + listIter li; + listNode *ln; + listRewind(RolesToLoad, &li); + while ((ln = listNext(&li)) != NULL) { + sds *aclrules = listNodeValue(ln); + sds rolename = aclrules[0]; + + if (ACLStringHasSpaces(rolename, sdslen(rolename))) { + serverLog(LL_WARNING, "Spaces not allowed in ACL role names"); + return C_ERR; + } + + user *r = ACLGetRoleByName(rolename, sdslen(rolename)); + /* Count number of rules */ + int argc = 0; + while (aclrules[argc + 1]) argc++; + + sds error = ACLStringSetRole(r, rolename, aclrules + 1, argc); + if (error) { + serverLog(LL_WARNING, "Error loading ACL role '%s': %s", rolename, error); + sdsfree(error); + return C_ERR; + } + } + return C_OK; +} + +/* Move role memberships of users not replaced by ACL LOAD, i.e. module users, to + * the new role of the same name, or drop them if it is gone. Called after the old + * users are freed, so only such survivors are left on the old member lists. */ +static void ACLRemapSurvivingRoleMembers(rax *old_roles) { + raxIterator ri; + raxStart(&ri, old_roles); + raxSeek(&ri, "^", NULL, 0); + while (raxNext(&ri)) { + user *old_role = ri.data; + if (!old_role->members || dictSize(old_role->members) == 0) continue; + + /* Snapshot the members before mutating the dict. */ + int count = 0, numsurvivors = dictSize(old_role->members); + user **survivors = zmalloc(sizeof(user *) * numsurvivors); + dictIterator *di = dictGetIterator(old_role->members); + dictEntry *de; + while ((de = dictNext(di))) survivors[count++] = dictGetVal(de); + dictReleaseIterator(di); + + user *new_role = ACLGetRoleByName(old_role->name, sdslen(old_role->name)); + for (int j = 0; j < numsurvivors; j++) { + user *u = survivors[j]; + listNode *ln = listSearchKey(u->roles, old_role); + serverAssert(ln != NULL); + dictDelete(old_role->members, u); + if (new_role) { + /* Swap the role in place so the user keeps its role order. */ + listNodeValue(ln) = new_role; + serverAssert(dictAdd(new_role->members, u, u) == DICT_OK); + } else { + listDelNode(u->roles, ln); + serverLog(LL_NOTICE, + "The ACL role '%s' held by the user '%s' no longer exists after reloading the ACLs, " + "the membership was dropped.", + old_role->name, u->name); + } + if (u->acl_string) { + decrRefCount(u->acl_string); + u->acl_string = NULL; + } + } + zfree(survivors); + } + raxStop(&ri); +} + /* This function loads the ACL from the specified filename: every line * is validated and should be either empty or in the format used to specify * users in the valkey.conf or in the ACL file, that is: @@ -2571,9 +3105,12 @@ static sds ACLLoadFromFile(const char *filename) { * so if there are errors loading the ACL file we can rollback to the * old version. */ rax *old_users = Users; + rax *old_roles = Roles; Users = raxNew(); + Roles = raxNew(); /* Load each line of the file. */ + /* First pass: load role definitions */ for (int i = 0; i < totlines; i++) { sds *argv; int argc; @@ -2597,8 +3134,74 @@ static sds ACLLoadFromFile(const char *filename) { continue; } - /* The line should start with the "user" keyword. */ - if (strcmp(argv[0], "user") || argc < 2) { + /* Only process role lines in first pass */ + if (strcmp(argv[0], "role") != 0) { + sdsfreesplitres(argv, argc); + continue; + } + + if (argc < 2) { + errors = sdscatprintf(errors, + "%s:%d: role line requires a role name. ", + server.acl_filename, linenum); + sdsfreesplitres(argv, argc); + continue; + } + + const char *nameerr = ACLRoleNameError(argv[1], sdslen(argv[1])); + if (nameerr) { + errors = sdscatprintf(errors, "%s:%d: invalid role name '%s': %s. ", + server.acl_filename, linenum, argv[1], nameerr); + sdsfreesplitres(argv, argc); + continue; + } + + user *r = ACLGetRoleByName(argv[1], sdslen(argv[1])); + if (r) { + errors = sdscatprintf(errors, "WARNING: Duplicate role '%s' found on line %d. ", argv[1], linenum); + sdsfreesplitres(argv, argc); + continue; + } + + sds error = ACLStringSetRole(r, argv[1], argv + 2, argc - 2); + if (error) { + errors = sdscatprintf(errors, "%s:%d: %s. ", server.acl_filename, linenum, error); + sdsfree(error); + } + + sdsfreesplitres(argv, argc); + } + + /* Second pass: load user definitions */ + for (int i = 0; i < totlines; i++) { + sds *argv; + int argc; + int linenum = i + 1; + + /* Re-trim is safe since lines were already trimmed */ + if (lines[i][0] == '\0') continue; + + /* Split into arguments */ + argv = sdssplitlen(lines[i], sdslen(lines[i]), " ", 1, &argc); + if (argv == NULL) continue; /* Error already reported in first pass */ + if (argc == 0) { + sdsfreesplitres(argv, argc); + continue; + } + + /* Only process user lines in second pass */ + if (strcmp(argv[0], "user") != 0) { + /* If it's not 'user' or 'role', report error */ + if (strcmp(argv[0], "role") != 0) { + errors = sdscatprintf(errors, + "%s:%d should start with user or role keyword. ", + server.acl_filename, linenum); + } + sdsfreesplitres(argv, argc); + continue; + } + + if (argc < 2) { errors = sdscatprintf(errors, "%s:%d should start with user keyword followed " "by the username. ", @@ -2639,7 +3242,7 @@ static sds ACLLoadFromFile(const char *filename) { for (int j = 0; j < merged_argc; j++) { acl_args[j] = sdstrim(acl_args[j], "\t\r\n"); if (ACLSetUser(u, acl_args[j], sdslen(acl_args[j])) != C_OK) { - const char *errmsg = ACLSetUserStringError(); + const char *errmsg = ACLSetStringError(); if (errno == ENOENT) { /* For missing commands, we print out more information since * it shouldn't contain any sensitive information. */ @@ -2722,11 +3325,15 @@ static sds ACLLoadFromFile(const char *filename) { if (user_channels) raxFreeWithCallback(user_channels, listReleaseVoid); raxFreeWithCallback(old_users, ACLFreeUserVoid); + ACLRemapSurvivingRoleMembers(old_roles); + raxFreeWithCallback(old_roles, ACLFreeUserVoid); sdsfree(errors); return NULL; } else { raxFreeWithCallback(Users, ACLFreeUserVoid); + raxFreeWithCallback(Roles, ACLFreeUserVoid); Users = old_users; + Roles = old_roles; errors = sdscat(errors, "WARNING: ACL errors detected, no change to the previously active ACL rules was performed"); return errors; @@ -2745,6 +3352,25 @@ static int ACLSaveToFile(const char *filename) { /* Let's generate an SDS string containing the new version of the * ACL file. */ raxIterator ri; + + /* Write roles first */ + raxStart(&ri, Roles); + raxSeek(&ri, "^", NULL, 0); + while (raxNext(&ri)) { + user *r = ri.data; + sds role = sdsnew("role "); + role = sdscatsds(role, r->name); + role = sdscatlen(role, " ", 1); + robj *descr = ACLDescribeUser(r); + role = sdscatsds(role, objectGetVal(descr)); + decrRefCount(descr); + acl = sdscatsds(acl, role); + acl = sdscatlen(acl, "\n", 1); + sdsfree(role); + } + raxStop(&ri); + + /* Write users */ raxStart(&ri, Users); raxSeek(&ri, "^", NULL, 0); while (raxNext(&ri)) { @@ -2826,6 +3452,23 @@ void ACLLoadUsersAtStartup(void) { exit(1); } + if (server.acl_filename[0] != '\0' && listLength(RolesToLoad) != 0) { + serverLog(LL_WARNING, + "Configuring %s with roles defined in valkey.conf and at " + "the same setting an ACL file path is invalid. This setup " + "is very likely to lead to configuration errors and security " + "holes, please define either an ACL file or declare roles " + "directly in your valkey.conf, but not both.", + SERVER_TITLE); + exit(1); + } + + /* Load roles before users so that users can reference them */ + if (ACLLoadConfiguredRoles() == C_ERR) { + serverLog(LL_WARNING, "Critical error while loading ACL roles. Exiting."); + exit(1); + } + if (ACLLoadConfiguredUsers() == C_ERR) { serverLog(LL_WARNING, "Critical error while loading ACLs. Exiting."); exit(1); @@ -2841,6 +3484,7 @@ void ACLLoadUsersAtStartup(void) { } } +/* Also provide a function to load roles at startup from config */ /* ============================================================================= * ACL log * ==========================================================================*/ @@ -3256,11 +3900,47 @@ void aclCommand(client *c) { int sfields = aclAddReplySelectorDescription(c, (aclSelector *)listNodeValue(ln)); setDeferredMapLen(c, slen, sfields); } + + /* Roles */ + addReplyBulkCString(c, "roles"); + addReplyArrayLen(c, u->roles ? listLength(u->roles) : 0); + fields++; + if (u->roles) { + listIter rli; + listNode *rln; + listRewind(u->roles, &rli); + while ((rln = listNext(&rli))) { + user *r = listNodeValue(rln); + addReplyBulkCBuffer(c, r->name, sdslen(r->name)); + } + } + setDeferredMapLen(c, ufields, fields); } else if ((!strcasecmp(sub, "list") || !strcasecmp(sub, "users")) && c->argc == 2) { int justnames = !strcasecmp(sub, "users"); - addReplyArrayLen(c, raxSize(Users)); + if (justnames) { + addReplyArrayLen(c, raxSize(Users)); + } else { + addReplyArrayLen(c, raxSize(Roles) + raxSize(Users)); + } raxIterator ri; + if (!justnames) { + /* List roles first in ACL LIST */ + raxStart(&ri, Roles); + raxSeek(&ri, "^", NULL, 0); + while (raxNext(&ri)) { + user *r = ri.data; + sds config = sdsnew("role "); + config = sdscatsds(config, r->name); + config = sdscatlen(config, " ", 1); + robj *descr = ACLDescribeUser(r); + config = sdscatsds(config, objectGetVal(descr)); + decrRefCount(descr); + addReplyBulkSds(c, config); + } + raxStop(&ri); + } + /* List users */ raxStart(&ri, Users); raxSeek(&ri, "^", NULL, 0); while (raxNext(&ri)) { @@ -3268,7 +3948,6 @@ void aclCommand(client *c) { if (justnames) { addReplyBulkCBuffer(c, u->name, sdslen(u->name)); } else { - /* Return information in the configuration file format. */ sds config = sdsnew("user "); config = sdscatsds(config, u->name); config = sdscatlen(config, " ", 1); @@ -3416,6 +4095,103 @@ void aclCommand(client *c) { addReplyBulkCString(c, "timestamp-last-updated"); addReplyLongLong(c, le->ctime); } + } else if (!strcasecmp(sub, "setrole") && c->argc >= 3) { + sds rolename = objectGetVal(c->argv[2]); + /* Check role name validity. */ + const char *nameerr = ACLRoleNameError(rolename, sdslen(rolename)); + if (nameerr) { + addReplyError(c, nameerr); + return; + } + + user *r = ACLGetRoleByName(rolename, sdslen(rolename)); + + sds *temp_argv = zmalloc(c->argc * sizeof(sds)); + for (int i = 3; i < c->argc; i++) temp_argv[i - 3] = objectGetVal(c->argv[i]); + + sds error = ACLStringSetRole(r, rolename, temp_argv, c->argc - 3); + zfree(temp_argv); + if (error == NULL) { + addReply(c, shared.ok); + } else { + addReplyErrorSdsSafe(c, error); + } + return; + } else if (!strcasecmp(sub, "delrole") && c->argc >= 3) { + for (int j = 2; j < c->argc; j++) { + sds rolename = objectGetVal(c->argv[j]); + user *r = ACLGetRoleByName(rolename, sdslen(rolename)); + if (r && dictSize(r->members) > 0) { + addReplyErrorFormat(c, "Role '%s' is assigned to one or more users. Remove it from them first.", + rolename); + return; + } + } + + int deleted = 0; + for (int j = 2; j < c->argc; j++) { + sds rolename = objectGetVal(c->argv[j]); + user *r; + if (raxRemove(Roles, (unsigned char *)rolename, sdslen(rolename), (void **)&r)) { + ACLFreeUser(r); + deleted++; + } + } + addReplyLongLong(c, deleted); + } else if (!strcasecmp(sub, "getrole") && c->argc == 3) { + sds rolename = objectGetVal(c->argv[2]); + user *r = ACLGetRoleByName(rolename, sdslen(rolename)); + if (!r) { + addReplyNull(c); + return; + } + + void *gfields = addReplyDeferredLen(c); + int fields = 0; + + /* Commands/keys/channels from root selector */ + aclSelector *root = listNodeValue(listFirst(r->selectors)); + fields += aclAddReplySelectorDescription(c, root); + + /* Additional selectors */ + addReplyBulkCString(c, "selectors"); + addReplyArrayLen(c, listLength(r->selectors) - 1); + fields++; + listIter li; + listNode *ln; + listRewind(r->selectors, &li); + listNext(&li); /* skip root */ + while ((ln = listNext(&li))) { + void *slen = addReplyDeferredLen(c); + int sfields = aclAddReplySelectorDescription(c, (aclSelector *)listNodeValue(ln)); + setDeferredMapLen(c, slen, sfields); + } + + /* Users holding this role */ + addReplyBulkCString(c, "users"); + addReplyArrayLen(c, dictSize(r->members)); + fields++; + { + dictIterator *di = dictGetIterator(r->members); + dictEntry *de; + while ((de = dictNext(di))) { + user *u = dictGetVal(de); + addReplyBulkCBuffer(c, u->name, sdslen(u->name)); + } + dictReleaseIterator(di); + } + + setDeferredMapLen(c, gfields, fields); + } else if (!strcasecmp(sub, "roles") && c->argc == 2) { + addReplyArrayLen(c, raxSize(Roles)); + raxIterator ri; + raxStart(&ri, Roles); + raxSeek(&ri, "^", NULL, 0); + while (raxNext(&ri)) { + user *r = ri.data; + addReplyBulkCBuffer(c, r->name, sdslen(r->name)); + } + raxStop(&ri); } else if (!strcasecmp(sub, "dryrun") && c->argc >= 4) { struct serverCommand *cmd; user *u = ACLGetUserByName(objectGetVal(c->argv[2]), sdslen(objectGetVal(c->argv[2]))); @@ -3460,6 +4236,14 @@ void aclCommand(client *c) { "GENPASS []", " Generate a secure 256-bit user password. The optional `bits` argument can", " be used to specify a different size.", + "SETROLE [ ...]", + " Create or modify a role with the specified rules.", + "DELROLE [ ...]", + " Delete one or more roles (each must not be assigned to any user).", + "GETROLE ", + " Get the role's details.", + "ROLES", + " List all the registered role names.", "LIST", " Show users details in config file format.", "LOAD", diff --git a/src/ae.c b/src/ae.c index 6d1fac8ca..edf6cad16 100644 --- a/src/ae.c +++ b/src/ae.c @@ -33,12 +33,14 @@ #include "ae.h" #include "anet.h" #include "serverassert.h" +#include "monotonic.h" #include #include #include #include #include +#include #include #include #include @@ -96,6 +98,12 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->aftersleep = NULL; eventLoop->custompoll = NULL; eventLoop->flags = 0; + eventLoop->priority_apidata = NULL; + eventLoop->priority_fd = -1; + eventLoop->priority_fired = NULL; + eventLoop->priority_events_last_poll = 0; + eventLoop->priority_events_preempt_check_interval_us = 0; + eventLoop->priority_events_stats_callback = NULL; /* Initialize the eventloop mutex with PTHREAD_MUTEX_ERRORCHECK type */ pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); @@ -150,6 +158,11 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { if (eventLoop->maxfd >= setsize) goto err; if (aeApiResize(eventLoop->apidata, setsize) == -1) goto err; + if (eventLoop->priority_apidata) { + if (aeApiResize(eventLoop->priority_apidata, setsize) == -1) goto err; + eventLoop->priority_fired = zrealloc(eventLoop->priority_fired, sizeof(aeFiredEvent) * setsize); + } + eventLoop->events = zrealloc(eventLoop->events, sizeof(aeFileEvent) * setsize); eventLoop->fired = zrealloc(eventLoop->fired, sizeof(aeFiredEvent) * setsize); eventLoop->setsize = setsize; @@ -167,6 +180,10 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { } void aeDeleteEventLoop(aeEventLoop *eventLoop) { + if (eventLoop->priority_apidata) { + aeApiFree(eventLoop->priority_apidata); + zfree(eventLoop->priority_fired); + } aeApiFree(eventLoop->apidata); zfree(eventLoop->events); zfree(eventLoop->fired); @@ -196,9 +213,32 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc } aeFileEvent *fe = &eventLoop->events[fd]; - int backend_add_mask = BACKEND_MASK(mask) & ~fe->mask; // just the meaningful additions - if (backend_add_mask) { - if (aeApiAddEvent(eventLoop->apidata, fd, BACKEND_MASK(fe->mask), backend_add_mask) == -1) goto done; + bool old_is_priority = (fe->mask & AE_HIGH_PRIORITY) && (eventLoop->priority_apidata != NULL); + bool new_is_priority = (mask & AE_HIGH_PRIORITY) && (eventLoop->priority_apidata != NULL); + + /* Handle dynamic cross-multiplexer migration if the priority of an active socket is changed */ + if (eventLoop->priority_apidata != NULL && old_is_priority != new_is_priority && fe->mask != AE_NONE) { + aeApiState *old_api = old_is_priority ? eventLoop->priority_apidata : eventLoop->apidata; + aeApiState *new_api = new_is_priority ? eventLoop->priority_apidata : eventLoop->apidata; + + /* 1. Remove active event mask from the old multiplexer */ + aeApiDelEvent(old_api, fd, BACKEND_MASK(fe->mask), BACKEND_MASK(fe->mask)); + + /* 2. Add combined event mask into the new multiplexer */ + int combined_backend_mask = BACKEND_MASK(fe->mask | mask); + if (combined_backend_mask) { + if (aeApiAddEvent(new_api, fd, 0, combined_backend_mask) == -1) goto done; + } + fe->mask = (fe->mask & ~AE_HIGH_PRIORITY) | (new_is_priority ? AE_HIGH_PRIORITY : 0); + } else { + bool is_priority = new_is_priority || old_is_priority; + if (!is_priority) mask &= ~AE_HIGH_PRIORITY; + aeApiState *target_api = is_priority ? eventLoop->priority_apidata : eventLoop->apidata; + + int backend_add_mask = BACKEND_MASK(mask) & ~fe->mask; // just the meaningful additions + if (backend_add_mask) { + if (aeApiAddEvent(target_api, fd, BACKEND_MASK(fe->mask), backend_add_mask) == -1) goto done; + } } fe->mask |= mask; if (mask & AE_READABLE) fe->rfileProc = proc; @@ -215,11 +255,15 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { AE_LOCK(eventLoop); + mask &= ~AE_HIGH_PRIORITY; // Priority flag is removed implicitly when read & write have been removed if (fd >= eventLoop->setsize) goto done; aeFileEvent *fe = &eventLoop->events[fd]; if (fe->mask == AE_NONE) goto done; + bool is_priority = (fe->mask & AE_HIGH_PRIORITY) && (eventLoop->priority_apidata != NULL); + aeApiState *target_api = is_priority ? eventLoop->priority_apidata : eventLoop->apidata; + /* We want to always remove AE_BARRIER if set when AE_WRITABLE * is removed. */ if (mask & AE_WRITABLE) mask |= AE_BARRIER; @@ -229,6 +273,9 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { int old_mask = fe->mask; fe->mask = fe->mask & ~mask; + if (!(fe->mask & (AE_READABLE | AE_WRITABLE))) { + fe->mask = AE_NONE; + } if (fd == eventLoop->maxfd && fe->mask == AE_NONE) { /* Update the max fd */ int j; @@ -243,7 +290,7 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { * touching the actual events. */ int backend_del_mask = BACKEND_MASK(mask); // just the meaningful deletions if (backend_del_mask) { - aeApiDelEvent(eventLoop->apidata, fd, BACKEND_MASK(old_mask), backend_del_mask); + aeApiDelEvent(target_api, fd, BACKEND_MASK(old_mask), backend_del_mask); } done: @@ -403,6 +450,98 @@ int aePoll(aeEventLoop *eventLoop, struct timeval *tvp) { return ret; } +/* Fire the readable and/or writable event handlers for a given file descriptor, + * respecting the AE_BARRIER flag and protecting against re-entrancy issues + * (e.g. event loop resize or event deletion within callbacks). */ +static inline void aeFireFileEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeFileEvent *fe = &eventLoop->events[fd]; + int fired = 0; /* Number of events fired for current fd. */ + + /* Normally we execute the readable event first, and the writable + * event later. This is useful as sometimes we may be able + * to serve the reply of a query immediately after processing the + * query. + * + * However if AE_BARRIER is set in the mask, our application is + * asking us to do the reverse: never fire the writable event + * after the readable. In such a case, we invert the calls. + * This is useful when, for instance, we want to do things + * in the beforeSleep() hook, like fsyncing a file to disk, + * before replying to a client. */ + int invert = fe->mask & AE_BARRIER; + + /* Note the "fe->mask & mask & ..." code: maybe an already + * processed event removed an element that fired and we still + * didn't processed, so we check if the event is still valid. + * + * Fire the readable event if the call sequence is not + * inverted. */ + if (!invert && (fe->mask & mask & AE_READABLE)) { + fe->rfileProc(eventLoop, fd, fe->clientData, mask); + fired++; + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + } + + /* Fire the writable event. */ + if (fe->mask & mask & AE_WRITABLE) { + if (!fired || fe->wfileProc != fe->rfileProc) { + fe->wfileProc(eventLoop, fd, fe->clientData, mask); + fired++; + } + } + + /* If we have to invert the call, fire the readable event now + * after the writable one. */ + if (invert) { + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + if ((fe->mask & mask & AE_READABLE) && (!fired || fe->wfileProc != fe->rfileProc)) { + fe->rfileProc(eventLoop, fd, fe->clientData, mask); + fired++; + } + } +} + +/* Process all high-priority events and invoke the stats callback + * with the elapsed time in microseconds if registered. */ +static int aeProcessQoSEvents(aeEventLoop *eventLoop) { + int processed = 0; + if (eventLoop->priority_apidata != NULL) { + monotime start = getMonotonicUs(); + struct timeval tv = {0, 0}; + int numevents = aeApiPoll(eventLoop->priority_apidata, eventLoop->priority_fired, eventLoop->events, + eventLoop->setsize, eventLoop->maxfd, &tv); + for (int j = 0; j < numevents; j++) { + int fd = eventLoop->priority_fired[j].fd; + int mask = eventLoop->priority_fired[j].mask; + aeFireFileEvent(eventLoop, fd, mask); + processed++; + } + eventLoop->priority_events_last_poll = getMonotonicUs(); + /* Update INFO stats if registered and high-priority events were processed */ + if (eventLoop->priority_events_stats_callback != NULL && processed > 0) { + eventLoop->priority_events_stats_callback(eventLoop, eventLoop->priority_events_last_poll - start); + } + } + return processed; +} + +/* Set the preemptive poll interval in microseconds for high-priority events. + * High-priority events are checked periodically during normal event processing + * when processing batches of normal events exceeds this interval. + * Setting this interval to 0 disables preemptive polling. */ +void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us) { + eventLoop->priority_events_preempt_check_interval_us = interval_us; +} + +/* Preemptively processes high-priority events if the elapsed time since the last poll + * exceeds the priority_events_preempt_check_interval_us threshold (0 disables preemption). */ +int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop) { + /* Skip if high-priority processing is disabled or preemptive polling is disabled */ + if (eventLoop->priority_apidata == NULL || eventLoop->priority_events_preempt_check_interval_us == 0) return 0; + if (elapsedUs(eventLoop->priority_events_last_poll) < eventLoop->priority_events_preempt_check_interval_us) return 0; + return aeProcessQoSEvents(eventLoop); +} + /* Process every pending file event, then every pending time event * (that may be registered by file event callbacks just processed). * Without special flags the function sleeps until some file event @@ -467,56 +606,27 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { /* After sleep callback. */ if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP) eventLoop->aftersleep(eventLoop, numevents); + /* Prioritize high-priority events: Always drain priority events + * immediately before processing normal events. */ + processed += aeProcessQoSEvents(eventLoop); + + /* Process normal file events */ for (j = 0; j < numevents; j++) { int fd = eventLoop->fired[j].fd; - aeFileEvent *fe = &eventLoop->events[fd]; + /* Skip processing high-priority FD events again as they were already processed above */ + if (fd == eventLoop->priority_fd) continue; int mask = eventLoop->fired[j].mask; - int fired = 0; /* Number of events fired for current fd. */ - - /* Normally we execute the readable event first, and the writable - * event later. This is useful as sometimes we may be able - * to serve the reply of a query immediately after processing the - * query. - * - * However if AE_BARRIER is set in the mask, our application is - * asking us to do the reverse: never fire the writable event - * after the readable. In such a case, we invert the calls. - * This is useful when, for instance, we want to do things - * in the beforeSleep() hook, like fsyncing a file to disk, - * before replying to a client. */ - int invert = fe->mask & AE_BARRIER; - - /* Note the "fe->mask & mask & ..." code: maybe an already - * processed event removed an element that fired and we still - * didn't processed, so we check if the event is still valid. - * - * Fire the readable event if the call sequence is not - * inverted. */ - if (!invert && fe->mask & mask & AE_READABLE) { - fe->rfileProc(eventLoop, fd, fe->clientData, mask); - fired++; - fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ - } - /* Fire the writable event. */ - if (fe->mask & mask & AE_WRITABLE) { - if (!fired || fe->wfileProc != fe->rfileProc) { - fe->wfileProc(eventLoop, fd, fe->clientData, mask); - fired++; - } - } + aeFireFileEvent(eventLoop, fd, mask); + processed++; - /* If we have to invert the call, fire the readable event now - * after the writable one. */ - if (invert) { - fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ - if ((fe->mask & mask & AE_READABLE) && (!fired || fe->wfileProc != fe->rfileProc)) { - fe->rfileProc(eventLoop, fd, fe->clientData, mask); - fired++; - } + /* Periodic Preemptive Polling of high priority events: This is to + * ensure that high priority events are processed in a timely manner + * even when there are long running normal events. */ + /* Sample monotonic clock once every (AE_QOS_PREEMPT_CHECK_MASK) to amortize vDSO overhead. */ + if ((j & AE_QOS_PREEMPT_CHECK_MASK) == 0) { + processed += aeProcessQoSEventsPreemptively(eventLoop); } - - processed++; } } /* Check time events */ @@ -579,3 +689,51 @@ void aeSetPollProtect(aeEventLoop *eventLoop, int protect) { eventLoop->flags &= ~AE_PROTECT_POLL; } } + +/* Actuate QoS event loop if supported: creates a secondary multiplexer state for internal + * connections (cluster bus, replication, and slot migration jobs). It installs + * a preemptive polling mechanism that wakes the main event loop and processes + * QoS events at regular interval. If qosPreemptPollIntervalUs is 0, standard + * event loop behavior is preserved. The provided stats callback is invoked with the + * elapsed duration of each QoS event processing cycle, enabling monitoring and + * statistics for QoS processing. + * Returns AE_OK on success, or AE_ERR if QoS eventloop actuation fails + * (e.g., unsupported platform, memory allocation failure, or I/O error). + */ +int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback) { + assert(eventLoop != NULL); + + /* Create high-priority polling state for internal connections (cluster bus, replication, and slot migration jobs) */ + aeApiState *priority_apidata = aeApiCreate(eventLoop->setsize); + if (priority_apidata == NULL) return AE_ERR; + + int priority_fd = aeApiGetPollFd(priority_apidata); + if (priority_fd == -1) { + aeApiFree(priority_apidata); + return AE_ERR; + } + + aeFiredEvent *priority_fired = zmalloc(sizeof(aeFiredEvent) * eventLoop->setsize); + if (priority_fired == NULL) { + aeApiFree(priority_apidata); + return AE_ERR; + } + + /* Register priority_fd with NULL callback: priority_fd is only used to wake up the main loop's + * multiplexer when high-priority traffic arrives. aeProcessEvents() checks for priority_fd and drains + * priority_apidata directly before dispatching normal events, and explicitly skips callback + * dispatch for priority_fd. */ + if (aeCreateFileEvent(eventLoop, priority_fd, AE_READABLE, NULL, NULL) == AE_ERR) { + zfree(priority_fired); + aeApiFree(priority_apidata); + return AE_ERR; + } + + eventLoop->priority_apidata = priority_apidata; + eventLoop->priority_fd = priority_fd; + eventLoop->priority_fired = priority_fired; + eventLoop->priority_events_stats_callback = qosStatsCallback; + eventLoop->priority_events_preempt_check_interval_us = qosPreemptPollIntervalUs; + + return AE_OK; +} diff --git a/src/ae.h b/src/ae.h index 86916bddc..bba180fd5 100644 --- a/src/ae.h +++ b/src/ae.h @@ -39,14 +39,18 @@ #define AE_OK 0 #define AE_ERR -1 -#define AE_NONE 0 /* No events registered. */ -#define AE_READABLE 1 /* Fire when descriptor is readable. */ -#define AE_WRITABLE 2 /* Fire when descriptor is writable. */ -#define AE_BARRIER 4 /* With WRITABLE, never fire the event if the \ - READABLE event already fired in the same event \ - loop iteration. Useful when you want to persist \ - things to disk before sending replies, and want \ - to do that in a group fashion. */ +#define AE_NONE 0 /* No events registered. */ +#define AE_READABLE 1 /* Fire when descriptor is readable. */ +#define AE_WRITABLE 2 /* Fire when descriptor is writable. */ +#define AE_BARRIER 4 /* With WRITABLE, never fire the event if the \ + READABLE event already fired in the same event \ + loop iteration. Useful when you want to persist \ + things to disk before sending replies, and want \ + to do that in a group fashion. */ +#define AE_HIGH_PRIORITY 8 /* Virtual routing mask flag: when set in aeCreateFileEvent(), \ + * the event is registered on priority_apidata if available. \ + * Stripped before passing to the underlying OS multiplexer. */ +#define AE_QOS_PREEMPT_CHECK_MASK 0x03 /* Mask to check high-priority preemption once every 4 iterations */ #define AE_FILE_EVENTS (1 << 0) #define AE_TIME_EVENTS (1 << 1) @@ -80,6 +84,8 @@ typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientDat typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop); typedef void aeAfterSleepProc(struct aeEventLoop *eventLoop, int numevents); typedef int aeCustomPollProc(struct aeEventLoop *eventLoop); +/* Callback invoked with elapsed microseconds after high-priority events are processed. */ +typedef void aeQoSStatsProc(struct aeEventLoop *eventLoop, uint64_t duration_us); /* File event structure */ typedef struct aeFileEvent { @@ -123,6 +129,17 @@ typedef struct aeEventLoop { aeCustomPollProc *custompoll; pthread_mutex_t poll_mutex; int flags; + + /* High-priority event processing: + * Sockets registered with AE_HIGH_PRIORITY are tracked in priority_apidata. + * priority_fd is registered into apidata to wake the main loop when high-priority traffic arrives. + * priority_fired holds fired events when draining high-priority channels. */ + aeApiState *priority_apidata; /* Dedicated high-priority polling state */ + int priority_fd; /* File descriptor of high-priority polling backend (-1 if disabled) */ + aeFiredEvent *priority_fired; /* Fired events buffer for high-priority polling */ + monotime priority_events_last_poll; /* Timestamp when high-priority events were last drained */ + uint64_t priority_events_preempt_check_interval_us; /* Preemptive check interval in microseconds (0 = disabled) */ + aeQoSStatsProc *priority_events_stats_callback; /* Callback invoked with elapsed microseconds after draining high-priority events */ } aeEventLoop; /* Prototypes */ @@ -152,4 +169,9 @@ int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); void aeSetDontWait(aeEventLoop *eventLoop, int noWait); +/* High-priority event loop prototypes */ +int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback); +int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop); +void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us); + #endif diff --git a/src/ae_epoll.c b/src/ae_epoll.c index 5ceda92b7..b0dc8a7d6 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -134,3 +134,7 @@ static int aeApiPoll(aeApiState *state, aeFiredEvent *fired, aeFileEvent *events static char *aeApiName(void) { return "epoll"; } + +static int aeApiGetPollFd(aeApiState *state) { + return state ? state->epfd : -1; +} diff --git a/src/ae_evport.c b/src/ae_evport.c index d730abb09..e6ab055a9 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -289,3 +289,8 @@ static int aeApiPoll(aeApiState *state, aeFiredEvent *fired, aeFileEvent *events static char *aeApiName(void) { return "evport"; } + +static int aeApiGetPollFd(aeApiState *state) { + UNUSED(state); + return -1; +} diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index 33615c2da..d518b88c4 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -180,3 +180,7 @@ static int aeApiPoll(aeApiState *state, aeFiredEvent *fired, aeFileEvent *events static char *aeApiName(void) { return "kqueue"; } + +static int aeApiGetPollFd(aeApiState *state) { + return state ? state->kqfd : -1; +} diff --git a/src/ae_select.c b/src/ae_select.c index df19617da..61c665b6c 100644 --- a/src/ae_select.c +++ b/src/ae_select.c @@ -107,3 +107,8 @@ static int aeApiPoll(aeApiState *state, aeFiredEvent *fired, aeFileEvent *events static char *aeApiName(void) { return "select"; } + +static int aeApiGetPollFd(aeApiState *state) { + UNUSED(state); + return -1; +} diff --git a/src/anet.c b/src/anet.c index 3d49b9266..b45ee7216 100644 --- a/src/anet.c +++ b/src/anet.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include "anet.h" @@ -869,3 +870,114 @@ int anetIsFifo(char *filepath) { if (stat(filepath, &sb) == -1) return 0; return S_ISFIFO(sb.st_mode); } + +/* anetParseSubnet parses a subnet token in CIDR notation (e.g. "192.168.1.0/24") + * or raw IP and populates the anetSubnet structure. + * Returns ANET_OK on success, ANET_ERR on parsing/validation error. */ +int anetParseSubnet(char *err, const char *token, anetSubnet *subnet) { + if (!token || !subnet) { + anetSetError(err, "Invalid token or subnet pointer"); + return ANET_ERR; + } + + const char *slash = strchr(token, '/'); + size_t ip_len = slash ? (size_t)(slash - token) : strlen(token); + if (ip_len == 0 || ip_len >= INET6_ADDRSTRLEN) { + anetSetError(err, "Invalid IP address length in subnet token: %s", token); + return ANET_ERR; + } + + char ip_part[INET6_ADDRSTRLEN]; + memcpy(ip_part, token, ip_len); + ip_part[ip_len] = '\0'; + + int family = strchr(ip_part, ':') ? AF_INET6 : AF_INET; + long max_prefix = (family == AF_INET) ? 32 : 128; + long prefix = max_prefix; + + if (slash) { + char *endptr; + prefix = strtol(slash + 1, &endptr, 10); + if (endptr == slash + 1 || *endptr != '\0' || prefix < 0 || prefix > max_prefix) { + anetSetError(err, "Invalid prefix length in subnet token: %s", token); + return ANET_ERR; + } + } + + if (family == AF_INET) { + if (inet_pton(AF_INET, ip_part, &subnet->addr.ipv4) != 1) { + anetSetError(err, "Invalid IPv4 address: %s", ip_part); + return ANET_ERR; + } + } else { + if (inet_pton(AF_INET6, ip_part, &subnet->addr.ipv6) != 1) { + anetSetError(err, "Invalid IPv6 address: %s", ip_part); + return ANET_ERR; + } + /* Normalize IPv4-mapped IPv6 subnet */ + if (IN6_IS_ADDR_V4MAPPED(&subnet->addr.ipv6)) { + family = AF_INET; + subnet->addr.ipv4.s_addr = *(uint32_t *)(&subnet->addr.ipv6.s6_addr[12]); + prefix = (prefix > 96) ? (prefix - 96) : 0; + } + } + + subnet->family = family; + subnet->prefix_len = (int)prefix; + return ANET_OK; +} + +/* anetMatchIpSubnet checks if the given IP matches any subnet in subnets[]. + * Returns 1 if matching, 0 otherwise. + * Note: ip can be NULL for non-IP transports (e.g. UNIX sockets), returning 0. */ +int anetMatchIpSubnet(const char *ip, const anetSubnet *subnets, int count) { + if (!ip || !subnets || count <= 0) return 0; + + int family; + union { + struct in_addr ipv4; + struct in6_addr ipv6; + } ip_addr; + + if (strchr(ip, ':')) { + family = AF_INET6; + if (inet_pton(AF_INET6, ip, &ip_addr.ipv6) != 1) return 0; + /* Normalize IPv4-mapped IPv6 address */ + if (IN6_IS_ADDR_V4MAPPED(&ip_addr.ipv6)) { + family = AF_INET; + ip_addr.ipv4.s_addr = *(uint32_t *)(&ip_addr.ipv6.s6_addr[12]); + } + } else { + family = AF_INET; + if (inet_pton(AF_INET, ip, &ip_addr.ipv4) != 1) return 0; + } + + for (int i = 0; i < count; i++) { + const anetSubnet *subnet = &subnets[i]; + if (subnet->family != family) continue; + + if (family == AF_INET) { + uint32_t subnet_val = ntohl(subnet->addr.ipv4.s_addr); + uint32_t ip_val = ntohl(ip_addr.ipv4.s_addr); + uint32_t mask = (subnet->prefix_len == 0) ? 0 : (0xFFFFFFFFU << (32 - subnet->prefix_len)); + if ((ip_val & mask) == (subnet_val & mask)) return 1; + } else { + int bytes = subnet->prefix_len / 8; + int bits = subnet->prefix_len % 8; + int match = 1; + for (int j = 0; j < bytes; j++) { + if (ip_addr.ipv6.s6_addr[j] != subnet->addr.ipv6.s6_addr[j]) { + match = 0; + break; + } + } + if (match && bits > 0) { + uint8_t mask = (uint8_t)(0xFF << (8 - bits)); + if ((ip_addr.ipv6.s6_addr[bytes] & mask) != (subnet->addr.ipv6.s6_addr[bytes] & mask)) + match = 0; + } + if (match) return 1; + } + } + return 0; +} diff --git a/src/anet.h b/src/anet.h index 79a4ecebf..7216cb8e2 100644 --- a/src/anet.h +++ b/src/anet.h @@ -32,6 +32,8 @@ #define ANET_H #include +#include +#include #define ANET_OK 0 #define ANET_ERR -1 @@ -51,6 +53,18 @@ #undef ip_len #endif +/* Represents an IP subnet (IPv4 or IPv6) and its prefix length. */ +typedef struct anetSubnet { + int family; /* AF_INET or AF_INET6 */ + union { + struct in_addr ipv4; + struct in6_addr ipv6; + } addr; + int prefix_len; +} anetSubnet; + +int anetParseSubnet(char *err, const char *token, anetSubnet *subnet); +int anetMatchIpSubnet(const char *ip, const anetSubnet *subnets, int count); int anetTcpNonBlockConnect(char *err, const char *addr, int port); int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr, int mptcp); int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags); diff --git a/src/aof.c b/src/aof.c index 2d18aae82..24753b852 100644 --- a/src/aof.c +++ b/src/aof.c @@ -27,12 +27,17 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include "entry.h" +#include "expire.h" +#include "listpack.h" +#include "sds.h" #include "server.h" #include "ordered_index.h" #include "bio.h" #include "rio.h" #include "functions.h" #include "module.h" +#include "util.h" #include #include @@ -965,7 +970,7 @@ int startAppendOnly(void) { serverAssert(server.aof_state == AOF_OFF); server.aof_state = AOF_WAIT_REWRITE; - if (hasActiveChildProcess() && server.child_type != CHILD_TYPE_AOF) { + if (hasActiveSaveOrChild() && server.child_type != CHILD_TYPE_AOF) { server.aof_rewrite_scheduled = 1; serverLog(LL_NOTICE, "AOF was enabled but there is already another background operation. An AOF background was " "scheduled to start when possible."); @@ -2090,37 +2095,44 @@ static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) { * The function returns 0 on error, 1 on success. */ int rewriteHashObject(rio *r, robj *key, robj *o) { hashTypeIterator hi; - long long count = 0, volatile_items = 0, non_volatile_items; + long long count = 0, non_volatile_items; + sds field, value; + /* First serialize volatile items if exist */ if (hashTypeHasVolatileFields(o)) { hashTypeInitVolatileIterator(o, &hi); while (hashTypeNext(&hi) != C_ERR) { - long long expiry = entryGetExpiry(hi.next); - sds field = entryGetField(hi.next); - size_t value_len; - char *value = entryGetValue(hi.next, &value_len); - if (rioWriteBulkCount(r, '*', 8) == 0) return 0; - if (rioWriteBulkString(r, "HSETEX", 6) == 0) return 0; - if (rioWriteBulkObject(r, key) == 0) return 0; - if (rioWriteBulkString(r, "PXAT", 4) == 0) return 0; - if (rioWriteBulkLongLong(r, expiry) == 0) return 0; - if (rioWriteBulkString(r, "FIELDS", 6) == 0) return 0; - if (rioWriteBulkLongLong(r, 1) == 0) return 0; - if (rioWriteBulkString(r, field, sdslen(field)) == 0) return 0; - if (rioWriteBulkString(r, value, value_len) == 0) return 0; - volatile_items++; + long long expiry = hashTypeCurrentExpiry(o, &hi); + field = hashTypeCurrentObjectNewSds(&hi, OBJ_HASH_FIELD); + value = hashTypeCurrentObjectNewSds(&hi, OBJ_HASH_VALUE); + if (expiry > commandTimeSnapshot()) { + if (rioWriteBulkCount(r, '*', 8) == 0) goto werr; + if (rioWriteBulkString(r, "HSETEX", 6) == 0) goto werr; + if (rioWriteBulkObject(r, key) == 0) goto werr; + if (rioWriteBulkString(r, "PXAT", 4) == 0) goto werr; + if (rioWriteBulkLongLong(r, expiry) == 0) goto werr; + if (rioWriteBulkString(r, "FIELDS", 6) == 0) goto werr; + if (rioWriteBulkLongLong(r, 1) == 0) goto werr; + if (rioWriteBulkString(r, field, sdslen(field)) == 0) goto werr; + if (rioWriteBulkString(r, value, sdslen(value)) == 0) goto werr; + } + sdsfree(field); + sdsfree(value); } hashTypeResetIterator(&hi); } - non_volatile_items = hashTypeLength(o) - volatile_items; - hashTypeInitIterator(o, &hi); - while (hashTypeNext(&hi) != C_ERR) { - if (volatile_items > 0 && entryHasExpiry(hi.next)) - continue; + /* Write the persistent (no-TTL) fields as HMSET batches. The batch + * header needs the count of fields the persistent iterator will emit: + * total fields minus ALL volatile ones (expired-unreaped included, + * since the iterator skips those too). */ + non_volatile_items = hashTypeLength(o) - hashTypeVolatileCount(o); + + hashTypeInitPersistentIterator(o, &hi); + while (hashTypeNext(&hi) != C_ERR) { + /* If new vector write the HMSET command first */ if (count == 0) { int cmd_items = (non_volatile_items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : non_volatile_items; - if (!rioWriteBulkCount(r, '*', 2 + cmd_items * 2) || !rioWriteBulkString(r, "HMSET", 5) || !rioWriteBulkObject(r, key)) { hashTypeResetIterator(&hi); @@ -2128,6 +2140,7 @@ int rewriteHashObject(rio *r, robj *key, robj *o) { } } + /* Iterate till we reach the batch size */ if (!rioWriteHashIteratorCursor(r, &hi, OBJ_HASH_FIELD) || !rioWriteHashIteratorCursor(r, &hi, OBJ_HASH_VALUE)) { hashTypeResetIterator(&hi); return 0; @@ -2135,9 +2148,14 @@ int rewriteHashObject(rio *r, robj *key, robj *o) { if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; non_volatile_items--; } - hashTypeResetIterator(&hi); return 1; + +werr: + sdsfree(field); + sdsfree(value); + hashTypeResetIterator(&hi); + return 0; } /* Helper for rewriteStreamObject() that generates a bulk string into the @@ -2598,7 +2616,7 @@ int rewriteAppendOnlyFile(char *filename) { int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - if (hasActiveChildProcess()) return C_ERR; + if (hasActiveSaveOrChild()) return C_ERR; if (dirCreateIfMissing(server.aof_dirname) == -1) { serverLog(LL_WARNING, "Can't open or create append-only dir %s: %s", server.aof_dirname, strerror(errno)); @@ -2668,7 +2686,7 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(client *c) { if (server.child_type == CHILD_TYPE_AOF) { addReplyError(c, "Background append only file rewriting already in progress"); - } else if (hasActiveChildProcess() || server.in_exec) { + } else if (hasActiveSaveOrChild() || server.in_exec) { server.aof_rewrite_scheduled = 1; /* When manually triggering AOFRW we reset the count * so that it can be executed immediately. */ diff --git a/src/bgiteration.c b/src/bgiteration.c new file mode 100644 index 000000000..c350bb2c6 --- /dev/null +++ b/src/bgiteration.c @@ -0,0 +1,2699 @@ +/* + * Copyright Valkey Contributors. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "fmacros.h" +#include "bgiteration.h" +#include "dict.h" +#include "fifo.h" +#include "kvstore.h" +#include "monotonic.h" +#include "mutexqueue.h" +#include "server.h" + + +static bool receiveItemsBackFromOneIterator(bgIterator *it); + + +// Returns true if the cmd is a script command that may replicate. +static bool isScriptCallWriteCmd(struct serverCommand *cmd) { + return ((cmd->proc == fcallCommand) || (cmd->proc == evalCommand) || (cmd->proc == evalShaCommand)); +} + +/* The PFCOUNT command (which does NOT have the CMD_WRITE flag) modifies the underlying string and + * is replicated as a write. So it needs to be detected and handled specially. */ +static bool isWriteCmd(struct serverCommand *cmd) { + return ((cmd->flags & CMD_WRITE) || (cmd->proc == pfcountCommand) || (cmd->proc == execCommand) || (isScriptCallWriteCmd(cmd))); +} + +// Returns true if the command is a deletion based command (DEL or UNLINK) +static bool isDeleteCmd(struct serverCommand *cmd) { + return ((cmd->proc == delCommand) || (cmd->proc == unlinkCommand)); +} + +/* This utility utilizes the main thread and background threads for processing. The API is split, + * with some of the functions intended for the main thread and others intended for the background + * clients. This sanity check ensures that we maintain thread safety, calling the API as intended. */ +static bool hasMainThreadExclusivity(void) { + /* Modules interact with the main thread using a mutex. If a module owns the mutex, consider + * that equivalent to being on the main thread. */ + bool mightBeInModule = (atomic_load_explicit(&server.module_gil_acquired, memory_order_relaxed) == 0); + return onServerMainThread() || mightBeInModule; +} + + +/* Parse a parameters robj, extracting a valid DBID. + * Returns FALSE if DBID isn't valid. */ +static bool getDbIdFromRobj(robj *obj, int *db_id) { + long long value; + if (getLongLongFromObject(obj, &value) != C_OK) return false; + if ((value < 0) || (value >= server.dbnum)) return false; + *db_id = (int)value; + return true; +} + +/* Parse the parameters of the COPY command, extracting the target DBID. + * Returns FALSE if the command would not run. */ +static bool getTargetDbIdForCopyCommand(int argc, robj **argv, int selected_dbid, int *target_dbid) { + const int COPY_COMMAND_OPTIONAL_ARG_START_INDEX = 3; + + *target_dbid = selected_dbid; + + for (int i = COPY_COMMAND_OPTIONAL_ARG_START_INDEX; i < argc; i++) { + if (!strcasecmp((char *)objectGetVal(argv[i]), "replace")) { + continue; + } else if (!strcasecmp((char *)objectGetVal(argv[i]), "db") && (i + 1 < argc)) { + /* Note the parsing here needs to perfectly match what we have in copyCommand. The + * following command is considered OK so we can't return here, but must continue to + * parse till the last db which is the one that's effectively used. + * COPY key1 key2 db 1 db 2 db 3 (This will use db 3) */ + if (!getDbIdFromRobj(argv[i + 1], target_dbid)) { + return false; // parse failure + } + i++; // Consume additional argument + } else { + return false; // parse failure + } + } + return true; +} + +/* Get parameters for the SWAPDB command. + * The optional permission_client allows for checking of a client's permission for swapdb. + * Returns true if command would be executed. */ +static bool getParamsForSwapdb(int argc, robj **argv, client *permission_client, int *id1_p, int *id2_p) { + static struct serverCommand *swapdb_cmd = NULL; + + // We don't need to check permissions in the replication phase + if (permission_client != NULL) { + if (swapdb_cmd == NULL) { + swapdb_cmd = lookupCommandByCString("swapdb"); + serverAssert(swapdb_cmd != NULL); + } + + int idxptr; + if (ACLCheckAllUserCommandPerm(permission_client->user, swapdb_cmd, argv, argc, + permission_client->db->id, &idxptr) != ACL_OK) return false; + } + + long long dbid1, dbid2; + if (argc != 3) return false; + if (server.cluster_enabled) return false; + if (getLongLongFromObject(argv[1], &dbid1) != C_OK) return false; + if (getLongLongFromObject(argv[2], &dbid2) != C_OK) return false; + if (dbid1 < 0 || dbid1 >= server.dbnum) return false; + if (dbid2 < 0 || dbid2 >= server.dbnum) return false; + if (dbid1 == dbid2) return false; // Valid, but doesn't do anything + + *id1_p = (int)dbid1; + *id2_p = (int)dbid2; + return true; +} + +/* Get parameters for the SELECT command. + * The optional permission_client allows for checking of a client's permission for select. + * Returns true if command would be executed. */ +static bool getParamsForSelect(int argc, robj **argv, client *permission_client, int *dbid_p) { + static struct serverCommand *select_cmd = NULL; + + // We don't need to check permissions in the replication phase + if (permission_client != NULL) { + if (select_cmd == NULL) { + select_cmd = lookupCommandByCString("select"); + serverAssert(select_cmd != NULL); + } + + int idxptr; + if (ACLCheckAllUserCommandPerm(permission_client->user, select_cmd, argv, argc, + permission_client->db->id, &idxptr) != ACL_OK) return false; + } + + long long dbid; + if (argc != 2) return false; + if (getLongLongFromObject(argv[1], &dbid) != C_OK) return false; + if (dbid < 0 || dbid >= server.dbnum) return false; + + *dbid_p = (int)dbid; + return true; +} + +static void pauseRehashForKvsHashtable(kvstore *kvs, int didx) { + hashtable *ht = kvstoreGetHashtable(kvs, didx); + if (ht != NULL) hashtablePauseRehashing(ht); +} + +static void resumeRehashForKvsHashtable(kvstore *kvs, int didx) { + hashtable *ht = kvstoreGetHashtable(kvs, didx); + if (ht != NULL) hashtableResumeRehashing(ht); +} + + +/* DictType for SDS->ptr. The SDS is referenced, no destructor. */ +static dictType sdsrefToPtrDictType = { + .entryGetKey = dictEntryGetKey, + .hashFunction = dictSdsHash, + .keyCompare = dictSdsKeyCompare, + .entryDestructor = zfree}; + + +/* Wrap decrRefCount() so that it can be used as a callback requiring void. */ +static void decrRefCountVoid(void *o) { + decrRefCount(o); +} + + +/* Concatenate argc/argv into a command string for debugging. */ +static sds createSdsFromClientArgv(int argc, robj **argv) { + sds cmd = sdsempty(); + for (int i = 0; i < argc; i++) { + robj *arg = getDecodedObject(argv[i]); // some objects are int encoded + cmd = sdscatprintf(cmd, "'%s' ", (char *)objectGetVal(arg)); + decrRefCount(arg); + } + return cmd; +} + + +// ########################################################################### + + +/* bgIteration internal (compile time) configuration values */ +enum { + BGITER_EARLY_ITERATE_DICT_INITIAL_SIZE = 16384, // Prevent initial rehashing + BGITER_MAX_CLONE_ITEM_BYTES = 512, // Max size item to clone + BGITER_MAX_CLONE_POOL_BYTES = (1 * 1024 * 1024), // Total limit for all cloned items + BGITER_QUEUE_INCREASE_INCR = 100, // Step size when increasing queue target + BGITER_QUEUE_MAX_LENGTH = 10000, // Max length for the dynamic queue + BGITER_CYCLE_DELAY_MS = 2, // Delay between calls on bgIteration timer + BGITER_CYCLE_BUDGET_MS = 1, // Normal time limit for timer processing + BGITER_CYCLE_BUDGET_MAX_MS = 10 // Maximum time limit when starvation seen +}; + +// dbEntry metadata +typedef struct { + uint32_t iterator_epoch; // iterator epoch of last modification +} bgIterationEntryMetadata; +static_assert(sizeof(bgIterationEntryMetadata) == BGITERATION_ENTRY_METADATA_SIZE, ""); + + +// These can be tweaked by unit tests +static int bgiter_max_clone_item_bytes = BGITER_MAX_CLONE_ITEM_BYTES; +static int bgiter_max_clone_pool_bytes = BGITER_MAX_CLONE_POOL_BYTES; + +void bgIteration_unitTestDisableCloning(void) { + bgiter_max_clone_item_bytes = 0; + bgiter_max_clone_pool_bytes = 0; +} +void bgIteration_unitTestEnableCloning(int item_bytes, int pool_bytes) { + bgiter_max_clone_item_bytes = item_bytes; + bgiter_max_clone_pool_bytes = pool_bytes; +} + +typedef enum { + BGITERATION_TYPE_NONE, + BGITERATION_TYPE_FULLSCAN, + BGITERATION_TYPE_CLUSTERSLOT +} bgIterationType; + + +/* Flag indicates that a consistent iteration is required. This is used to create a point-in-time + * iteration. The iteration client will see all keys AS THEY EXISTED at the time when the iterator + * was created. + * Note: The DBID provided with the DICTENTRY events is the original DBID (at the time of iteration + * start). SWAPDB events are NOT provided during a consistent iteration. */ +#define BGITERATOR_FLAG_CONSISTENT (1 << 0) + +/* Flag indicating that the replication stream for keys which have already been processed should be + * forwarded to the iteration client. Used for non-consistent iteration to track changes + * to keys already processed. By tracking changes, this allows an non-consistent iteration client + * to achieve a consistent view at the END of the iteration. + * NOTE: Replication events will be provided ordered and synchronized with any SWAPDB events. */ +#define BGITERATOR_FLAG_REPLICATION (1 << 1) + + +/* Extensions to bgIteratorItemType. These enumerations are used internally, and are not part of + * the published interface. These allow for extensibility in the internal information-passing + * between the Valkey main thread and the iteration client thread. */ +typedef enum { + /* Indicates that the iteration client has completed use of the bgIterator and that the + * bgIterator should be cleaned up and freed by the Valkey main thread. */ + BGITERATOR_ITEMEXT_ITER_CLOSED = 10 +} bgIteratorItemTypeExtended; + +// Static bgIterator items for items which carry no data +static const bgIteratorItem STATIC_ITEM_TERMINATED = {.type = (bgIteratorItemType)BGITERATOR_ITEM_TERMINATED}; +static const bgIteratorItem STATIC_ITEM_ITER_CLOSED = {.type = (bgIteratorItemType)BGITERATOR_ITEMEXT_ITER_CLOSED}; + + +/* A dictionary with a pointer (itself) as a key (the address pointed to is NOT referenced). + * Nothing is duplicated, this is a very fast dictionary, but potentially unsafe if the original + * items are deleted or moved. + * WARNING: This needs to maintain safety with things that may move the object. + * + In db.c, if the object is reallocated, bgIteration_updateDbEntryPtr() is called. + * + In defrag.c, we don't defrag if there are multiple references (and we incr the refcount). */ + +// Thomas Wang's 64-bit mix +static uint64_t pointerHash(const void *key) { + uint64_t h = (uint64_t)(uintptr_t)key; + h = (~h) + (h << 21); // h = (h << 21) - h - 1; + h = h ^ (h >> 24); + h = (h + (h << 3)) + (h << 8); // h * 265 + h = h ^ (h >> 14); + h = (h + (h << 2)) + (h << 4); // h * 21 + h = h ^ (h >> 28); + h = h + (h << 31); + return h; +} + +static int pointerCompare(const void *key1, const void *key2) { + return key1 == key2; +} + +// This dict grows and shrinks constantly during the iteration. Avoid constant rehashing. +static int onlyAllowExpansion(size_t moreMem, double usedRatio) { + UNUSED(moreMem); + return (usedRatio > 0.5); // Return true only if expanding +} + +static dictType dictEntryPtrDictType = { + .entryGetKey = dictEntryGetKey, + .hashFunction = pointerHash, + .keyCompare = pointerCompare, + .resizeAllowed = onlyAllowExpansion, + .entryDestructor = zfree}; + +static hashtableType dbEntryPtrHashtableType = { + .hashFunction = pointerHash, + .keyCompare = pointerCompare, + .resizeAllowed = onlyAllowExpansion}; + + +// A free list for bgIteratorItem's - avoids churning zmalloc calls +typedef struct itemListNode { + struct itemListNode *next; +} itemListNode; + +static const int FREE_ITEM_MAX = 500; +static itemListNode *freeItemStackHead = NULL; +static int freeItemStackCount = 0; + +static void itemFreeList_returnItemBackToFreeList(bgIteratorItem *item) { + itemListNode *freedNode = (itemListNode *)item; + if (freeItemStackCount < FREE_ITEM_MAX) { + freedNode->next = freeItemStackHead; + freeItemStackHead = freedNode; + freeItemStackCount++; + } else { + zfree(freedNode); + } +} + +// Pop a free node from the free list or allocate if none free +static bgIteratorItem *itemFreeList_getElementOrAllocate(void) { + bgIteratorItem *item; + if (freeItemStackHead) { + item = (bgIteratorItem *)freeItemStackHead; + freeItemStackHead = freeItemStackHead->next; + freeItemStackCount--; + if (freeItemStackHead) valkey_prefetch(freeItemStackHead); + } else { + serverAssert(freeItemStackCount == 0); + // Create new listNode and item + item = zmalloc(sizeof(bgIteratorItem)); + } + return item; +} + +static void itemFreeList_release(void) { + while (freeItemStackHead) { + itemListNode *node = freeItemStackHead; + freeItemStackHead = node->next; + freeItemStackCount--; + zfree(node); + } + serverAssert(freeItemStackCount == 0); +} + + +/* A TEMPORARY set of robj's (of type sds). This is only for temporary sets as the robj's are not + * ref-counted at insertion/deletion. */ +static hashtableType tempKeysetHashtableType = { + .hashFunction = dictObjHash, + .keyCompare = dictObjKeyCompare}; + + +typedef struct genericIterator genericIterator; +typedef void (*iteratorReleaseFunc)(genericIterator *genIt); +typedef fifo *(*iteratorGetEntriesFunc)(genericIterator *genIt, int *orig_dbid, int *cur_dbid); +typedef void (*iteratorSwapDbFunc)(genericIterator *genIt, int db1, int db2); +typedef void (*iteratorFlushDbFunc)(genericIterator *genIt, int cur_dbid); +typedef bool (*iteratorHasPassedItemFunc)(genericIterator *genIt, const_sds key, int cur_dbid); +typedef int (*iteratorOriginalDbFunc)(genericIterator *genIt, int cur_dbid); +typedef bool (*iteratorIsKeyInScopeFunc)(genericIterator *genIt, const_sds key); + +// Function pointers supporting polymorphic iterator implementation +struct genericIterator { + iteratorReleaseFunc release; + iteratorGetEntriesFunc getEntries; + iteratorSwapDbFunc swapDb; + iteratorFlushDbFunc flushDb; + iteratorHasPassedItemFunc hasPassedItem; + iteratorOriginalDbFunc originalDb; + iteratorIsKeyInScopeFunc isKeyInScope; +}; + + +/* This struct is used across threads. Unless otherwise noted, the fields are initialized at + * iterator creation (within the main thread) and are read-only by the client thread. */ +struct bgIterator { + sds name; // Iterator name + bgIteratorReplDoneFunc repldone; // Optional repldone function to be run on the main thread + bgIteratorCleanupFunc cleanup; // Optional cleanup function to be run on main thread + void *privdata; // Client's private data to be passed to cleanup function + + int iteration_flags; // Consistent and/or Replication + int iteration_type; // Full scan or cluster slot + uint32_t consistent_modification_id; // iterator epoch at time of iterator creation + + genericIterator *keyset_iter; // Low-level iterator (polymorphic) + + /* A set of dbEntry, compared by pointer. Used to track items which have already been iterated + * over by out-of-order expedited processing. Ensures a bgIterator does not try to reprocess + * items. Used only by main thread. */ + hashtable *early_iterate_entries; + + mutexQueue *items_for_iterator; // Created/Destroyed in main thread, used in both (threadsafe) + + mutexQueue *return_to_main_thread; // Queue of items to be returned to the Valkey main thread (threadsafe) + + unsigned int item_count_target; // Used only by main thread + + bgIteratorItem *current_item; // Used in client thread, validated in main after iterator complete + + bool client_is_active; // Set to true when client performs 1st read + + /* Set to true in main thread when last item from iteration has been queued to the client. No + * additional items will be enqueued to the client after this has been set. */ + bool completed; + + /* Set to true in main thread when iteration is to be killed. + * Set to true in iteration client when it decides to end early. */ + volatile bool terminated; + + bool cur_cmd_may_replicate; // Used only in main thread during command processing + + // Variables maintaining runtime statistics + unsigned long dbentries_queued; // Updated by main thread + unsigned long dbentries_processed; // Updated by client thread + unsigned long replication_queued; // Updated by main thread + unsigned long replication_processed; // Updated by client thread + unsigned long swapdb_queued; // Updated by main thread + unsigned long swapdb_processed; // Updated by client thread + unsigned long flushdb_queued; // Updated by main thread + unsigned long flushdb_processed; // Updated by client thread + unsigned long dbentry_clones_queued; // Updated by main thread + unsigned long dbentry_clones_processed; // Updated by client thread + monotime monotonic_start_time; // Time iteration started + + /* FLUSHDB and SWAPDB are special in that they affect all keys. When expediting a key, it's + * preferable to put it at the front of the queue. However, if there is a FLUSHDB or SWAPDB in + * the queue, we must maintain strict ordering. + * This value is equivalent to (flushdb_queued-flushdb_processed)+(swapdb_queued-swapdb_processed) */ + int barrier_items; + + /* The item start time is set in the iteration client. It is marked volatile as it can be read + * from the main thread by bgIteratorGetStatus. If 0, this indicates that the iteration client + * is waiting for an item to process. */ + volatile monotime monotonic_item_start_time; +}; + + +// These static values are only accessed from the main Valkey thread. + +static list *allIterators; // list of bgIterator +static dict *nameToIterator; // bgIterator->name -> bgIterator + +// Global, across all iterators, dict contains a dbEntry pointer -> ref count +static dict *inUseEntries; // dbEntry -> ref count + +/* Key values in the current command which don't exist in the DB yet. Needed for determination of + * replication for NON-consistent iterations. */ +static list *curCmdMissingKeys; // list of robj + +/* A counter of the total amount of memory used for buffered replication data. This amount is + * excluded when computing the need for evictions. */ +static ssize_t bufferedReplicationBytes; + +// Memory pool to track current allocated memory of cloned items (in bytes) +static ssize_t bgiteration_current_clone_memory_pool_size; + +/* Snapshot of the last queue size to seed the next queue. We assume all bgIterators consume items + * at roughly the same rate. */ +static int last_item_count_target; + +// Eventloop ID of the timerproc (or AE_DELETED_EVENT_ID) +static long long bgIterator_timeproc_id; + +// Incremented on each new iteration, this is updated in dbEntry metadata whenever an entry is modified. +static uint32_t bgIteration_epoch = 1; + +/* If true, the iterators' cur_cmd_may_replicate flag was determined in the last call to + * blockClientIfRequired. Otherwise, we skipped over computing this flag (maybe because it was a + * READ command). + * If this is true, AND we are in the context of executing a command inside of call(), then we + * should respect the setting of cur_cmd_may_replicate. */ +static bool iteratorReplicationFlagsWereUpdated; + +/* When a key is deleted (expire/evict): + * 1. bgIteration_keyDelete() is called + * 2. the key is physically deleted + * 3. replication is generated + * At the time of replication, we need the (deleted) dbEntry pointer to be able to check + * early_iterated_entries. This variable stores the pointer from the last call of keyDelete() */ +static dbEntry *dbEntryPtrOfLastKeyDelete; + +/* BgIteration debug captures BgIteration activity to a large sds buffer. When an iterator is + * completed, the entire buffer is written to a file in the current working directory. Note that + * memory must be available for the ENTIRE debug in memory. This isn't captured incrementally to + * a file as the file I/O is more likely to affect timing. + * + * Future implementation: the current design is most useful for a single iterator. When items are + * queued to an iterator, the iterator name is not recorded (to save space). + * + * Developer note: using a CONST value here allows the compiler to completely remove all of the + * debugging code at compile time. There is no run-time performance overhead when set to FALSE. + * This is essentially like an IFDEF, however, it's better as it forces the compiler to validate + * syntax. */ +static const bool BGITERATION_DEBUG = false; // DO NOT SUBMIT WITH THIS SYMBOL SET TO TRUE! +static sds debugBuffer; + + +/* ============================================================================================= + * Full Scan Iterator + * ============================================================================================= + * The full scan iterator performs the actual iteration over the Valkey keyset. The iterator is + * only used from within the Valkey main thread. Iteration proceeds one DB at a time, based on + * the DB ordering at the time of iterator creation. Each time the iterator returns items, all + * of the dictionary entries from a single hash bucket are returned. */ + +struct fullScanIterator { + genericIterator callbacks; // (must be first item) + + /* Array of mapping from original DB ID (at the time of iteration start) to that DB's current + * index. So, if the DB which was DB-0 is now at index 6, orig_to_cur_db[0]==6. */ + int *orig_to_cur_db; + + /* The reverse of the above array. This maps a current DB index to its original index (at the + * time of iteration start). */ + int *cur_to_orig_db; + + /* This is the DB we are currently iterating over. This is relative to the ORIGINAL DB + * ordering, at the time of iterator creation. Iteration proceeds from 0..N based on the + * original ordering. */ + int iter_db; + + // Iterator for the DB orig_to_cur_db[iter_db] + kvstore *kvs; // keep track of kvs associated with iter_dbi + int kvs_didx; // hashtable index within the kvstore + size_t ht_cursor; // cursor for scanning hashtable +}; + +static void fullScanIteratorRelease(genericIterator *genIt) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + if (it->kvs) resumeRehashForKvsHashtable(it->kvs, it->kvs_didx); + zfree(it->orig_to_cur_db); + zfree(it->cur_to_orig_db); + zfree(it); +} + +/* Scan callback used by fullScanIteratorGetEntries2 to collect entries into a fifo. */ +static void fullScanIteratorScanCallback(void *privdata, void *entry) { + fifo *dbEntryFifo = (fifo *)privdata; + dbEntry *de = (dbEntry *)entry; + fifoPush(dbEntryFifo, de); +} + +static fifo *fullScanIteratorGetEntries(genericIterator *genIt, int *orig_dbid, int *cur_dbid) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + if (it->iter_db >= server.dbnum) return NULL; // Finished scanning + + fifo *dbEntryFifo = fifoCreate(); + while (fifoLength(dbEntryFifo) == 0) { + while (it->kvs == NULL) { + if (++it->iter_db >= server.dbnum) { + fifoRelease(dbEntryFifo); + return NULL; // Iteration complete + } + serverDb *db = server.db[it->orig_to_cur_db[it->iter_db]]; + if (db != NULL) { + it->kvs = db->keys; + it->kvs_didx = kvstoreGetFirstNonEmptyHashtableIndex(it->kvs); + it->ht_cursor = 0; + if (it->kvs_didx == KVSTORE_INDEX_NOT_FOUND) it->kvs = NULL; + if (it->kvs != NULL) pauseRehashForKvsHashtable(it->kvs, it->kvs_didx); + } + } + + hashtable *ht = kvstoreGetHashtable(it->kvs, it->kvs_didx); + if (ht) { + it->ht_cursor = hashtableScan(ht, it->ht_cursor, fullScanIteratorScanCallback, dbEntryFifo); + } else { + it->ht_cursor = 0; + } + + if (it->ht_cursor == 0) { + /* Done with this hashtable, move to next. */ + resumeRehashForKvsHashtable(it->kvs, it->kvs_didx); + it->kvs_didx = kvstoreGetNextNonEmptyHashtableIndex(it->kvs, it->kvs_didx); + if (it->kvs_didx == KVSTORE_INDEX_NOT_FOUND) it->kvs = NULL; + if (it->kvs != NULL) pauseRehashForKvsHashtable(it->kvs, it->kvs_didx); + } + } + *orig_dbid = it->iter_db; + *cur_dbid = it->orig_to_cur_db[*orig_dbid]; + return dbEntryFifo; +} + +static void fullScanIteratorSwapDb(genericIterator *genIt, int db1, int db2) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + int temp = it->cur_to_orig_db[db1]; + it->cur_to_orig_db[db1] = it->cur_to_orig_db[db2]; + it->cur_to_orig_db[db2] = temp; + + it->orig_to_cur_db[it->cur_to_orig_db[db1]] = db1; + it->orig_to_cur_db[it->cur_to_orig_db[db2]] = db2; +} + +static void fullScanIteratorFlushDb(genericIterator *genIt, int cur_dbid) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + int orig_db = (cur_dbid == -1) ? it->iter_db : it->cur_to_orig_db[cur_dbid]; + if (orig_db == it->iter_db) { + // We are currently iterating on the DB that's being flushed. + if (it->kvs) { + // If it->kvs is set, we're actively scanning and have paused rehash + resumeRehashForKvsHashtable(it->kvs, it->kvs_didx); + it->kvs = NULL; + } + // Iteration will continue with the next DB. + } +} + +static bool fullScanIteratorHasPassedItem(genericIterator *genIt, const_sds key, int cur_dbid) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + int orig_dbid = it->cur_to_orig_db[cur_dbid]; + + if (orig_dbid < it->iter_db) return true; // Entire DB has already been processed + if (orig_dbid > it->iter_db) return false; // Haven't started this DB yet + // Now, orig_dbid == it->iter_db + + if (it->kvs == NULL) return true; // just finished this DB + + /* We're in the middle of processing a DB. In cluster-mode, the DB is divided into 1 hashtable + * per slot. In cluster-mode-disabled, we treat all keys as in slot 0. */ + int keySlot = server.cluster_enabled ? getKVStoreIndexForKey((sds)key) : 0; + if (keySlot < it->kvs_didx) return true; + if (keySlot > it->kvs_didx) return false; + + // At this point, we're down to a specific hashtable. + + hashtable *ht = kvstoreGetHashtable(it->kvs, keySlot); + if (hashtableScanHasPassedKey(ht, key, it->ht_cursor)) return true; + + return false; +} + +static int fullScanIteratorOriginalDb(genericIterator *genIt, int cur_dbid) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + return it->cur_to_orig_db[cur_dbid]; +} + +static bool fullScanIteratorIsKeyInScope(genericIterator *genIt, const_sds key) { + UNUSED(genIt); + UNUSED(key); + return true; // All keys are in scope +} + +static genericIterator *fullScanIteratorCreate(void) { + struct fullScanIterator *it = zmalloc(sizeof(struct fullScanIterator)); + it->orig_to_cur_db = zmalloc(sizeof(int) * server.dbnum); + it->cur_to_orig_db = zmalloc(sizeof(int) * server.dbnum); + for (int i = 0; i < server.dbnum; i++) { + it->orig_to_cur_db[i] = i; + it->cur_to_orig_db[i] = i; + } + it->iter_db = -1; + it->kvs = NULL; + + it->callbacks.release = fullScanIteratorRelease; + it->callbacks.getEntries = fullScanIteratorGetEntries; + it->callbacks.swapDb = fullScanIteratorSwapDb; + it->callbacks.flushDb = fullScanIteratorFlushDb; + it->callbacks.hasPassedItem = fullScanIteratorHasPassedItem; + it->callbacks.originalDb = fullScanIteratorOriginalDb; + it->callbacks.isKeyInScope = fullScanIteratorIsKeyInScope; + + return (genericIterator *)it; +} + + +/* ============================================================================================= + * Cluster Slot Iterator + * ============================================================================================= + * The cluster slot iterator performs iteration over one cluster slot of the Valkey keyset. The + * iterator is only used from within the Valkey main thread. */ +struct clusterSlotIterator { + genericIterator callbacks; // (must be first item) +}; + +static void clusterSlotIteratorRelease(genericIterator *genIt) { + UNUSED(genIt); + serverAssert(false); // Not yet implemented +} + +static fifo *clusterSlotIteratorGetEntries(genericIterator *genIt, int *orig_dbid, int *cur_dbid) { + UNUSED(genIt); + UNUSED(orig_dbid); + UNUSED(cur_dbid); + serverAssert(false); // Not yet implemented +} + +static void clusterSlotIteratorSwapDb(genericIterator *genIt, int db1, int db2) { + UNUSED(genIt); + UNUSED(db1); + UNUSED(db2); + serverAssert(false); // swap not valid in cluster mode +} + +static void clusterSlotIteratorFlushDb(genericIterator *genIt, int cur_dbid) { + UNUSED(genIt); + UNUSED(cur_dbid); + serverAssert(false); // Not yet implemented +} + +static bool clusterSlotIteratorHasPassedItem(genericIterator *genIt, const_sds key, int cur_dbid) { + UNUSED(genIt); + UNUSED(key); + UNUSED(cur_dbid); + serverAssert(false); // Not yet implemented +} + +static int clusterSlotIteratorOriginalDb(genericIterator *genIt, int cur_dbid) { + UNUSED(genIt); + UNUSED(cur_dbid); + return cur_dbid; // swap not supported in cluster mode +} + +/* When checking if a command is in scope for this iterator, all of its keys should be either in + * scope or not. In cluster mode enabled a command cannot reference keys from different slots, so + * this assumption will always be true. */ +static bool clusterSlotIteratorIsKeyInScope(genericIterator *genIt, const_sds key) { + UNUSED(genIt); + UNUSED(key); + serverAssert(false); // Not yet implemented +} + +static genericIterator *clusterSlotIteratorCreate(const int *slots, size_t slots_count) { + struct clusterSlotIterator *it = zmalloc(sizeof(struct clusterSlotIterator)); + it->callbacks.release = clusterSlotIteratorRelease; + it->callbacks.getEntries = clusterSlotIteratorGetEntries; + it->callbacks.swapDb = clusterSlotIteratorSwapDb; + it->callbacks.flushDb = clusterSlotIteratorFlushDb; + it->callbacks.hasPassedItem = clusterSlotIteratorHasPassedItem; + it->callbacks.originalDb = clusterSlotIteratorOriginalDb; + it->callbacks.isKeyInScope = clusterSlotIteratorIsKeyInScope; + + UNUSED(slots); + UNUSED(slots_count); + serverAssert(false); // Not yet implemented + + return (genericIterator *)it; +} + + +/* ============================================================================================= + * General iteration support (across all iterators) + * ============================================================================================= */ + +/* While an item is potentially in use by a background thread, we can't have rehashing by the main + * thread. Returns true if rehashing was paused. */ +static bool pauseRehashing(dbEntry *de) { + switch (de->encoding) { + case OBJ_ENCODING_HASHTABLE: { // SET or HASH + hashtable *ht = objectGetVal(de); + hashtablePauseRehashing(ht); + return true; + } + case OBJ_ENCODING_BTREE: { // SORTED SET + zset *zs = objectGetVal(de); + hashtablePauseRehashing(zs->ht); + return true; + } + default: + return false; + } +} + +static void resumeRehashing(dbEntry *de) { + switch (de->encoding) { + case OBJ_ENCODING_HASHTABLE: { // SET or HASH + hashtable *ht = objectGetVal(de); + hashtableResumeRehashing(ht); + break; + } + case OBJ_ENCODING_BTREE: { // SORTED SET + zset *zs = objectGetVal(de); + hashtableResumeRehashing(zs->ht); + break; + } + default: + break; + } +} + +// Maintain a list of entries which are currently in-use. These items should not be modified. +static void incrementEntryInuse(dbEntry *de) { + dictEntry *existingEntry; + dictEntry *newEntry = dictAddRaw(inUseEntries, de, &existingEntry); + if (newEntry) { + incrRefCount(de); + dictSetSignedIntegerVal(newEntry, 1); + } else { + dictSetSignedIntegerVal(existingEntry, dictGetSignedIntegerVal(existingEntry) + 1); + } +} + + +static void decrementEntryInuse(dbEntry *de) { + dictEntry *entry = dictFind(inUseEntries, de); + if (dictGetSignedIntegerVal(entry) == 1) { + dictDelete(inUseEntries, de); + decrRefCount(de); + } else { + serverAssert(dictGetSignedIntegerVal(entry) > 1); + dictSetSignedIntegerVal(entry, dictGetSignedIntegerVal(entry) - 1); + } +} + +static bool isEntryInuseBySingleIterator(dbEntry *de) { + dictEntry *entry = dictFind(inUseEntries, de); + return dictGetSignedIntegerVal(entry) == 1; +} + +static bool isEntryInuseByAnyIterator(dbEntry *de) { + return (dictFind(inUseEntries, de) != NULL); +} + + +static ssize_t computeStringDbEntrySize(dbEntry *de) { + sds key = objectGetKey(de); + size_t valueSize = stringObjectLen(de); + + return sdslen(key) + valueSize; // ignore the rest of the overhead, it's minor & transient +} + + +static dbEntry *tryCloneDbEntry(dbEntry *de) { + if (bgiteration_current_clone_memory_pool_size + bgiter_max_clone_item_bytes > + bgiter_max_clone_pool_bytes) return NULL; + + /* Future optimization: Incorporate small ziplists, sorted sets, etc. + * OBJ_ENCODING_INT is omitted only because there isn't a good API for cloning it yet. */ + if (de->type == OBJ_STRING && de->encoding != OBJ_ENCODING_INT) { + ssize_t itemSize = computeStringDbEntrySize(de); + + if (itemSize <= bgiter_max_clone_item_bytes) { + bgiteration_current_clone_memory_pool_size += itemSize; + dbEntry *clone = createStringObjectWithKeyAndExpire((char *)objectGetVal(de), + sdslen(objectGetVal(de)), + objectGetKey(de), + objectGetExpire(de)); + ((bgIterationEntryMetadata *)objectGetMetadata(clone))->iterator_epoch = + ((bgIterationEntryMetadata *)objectGetMetadata(de))->iterator_epoch; + return clone; + } + } + + return NULL; +} + +static void freeClonedDictEntry(dbEntry *clonedEntry) { + serverAssert(clonedEntry->type == OBJ_STRING); + + bgiteration_current_clone_memory_pool_size -= computeStringDbEntrySize(clonedEntry); + + decrRefCount(clonedEntry); +} + + +static bgIteratorItem *makeDbEntryItem(dbEntry *de, int dbid, bool isCloned) { + if (!isCloned) incrementEntryInuse(de); + + bgIteratorItem *item = itemFreeList_getElementOrAllocate(); + item->type = BGITERATOR_ITEM_DBENTRY; + item->dbid = dbid; + item->u.dbe.de = de; + item->u.dbe.is_cloned = isCloned; + item->u.dbe.is_rehashing_paused = pauseRehashing(de); + + return item; +} + +static robj **cloneRobjArray(int argc, robj **argv) { + robj **newarray = zmalloc(sizeof(robj *) * argc); + for (int i = 0; i < argc; i++) { + newarray[i] = argv[i]; + incrRefCount(argv[i]); + } + return newarray; +} + + +static void freeRobjArray(int argc, robj **argv) { + for (int i = 0; i < argc; i++) { + decrRefCount(argv[i]); + } + zfree(argv); +} + + +// Called by iterator thread to release an item. +static void returnCurrentItemToMainThread(bgIterator *it) { + bgIteratorItem *item = it->current_item; + if (item == NULL) return; + + switch (item->type) { + case BGITERATOR_ITEM_DBENTRY: + it->dbentries_processed++; + if (item->u.dbe.is_cloned) it->dbentry_clones_processed++; + mutexQueueAdd(it->return_to_main_thread, item); + break; + case BGITERATOR_ITEM_REPLICATION: + it->replication_processed++; + mutexQueueAdd(it->return_to_main_thread, item); + break; + case BGITERATOR_ITEM_SWAPDB: + it->swapdb_processed++; + mutexQueueAdd(it->return_to_main_thread, item); + break; + case BGITERATOR_ITEM_FLUSHDB: + it->flushdb_processed++; + mutexQueueAdd(it->return_to_main_thread, item); + break; + + case BGITERATOR_ITEM_COMPLETE: + case BGITERATOR_ITEM_TERMINATED: + // These are static and just used to wake the iterator - they should never be returned. + serverAssert(false); + break; + + default: + serverAssert(false); + } + + it->current_item = NULL; +} + + +/* ============================================================================================= + * Background Iterator (private) + * ============================================================================================= */ + +static void bgIteratorRelease(bgIterator *it) { + serverAssert(hasMainThreadExclusivity()); + serverAssert(it->current_item == NULL); + serverAssert(mutexQueueLength(it->items_for_iterator) == 0); + serverAssert(mutexQueueLength(it->return_to_main_thread) == 0); + + dictDelete(nameToIterator, it->name); + listDelNode(allIterators, listSearchKey(allIterators, it)); + + mutexQueueRelease(it->items_for_iterator); + it->items_for_iterator = NULL; + + mutexQueueRelease(it->return_to_main_thread); + it->return_to_main_thread = NULL; + + it->keyset_iter->release(it->keyset_iter); + it->keyset_iter = NULL; + + hashtableRelease(it->early_iterate_entries); + it->early_iterate_entries = NULL; + + sdsfree(it->name); + zfree(it); +} + + +static bool shouldFeedIteratorMore(bgIterator *it) { + return (!it->completed && + !it->terminated && + mutexQueueLength(it->items_for_iterator) < it->item_count_target); +} + + +// Debugging routine +static sds createEntryString(int dbid, dbEntry *de) { + sds key = objectGetKey(de); + + sds entrySds = sdsempty(); + entrySds = sdscatprintf(entrySds, "(%d)'%s'", dbid, key); + if (de->type == OBJ_STRING) { + robj *o = getDecodedObject(de); // might be encoded as int + const unsigned valuePrintLen = 20; + entrySds = sdscatprintf(entrySds, " : '%.*s'", valuePrintLen, (char *)objectGetVal(o)); + if (sdslen((sds)objectGetVal(o)) > valuePrintLen) entrySds = sdscat(entrySds, "..."); + decrRefCount(o); + } else { + entrySds = sdscatprintf(entrySds, " : type(%d)", de->type); + } + return entrySds; +} + + +static void feedIterator(bgIterator *it, monotime end_time_us) { + unsigned int initial_queue_len = mutexQueueLength(it->items_for_iterator); + + /* The queue size dynamically adjusts using an AIMD approach. If we have left over stuff from + * the prior call to feedIterator, reduce by half the remaining size. If the queue ran dry + * and we have time left (at the end of this function), additively increase the queue length. */ + if (initial_queue_len > 2 && it->item_count_target >= initial_queue_len) { + it->item_count_target -= initial_queue_len / 2; + } + + // Now do some feeding + bool have_time = (getMonotonicUs() < end_time_us); + int timeCheckCounter = 0; + while (shouldFeedIteratorMore(it) && have_time) { + int orig_dbid, cur_dbid; + fifo *dbEntryFifo = it->keyset_iter->getEntries(it->keyset_iter, &orig_dbid, &cur_dbid); + + if (dbEntryFifo == NULL) { + // Iteration of items is complete for this iterator + serverAssert(it->dbentries_queued >= it->dbentries_processed); + serverAssert(it->replication_queued >= it->replication_processed); + serverAssert(it->swapdb_queued >= it->swapdb_processed); + serverAssert(it->flushdb_queued >= it->flushdb_processed); + serverAssert(it->dbentry_clones_queued >= it->dbentry_clones_processed); + + // Snapshot queue size to seed next iterator when terminated + last_item_count_target = it->item_count_target; + + if (it->iteration_flags & BGITERATOR_FLAG_REPLICATION) { + if (!it->client_is_active || (it->dbentries_queued > it->dbentries_processed)) { + /* Even though we have sent all of the dbEntries, we continue sending + * replication until the iterator has consumed all of the dbEntries. + * client_is_active prevents race conditions in the case of an empty DB. */ + break; + } + if (it->repldone) { + bool clientWantsMoreReplication = (!it->repldone(it->privdata)); + if (clientWantsMoreReplication) break; + } + } + bgIteratorItem *completionItem = itemFreeList_getElementOrAllocate(); + *completionItem = (bgIteratorItem){.type = BGITERATOR_ITEM_COMPLETE}; + if (it->iteration_flags & BGITERATOR_FLAG_REPLICATION) { + rdbSaveInfo rsi; + completionItem->dbid = (rdbPopulateSaveInfo(&rsi)) ? rsi.repl_stream_db : 0; + completionItem->u.master_repl_offset = server.primary_repl_offset; + if (BGITERATION_DEBUG) { + debugBuffer = sdscat(debugBuffer, "REPLDONE FN\n"); + } + } + + if (BGITERATION_DEBUG) { + debugBuffer = sdscat(debugBuffer, "SENDING COMPLETE\n"); + } + + mutexQueueAdd(it->items_for_iterator, completionItem); + it->completed = true; + break; + } + + int dbid = (it->iteration_flags & BGITERATOR_FLAG_CONSISTENT) ? orig_dbid : cur_dbid; + + fifo *itemsToAdd = fifoCreate(); + while (fifoLength(dbEntryFifo) > 0) { + dbEntry *de; + fifoPop(dbEntryFifo, (void **)&de); + + // Remove new/modified items during consistent iteration. + if (it->iteration_flags & BGITERATOR_FLAG_CONSISTENT && + ((bgIterationEntryMetadata *)objectGetMetadata(de))->iterator_epoch > it->consistent_modification_id) { + continue; + } + + // Remove any items which have been processed early + if (hashtableDelete(it->early_iterate_entries, de)) { + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(dbid, de); + debugBuffer = sdscatprintf(debugBuffer, "SKIPPING ITEM(early iterate): %s\n", entryString); + sdsfree(entryString); + } + continue; + } + + // For items which are left, convert them from dbEntry to iteratorItem + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(dbid, de); + debugBuffer = sdscatprintf(debugBuffer, "ITEM: %s\n", entryString); + sdsfree(entryString); + } + + bgIteratorItem *item = makeDbEntryItem(de, dbid, false); + fifoPush(itemsToAdd, item); + } + fifoRelease(dbEntryFifo); + + if (fifoLength(itemsToAdd) > 0) { + it->dbentries_queued += fifoLength(itemsToAdd); + mutexQueueAddMultiple(it->items_for_iterator, itemsToAdd); + } + fifoRelease(itemsToAdd); + + // This is a predictably fast loop. We don't need to check the time on every pass. + if (++timeCheckCounter % 32 == 0) { + have_time = (getMonotonicUs() < end_time_us); + } + } + + // Smart logic to dynamically adjust the size of the queue + if (initial_queue_len == 0 && have_time && it->item_count_target < BGITER_QUEUE_MAX_LENGTH) { + it->item_count_target += BGITER_QUEUE_INCREASE_INCR; + } +} + + +static bool addEarlyIterationKey(bgIterator *it, dbEntry *earlyEntry, int cur_dbid) { + bool wasAdded = hashtableAdd(it->early_iterate_entries, earlyEntry); + serverAssert(wasAdded); + + int dbid = (it->iteration_flags & BGITERATOR_FLAG_CONSISTENT) + ? it->keyset_iter->originalDb(it->keyset_iter, cur_dbid) + : cur_dbid; + + dbEntry *cloneEntry = tryCloneDbEntry(earlyEntry); + bool isClonedEntry = (cloneEntry != NULL); + bgIteratorItem *item = makeDbEntryItem(isClonedEntry ? cloneEntry : earlyEntry, dbid, isClonedEntry); + + it->dbentries_queued++; + if (isClonedEntry) it->dbentry_clones_queued++; + + if (it->barrier_items == 0) { + // If there are no barrier items, we can add the key right to the front of the queue. + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(dbid, item->u.dbe.de); + debugBuffer = sdscatprintf(debugBuffer, "EARLY_1: %s\n", entryString); + sdsfree(entryString); + } + mutexQueuePushPriority(it->items_for_iterator, item); + } else { + // With barrier items, the key must be added to the end, and processed in order. + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(dbid, item->u.dbe.de); + debugBuffer = sdscatprintf(debugBuffer, "EARLY: %s\n", entryString); + sdsfree(entryString); + } + mutexQueueAdd(it->items_for_iterator, item); + } + return !isClonedEntry; // Block if the entry will be used by the background thread +} + + +static bool iteratorHasPassedKey(bgIterator *it, int dbid, const_sds key, dbEntry *de) { + if (it->completed || it->terminated) return true; + + if (it->keyset_iter->hasPassedItem(it->keyset_iter, key, dbid)) return true; + + if (de && hashtableFind(it->early_iterate_entries, de, NULL)) return true; + + return false; +} + + +// This expedites a single key and doesn't attempt to avoid expediting through optimization. +static bool expediteSingleKeyWithoutOptimization(bgIterator *it, + int dbid, + robj *oKey, + hashtable *waitingOnKeys) { + bool mustBlock = false; + + sds key = objectGetVal(oKey); + dbEntry *de = (server.db[dbid]) ? dbFind(server.db[dbid], key) : NULL; + if (de != NULL) { + if (!iteratorHasPassedKey(it, dbid, key, de)) { + if (addEarlyIterationKey(it, de, dbid)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } else { + if (isEntryInuseByAnyIterator(de)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } + } + + return mustBlock; +} + + +// MOVE/COPY are unfortunate special commands. They work on 2 DBs at once. +const int MOVE_COMMAND_DBID_ARG_INDEX = 2; +static bool expediteKeysForMove(bgIterator *it, + int dbid, + int argc, + robj **argv, + hashtable *waitingOnKeys) { + if (argc <= MOVE_COMMAND_DBID_ARG_INDEX) return false; + + int destDbid; + if (!getDbIdFromRobj(argv[MOVE_COMMAND_DBID_ARG_INDEX], &destDbid)) return false; + + bool mustBlock = false; + robj *key = argv[1]; + + /* Not looking for special cases to optimize here. Just try to expedite both src and dest + * keys. Note that the dest key might exist (and need iteration) but could be expired and + * could be overwritten by MOVE. In this case, a DEL would replicate due to the expiry. So + * even if the target is expired, we need to replicate it before executing the command. */ + if (expediteSingleKeyWithoutOptimization(it, dbid, key, waitingOnKeys)) mustBlock = true; + if (expediteSingleKeyWithoutOptimization(it, destDbid, key, waitingOnKeys)) mustBlock = true; + + it->cur_cmd_may_replicate = true; + return mustBlock; +} + + +// MOVE/COPY are unfortunate special commands. They work on 2 DBs at once. +static bool expediteKeysForCopy(bgIterator *it, + int dbid, + int argc, + robj **argv, + hashtable *waitingOnKeys) { + int destDbid; + if (!getTargetDbIdForCopyCommand(argc, argv, dbid, &destDbid)) return false; + + bool mustBlock = false; + robj *srcKey = argv[1]; + robj *destKey = argv[2]; + + /* Not trying to optimize COPY. Just expedite source and destination (if it exists). We + * don't really care if the value is overwritten or not (so no need to parse REPLACE option). */ + if (expediteSingleKeyWithoutOptimization(it, dbid, srcKey, waitingOnKeys)) mustBlock = true; + if (expediteSingleKeyWithoutOptimization(it, destDbid, destKey, waitingOnKeys)) mustBlock = true; + + it->cur_cmd_may_replicate = true; + return mustBlock; +} + + +/* There are several cases where a client must be blocked on write operations. (Clients never need + * to be blocked for read operations.) + * + * Note: The CMD_WRITE_FIRSTKEY_ONLY flag allows us to identify commands where the first key is for + * write and the rest are for read. This allows us to make the following optimizations: + * - For keys which are read only, there's no need to block if the key is in-use by an iterator + * - Without replication, there's no need to immediately queue read keys on a consistent iteration + * + * Iterator: CONSISTENT = NO, REPLICATION = NO + * - Block if any write-key is in use by an iterator + * + * Iterator: CONSISTENT = NO, REPLICATION = YES + * - Block if any write-key is in use by an iterator + * - If ANY key has already been iterated (but some keys have not), then + * - Block and immediately queue any key (read or write) that has not + * already been iterated + * Example: SDIFFSTORE KEY_A KEY_B KEY_C + * In this case, KEY_A is written, KEY_B and KEY_C are read. If KEY_A has already been + * iterated over, the replication stream will contain this command. The receiver of this + * replication will need KEY_B and KEY_C in order to process the replication stream. So + * these need to be iterated and the client blocked. + * + * Iterator: CONSISTENT = YES, REPLICATION = NO + * - Block if any write-key is in use by an iterator + * - Block and immediately queue any WRITE-key that has not already been iterated + * + * Iterator: CONSISTENT = YES, REPLICATION = YES + * (Combination only valid in cluster mode - no SWAPDB possible) + * - Block if any write-key is in use by an iterator + * - Block and immediately queue any key (read or write) that has not already been iterated */ +static bool expediteKeysForWrite(bgIterator *it, + int dbid, + struct serverCommand *cmd, + int argc, + robj **argv, + keyReference *keyrefs, + int numKeys, + hashtable *waitingOnKeys) { + serverAssert(numKeys > 0); + + bool mustBlock = false; + + /* All keys of the command should either be in scope or not since in cluster mode enabled they + * should all be in the same slot. So we just check the first key. */ + robj *oKey = argv[keyrefs[0].pos]; + sds key = objectGetVal(oKey); + /* If it's not in the iteration scope for the current iterator, then we don't need to do + * anything with this command. */ + if (!it->keyset_iter->isKeyInScope(it->keyset_iter, key)) return false; + + if ((cmd->flags & CMD_WRITE_FIRSTKEY_ONLY) && + !(it->iteration_flags & BGITERATOR_FLAG_REPLICATION)) { + /* If this write command only modifies the 1st key, we don't need to expedite others + * unless replication enabled. */ + numKeys = 1; + } + + if (cmd->proc == moveCommand) { + // Special case for MOVE + return expediteKeysForMove(it, dbid, argc, argv, waitingOnKeys); + } + + if (cmd->proc == copyCommand) { + // Similar special case for COPY + return expediteKeysForCopy(it, dbid, argc, argv, waitingOnKeys); + } + + if (it->iteration_flags & BGITERATOR_FLAG_CONSISTENT) { + // CONSISTENT = YES, REPLICATION = YES / NO + for (int i = 0; i < numKeys; i++) { + robj *oKey = argv[keyrefs[i].pos]; + sds key = objectGetVal(oKey); + dbEntry *de = (server.db[dbid]) ? dbFind(server.db[dbid], key) : NULL; + if (de == NULL) continue; // New key, no need to expedite + if (!iteratorHasPassedKey(it, dbid, key, de) && + ((bgIterationEntryMetadata *)objectGetMetadata(de))->iterator_epoch <= it->consistent_modification_id) { + if (addEarlyIterationKey(it, de, dbid)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } else { + if (isEntryInuseByAnyIterator(de)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } + } + it->cur_cmd_may_replicate = true; // Will replicate only if replication enabled + } else { + /* Identification of missing keys is only needed for non-consistent iteration. This only + * needs to be collected once (on the 1st non-consistent iteration). */ + bool collectMissing = (listLength(curCmdMissingKeys) == 0); + + if (it->iteration_flags & BGITERATOR_FLAG_REPLICATION) { + // CONSISTENT = NO, REPLICATION = YES + bool someIterated = false; + /* dict containing the keys that have not been iterated yet. + * Using a dict dedupes the keys in case the command contains duplicated keys. */ + dict *notIteratedKeys = dictCreate(&dictEntryPtrDictType); // dict of dbEntry* -> robj* + + for (int i = 0; i < numKeys; i++) { + robj *oKey = argv[keyrefs[i].pos]; + sds key = objectGetVal(oKey); + dbEntry *de = (server.db[dbid]) ? dbFind(server.db[dbid], key) : NULL; + if (de == NULL) { + if (collectMissing) { + incrRefCount(oKey); + listAddNodeHead(curCmdMissingKeys, oKey); + } + continue; + } + if (iteratorHasPassedKey(it, dbid, key, de)) { + someIterated = true; + } else { + dictAdd(notIteratedKeys, de, oKey); + } + if (isEntryInuseByAnyIterator(de)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } + + /* Since missing keys are considered as already iterated, if there are any missing keys + * we must consider that some keys have been iterated, and make sure all other keys + * will be expedited if needed. */ + if (listLength(curCmdMissingKeys) > 0) someIterated = true; + + /* This command may be executing as part of a larger transaction. If some parts of the + * transaction have already been identified to replicate, we must wait on all keys and + * replicate here as well. (Take care not to set cur_cmd_may_replicate to false.) */ + if (someIterated) { + if (server.in_exec) { + /* We are now executing the commands in a multi-exec block. + * + * Regarding MULTI/EXEC: Remember that this code is executed twice for commands + * within a MULTI/EXEC block. First, we parse all the commands when deciding + * if the EXEC should be blocked. Then, as each command is executed, it's + * re-parsed so that we can maintain the early iterated list as the commands + * execute. In this second pass, as each command is executed, we can't change + * the replication decision which was made earlier (when the EXEC was processed). + * We don't want to get tricked (by a key being removed and recreated) into + * starting to replicate in the middle of a MULTI/EXEC block. */ + } else { + it->cur_cmd_may_replicate = true; + } + } + if (it->cur_cmd_may_replicate) { + dictEntry *de; + dictIterator *di = dictGetIterator(notIteratedKeys); + while ((de = dictNext(di)) != NULL) { + dbEntry *notIteratedEntry = dictGetKey(de); + robj *oKey = dictGetVal(de); + + if (addEarlyIterationKey(it, notIteratedEntry, dbid)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } + dictReleaseIterator(di); + } + dictRelease(notIteratedKeys); + } else { + // CONSISTENT = NO, REPLICATION = NO + for (int i = 0; i < numKeys; i++) { + robj *oKey = argv[keyrefs[i].pos]; + sds key = objectGetVal(oKey); + dbEntry *de = (server.db[dbid]) ? dbFind(server.db[dbid], key) : NULL; + if (de == NULL) { + if (collectMissing) { + incrRefCount(oKey); + listAddNodeHead(curCmdMissingKeys, oKey); + } + continue; + } + if (isEntryInuseByAnyIterator(de)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } + } + } + + return mustBlock; +} + + +/* Called when an iterator is terminated. Pulls everything out of the queue + * and returns the items to the main thread (before they hit the iterator). */ +static void returnAllItemsToMainThread(bgIterator *it) { + serverAssert(hasMainThreadExclusivity()); + + fifo *poppedFifo = mutexQueuePopAll(it->items_for_iterator, false); + if (poppedFifo == NULL) return; // Nothing to return + + // Release non-dictentry items first... + fifo *itemsToReturn = fifoCreate(); + while (fifoLength(poppedFifo) > 0) { + bgIteratorItem *item; + fifoPop(poppedFifo, (void **)&item); + switch (item->type) { + // back out the "queued" statistic + case BGITERATOR_ITEM_DBENTRY: + it->dbentries_queued--; + if (item->u.dbe.is_cloned) it->dbentry_clones_queued--; + break; + case BGITERATOR_ITEM_REPLICATION: + it->replication_queued--; + break; + case BGITERATOR_ITEM_SWAPDB: + it->swapdb_queued--; + it->barrier_items--; + break; + case BGITERATOR_ITEM_FLUSHDB: + it->flushdb_queued--; + it->barrier_items--; + break; + + case BGITERATOR_ITEM_COMPLETE: + /* This can only happen if the completion item has been enqueued and + * the iterator is terminated before reaching the completion item. */ + itemFreeList_returnItemBackToFreeList(item); + continue; // Skip pushing this onto itemsToReturn + + case BGITERATOR_ITEM_TERMINATED: + /* This can only happen if there is a race when terminating between + * the iteration client and main thread. */ + serverAssert(item == &STATIC_ITEM_TERMINATED); + continue; // Skip pushing this onto itemsToReturn + + default: + serverAssert(false); + } + + fifoPush(itemsToReturn, item); + } + fifoRelease(poppedFifo); + + // Now release items all at once... + if (fifoLength(itemsToReturn) > 0) { + mutexQueueAddMultiple(it->return_to_main_thread, itemsToReturn); + } + fifoRelease(itemsToReturn); +} + + +/* ============================================================================================= + * Foreground support functions (private) + * ============================================================================================= */ + +static size_t replicationItemSize(bgIteratorItem *item) { + serverAssert(item->type == BGITERATOR_ITEM_REPLICATION); + size_t itemSize = sizeof(bgIteratorItem); + for (int i = 0; i < item->u.repl.argc; i++) { + itemSize += objectComputeSize(NULL, item->u.repl.argv[i], 0, 0); + } + return itemSize; +} + +static void processReturnOfItemToMainThread(bgIterator *it, bgIteratorItem *item) { + serverAssert(hasMainThreadExclusivity()); + switch ((int)item->type) { + case BGITERATOR_ITEM_REPLICATION: + bufferedReplicationBytes -= item->u.repl.replication_size; + freeRobjArray(item->u.repl.argc, item->u.repl.argv); + break; + + case BGITERATOR_ITEM_DBENTRY: + if (item->u.dbe.is_cloned) { + freeClonedDictEntry(item->u.dbe.de); + } else { + if (isEntryInuseBySingleIterator(item->u.dbe.de)) { + /* This blocking mechanism assumes a single DB so if the same key appears in + * multiple DBs, commands might get unblocked only to get blocked again. (This + * would happen only rarely, and with minimal impact.) */ + robj *key = createStringObjectFromSds(objectGetKey(item->u.dbe.de)); + unblockClientsInUseOnKey(key); + decrRefCount(key); + } + // resumeRehashing must be called before decrementEntryInuse, since decrementEntryInuse can free + if (item->u.dbe.is_rehashing_paused) resumeRehashing(item->u.dbe.de); + decrementEntryInuse(item->u.dbe.de); + } + break; + + case BGITERATOR_ITEM_SWAPDB: + case BGITERATOR_ITEM_FLUSHDB: + it->barrier_items--; + break; + + case BGITERATOR_ITEMEXT_ITER_CLOSED: { + if (it->terminated) { + /* Abnormal termination + * Normally the item is TERMINATED, but might be COMPLETE in race */ + serverAssert(it->current_item->type == BGITERATOR_ITEM_TERMINATED || + it->current_item->type == BGITERATOR_ITEM_COMPLETE); + // Release any items stranded on the iterator after early termination + returnAllItemsToMainThread(it); + receiveItemsBackFromOneIterator(it); + } else { + // Normal completion + serverAssert(it->current_item->type == BGITERATOR_ITEM_COMPLETE); + } + if (it->current_item != &STATIC_ITEM_TERMINATED) itemFreeList_returnItemBackToFreeList(it->current_item); + it->current_item = NULL; + + serverAssert(mutexQueueLength(it->items_for_iterator) == 0); + serverAssert(it->barrier_items == 0); + serverAssert(it->dbentries_queued == it->dbentries_processed); + serverAssert(it->replication_queued == it->replication_processed); + serverAssert(it->swapdb_queued == it->swapdb_processed); + serverAssert(it->flushdb_queued == it->flushdb_processed); + serverAssert(it->dbentry_clones_queued == it->dbentry_clones_processed); + + listEmpty(curCmdMissingKeys); // Just in case any remain + + bool terminated = it->terminated; + void *privdata = it->privdata; + bgIteratorCleanupFunc cleanup = it->cleanup; + bgIteratorRelease(it); // Fully release the iterator before calling cleanup + + if (BGITERATION_DEBUG) { + if (cleanup) debugBuffer = sdscatprintf(debugBuffer, "CLEANUP FN (%s)\n", + (terminated) ? "terminated" : "success"); + + sds filename = sdscatprintf(sdsempty(), "bgiteration_debug.%d", getpid()); + FILE *f = fopen(filename, "w"); + sdsfree(filename); + + fputs(debugBuffer, f); + + fclose(f); + sdsfree(debugBuffer); + debugBuffer = sdsempty(); + } + + if (cleanup) cleanup(terminated, privdata); + item = NULL; // Prevent return of static item to free list + } break; + + default: + serverAssert(false); // Not expecting any other type of item! + } + + if (item) itemFreeList_returnItemBackToFreeList(item); +} + +static void prepareAndProcessReturnedItems(bgIterator *it, int n, bgIteratorItem **items) { + for (int i = 0; i < n; i++) valkey_prefetch(items[i]); + for (int i = 0; i < n; i++) { + if (items[i]->type != BGITERATOR_ITEM_DBENTRY) continue; + valkey_prefetch(items[i]->u.dbe.de); + } + for (int i = 0; i < n; i++) { + if (items[i]->type != BGITERATOR_ITEM_DBENTRY) continue; + valkey_prefetch(objectGetKey(items[i]->u.dbe.de)); + } + for (int i = 0; i < n; i++) processReturnOfItemToMainThread(it, items[i]); +} + +#define PREFETCH_BATCH_SIZE 16 + +// Returns true if we process at least one item from a given iterator's return_to_main_thread queue. +static bool receiveItemsBackFromOneIterator(bgIterator *it) { + bgIteratorItem *batchPool[PREFETCH_BATCH_SIZE]; + int n = 0; + fifo *poppedFifo = mutexQueuePopAll(it->return_to_main_thread, false); + if (poppedFifo != NULL) { + while (fifoLength(poppedFifo) > 0) { + fifoPop(poppedFifo, (void **)&batchPool[n++]); + if (n == PREFETCH_BATCH_SIZE) { + prepareAndProcessReturnedItems(it, n, batchPool); + n = 0; + } + } + if (n > 0) { + prepareAndProcessReturnedItems(it, n, batchPool); + } + fifoRelease(poppedFifo); + return true; + } + return false; +} + +/* Process each iterator's return_to_main_thread queue + * If `blocking` is true, continue reading until at least one queue was not empty. */ +static void receiveItemsBackFromIterators(bool blocking) { + serverAssert(hasMainThreadExclusivity()); + listIter li; + listNode *node; + bool processedItems = false; + do { + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + processedItems |= receiveItemsBackFromOneIterator(it); + } + if (blocking && !processedItems) usleep(100); // Short sleep before retry + } while (blocking && !processedItems); +} + + +static long long bgIteration_feedIterators_task(struct aeEventLoop *eventLoop, + long long id, + void *clientData) { + UNUSED(eventLoop); + UNUSED(id); + UNUSED(clientData); + serverAssert(hasMainThreadExclusivity()); + + static monotime lastFeedEndTime; // STATIC: Persists For checking starvation + monotime startTime = getMonotonicUs(); + + if (!bgIteration_iterationActive()) { + // No more iterators exist. Self-check, and terminate the "feed" task. + serverAssert(dictSize(nameToIterator) == 0); + serverAssert(dictSize(inUseEntries) == 0); + serverAssert(bufferedReplicationBytes == 0); + + // Shrink dict back to zero (doesn't normally shrink) + dictRelease(inUseEntries); + inUseEntries = dictCreate(&dictEntryPtrDictType); + + itemFreeList_release(); + + bgIterator_timeproc_id = AE_DELETED_EVENT_ID; + lastFeedEndTime = 0; + return AE_NOMORE; + } + + long dutyTimeUs = BGITER_CYCLE_BUDGET_MS * 1000; + if (lastFeedEndTime > 0) { + /* If the timer was delayed, compute the proportional time we should have had, and increase + * the duty cycle to compensate (up to a limit). */ + long starvationUs = (startTime - lastFeedEndTime) - BGITER_CYCLE_DELAY_MS * 1000; + if (starvationUs > 0) { + long starvationCompensationUs = starvationUs * BGITER_CYCLE_BUDGET_MS / + (BGITER_CYCLE_BUDGET_MS + BGITER_CYCLE_DELAY_MS); + dutyTimeUs += starvationCompensationUs; + dutyTimeUs = MIN(dutyTimeUs, BGITER_CYCLE_BUDGET_MAX_MS * 1000); + } + } + monotime endTime = startTime + dutyTimeUs; + + // Test path (manual feed, no timer thread): ignore the budget so a slow env can't starve the blocking read. + if (eventLoop == NULL) endTime = UINT64_MAX; + + // Run this part regardless of time limit... + receiveItemsBackFromIterators(false); + + // Feeding iterators (below) respects endTime. The stuff above always runs to completion. + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL && getMonotonicUs() < endTime) { + bgIterator *it = listNodeValue(node); + if (it->completed || it->terminated) continue; + feedIterator(it, endTime); + } + + lastFeedEndTime = getMonotonicUs(); + return BGITER_CYCLE_DELAY_MS; +} + + +// Not static, but not API. Intended for unit tests where the event loop may not be active. +void bgIteration_feedIterators(void) { + /* For unit testing, force the item_count_target to 1 in each call. This ensures that we only + * feed a minimal amount to the iterators rather than a non-deterministic amount. */ + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + it->item_count_target = 1; + } + + // Invoke the feeding task (normally invoked by timer). + bgIteration_feedIterators_task(NULL, 0, NULL); +} + + +static void resetReplicationFlagForIterators(client *c) { + /* For any given command, the command may or may not need to be replicated based on the status + * and flags of each iterator. Furthermore, if a command does need to be replicated, this + * replication must occur for an entire atomic unit; we can't replicate only part of a script + * or multi/exec. + * This function is the only place where the replication flag is cleared. */ + + if (c->flag.multi || c->flag.script) { + /* REGARDING MULTI/EXEC + * -------------------- + * When processing a MULTI/EXEC, blockClientIfRequired is called first for the MULTI. Then, + * all of the commands are queued up in server.c:processCommand(). It's only when EXEC is + * encountered, that server.c:call() is fired to begin execution. + * + * AFTER the EXEC is processed by call(), then each of the commands in the MULTI/EXEC block + * will be processed through call(). + * + * If write commands are present, MULTI & EXEC will be passed to the replication stream + * before/after the transaction commands. Note that MULTI & EXEC are not actually + * "executed" at the time when their replication is passed to the replication stream. + * + * Example: MULTI; SET A B; EXEC + * 1. blockClientIfRequired() called for MULTI. MULTI flag IS NOT set. (Won't block.) + * 2. blockClientIfRequired() called for EXEC. MULTI flag IS set. (Might block.) + * 3. blockClientIfRequired() called for SET. MULTI flag IS set. (Won't block.) + * 4. handleCommandReplication() is called for MULTI. + * 5. handleCommandReplication() is called for SET. + * 6. handleCommandReplication() is called for EXEC. + * + * SO - if the MULTI flag is set, we DON'T clear the flag. It should only be cleared at the + * start of the transaction, when MULTI is received - and the flag isn't set yet. */ + + /* REGARDING SCRIPTS + * ----------------- + * When processing a script, blockClientIfRequired is called first for the EVAL/EVALSHA/FCALL. + * Then, all of the commands are processed using a special script client. The script + * client has the CLIENT_SCRIPT flag set. For scripts, the replication flag is set when + * processing the EVAL/EVALSHA/FCALL and should not be cleared when executing individual + * commands in the script. */ + + /* If it's the EXEC command, we fall through and clear the flag below. But for all other + * commands within the transaction, we don't clear the flag. */ + if (c->cmd->proc != execCommand) return; + } + + /* For most commands, the replication flag is cleared and we determine if replication is needed + * based on the keys being used and their state in each iterator. If a modified key hasn't been + * processed yet, there's no need to expedite the key or send the replication. The key will be + * sent later, when reached by the iterator. + * + * However, for scripts, it is not possible to perform this optimization. There is no way to + * know if an undeclared key might be modified. Since the entire script needs to be replicated + * (or not replicated) atomically, we can't take the chance that an undeclared key might be + * hit which requires replication. */ + bool isScript = isScriptCallWriteCmd(c->cmd); + + sds firstScriptKey = NULL; + if (isScript) { + /* If it's a script, we will normally replicate. But if the keys are out of scope for the + * iteration, we shouldn't. The use-case for this is with slot iteration, when the script + * is acting on keys from a different slot. Here, we just check the first declared key, and + * if it's out of scope for the iteration, we won't replicate it. This might cause issues + * for cross-slot scripts (anti-pattern), but the alternative is replicating all scripts, + * regardless of slot. */ + getKeysResult result; + initGetKeysResult(&result); + getKeysFromCommand(c->cmd, c->argv, c->argc, &result); + if (result.numkeys > 0) firstScriptKey = objectGetVal(c->argv[result.keys[0].pos]); + getKeysFreeResult(&result); + } + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (it->completed || it->terminated) { + it->cur_cmd_may_replicate = false; + } else { + /* For normal commands, the flag is initialized to false (not to replicate). For these + * commands, we decide later based on the actual commands. + * + * However, for scripts, we don't know what commands will be executed. So IF it's a + * script, and the keys are in scope (on the right slot) we initialize the replication + * flag to true. */ + it->cur_cmd_may_replicate = isScript && firstScriptKey && + it->keyset_iter->isKeyInScope(it->keyset_iter, firstScriptKey); + } + } +} + + +static void handleSwapdb(int db1, int db2) { + serverAssert(hasMainThreadExclusivity()); + serverAssert(bgIteration_iterationActive()); + serverAssert(!server.cluster_enabled); + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (it->completed || it->terminated) continue; + + // Let the iterator internal mechanism know + it->keyset_iter->swapDb(it->keyset_iter, db1, db2); + + // Let the background client know + if (!(it->iteration_flags & BGITERATOR_FLAG_CONSISTENT)) { + if (BGITERATION_DEBUG) { + debugBuffer = sdscatprintf(debugBuffer, "SWAP: %d %d\n", db1, db2); + } + + bgIteratorItem *item = itemFreeList_getElementOrAllocate(); + item->type = BGITERATOR_ITEM_SWAPDB; + item->dbid = db1; + item->u.dbid2 = db2; + it->swapdb_queued++; + it->barrier_items++; + mutexQueueAdd(it->items_for_iterator, item); + } + } +} + + +static bool isDbSignificant(int dbid) { + unsigned long long totalKeys = 0; + for (int i = 0; i < server.dbnum; i++) { + totalKeys += (server.db[i]) ? dbSize(server.db[i]) : 0; + } + return (server.db[dbid]) ? (dbSize(server.db[dbid]) > totalKeys / 2) : false; +} + + +static void handleFlushdb(int dbid) { + // Invoked BEFORE the actual flush. -1 indicates FLUSHALL. + bool should_abort_iterators = (dbid == -1 || isDbSignificant(dbid)); + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + + // Let the low-level iterator know the DB is being flushed + it->keyset_iter->flushDb(it->keyset_iter, dbid); + + if (should_abort_iterators || it->iteration_flags & BGITERATOR_FLAG_CONSISTENT) { + if (!it->terminated) bgIteratorTerminate(it); + } else { + /* In this (limited) case, we're only flushing a single DB that contains < half the + * keys. We don't want to kill a full-sync replication. We will just continue with + * iteration, knowing that a replication client will also receive the FLUSHDB on the + * replication stream. There's no need to worry about the items themselves. Since + * we've incremented the refcount, the items still in queue won't be physically deleted. */ + + // Send a flushdb event to notify the client + if (BGITERATION_DEBUG) { + debugBuffer = sdscatprintf(debugBuffer, "FLUSH: %d\n", dbid); + } + bgIteratorItem *item = itemFreeList_getElementOrAllocate(); + item->type = BGITERATOR_ITEM_FLUSHDB; + item->dbid = dbid; + it->flushdb_queued++; + it->barrier_items++; + mutexQueueAdd(it->items_for_iterator, item); + } + } + receiveItemsBackFromIterators(false); // Receive items back before flushing the items +} + + +static bool expediteKeysForWriteOnAllIterators(int dbid, + struct serverCommand *cmd, + int argc, + robj **argv, + keyReference *keyrefs, + int numKeys, + hashtable *waitingOnKeys) { + bool mustBlock = false; + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (expediteKeysForWrite(it, dbid, cmd, argc, argv, keyrefs, numKeys, waitingOnKeys)) + mustBlock = true; + } + + return mustBlock; +} + + +static bool anIteratorWillReplicateForThisCommand(void) { + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (it->cur_cmd_may_replicate) return true; + } + return false; +} + + +static bool expediteKeysForMultiExec(client *c, hashtable *waitingOnKeys) { + serverAssert(c->cmd->proc == execCommand); + + /* For MULTI/EXEC, Valkey buffers all of the commands until hitting the EXEC. + * At this point, the client holds all of the commands to be executed. This function searches + * for all of the keys used by any of the buffered write commands. In addition, if SWAPDB or + * SELECT is used, this tracks the DBIDs through various swap/select operations. */ + + /* There's a special concern for a NON-consistent iteration with replication. If the keys are + * all "future" keys (which haven't been processed by the iterator yet), then we don't expedite + * the keys or replicate. However, if some keys have already been processed, we need to + * expedite the remaining keys and replicate everything. + * + * When processing a single command, this is all handled. But in this function, for MULTI/EXEC, + * we process 1 command at a time. There's an issue if the first command modifies a "future" + * key, we don't know (without reading ahead) if a later command will modify a prior key. This + * would require the future key to be expedited. + * + * This COULD be addressed by collecting all of the keys into a single structure and then + * analyzing them all at once. However, this won't share code well with the single commands. + * Also, building this structure is a little complex/time-consuming as we need to track both + * key AND dictID. One way to do this might be with a dict of dicts, where the first dict maps + * a dictID to a dict of keys. + * + * ALTERNATIVELY (and it's the simpler approach that's taken here) we can just check if the + * MULTI will be replicated. If so, we re-process the MULTI, just in case there were commands + * prior to deciding that replication was required that might have missed expediting. If so, + * these will be caught on the 2nd time around. + * + * Checking replication status before/after ensures that there can only be a single recursive + * call. */ + bool initiallyAnIteratorWillReplicate = anIteratorWillReplicateForThisCommand(); + + bool mustBlock = false; + int *cur_to_orig_db = NULL; + + int curDb = c->db->id; + for (int cmdNum = 0; cmdNum < c->mstate->count; cmdNum++) { + struct serverCommand *cmd = c->mstate->commands[cmdNum].cmd; + robj **argv = c->mstate->commands[cmdNum].argv; + int argc = c->mstate->commands[cmdNum].argc; + + if (cmd->proc == swapdbCommand) { + int id1, id2; + if (getParamsForSwapdb(argc, argv, c, &id1, &id2)) { + if (cur_to_orig_db == NULL) { + cur_to_orig_db = zmalloc(sizeof(int) * server.dbnum); + for (int i = 0; i < server.dbnum; i++) cur_to_orig_db[i] = i; + } + int temp = cur_to_orig_db[id1]; + cur_to_orig_db[id1] = cur_to_orig_db[id2]; + cur_to_orig_db[id2] = temp; + } + continue; + } + + if (cmd->proc == selectCommand) { + int id; + if (getParamsForSelect(argc, argv, c, &id)) { + curDb = id; + } + continue; + } + + if (!isWriteCmd(cmd)) continue; + + getKeysResult result; + initGetKeysResult(&result); + int numkeys = getKeysFromCommand(cmd, argv, argc, &result); + keyReference *keyrefs = result.keys; + if (numkeys == 0) { + getKeysFreeResult(&result); + continue; // Write command with no keys - like FLUSHDB + } + + if (expediteKeysForWriteOnAllIterators( + cur_to_orig_db ? cur_to_orig_db[curDb] : curDb, + cmd, argc, argv, keyrefs, numkeys, waitingOnKeys)) { + mustBlock = true; + } + getKeysFreeResult(&result); + } + + zfree(cur_to_orig_db); + + if (!initiallyAnIteratorWillReplicate && anIteratorWillReplicateForThisCommand()) { + /* We've decided to replicate. Re-process the MULTI/EXEC just once more to make sure that + * we didn't miss any keys at the beginning. This can't continue to recurse because + * `initiallyAnIteratorWillReplicate` will be TRUE in the recursive call. Note that the + * recursive call may add additional entries to `waitingOnKeys`. */ + if (expediteKeysForMultiExec(c, waitingOnKeys)) mustBlock = true; + } + + return mustBlock; +} + + +static bgIterator *bgIteratorCreate(const char *name, + bgIteratorConsistency consistency, + bgIteratorReplDoneFunc repldone, + bgIteratorCleanupFunc cleanup, + void *privdata, + bgIterationType iter_type, + genericIterator *keyset_iter) { + serverAssert(server.forkless_infrastructure_enabled); + serverAssert(hasMainThreadExclusivity()); + serverAssert(server.cluster_enabled || iter_type == BGITERATION_TYPE_FULLSCAN); + + int flags; + switch (consistency) { + case BGITERATOR_CONSISTENCY_NONE: flags = 0; break; + case BGITERATOR_CONSISTENCY_START: flags = BGITERATOR_FLAG_CONSISTENT; break; + case BGITERATOR_CONSISTENCY_EVENTUAL: flags = BGITERATOR_FLAG_REPLICATION; break; + default: serverAssert(false); + } + // Consistent, with replication - doesn't make sense. + serverAssert(!((flags & BGITERATOR_FLAG_CONSISTENT) && (flags & BGITERATOR_FLAG_REPLICATION))); + + bgIterator *it = zmalloc(sizeof(bgIterator)); + it->name = sdsnew(name); + it->repldone = repldone; + it->cleanup = cleanup; + it->privdata = privdata; + it->items_for_iterator = mutexQueueCreate(); + it->return_to_main_thread = mutexQueueCreate(); + + // Floor queue size to bgiteration_queue_increase_incr or use last queue size value + if (last_item_count_target < BGITER_QUEUE_INCREASE_INCR) { + last_item_count_target = BGITER_QUEUE_INCREASE_INCR; + } + it->item_count_target = last_item_count_target; + it->iteration_flags = flags; + it->iteration_type = iter_type; + it->consistent_modification_id = bgIteration_epoch++; + it->keyset_iter = keyset_iter; + it->early_iterate_entries = hashtableCreate(&dbEntryPtrHashtableType); + hashtableExpand(it->early_iterate_entries, BGITER_EARLY_ITERATE_DICT_INITIAL_SIZE); + it->current_item = NULL; + it->client_is_active = false; + it->completed = false; + it->terminated = false; + it->cur_cmd_may_replicate = false; + + it->dbentries_queued = 0; + it->dbentries_processed = 0; + it->replication_queued = 0; + it->replication_processed = 0; + it->swapdb_queued = 0; + it->swapdb_processed = 0; + it->flushdb_queued = 0; + it->flushdb_processed = 0; + it->dbentry_clones_queued = 0; + it->dbentry_clones_processed = 0; + + it->barrier_items = 0; + + elapsedStart(&it->monotonic_start_time); + it->monotonic_item_start_time = 0; + + + if (bgIterator_timeproc_id <= 0) { + // If iteration is not currently active, start the feeding task. (Runs in main thread.) + bgIterator_timeproc_id = aeCreateTimeEvent(server.el, 0, bgIteration_feedIterators_task, NULL, NULL); + serverAssert(bgIterator_timeproc_id != AE_ERR); + } + + if (dictAdd(nameToIterator, it->name, it) != DICT_OK) { + // Can't have 2 iterators with the same name! + serverAssert(false); + } + + listAddNodeTail(allIterators, it); + + dictExpand(inUseEntries, listLength(allIterators) * it->item_count_target); + + return it; +} + + +/* ============================================================================================= + * PUBLIC INTERFACE: Iterator creation and use + * ============================================================================================= */ + +// PUBLIC API +bgIterator *bgIteratorCreateFullScanIter(const char *name, + bgIteratorConsistency consistency, + bgIteratorReplDoneFunc repldone, + bgIteratorCleanupFunc cleanup, + void *privdata) { + return bgIteratorCreate(name, consistency, repldone, cleanup, privdata, + BGITERATION_TYPE_FULLSCAN, fullScanIteratorCreate()); +} + +// PUBLIC API +bgIterator *bgIteratorCreateSlotsIter(const char *name, + bgIteratorConsistency consistency, + const int *slots, + int slots_count, + bgIteratorReplDoneFunc repldone, + bgIteratorCleanupFunc cleanup, + void *privdata) { + return bgIteratorCreate(name, consistency, repldone, cleanup, privdata, + BGITERATION_TYPE_CLUSTERSLOT, clusterSlotIteratorCreate(slots, slots_count)); +} + +// PUBLIC API +bgIterator *bgIteratorFind(const char *name) { + serverAssert(hasMainThreadExclusivity()); + + sds sdsname = sdsnew(name); + bgIterator *it = dictFetchValue(nameToIterator, sdsname); + sdsfree(sdsname); + + return it; +} + + +// PUBLIC API +const char *bgIteratorName(bgIterator *it) { + return it->name; +} + + +// PUBLIC API +void bgIteratorGetStatus(bgIterator *it, bgIteratorStatus *status) { + status->dbentries_queued = it->dbentries_queued; + status->dbentries_processed = it->dbentries_processed; + status->replication_queued = it->replication_queued; + status->replication_processed = it->replication_processed; + status->swapdb_queued = it->swapdb_queued; + status->swapdb_processed = it->swapdb_processed; + status->flushdb_queued = it->flushdb_queued; + status->flushdb_processed = it->flushdb_processed; + status->dbentry_clones_queued = it->dbentry_clones_queued; + status->dbentry_clones_processed = it->dbentry_clones_processed; + + status->queue_length = mutexQueueLength(it->items_for_iterator); + status->queue_length_target = it->item_count_target; + + status->runtime_ms = elapsedMs(it->monotonic_start_time); + + monotime nonvolatile_item_start_time = it->monotonic_item_start_time; + status->current_item_ms = (nonvolatile_item_start_time == 0) + ? 0 + : elapsedMs(nonvolatile_item_start_time); +} + + +// PUBLIC API +void bgIteratorTerminate(bgIterator *it) { + serverAssert(hasMainThreadExclusivity()); + + // Remove any items in the queue, but doesn't affect the 1 item that's being processed. + returnAllItemsToMainThread(it); + + // We have to add an item, just in case the READER is waiting on the mutex. + if (BGITERATION_DEBUG) { + debugBuffer = sdscat(debugBuffer, "SENDING TERMINATE\n"); + } + + mutexQueueAdd(it->items_for_iterator, (void *)&STATIC_ITEM_TERMINATED); + + it->terminated = true; +} + + +// PUBLIC API +bool bgIteratorIsTerminating(bgIterator *it) { + return it->terminated; +} + + +// PUBLIC API +bgIteratorItem *bgIteratorRead(bgIterator *it) { + serverAssert(it->current_item == NULL || + (it->current_item->type != BGITERATOR_ITEM_COMPLETE && + it->current_item->type != BGITERATOR_ITEM_TERMINATED)); + + // First, clean up the previous item read + if (it->current_item != NULL) { + returnCurrentItemToMainThread(it); + + /* To support unit tests. Normal clients call bgIteratorRead from an alternate thread. + * Without this, a unit test could get stuck waiting on the completion event because + * feed won't get invoked. For production, feed is called regularly from the main thread. + * Note - this is checking that the exact same thread is used and shouldn't count modules. */ + if (onServerMainThread()) bgIteration_feedIterators_task(NULL, 0, NULL); + } else { + it->client_is_active = true; + } + + it->monotonic_item_start_time = 0; // idle until blocking pop returns + it->current_item = mutexQueuePop(it->items_for_iterator, true); + it->monotonic_item_start_time = getMonotonicUs(); + + return it->current_item; +} + + +// PUBLIC API +void bgIteratorClose(bgIterator *it) { + if (it->current_item != NULL) { + if (it->current_item->type == BGITERATOR_ITEM_COMPLETE || + it->current_item->type == BGITERATOR_ITEM_TERMINATED) { + // Normal confirmation of background completion + } else { + // Client is initiating the termination + it->terminated = true; + returnCurrentItemToMainThread(it); + + it->current_item = (bgIteratorItem *)&STATIC_ITEM_TERMINATED; + } + } else { + // terminated before first item read + it->terminated = true; + it->current_item = (bgIteratorItem *)&STATIC_ITEM_TERMINATED; + } + + mutexQueueAdd(it->return_to_main_thread, (void *)&STATIC_ITEM_ITER_CLOSED); +} + + +/* ============================================================================================= + * PUBLIC INTERFACE: Valkey main-thread support hooks + * ============================================================================================= */ + +// PUBLIC API +void bgIteration_init(void) { + serverAssert(hasMainThreadExclusivity()); + + /* This should be called once and only once from the Valkey main thread. However to support + * unit tests, this is not validated, and multiple invocations are ignored. */ + if (nameToIterator) return; // If already initialized, ignore (unit tests) + + nameToIterator = dictCreate(&sdsrefToPtrDictType); + serverAssert(nameToIterator != NULL); + + allIterators = listCreate(); + serverAssert(allIterators != NULL); + + inUseEntries = dictCreate(&dictEntryPtrDictType); + serverAssert(inUseEntries != NULL); + + curCmdMissingKeys = listCreate(); + serverAssert(curCmdMissingKeys != NULL); + listSetFreeMethod(curCmdMissingKeys, decrRefCountVoid); + + bufferedReplicationBytes = 0; + + if (BGITERATION_DEBUG) { + debugBuffer = sdsMakeRoomFor(sdsempty(), SDS_MAX_PREALLOC); + } +} + + +// PUBLIC API +bool bgIteration_iterationActive(void) { + return (allIterators != NULL && listLength(allIterators) > 0); +} + + +// PUBLIC API +void bgIteration_beforeSleep(void) { + if (!bgIteration_iterationActive()) return; + receiveItemsBackFromIterators(false); +} + + +// PUBLIC API +void bgIteration_keyDelete(int dbid, const_sds key) { + if (!bgIteration_iterationActive()) return; + serverAssert(hasMainThreadExclusivity()); + + if (BGITERATION_DEBUG) { + debugBuffer = sdscatprintf(debugBuffer, "KEYDEL: (%d)%s\n", dbid, key); + } + + dbEntry *de = dbFind(server.db[dbid], (sds)key); + serverAssert(de != NULL); // This API should be called BEFORE removal from main dict + + dbEntryPtrOfLastKeyDelete = de; // save for check at replication time + + // For consistent iterators, we need to make sure the item gets written before delete + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (it->completed || it->terminated || !it->keyset_iter->isKeyInScope(it->keyset_iter, key)) continue; + + if (it->iteration_flags & BGITERATOR_FLAG_CONSISTENT && + ((bgIterationEntryMetadata *)objectGetMetadata(de))->iterator_epoch <= it->consistent_modification_id) { + if (!iteratorHasPassedKey(it, dbid, key, de)) { + addEarlyIterationKey(it, de, dbid); // (may also add to inUseEntries) + } + } + } + + /* We might be within the context of a command execution. This happens if the key is found to + * be expired when attempting to execute the command. In this case, we should treat the key as + * missing. If the key exists after the command executes, we can treat it like a new key. */ + if (server.in_call) { + robj *oKey = createObject(OBJ_STRING, sdsdup(key)); + listAddNodeHead(curCmdMissingKeys, oKey); + } +} + + +// PUBLIC API +void bgIteration_flushall(void) { + handleFlushdb(-1); +} + + +// PUBLIC API +bool bgIteration_blockClientIfRequired(client *c) { + serverAssert(hasMainThreadExclusivity()); + iteratorReplicationFlagsWereUpdated = false; + if (!bgIteration_iterationActive()) return false; + if (!isWriteCmd(c->cmd)) return false; + + if (BGITERATION_DEBUG) { + sds sdsArgv = createSdsFromClientArgv(c->argc, c->argv); + debugBuffer = sdscatprintf(debugBuffer, "BLCK?: (%d)%s\n", c->db->id, sdsArgv); + sdsfree(sdsArgv); + } + + /* Before executing a command or atomic transaction, the replication flag is cleared for each + * iterator. If it's determined that the command should replicate, the flag will be set + * as the command and keys are examined for expedite. */ + resetReplicationFlagForIterators(c); + iteratorReplicationFlagsWereUpdated = true; + + if (c->cmd->proc == flushdbCommand || c->cmd->proc == flushallCommand) { + // Handle flush commands prior to execution + int flags; + if (parseFlushCommandFlags(c, &flags) == C_OK) { + // The command parsed ok - we WILL flush + handleFlushdb((c->cmd->proc == flushdbCommand) ? c->db->id : -1); + } + } + + bool mustBlock = false; + hashtable *waitOnKeys = hashtableCreate(&tempKeysetHashtableType); // set of robj(sds) + listEmpty(curCmdMissingKeys); + + if (c->cmd->proc == execCommand) { + mustBlock = expediteKeysForMultiExec(c, waitOnKeys); + } else { + getKeysResult result; + initGetKeysResult(&result); + int numkeys = getKeysFromCommand(c->cmd, c->argv, c->argc, &result); + keyReference *keyrefs = result.keys; + if (numkeys > 0) { + mustBlock = expediteKeysForWriteOnAllIterators( + c->db->id, c->cmd, c->argc, c->argv, keyrefs, numkeys, waitOnKeys); + // We shouldn't need to block on a command within a multi (that's not a script) + serverAssert(!(mustBlock && c->flag.multi && !c->flag.script)); + + if (mustBlock && (c->flag.script)) { + /* For scripts, we will block for keys declared in EVAL/EVALSHA/FCALL. + * However, scripts are NOT required to declare keys. Even if it declares keys, + * it's not declaring the DB for the key. After a SELECT or SWAPDB, we might be on + * a key we haven't blocked for. In this case, there is no option but to execute a + * synchronous block and wait for the iterator(s) to be done with the key(s). + * (Yuck.) */ + static const mstime_t SYNC_BLOCKING_LOG_INTERVAL = 60000; + static mstime_t last_log = 0; // STATIC: persists to prevent log spamming + static int blocked_count = 0; // STATIC: persistent count since last log + blocked_count++; + if (server.mstime - last_log > SYNC_BLOCKING_LOG_INTERVAL) { + serverLog(LL_WARNING, + "Forkless operation synchronously blocked %d times for scripts with undeclared keys", + blocked_count); + last_log = server.mstime; + blocked_count = 0; + } + + while (mustBlock) { + receiveItemsBackFromIterators(true); // Blocking + hashtableEmpty(waitOnKeys, NULL); + mustBlock = expediteKeysForWriteOnAllIterators( + c->db->id, c->cmd, c->argc, c->argv, keyrefs, numkeys, waitOnKeys); + } + } + } else { + // WRITE commands with no keys should always be replicated. SWAPDB, FLUSH, FUNCTION, etc. + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + it->cur_cmd_may_replicate = true; + } + } + getKeysFreeResult(&result); + } + + if (mustBlock) { + serverAssert(hashtableSize(waitOnKeys) > 0); + robj **waitKeysArgv = zmalloc(sizeof(robj *) * hashtableSize(waitOnKeys)); + + robj *key; + hashtableIterator hi; + hashtableInitIterator(&hi, waitOnKeys, 0); + unsigned long argvCount = 0; + while (hashtableNext(&hi, (void **)&key)) { + waitKeysArgv[argvCount++] = key; + } + hashtableCleanupIterator(&hi); + serverAssert(argvCount == hashtableSize(waitOnKeys)); + + blockClientInUseOnKeys(c, argvCount, waitKeysArgv); + + zfree(waitKeysArgv); + } + + hashtableRelease(waitOnKeys); + + if (BGITERATION_DEBUG) { + if (mustBlock) debugBuffer = sdscat(debugBuffer, " (blocked)\n"); + } + + return mustBlock; +} + + +// PUBLIC API +void bgIteration_handleCommandReplication(int dbid, + struct serverCommand *cmd, + int argc, + robj **argv) { + if (BGITERATION_DEBUG) { + // DEBUG - enable this to capture replication not queued because iteration is inactive + if (0 && !bgIteration_iterationActive() && (isWriteCmd(cmd) || cmd->proc == multiCommand)) { + sds sdsArgv = createSdsFromClientArgv(argc, argv); + debugBuffer = sdscatprintf(debugBuffer, "REPL? INACT: (%d)%s\n", dbid, sdsArgv); + sdsfree(sdsArgv); + } + } + + if (!bgIteration_iterationActive()) return; + serverAssert(onServerMainThread()); + + /* Some commands are replicated which are not writes (like publish) these can be ignored. + * Be careful with MULTI which is not a write command, but must be replicated. */ + if (!isWriteCmd(cmd) && cmd->proc != multiCommand) return; + + if (BGITERATION_DEBUG) { + sds sdsArgv = createSdsFromClientArgv(argc, argv); + debugBuffer = sdscatprintf(debugBuffer, "REPL?: (%d)%s\n", dbid, sdsArgv); + sdsfree(sdsArgv); + } + + if (cmd->proc == swapdbCommand) { + // All iterators and clients must be informed of swapdb + int id1, id2; + // command has been processed, but Valkey allows "swapdb 0 0" (which can be ignored) + if (getParamsForSwapdb(argc, argv, NULL, &id1, &id2)) + handleSwapdb(id1, id2); + } + + /* In the case that a key is touched in a different DB (COPY/MOVE) the key is recorded as + * a "special" key and than handled below. */ + int special_dbid = 0; + sds special_key = NULL; + dbEntry *special_dbEntry = NULL; + if (cmd->proc == moveCommand) { + /* The MOVE command succeeded. However MOVE requires special handling as it creates a new + * key in a different database. We need to make sure that we don't later try to iterate + * on the key as it would be a duplicate key at that point. So, instead, we will mark the + * newly created key as "early iterated". */ + bool success = getDbIdFromRobj(argv[MOVE_COMMAND_DBID_ARG_INDEX], &special_dbid); + serverAssert(success); // the command already succeeded, so this should work! + + robj *oKey = argv[1]; + special_key = (sds)objectGetVal(oKey); + + special_dbEntry = dbFind(server.db[special_dbid], special_key); + } + if (cmd->proc == copyCommand) { + // The COPY command succeeded. However COPY requires special handling (like MOVE). + bool success = getTargetDbIdForCopyCommand(argc, argv, dbid, &special_dbid); + serverAssert(success); // the command already succeeded, so this should work! + + // Find the newly created entry. + robj *oKey = argv[2]; + special_key = (sds)objectGetVal(oKey); + + special_dbEntry = dbFind(server.db[special_dbid], special_key); + } + + /* Implementation note regarding LUA and MULTI: LUA scripts and MULTI-EXEC blocks must be + * treated atomically. We need to ensure that either ALL of the replication (or none of the + * replication) for the atomic operation is processed by the iterator(s). This is handled + * naturally as we can only "complete" the iteration during the feeding process - and feeding + * is only performed when handling timer events (after the LUA/MULTI has completed). */ + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (it->completed || it->terminated) continue; + + /* For consistent iteration, we only iterate values based on version. But for + * non-consistent iteration, we don't need to explicitly iterate any values newly created + * during the iteration. So we mark them as expedited. We know we have a new key if it + * was missing before the command, and exists now. */ + + if (!(it->iteration_flags & BGITERATOR_FLAG_CONSISTENT)) { + // Handle the special case of a key moved to a different DB + if (special_dbEntry != NULL) { + if (it->cur_cmd_may_replicate && + !it->keyset_iter->hasPassedItem(it->keyset_iter, special_key, special_dbid)) { + hashtableAdd(it->early_iterate_entries, special_dbEntry); + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(special_dbid, special_dbEntry); + debugBuffer = sdscatprintf(debugBuffer, "EARLY(special): %s\n", entryString); + sdsfree(entryString); + } + } + + /* Note: In the cases where there's a special command, we are copying or moving an + * item to a different DB. In these limited cases, we can only possibly be + * creating a single key. And if we've handled it here, we don't need to + * handle it as a "missing key" below. If we were to try to handle it as a + * standard "missing key", we would get the DBID incorrect. */ + + } else if (listLength(curCmdMissingKeys) > 0) { + listIter missingIt; + listNode *missingNode; + listRewind(curCmdMissingKeys, &missingIt); + while ((missingNode = listNext(&missingIt)) != NULL) { + robj *oKey = listNodeValue(missingNode); + const_sds key = objectGetVal(oKey); + dbEntry *de = dbFind(server.db[dbid], (sds)key); + if (de != NULL) { + // It exists now! + if (it->cur_cmd_may_replicate && + !it->keyset_iter->hasPassedItem(it->keyset_iter, key, dbid)) { + /* If the current command is allowed to replicate, and there is a new + * key which we haven't yet reached in iteration, it needs to be added + * to the set of early iterate entries. (We know that it's not already + * in that set because it's a newly created key!) */ + bool wasAdded = hashtableAdd(it->early_iterate_entries, de); + serverAssert(wasAdded); + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(dbid, de); + debugBuffer = sdscatprintf(debugBuffer, "EARLY(NEW): %s\n", entryString); + sdsfree(entryString); + } + } + } + } + } + } + + /* Deletes (and unlinks) are special. + * Developer context: For most commands, we call bgIteration_blockClientIfRequired before + * the command and then call bgIteration_handleCommandReplication after the command. While + * the "before" logic is determining the need to block, it can also determine (mostly) the + * need for replication (on each iterator). Doing this all in one place saves us from + * performing some of the same logic twice. When we get to this point in the code, we just + * use the previously determined information regarding replication. This works because + * Valkey is single-threaded and only processes one command at a time. + * + * But deletes (and unlinks) happen multiple ways - and occur outside the normal + * before/after logic for commands. These situations must be handled: + * - A normal (client-driven) DEL/UNLINK command will use the standard before/after + * logic. If the key is in use by bgIteration, the command will be blocked. + * - An EVICTION generates a DEL/UNLINK which happens outside of the context of a client + * issued command. The replication flags on the iterators are stale and relate to the + * prior command executed. + * - An EXPIRATION in the context of a client-driven WRITE command occurs when the client + * command attempts to access a key and it is found to be expired. In this case, the + * client-command has already gone through the blocking process, so it should be OK to + * use it->cmd_may_replicate. + * - An EXPIRATION in the context of a client-driven READ command occurs when the client + * command attempts to access a key and it is found to be expired. In this case, the + * client-command has NOT gone through the blocking process. The replication flags on + * the iterators are stale and relate to the prior (write) command executed. + * - An EXPIRATION outside of a client-driven command occurs due to active expiry. In + * this case, the replication flags on the iterator are stale and relate to the prior + * command executed. + * + * In the case of EXPIRE/EVICT occurring outside the context of a write command, this is + * handled. If the key is in-use by bgIterator, increment of robj's refcount prevents the + * key from deletion. In this case the key will be removed from the main dictionary, but + * held by bgIteration until no longer needed. + * Even though the entry is not physically deleted yet, it is logically deleted and it is + * safe to replicate the DEL/UNLINK. Since iterators process items FIFO, the replication + * for DEL/UNLINK won't actually get processed until other queued replication is processed. + * + * In the case of a client driven DEL command, the key will have already been deleted when + * we hit this routine. In the case of EXPIRE/EVICT, they propagate happens before the key + * is deleted. So if the key is missing, we can use the cached replication decision. But + * if the key still exists (indicating EXPIRE/EVICT) we evaluate it specially. */ + bool shouldReplicateDelCommand = false; + bool isDelCommand = isDeleteCmd(cmd); + if (isDelCommand) { + sds key = objectGetVal(argv[1]); + dbEntry *de = dbFind(server.db[dbid], key); + serverAssert(de == NULL); // dbEntry should be removed before replication (self-check) + if (it->keyset_iter->isKeyInScope(it->keyset_iter, key)) { + bool blockClientIfRequiredWasCalled = (server.in_call > 0); + if (blockClientIfRequiredWasCalled && iteratorReplicationFlagsWereUpdated) { + // Here we know that the DEL is related to the running command + shouldReplicateDelCommand = it->cur_cmd_may_replicate; + } else { + // Otherwise, it's something like active expiration or eviction (unrelated) + if (iteratorHasPassedKey(it, dbid, key, dbEntryPtrOfLastKeyDelete)) { + shouldReplicateDelCommand = true; + } + } + hashtableDelete(it->early_iterate_entries, dbEntryPtrOfLastKeyDelete); // just try delete (might not be here) + } + } + + bool replicate = (it->iteration_flags & BGITERATOR_FLAG_REPLICATION && + ((!isDelCommand && it->cur_cmd_may_replicate) || shouldReplicateDelCommand)); + + if (replicate) { + /* We will replicate the command in these cases: + * 1) For consistent iteration - it->cur_cmd_may_replicate is always true + * 2) For non-consistent, if any of the keys have been processed, expediteKeysForWrite + * will ensure that ALL of the keys have been expedited - and we should replicate + * 3) For non-consistent, if NONE of the keys have been processed, no need to replicate */ + if (BGITERATION_DEBUG) { + debugBuffer = sdscat(debugBuffer, " (queued)\n"); + } + + bgIteratorItem *item = itemFreeList_getElementOrAllocate(); + item->type = BGITERATOR_ITEM_REPLICATION; + item->dbid = dbid; + item->u.repl.cmd = cmd; + item->u.repl.argv = cloneRobjArray(argc, argv); + item->u.repl.argc = argc; + item->u.repl.replication_size = replicationItemSize(item); + bufferedReplicationBytes += item->u.repl.replication_size; + it->replication_queued++; + mutexQueueAdd(it->items_for_iterator, item); + } + } // allIterators loop +} + + +// PUBLIC API +size_t bgIteration_memoryInuseForReplication(void) { + return bufferedReplicationBytes; +} + + +// PUBLIC API +bool bgIteration_isEntryInuse(dbEntry *de) { + serverAssert(hasMainThreadExclusivity()); + if (!bgIteration_iterationActive()) return false; + return isEntryInuseByAnyIterator(de); +} + + +// PUBLIC API +void bgIteration_dbEntryModified(dbEntry *de) { + if (bgIteration_iterationActive()) { + bgIterationEntryMetadata *md = (bgIterationEntryMetadata *)objectGetMetadata(de); + if (md) md->iterator_epoch = bgIteration_epoch; + } +} + + +// PUBLIC API +void bgIteration_keyModified(int dbid, const_sds key) { + if (bgIteration_iterationActive()) { + dbEntry *de = dbFind(server.db[dbid], (sds)key); + if (de) bgIteration_dbEntryModified(de); + } +} + + +// PUBLIC API +void bgIteration_updateDbEntryPtr(dbEntry *old, dbEntry *new) { + if (!bgIteration_iterationActive() || old == new) return; + serverAssert(hasMainThreadExclusivity()); + serverAssert(!isEntryInuseByAnyIterator(old)); + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (hashtableDelete(it->early_iterate_entries, old)) { + if (BGITERATION_DEBUG) { + debugBuffer = sdscatprintf(debugBuffer, "EARLY LIST UPDATE %p -> %p\n", (void *)old, (void *)new); + } + bool wasAdded = hashtableAdd(it->early_iterate_entries, new); + serverAssert(wasAdded); + } + } +} diff --git a/src/bgiteration.h b/src/bgiteration.h new file mode 100644 index 000000000..f40e7ddde --- /dev/null +++ b/src/bgiteration.h @@ -0,0 +1,366 @@ +/* + * Copyright Valkey Contributors. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __BGITERATION_H +#define __BGITERATION_H + +#include +#include "sds.h" + +/* A mechanism for creating iteration clients which iterate over the main dictionary in a + * background thread. + * + * This mechanism passes keys to the iteration client, while blocking the keys from write by the + * Valkey main thread. Once an iteration client is done with a key, it is returned to the Valkey + * main thread and any pending writers are unblocked. + * + * A bgIterator must be created on the main Valkey thread, and then passed to another thread which + * implements the logic of the iteration client. + * + * Iteration clients are expected to read through the keyspace until the iteration is complete or + * terminated. An iteration client may not perform modifications on a key. */ + +/* Avoids dependency on server.h */ +typedef struct serverObject dbEntry; // An object with key/value inserted into main dictionary +typedef struct serverObject robj; // An object with a value used for command parameters +typedef struct client client; + +/* The bgIterator is an opaque structure. */ +typedef struct bgIterator bgIterator; + + +/* Consistency type for iteration. */ +typedef enum { + /* With no consistency requirements, dbEntries are provided to the iteration client as they + * appear at the time of iteration. No replication is provided. The only guarantee is that + * dbEntries which existed at the start of iteration, and remained through the duration of + * iteration, will be provided to the iteration client once (and only once). If a dbEntry is + * modified during iteration, either the old or the new value may be provided. */ + BGITERATOR_CONSISTENCY_NONE = 0, + + /* With consistency at the start of iteration, a point-in-time iteration is performed. The + * iteration client will see all keys AS THEY EXISTED at the time when the iterator was created. + * Note: The DBID provided with the DICTENTRY events is the original DBID (at the time of iteration + * start). SWAPDB events will not be provided. */ + BGITERATOR_CONSISTENCY_START = 1, + + /* With an eventually consistent iteration, dbEntries will be followed by relevant replication. + * This will allow a client to achieve a consistent state at the END of the iteration. Once a + * dbEntry has been provided to the iteration client, any replication related to that entry will + * also be forwarded to the iteration client. With eventual consistency, keys are provided as + * they are at the time of iteration. This mode requires that the iteration client be aware of + * SWAPDB events. If a SWAPDB is performed, the client will receive a SWAPDB event. + * Replication events will be provided ordered and synchronized with any SWAPDB events. */ + BGITERATOR_CONSISTENCY_EVENTUAL = 2 +} bgIteratorConsistency; + + +/* When running an iterator with replication, a replication-done function (callback) may be + * provided. This function will be executed after the last replication item has been fed into the + * queue for the client. This function will be run on the Valkey main thread, and allows a client + * to recognize the point where no additional replication data will be sent for processing. + * + * PRIVDATA: this pointer is for data private to the iteration client. + * + * Returns true when an iterator stops accepting any replication item into the queue for the client. + * If false is returned, replication will continue, and bgiteration will periodically call the callback + * until true is returned. In this context, returning false indicates that the client is not ready to + * stop receiving replication, it is requesting that replication be continued. */ +typedef bool (*bgIteratorReplDoneFunc)(void *privdata); + + +/* When creating a bgIterator, a cleanup function (callback) may be provided. This function will be + * executed once iteration has completed and this will run on the Valkey main thread. + * + * TERMINATED: will be passed as TRUE if the iteration process was terminated early (either by + * the main thread calling bgIteratorTerminate() or the iteration client calling + * bgIteratorClose()). + * PRIVDATA: this pointer is for data private to the iteration client. */ +typedef void (*bgIteratorCleanupFunc)(bool terminated, void *privdata); + + +/* Create a background full-scan iterator (bgIterator). + * This bgIterator will iterate through the entire keyspace (across all DBs). + * + * NAME: a human readable name for the iterator (must be unique) + * CONSISTENCY: the consistency guarantee for the iteration + * REPLDONE: if provided, called after the last replication item has been queued (on the Valkey main thread) + * CLEANUP: if provided, called at the end of iteration (on the Valkey main thread) + * PRIVDATA: passed to cleanup function + * + * This method creates and initializes the bgIterator. It does not perform any thread management. + * It is expected that the main Valkey thread will call this method, and then start a new thread to + * to implement the iteration client which will read from the returned bgIterator. + * + * There is no need to delete/destroy a bgIterator. It will automatically be cleaned up after the + * last item is read. */ +bgIterator *bgIteratorCreateFullScanIter( + const char *name, + bgIteratorConsistency consistency, + bgIteratorReplDoneFunc repldone, + bgIteratorCleanupFunc cleanup, + void *privdata); + + +/* Create a background slots iterator (bgIterator). + * This bgIterator will iterate through the keys belonging to a set of cluster slots. + * + * NAME: a human readable name for the iterator (must be unique) + * CONSISTENCY: the consistency guarantee for the iteration + * SLOTS: array of cluster slots to iterate over + * SLOTS_COUNT: size of the array of slots + * REPLDONE: if provided, called after the last replication item has been queued (on the Valkey main thread) + * CLEANUP: if provided, called at the end of iteration (on the Valkey main thread) + * PRIVDATA: passed to cleanup function + * + * This method creates and initializes the bgIterator. It does not perform any thread management. + * It is expected that the main Valkey thread will call this method, and then start a new thread to + * to implement the iteration client which will read from the returned bgIterator. + * + * The caller of this function has the ownership of the `slots` array's memory. This function will + * just copy its data and leave the array untouched. + * + * There is no need to delete/destroy a bgIterator. It will automatically be cleaned up after the + * last item is read. */ +bgIterator *bgIteratorCreateSlotsIter( + const char *name, + bgIteratorConsistency consistency, + const int *slots, + int slots_count, + bgIteratorReplDoneFunc repldone, + bgIteratorCleanupFunc cleanup, + void *privdata); + + +/* Find an existing bgIterator by name. + * Returns NULL if the iterator does not exist (or has completed). */ +bgIterator *bgIteratorFind(const char *name); + + +/* Get the name of an existing iterator. */ +const char *bgIteratorName(bgIterator *iter); + + +/* Struct to retrieve status information for an active iteration client. */ +typedef struct { + unsigned long dbentries_queued; // Cumulative BGITERATOR_ITEM_DBENTRY queued + unsigned long dbentries_processed; // Cumulative BGITERATOR_ITEM_DBENTRY processed + unsigned long replication_queued; // Cumulative BGITERATOR_ITEM_REPLICATION queued + unsigned long replication_processed; // Cumulative BGITERATOR_ITEM_REPLICATION processed + unsigned long swapdb_queued; // Cumulative BGITERATOR_ITEM_SWAPDB queued + unsigned long swapdb_processed; // Cumulative BGITERATOR_ITEM_SWAPDB processed + unsigned long flushdb_queued; // Cumulative BGITERATOR_ITEM_FLUSHDB queued + unsigned long flushdb_processed; // Cumulative BGITERATOR_ITEM_FLUSHDB processed + unsigned long dbentry_clones_queued; // A subset of dbentries_queued for cloned entries + unsigned long dbentry_clones_processed; // A subset of dbentries_processed for cloned entries + unsigned long queue_length; // Current length of queue to iteration client + unsigned long queue_length_target; // Dynamic target length for queue to iteration client + unsigned long runtime_ms; // Time, in milliseconds, that iterator has been running + unsigned long current_item_ms; // Time, in milliseconds, spent processing current item +} bgIteratorStatus; + + +/* Get the status of a background iteration. + * + * The caller-provided bgIteratorStatus will be populated. */ +void bgIteratorGetStatus(bgIterator *iter, bgIteratorStatus *status); + + +/* Terminate a background iteration. + * + * An iteration is terminated by the Valkey main thread. It is expected that the iteration client + * will continue to read, receiving BGITERATOR_ITEM_TERMINATED or BGITERATOR_ITEM_COMPLETE to + * complete the iteration. (This is necessary to ensure proper cleanup.) + * NOTE: If the iteration client wants to terminate iteration, it may call bgIteratorClose(). */ +void bgIteratorTerminate(bgIterator *iter); + + +/* Check if an iterator is being terminated. + * + * This checks if the iterator is in the process of terminating. For the Valkey main thread, this + * can be used to determine if a call has already been made to bgIteratorTerminate. For an + * iteration client, it normally learns about terminate by reading the next item, this allows + * out-of-band detection of termination which can be useful when processing a large key. */ +bool bgIteratorIsTerminating(bgIterator *iter); + + +typedef enum { + /* Indicates that the iteration has completed normally. No more items to read. + * If replication is enabled, on completion, the final replication offset is recorded in + * 'u.master_repl_offset' and 'dbid' is set to the selected replication db. The iteration + * client will have received all *applicable* replication data to this point. */ + BGITERATOR_ITEM_COMPLETE = 1, + + /* Indicates that the iteration has been terminated before completion. No more items to read.*/ + BGITERATOR_ITEM_TERMINATED, + + /* A dbEntry for DB=dbid. + * NOTE: The dbEntry MAY be expired. It is up to the client to decide how to handle + * expired entries. */ + BGITERATOR_ITEM_DBENTRY, + + /* A replication command for DB=dbid. cmd, argv, & argc provided. + * NOTE: The command may have been re-written before replication. */ + BGITERATOR_ITEM_REPLICATION, + + /* A SWAPDB event. dbid swapped with dbid2. + * Note that SWAPDB events are not provided during consistent iteration. */ + BGITERATOR_ITEM_SWAPDB, + + /* A FLUSHDB event. In most cases, iteration will be terminated, and this event will NOT be + * sent. However, in the case of a single minor DB being flushed, non-consistent iteration is + * permitted to continue. */ + BGITERATOR_ITEM_FLUSHDB +} bgIteratorItemType; + + +typedef struct { + dbEntry *de; + bool is_cloned; + bool is_rehashing_paused; +} dbEntryData; + +typedef struct { + struct serverCommand *cmd; + robj **argv; + int argc; + size_t replication_size; +} replicationData; + +typedef struct { + bgIteratorItemType type; + int dbid; // orig DB ID for CONSISTENT, queue-time DB ID for !CONSISTENT. + union { + dbEntryData dbe; // for BGITERATOR_ITEM_DBENTRY + replicationData repl; // for BGITERATOR_ITEM_REPLICATION + long long master_repl_offset; // for BGITERATOR_ITEM_COMPLETE + int dbid2; // for BGITERATOR_ITEM_SWAPDB + } u; +} bgIteratorItem; + + +/* Read the next bgIteratorItem from the bgIterator. + * + * The iteration client is expected to call this function in a loop. After reading + * BGITERATOR_ITEM_COMPLETE or BGITERATOR_ITEM_TERMINATED, the iteration client must call + * bgIteratorClose to finalize the iteration process. + * + * This is a blocking call. If the main Valkey thread has been too busy to send items to the + * iterator, the iteration client's queue may run dry and this call will block until data is + * available. + * + * NOTE: Reading an item returns previously read items to the main thread. It is unsafe to + * reference an item previously read. + * + * (All memory management is the responsibility of the bgIterator - not the reader.) */ +bgIteratorItem *bgIteratorRead(bgIterator *iter); + + +/* Close the bgIterator, allowing the bgIterator to be deallocated. + * + * This must be called by an iteration client to release the bgIterator. + * + * It is required that this is called after receiving BGITERATOR_ITEM_COMPLETE or + * BGITERATOR_ITEM_TERMINATED and signals that the background activity is complete. + * + * This may also be called by the iteration client to force terminate an iteration early. The + * bgIterator will be marked as terminated. */ +void bgIteratorClose(bgIterator *iter); + + +/******************************************************************************************** + * BGITERATION HOOKS REQUIRED TO SUPPORT ITERATION - CALLS INSERTED INTO MAIN VALKEY CODE + ********************************************************************************************/ + +/* Size of bgIterationEntryMetadata (internal to bgiteration.c) */ +#define BGITERATION_ENTRY_METADATA_SIZE 4 + +/* Must be called once (and only once) at server startup. */ +void bgIteration_init(void); + + +/* Returns true if any iterators are currently active. */ +bool bgIteration_iterationActive(void); + + +/* Called as a beforeSleep action, receives items back from bgIteration. This is just a little + * quicker than waiting for bgIteration's internal timer. */ +void bgIteration_beforeSleep(void); + + +/* Notify bgIteration that a key is about to be deleted. This call must happen before the removal + * from the main dictionary. In Valkey, key deletion can occur in a READ command if the key is + * expired. Note that this notification is more about status than memory. Since the dbEntry is a + * reference counted object, the dbEntry can't be physically deleted if bgIteration is still + * actively using it. */ +void bgIteration_keyDelete(int dbid, const_sds key); + + +/* Iteration needs to know if a FLUSHALL is being performed. For normal clients, this comes through + * the standard "blockClientIfRequired" interface. This interface is for cases where Valkey + * performs the FLUSHALL operation independently of clients (e.g. when syncing with master). */ +void bgIteration_flushall(void); + + +/* Updating value or expiration of an existing key may lead to reallocation of the dbEntry (robj). + * BgIteration keeps track of expedited keys (by pointer) to avoid repeated iteration. BgIteration + * must be notified when dbEntries are reallocated. BgIteration will not dereference the pointers; + * it is safe to have deallocated the old dbEntry before calling this function. + * + * We can't update the dbEntry if the entry is actually in use (bgIteration_isEntryInuse)! + * + * To simplify calling code, this function does nothing if old_entry == new_entry. */ +void bgIteration_updateDbEntryPtr(dbEntry *old_entry, dbEntry *new_entry); + + +/* Before executing any command, the Valkey main thread must call this function. If the key(s) are + * blocked for writes by an iterator, the function returns true and the client is blocked. A + * blocked client will be unblocked once the key becomes available for write. + * + * This should be called for all commands - even commands which are executed as part of a MULTI/EXEC + * or LUA script. + * + * For MULTI/EXEC - This function is called when hitting the EXEC - after all of the commands + * have been queued. This may block the EXEC, but will NOT block individual + * commands as they are executed in the MULTI/EXEC block. + * + * For LUA script - This function is first called for EVAL/EVALSHA. It may block the script while + * waiting on declared keys. However, if the script accesses undeclared keys or + * performs SWAPDB, a synchronous block may be performed (returning false) on + * individual commands within the script. + * + * Note: this function should be called for all commands (not just writes). */ +bool bgIteration_blockClientIfRequired(client *c); + + +/* After execution of a write command, the Valkey main thread must provide the command to iterators + * which are interested in the replication feed. It is required that all commands have been passed + * through bgIteration_blockClientIfRequired(), however, it is permitted that the command can be + * re-written for propagation. */ +void bgIteration_handleCommandReplication( + int dbid, + struct serverCommand *cmd, + int argc, + robj **argv); + + +/* The memory that bgIteration uses while temporarily buffering replication data is not included in + * the maxmemory computation used for eviction. This function provides insight into the current + * amount of memory used for buffered replication data. */ +size_t bgIteration_memoryInuseForReplication(void); + + +/* Check if a dbEntry is currently in-use/locked by bgIteration. */ +bool bgIteration_isEntryInuse(dbEntry *de); + + +/* Notify bgIteration that a dbEntry has been added/modified. + * - If caller has a dbEntry*, dbEntryModified is more efficient + * - If caller has a dbid/key, a lookup is performed to find the dbEntry */ +void bgIteration_dbEntryModified(dbEntry *de); +void bgIteration_keyModified(int dbid, const_sds key); + +#endif diff --git a/src/bio.c b/src/bio.c index 801f09b50..aaf17b151 100644 --- a/src/bio.c +++ b/src/bio.c @@ -68,6 +68,7 @@ #include "server.h" #include "connection.h" +#include "cluster.h" #include "bio.h" #include "mutexqueue.h" #include "tls.h" @@ -80,6 +81,7 @@ static unsigned int bio_job_to_worker[] = { [BIO_LAZY_FREE] = 2, [BIO_RDB_SAVE] = 3, [BIO_TLS_RELOAD] = 4, /* only used when BUILD_TLS=yes */ + [BIO_CLUSTER_SAVE] = 5, }; typedef struct { @@ -94,6 +96,7 @@ static bio_worker_data bio_workers[] = { {"bio_lazy_free"}, {"bio_rdb_save"}, {"bio_tls_reload"}, /* only used when BUILD_TLS=yes */ + {"bio_cluster_config_save"}, }; static const bio_worker_data *const bio_worker_end = bio_workers + (sizeof bio_workers / sizeof *bio_workers); @@ -140,6 +143,12 @@ typedef union bio_job { struct { int type; } tls_reload_args; + + struct { + int type; + sds content; /* Cluster config file content. */ + bool do_fsync; /* A flag to indicate that a fsync is required. */ + } cluster_save_args; } bio_job; void *bioProcessBackgroundJobs(void *arg); @@ -238,6 +247,14 @@ void bioCreateTlsReloadJob(void) { bioSubmitJob(BIO_TLS_RELOAD, job); } +void bioCreateClusterConfigSaveJob(sds content, bool do_fsync) { + bio_job *job = allocBioJob(0); + job->cluster_save_args.content = content; + job->cluster_save_args.do_fsync = do_fsync; + bioSubmitJob(BIO_CLUSTER_SAVE, job); +} + +#define CONFIG_SAVE_LOG_ERROR_RATE 30 /* Seconds between errors logging. */ void *bioProcessBackgroundJobs(void *arg) { bio_worker_data *const bwd = arg; sigset_t sigset; @@ -307,6 +324,21 @@ void *bioProcessBackgroundJobs(void *arg) { #else serverPanic("BIO_TLS_RELOAD job type requires built-in TLS (BUILD_TLS=yes)."); #endif + } else if (job_type == BIO_CLUSTER_SAVE) { + static time_t last_save_error_log = 0; + if (clusterSaveConfigFromBio(job->cluster_save_args.content, job->cluster_save_args.do_fsync) == C_ERR) { + /* Limit logging rate to 1 line per CONFIG_SAVE_LOG_ERROR_RATE seconds. */ + if ((server.unixtime - last_save_error_log) > CONFIG_SAVE_LOG_ERROR_RATE) { + serverLog(LL_WARNING, "Failed to save the cluster config file in background. Cluster config " + "updated even though writing the cluster config file to disk failed."); + last_save_error_log = server.unixtime; + } + atomic_store_explicit(&server.cluster_config_save_status, C_ERR, memory_order_relaxed); + } else { + atomic_store_explicit(&server.cluster_config_save_status, C_OK, memory_order_relaxed); + atomic_store_explicit(&server.cluster_config_last_save_time, time(NULL), memory_order_relaxed); + last_save_error_log = 0; + } } else { serverPanic("Wrong job type in bioProcessBackgroundJobs()."); } diff --git a/src/bio.h b/src/bio.h index 70d77cf0e..2ebfe05be 100644 --- a/src/bio.h +++ b/src/bio.h @@ -43,6 +43,7 @@ void bioCreateFsyncJob(int fd, long long offset, int need_reclaim_cache); void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...); void bioCreateSaveRDBToDiskJob(connection *conn, int is_dual_channel); void bioCreateTlsReloadJob(void); +void bioCreateClusterConfigSaveJob(sds content, bool do_fsync); int inBioThread(void); /* Background job opcodes */ @@ -53,6 +54,7 @@ enum { BIO_CLOSE_AOF, /* Deferred close for AOF files. */ BIO_RDB_SAVE, /* Deferred save RDB to disk on replica */ BIO_TLS_RELOAD, /* Deferred TLS reload. */ + BIO_CLUSTER_SAVE, /* Deferred cluster config file save and fsync. */ BIO_NUM_OPS }; diff --git a/src/blocked.c b/src/blocked.c index 4c3a856bc..e8b1c8176 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -77,6 +77,7 @@ static void handleClientsBlockedOnKey(readyList *rl); static void unblockClientOnKey(client *c, robj *key); static void moduleUnblockClientOnKey(client *c, robj *key); static void releaseBlockedEntry(client *c, dictEntry *de, int remove_key); +static void unlinkBlockInUseClient(client *c); void initClientBlockingState(client *c) { if (c->bstate) return; @@ -164,7 +165,9 @@ void processUnblockedClients(void) { serverAssert(ln != NULL); c = ln->value; listDelNode(server.unblocked_clients, ln); + serverAssert(c->flag.module || !c->flag.blocked); c->flag.unblocked = 0; + serverAssert(!c->flag.throttled); if (c->flag.module) { if (!c->flag.blocked) { @@ -173,16 +176,16 @@ void processUnblockedClients(void) { continue; } - /* Process remaining data in the input buffer, unless the client - * is blocked again. Actually processInputBuffer() checks that the - * client is not blocked before to proceed, but things may change and - * the code is conceptually more correct this way. */ - if (!c->flag.blocked) { - /* If we have a queued command, execute it now. */ - if (processPendingCommandAndInputBuffer(c) == C_ERR) { + if (c->conn && !connHasReadHandler(c->conn)) { + if (connSetReadHandler(c->conn, readQueryFromClient) == C_ERR) { + freeClient(c); continue; } } + /* If we have a queued command, execute it now. */ + if (processPendingCommandAndInputBuffer(c) == C_ERR) { + continue; + } beforeNextClient(c); } } @@ -215,20 +218,31 @@ void queueClientForReprocessing(client *c) { /* Unblock a client calling the right function depending on the kind * of operation the client is blocking for. */ void unblockClient(client *c, int queue_for_reprocessing) { - if (c->bstate->btype == BLOCKED_LIST || c->bstate->btype == BLOCKED_ZSET || c->bstate->btype == BLOCKED_STREAM) { + switch (c->bstate->btype) { + case BLOCKED_LIST: + case BLOCKED_ZSET: + case BLOCKED_STREAM: unblockClientWaitingData(c); - } else if (c->bstate->btype == BLOCKED_WAIT) { + break; + case BLOCKED_WAIT: unblockClientWaitingReplicas(c); - } else if (c->bstate->btype == BLOCKED_MODULE) { + break; + case BLOCKED_MODULE: if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c); unblockClientFromModule(c); - } else if (c->bstate->btype == BLOCKED_POSTPONE) { + break; + case BLOCKED_POSTPONE: serverAssert(c->bstate->postponed_list_node); listDelNode(server.postponed_clients, c->bstate->postponed_list_node); c->bstate->postponed_list_node = NULL; - } else if (c->bstate->btype == BLOCKED_SHUTDOWN) { + break; + case BLOCKED_SHUTDOWN: /* No special cleanup. */ - } else { + break; + case BLOCKED_INUSE: + unlinkBlockInUseClient(c); + break; + default: serverPanic("Unknown btype in unblockClient()."); } @@ -338,6 +352,10 @@ void disconnectOrRedirectAllBlockedClients(void) { * which the command is already in progress in a way. */ if (c->bstate->btype == BLOCKED_POSTPONE) continue; + /* BLOCKED_INUSE clients will reprocess their command when unblocked + * by the caller. Sending error replies here would be incorrect. */ + if (c->bstate->btype == BLOCKED_INUSE) continue; + if (server.cluster_enabled) { if (clusterRedirectBlockedClientIfNeeded(c)) unblockClientOnError(c, NULL); @@ -478,10 +496,10 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo } } c->bstate->unblock_on_nokey = unblock_on_nokey; - /* Currently we assume key blocking will require reprocessing the command. - * However in case of modules, they have a different way to handle the reprocessing - * which does not require setting the pending command flag */ - if (btype != BLOCKED_MODULE) c->flag.pending_command = 1; + /* Key-blocked clients require pending_command for reprocessing on unblock. + * The caller must have set it (processInputBuffer for real clients, + * RM_Call for module fake clients). */ + serverAssert(c->flag.pending_command == 1); blockClient(c, btype); } @@ -692,8 +710,7 @@ void blockPostponeClient(client *c) { listAddNodeTail(server.postponed_clients, c); serverAssert(c->bstate->postponed_list_node == NULL); c->bstate->postponed_list_node = listLast(server.postponed_clients); - /* Mark this client to execute its command */ - c->flag.pending_command = 1; + serverAssert(c->flag.pending_command == 1); } /* Block client due to shutdown command */ @@ -724,7 +741,6 @@ static void unblockClientOnKey(client *c, robj *key) { /* In case this client was blocked on keys during command * we need to re process the command again */ if (c->flag.pending_command) { - c->flag.pending_command = 0; c->flag.reexecuting_command = 1; /* We want the command processing and the unblock handler (see RM_Call 'K' option) * to run atomically, this is why we must enter the execution unit here before @@ -803,6 +819,197 @@ void unblockClientOnError(client *c, const char *err_str) { unblockClient(c, 1); } +/* ========================== BlockInUse ==================================== + * + * Client blocking mechanism for keys currently being processed by a + * background thread. + * + * Note: All blockInUse APIs must be called from the main thread only. + * + * This uses the BLOCKED_INUSE blocking type (via blockClient()) and tracks + * blocked keys per client in c->bstate->keys and a key→clients mapping in + * a static hashtable (inuse_key_to_clients). + * + * Workflow: + * 1. blockClientInUseOnKeys() blocks the client via + * blockClient(c, BLOCKED_INUSE) and records mappings in both + * c->bstate->keys and inuse_key_to_clients. + * 2. unblockClientsInUseOnKey() unblocks a single key. A client remains + * blocked until all its keys are unblocked. + * 3. A client is fully unblocked only when it has no remaining keys in its + * c->bstate->keys dict. + * 4. processUnblockedClients() restores the read handler and resumes the + * pending command. + */ + +/* Internal blockInUse data structures. + * + * Clients are blocked on key name, regardless of DB. This avoids complexity + * with DB swaps. A BLOCKED_INUSE client may get unblocked early due to + * unblocking a key with the same name on a different DB. In this case, the + * client will get reblocked when attempting to reprocess the command. */ +static hashtable *inuse_key_to_clients; /* Maps keys to keyToClientsEntry. */ + +/* ----------------------------- key_to_clients Hashtable Util ------------------------- */ + +typedef struct { + robj *key; + list *clients; +} keyToClientsEntry; + +// hashtable callback, returns an robj containing a string +static const void *keyToClientsGetKey(const void *entry) { + return ((keyToClientsEntry *)entry)->key; +} + +// hashtable callback +static void keyToClientsDestructor(void *entry) { + keyToClientsEntry *e = entry; + decrRefCount(e->key); + listRelease(e->clients); + zfree(e); +} + +static hashtableType keyToClientsHashtableType = { + .entryGetKey = keyToClientsGetKey, + .hashFunction = dictEncObjHash, + .keyCompare = dictEncObjKeyCompare, + .entryDestructor = keyToClientsDestructor, +}; + +// Return the list of clients blocked on key, or NULL if none exist. +static list *keyToClients_getBlockedClientsList(robj *key) { + if (!inuse_key_to_clients) return NULL; + keyToClientsEntry *entry; + if (hashtableFind(inuse_key_to_clients, key, (void **)&entry)) { + return entry->clients; + } + return NULL; +} + +/* Create a new keyToClientsEntry for key, add it to key_to_clients, + * and return its clients list. Precondition: the key must not already exist. */ +static list *keyToClients_addEntry(robj *key) { + keyToClientsEntry *entry = zcalloc(sizeof(keyToClientsEntry)); + entry->key = key; + incrRefCount(key); + entry->clients = listCreate(); + serverAssert(hashtableAdd(inuse_key_to_clients, entry)); + return entry->clients; +} + +/* ----------------------------- blockInUse API ----------------------------- */ + +static bool isClientBlockedInUse(client *c) { + return c->flag.blocked && c->bstate->btype == BLOCKED_INUSE; +} + +/* Block a client on a set of keys. Duplicate keys are deduplicated. + * + * Each key robj must contain an sds string value. Keys are simple names, + * independent of DB — a client may be unblocked early if the same key name + * in another DB is unblocked. + * + * The client remains blocked until ALL of its keys are unblocked via + * unblockClientsInUseOnKey(). + * + * The caller should return without executing the command after calling this. */ +void blockClientInUseOnKeys(client *c, int num_keys, robj *keys[]) { + serverAssert(!c->flag.blocked && !c->flag.unblocked); + serverAssert(c->flag.pending_command == 1); + serverAssert(num_keys > 0); + serverAssert(!c->flag.replica); + + if (!inuse_key_to_clients) inuse_key_to_clients = hashtableCreate(&keyToClientsHashtableType); + + initClientBlockingState(c); + c->bstate->timeout = 0; + serverAssert(dictSize(c->bstate->keys) == 0); + + for (int i = 0; i < num_keys; ++i) { + robj *key = keys[i]; + serverAssert(key->type == OBJ_STRING); + + /* Deduplicate via bstate->keys dict */ + if (dictAdd(c->bstate->keys, key, NULL) != DICT_OK) continue; + incrRefCount(key); + + list *blockedClientsList = keyToClients_getBlockedClientsList(key); + if (!blockedClientsList) blockedClientsList = keyToClients_addEntry(key); + listAddNodeTail(blockedClientsList, c); + } + + serverAssert(dictSize(c->bstate->keys) > 0); + blockClient(c, BLOCKED_INUSE); + + /* Disable client's Read Handler to prevent reading commands while blocked */ + if (c->conn) { + connSetReadHandler(c->conn, NULL); + } +} + +/* Unblock clients blocked on the given key. + * + * A client is fully unblocked only when it has no remaining keys in its + * bstate->keys dict. Such clients are queued for reprocessing and resumed + * later during processUnblockedClients(). */ +void unblockClientsInUseOnKey(robj *key) { + list *blockedClientsList = keyToClients_getBlockedClientsList(key); + if (blockedClientsList == NULL) return; + + serverAssert(listLength(blockedClientsList) > 0); + + while (listLength(blockedClientsList) > 0) { + listNode *ln = listFirst(blockedClientsList); + client *c = listNodeValue(ln); + serverAssert(isClientBlockedInUse(c) && c->flag.unblocked == 0); + listDelNode(blockedClientsList, ln); + dictDelete(c->bstate->keys, key); + + if (dictSize(c->bstate->keys) == 0) { + unblockClient(c, 1); + } + } + + hashtableDelete(inuse_key_to_clients, key); +} + +/* Unblock all clients that are currently blocked by blockInUse, across all + * keys. Unblocked clients are queued for reprocessing and resumed during + * processUnblockedClients(). After this call, no clients remain blocked + * by blockInUse. */ +void unblockClientsInUseOnAllKeys(void) { + if (!inuse_key_to_clients) return; + hashtableIterator iter; + hashtableInitIterator(&iter, inuse_key_to_clients, HASHTABLE_ITER_SAFE); + keyToClientsEntry *e; + while (hashtableNext(&iter, (void **)&e)) { + unblockClientsInUseOnKey(e->key); + } + hashtableCleanupIterator(&iter); + serverAssert(server.blocked_clients_by_type[BLOCKED_INUSE] == 0); + serverAssert(hashtableSize(inuse_key_to_clients) == 0); +} + +/* Remove a client from all blockInUse key-to-clients mappings. + * Called from unblockClient() for BLOCKED_INUSE cleanup. */ +static void unlinkBlockInUseClient(client *c) { + if (!c->bstate->keys || dictSize(c->bstate->keys) == 0) return; + dictIterator *di = dictGetIterator(c->bstate->keys); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + robj *key = dictGetKey(de); + list *clientList = keyToClients_getBlockedClientsList(key); + serverAssert(clientList != NULL); + listNode *ln = listSearchKey(clientList, c); + serverAssert(ln != NULL); + listDelNode(clientList, ln); + if (listLength(clientList) == 0) hashtableDelete(inuse_key_to_clients, key); + } + dictReleaseIterator(di); + dictEmpty(c->bstate->keys, NULL); +} + void blockedBeforeSleep(void) { /* Handle precise timeouts of blocked clients. */ handleBlockedClientsTimeout(); @@ -826,3 +1033,21 @@ void blockedBeforeSleep(void) { /* Try to process pending commands for clients that were just unblocked. */ if (listLength(server.unblocked_clients)) processUnblockedClients(); } + +/* -------------------------------------------------------------------------- + * Test-only APIs for blockInUse + * -------------------------------------------------------------------------- */ + +/* Test-only: get the current number of blocked keys by blockInUse. */ +int getBlockInUseKeyCount(void) { + return inuse_key_to_clients ? hashtableSize(inuse_key_to_clients) : 0; +} + +/* Test-only: release the blockInUse hashtable. */ +void releaseBlockInUse(void) { + unblockClientsInUseOnAllKeys(); + if (inuse_key_to_clients) { + hashtableRelease(inuse_key_to_clients); + inuse_key_to_clients = NULL; + } +} diff --git a/src/cli_commands.h b/src/cli_commands.h index a8174e430..a1f8d3c8b 100644 --- a/src/cli_commands.h +++ b/src/cli_commands.h @@ -38,7 +38,8 @@ struct commandDocs { int numargs; cliCommandArg *args; /* An array of the command arguments. */ struct commandDocs *subcommands; - char *params; /* A string describing the syntax of the command arguments. */ + int member_arg_index; /* unused in CLI, present for commands.def compatibility */ + char *params; /* A string describing the syntax of the command arguments. */ }; extern struct commandDocs serverCommandTable[]; diff --git a/src/cli_common.c b/src/cli_common.c index d3a058e73..055b8105b 100644 --- a/src/cli_common.c +++ b/src/cli_common.c @@ -448,6 +448,23 @@ valkeyContext *valkeyConnectWrapper(enum valkeyConnectionType ct, const char *ip return valkeyConnectWithOptions(&options); } +/* 62-bit thread-local PRNG. glibc random() serializes every caller on a + * process-wide lock, which makes it a scalability bottleneck when called + * per command from many threads (valkey-benchmark placeholder replacement, + * fuzzer worker threads). Each thread runs an independent splitmix64 + * stream instead, seeded on first use from the global (srandom() seedable) + * generator. With a fixed srandom() seed, per-thread streams are seeded + * deterministically, but which stream a given thread receives depends on + * the scheduling order of first use. */ +static _Thread_local uint64_t rand62_state = 0; + uint64_t rand62(void) { - return ((uint64_t)random() << 31) | (uint64_t)random(); + if (rand62_state == 0) { + rand62_state = ((uint64_t)random() << 31) ^ (uint64_t)random(); + if (rand62_state == 0) rand62_state = 0x9E3779B97F4A7C15ULL; + } + uint64_t z = (rand62_state += 0x9E3779B97F4A7C15ULL); + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; + return (z ^ (z >> 31)) & ((1ULL << 62) - 1); } diff --git a/src/cli_common.h b/src/cli_common.h index 868015ab1..9e0eec188 100644 --- a/src/cli_common.h +++ b/src/cli_common.h @@ -54,8 +54,8 @@ sds cliVersion(void); valkeyContext *valkeyConnectWrapper(enum valkeyConnectionType ct, const char *ip_or_path, int port, const struct timeval tv, int nonblock, int multipath); -/* Two random() calls combined for 62-bit range. Not genrand64_int64: its - * state is unlocked and callers are multi-threaded. Seed via srandom(). */ +/* 62-bit per-thread PRNG (splitmix64), lock-free. Each thread's stream is + * seeded on first use from the srandom()-seedable global generator. */ uint64_t rand62(void); #endif /* __CLICOMMON_H */ diff --git a/src/cluster.c b/src/cluster.c index 6e524a1a7..8a9c7c633 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -365,8 +365,10 @@ migrateCachedSocket *migrateGetSocket(client *c, robj *host, robj *port, long ti dictDelete(server.migrate_cached_sockets, dictGetKey(de)); } - /* Create the connection */ + /* Create the connection and tag as high-priority so key/slot migration + * packets are not delayed by normal tenant commands. */ conn = connCreate(connTypeOfCluster()); + connSetPriority(conn, true); if (connBlockingConnect(conn, objectGetVal(host), atoi(objectGetVal(port)), timeout) != C_OK) { addReplyError(c, "-IOERR error or timeout connecting to the client"); connClose(conn); @@ -1068,8 +1070,14 @@ clusterNode *getNodeByQuery(client *c, int *error_code) { * distributed system. */ /* Determine transaction slot and return early on cross-slot. */ - if (c->cmd->proc == execCommand && c->flag.multi) { - int slot = -1; + if (c->cmd->proc == execCommand) { + if (!c->flag.multi || c->flag.dirty_exec) return myself; + + int slot = c->slot; + if (c->read_flags & READ_FLAGS_CROSSSLOT) { + if (error_code) *error_code = CLUSTER_REDIR_CROSS_SLOT; + return NULL; + } for (i = 0; i < c->mstate->count; i++) { if (slot == -1) { slot = c->mstate->commands[i].slot; @@ -1130,9 +1138,6 @@ clusterNode *getNodeByQuery(client *c, int *error_code) { /* We handle all the cases as if they were EXEC commands, so we have * a common code path for everything */ if (c->cmd->proc == execCommand) { - /* If CLIENT_MULTI flag is not set EXEC is just going to return an - * error. */ - if (!c->flag.multi) return myself; ms = c->mstate; } else { /* In order to have a single codepath create a fake Multi State @@ -1150,15 +1155,21 @@ clusterNode *getNodeByQuery(client *c, int *error_code) { serverDb *currentDb = origDb; /* Check for multiple keys, existing keys, missing keys. */ - for (i = 0; i < ms->count; i++) { + for (i = c->cmd->proc == execCommand ? -1 : 0; i < ms->count; i++) { struct serverCommand *mcmd; robj **margv; int margc, numkeys, j; keyReference *keyindex; - mcmd = ms->commands[i].cmd; - margc = ms->commands[i].argc; - margv = ms->commands[i].argv; + if (i == -1) { + mcmd = c->cmd; + margc = c->argc; + margv = c->argv; + } else { + mcmd = ms->commands[i].cmd; + margc = ms->commands[i].argc; + margv = ms->commands[i].argv; + } getKeysResult result; initGetKeysResult(&result); @@ -1223,7 +1234,7 @@ clusterNode *getNodeByQuery(client *c, int *error_code) { * slot migration, the channel will be served from the source * node until the migration completes with CLUSTER SETSLOT * NODE . */ - int flags = LOOKUP_NOTOUCH | LOOKUP_NOSTATS | LOOKUP_NONOTIFY | LOOKUP_NOEXPIRE; + int flags = LOOKUP_NOEFFECTS; /* not client key access */ if (!pubsubshard_included && (!c->flag.multi || (c->flag.multi && c->cmd->proc == execCommand))) { /* Multi/Exec validation happens on exec */ @@ -1636,6 +1647,8 @@ void resetClusterStats(void) { server.cluster->stats_bus_module_bytes_sent = 0; server.cluster->stats_bus_module_bytes_received = 0; server.cluster->stat_cluster_links_buffer_limit_exceeded = 0; + server.cluster->stat_cluster_links_established_inbound = 0; + server.cluster->stat_cluster_links_established_outbound = 0; } void clusterCommandFlushslot(client *c) { diff --git a/src/cluster.h b/src/cluster.h index 37cb5a22c..b7117130b 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -152,9 +152,9 @@ sds aggregateClientOutputBuffer(client *c); void resetClusterStats(void); unsigned int delKeysInSlot(unsigned int hashslot, int lazy, bool propagate_del, bool send_del_event); -unsigned int propagateSlotDeletionByKeys(unsigned int hashslot); void clusterUpdateState(void); -void clusterSaveConfigOrDie(int do_fsync); +int clusterSaveConfigFromBio(sds content, bool do_fsync); +void clusterSaveConfigOrDie(bool do_fsync); int clusterDelSlot(int slot); int clusterAddSlot(clusterNode *n, int slot); int clusterBumpConfigEpochWithoutConsensus(void); diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index 2778a03f0..902c7bd95 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -37,13 +37,17 @@ */ #include "server.h" +#include "hotkeys.h" #include "cluster.h" #include "cluster_legacy.h" #include "cluster_slot_stats.h" #include "cluster_migrateslots.h" #include "endianconv.h" #include "connection.h" +#include "connhelpers.h" #include "module.h" +#include "io_threads.h" +#include "bio.h" #include #include @@ -67,6 +71,9 @@ void clusterReadHandler(connection *conn); void clusterSendPing(clusterLink *link, int type); void clusterSendFail(char *nodename); void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request); +void clusterProcessFailoverAuthNack(clusterNode *sender, clusterMsg *request); +void clusterSendFailoverNack(clusterNode *node, uint8_t reason); +static const char *clusterNackReasonString(uint8_t reason); void clusterUpdateState(void); list *clusterGetNodesInMyShard(clusterNode *node); int clusterNodeAddReplica(clusterNode *primary, clusterNode *replica); @@ -134,7 +141,7 @@ sds auxReplicaPriorityGetter(clusterNode *n, sds s); int auxReplicaPriorityPresent(clusterNode *n); static void clusterBuildMessageHdrLight(clusterMsgLight *hdr, int type, size_t msglen); static void clusterBuildMessageHdr(clusterMsg *hdr, int type, size_t msglen); -void freeClusterLink(clusterLink *link); +int freeClusterLink(clusterLink *link); int verifyClusterNodeId(const char *name, int length); sds clusterEncodeOpenSlotsAuxField(int rdbflags); int clusterDecodeOpenSlotsAuxField(int rdbflags, sds s); @@ -241,13 +248,10 @@ static inline char *clusterLinkGetHumanNodeName(clusterLink *link) { #define CLUSTER_SLOT_WORDS (CLUSTER_SLOTS / 64) #define SLOT_WORD_OFFSET(w) ((w) << 3) -#define RCVBUF_INIT_LEN 1024 #define RCVBUF_MIN_READ_LEN 14 static_assert(offsetof(clusterMsg, type) + sizeof(uint16_t) == RCVBUF_MIN_READ_LEN, "Incorrect length to read to identify type"); -#define RCVBUF_MAX_PREALLOC (1 << 20) /* 1MB */ - /* Cluster nodes hash table, mapping nodes addresses 1.2.3.4:6379 to * clusterNode structures. */ dictType clusterNodesDictType = { @@ -1083,6 +1087,16 @@ int clusterLoadConfig(char *filename) { serverPanic("Unrecoverable error: corrupted cluster config file \"%s\".", line); } +/* Get the nodes description and concatenate our "vars" directive to + * save currentEpoch and lastVoteEpoch. */ +static sds clusterGenNodesConfContent(void) { + sds content = clusterGenNodesDescription(NULL, CLUSTER_NODE_HANDSHAKE, 0); + content = sdscatfmt(content, "vars currentEpoch %U lastVoteEpoch %U\n", + (unsigned long long)server.cluster->currentEpoch, + (unsigned long long)server.cluster->lastVoteEpoch); + return content; +} + /* Cluster node configuration is exactly the same as CLUSTER NODES output. * * This function writes the node config and returns C_OK, on error C_ERR @@ -1094,24 +1108,21 @@ int clusterLoadConfig(char *filename) { * new one. Since we have the full payload to write available we can use * a single write to write the whole file. If the preexisting file was * bigger we pad our payload with newlines that are anyway ignored and truncate - * the file afterward. */ -int clusterSaveConfig(int do_fsync) { - sds ci, tmpfilename; + * the file afterward. + * + * This function will be called by either the main thread or a bio thread. + * When called from a bio thread, latency is not recorded because it is not + * thread-safe. + * + * This function take ownership of the 'content' SDS string and will free it. */ +static int clusterSaveConfigImpl(sds content, bool from_bio, bool do_fsync) { + sds tmpfilename; size_t content_size, offset = 0; ssize_t written_bytes; int fd = -1; int retval = C_ERR; mstime_t latency; - - server.cluster->todo_before_sleep &= ~CLUSTER_TODO_SAVE_CONFIG; - - /* Get the nodes description and concatenate our "vars" directive to - * save currentEpoch and lastVoteEpoch. */ - ci = clusterGenNodesDescription(NULL, CLUSTER_NODE_HANDSHAKE, 0); - ci = sdscatfmt(ci, "vars currentEpoch %U lastVoteEpoch %U\n", - (unsigned long long)server.cluster->currentEpoch, - (unsigned long long)server.cluster->lastVoteEpoch); - content_size = sdslen(ci); + content_size = sdslen(content); /* Create a temp file with the new content. */ tmpfilename = sdscatfmt(sdsempty(), "%s.tmp-%i-%I", server.cluster_configfile, (int)getpid(), mstime()); @@ -1121,11 +1132,11 @@ int clusterSaveConfig(int do_fsync) { goto cleanup; } latencyEndMonitor(latency); - latencyAddSampleIfNeeded("cluster-config-open", latency); - latencyTraceIfNeeded(cluster, cluster_config_open, latency); + if (!from_bio) latencyAddSampleIfNeeded("cluster-config-open", latency); + if (!from_bio) latencyTraceIfNeeded(cluster, cluster_config_open, latency); latencyStartMonitor(latency); while (offset < content_size) { - written_bytes = write(fd, ci + offset, content_size - offset); + written_bytes = write(fd, content + offset, content_size - offset); if (written_bytes <= 0) { if (errno == EINTR) continue; serverLog(LL_WARNING, "Failed after writing (%zd) bytes to tmp cluster config file: %s", offset, @@ -1135,18 +1146,17 @@ int clusterSaveConfig(int do_fsync) { offset += written_bytes; } latencyEndMonitor(latency); - latencyAddSampleIfNeeded("cluster-config-write", latency); - latencyTraceIfNeeded(cluster, cluster_config_write, latency); + if (!from_bio) latencyAddSampleIfNeeded("cluster-config-write", latency); + if (!from_bio) latencyTraceIfNeeded(cluster, cluster_config_write, latency); if (do_fsync) { latencyStartMonitor(latency); - server.cluster->todo_before_sleep &= ~CLUSTER_TODO_FSYNC_CONFIG; if (valkey_fsync(fd) == -1) { serverLog(LL_WARNING, "Could not sync tmp cluster config file: %s", strerror(errno)); goto cleanup; } latencyEndMonitor(latency); - latencyAddSampleIfNeeded("cluster-config-fsync", latency); - latencyTraceIfNeeded(cluster, cluster_config_fsync, latency); + if (!from_bio) latencyAddSampleIfNeeded("cluster-config-fsync", latency); + if (!from_bio) latencyTraceIfNeeded(cluster, cluster_config_fsync, latency); } latencyStartMonitor(latency); @@ -1155,8 +1165,8 @@ int clusterSaveConfig(int do_fsync) { goto cleanup; } latencyEndMonitor(latency); - latencyAddSampleIfNeeded("cluster-config-rename", latency); - latencyTraceIfNeeded(cluster, cluster_config_rename, latency); + if (!from_bio) latencyAddSampleIfNeeded("cluster-config-rename", latency); + if (!from_bio) latencyTraceIfNeeded(cluster, cluster_config_rename, latency); if (do_fsync) { latencyStartMonitor(latency); if (fsyncFileDir(server.cluster_configfile) == -1) { @@ -1164,8 +1174,8 @@ int clusterSaveConfig(int do_fsync) { goto cleanup; } latencyEndMonitor(latency); - latencyAddSampleIfNeeded("cluster-config-dir-fsync", latency); - latencyTraceIfNeeded(cluster, cluster_config_dir_fsync, latency); + if (!from_bio) latencyAddSampleIfNeeded("cluster-config-dir-fsync", latency); + if (!from_bio) latencyTraceIfNeeded(cluster, cluster_config_dir_fsync, latency); } retval = C_OK; /* If we reached this point, everything is fine. */ @@ -1174,41 +1184,65 @@ int clusterSaveConfig(int do_fsync) { latencyStartMonitor(latency); close(fd); latencyEndMonitor(latency); - latencyAddSampleIfNeeded("cluster-config-close", latency); - latencyTraceIfNeeded(cluster, cluster_config_close, latency); + if (!from_bio) latencyAddSampleIfNeeded("cluster-config-close", latency); + if (!from_bio) latencyTraceIfNeeded(cluster, cluster_config_close, latency); } if (retval == C_ERR) { latencyStartMonitor(latency); unlink(tmpfilename); latencyEndMonitor(latency); - latencyAddSampleIfNeeded("cluster-config-unlink", latency); - latencyTraceIfNeeded(cluster, cluster_config_unlink, latency); + if (!from_bio) latencyAddSampleIfNeeded("cluster-config-unlink", latency); + if (!from_bio) latencyTraceIfNeeded(cluster, cluster_config_unlink, latency); } sdsfree(tmpfilename); - sdsfree(ci); + sdsfree(content); return retval; } +/* Save cluster config file. + * + * This function writes the node config and returns C_OK, on error C_ERR + * is returned. It is possible to use bio, which can move I/O latency into + * the bio thread. If bio is used, it always returns C_OK. */ +static int clusterSaveConfig(bool use_bio, bool do_fsync) { + server.cluster->todo_before_sleep &= ~CLUSTER_TODO_SAVE_CONFIG; + if (do_fsync) server.cluster->todo_before_sleep &= ~CLUSTER_TODO_FSYNC_CONFIG; + + /* The subsequent function will take ownership of the string and be responsible for freeing it. */ + sds content = clusterGenNodesConfContent(); + if (use_bio) { + /* We can actually always fsync the file in bio, but anyway lets follow the old code. */ + bioCreateClusterConfigSaveJob(content, do_fsync); + return C_OK; + } else { + int res = clusterSaveConfigImpl(content, false, do_fsync); + if (res == C_OK) { + atomic_store_explicit(&server.cluster_config_save_status, C_OK, memory_order_relaxed); + atomic_store_explicit(&server.cluster_config_last_save_time, time(NULL), memory_order_relaxed); + } else { + atomic_store_explicit(&server.cluster_config_save_status, C_ERR, memory_order_relaxed); + } + return res; + } +} + +/* Save the cluster file, it is called from the bio thread. */ +int clusterSaveConfigFromBio(sds content, bool do_fsync) { + return clusterSaveConfigImpl(content, true, do_fsync); +} + /* Save the cluster configuration file. If the save fails, exit the process. */ -void clusterSaveConfigOrDie(int do_fsync) { - if (clusterSaveConfig(do_fsync) == C_ERR) { +void clusterSaveConfigOrDie(bool fsync) { + if (clusterSaveConfig(false, fsync) == C_ERR) { serverLog(LL_WARNING, "Fatal: can't update cluster config file."); exit(1); } } -/* Save the cluster configuration file. If the save fails, print the log. */ -#define CONFIG_SAVE_LOG_ERROR_RATE 30 /* Seconds between errors logging. */ -void clusterSaveConfigOrLog(int do_fsync) { - if (clusterSaveConfig(do_fsync) == C_ERR) { - static time_t last_save_error_log = 0; - /* Limit logging rate to 1 line per CONFIG_SAVE_LOG_ERROR_RATE seconds. */ - if ((server.unixtime - last_save_error_log) > CONFIG_SAVE_LOG_ERROR_RATE) { - serverLog(LL_WARNING, "Cluster config updated even though writing " - "the cluster config file to disk failed."); - last_save_error_log = server.unixtime; - } - } +/* Save the cluster configuration file in bio thread. */ +static void clusterSaveConfigBackground(bool do_fsync) { + int res = clusterSaveConfig(true, do_fsync); + serverAssert(res == C_OK); } /* Lock the cluster config using flock(), and retain the file descriptor used to @@ -1295,13 +1329,14 @@ void deriveAnnouncedPorts(int *announced_tcp_port, void clusterUpdateMyselfFlags(void) { if (!myself) return; int oldflags = myself->flags; - int nofailover = server.cluster_replica_no_failover ? CLUSTER_NODE_NOFAILOVER : 0; + int nofailover = server.cluster_replica_no_failover == CLUSTER_REPLICA_NO_FAILOVER_YES ? CLUSTER_NODE_NOFAILOVER : 0; myself->flags &= ~CLUSTER_NODE_NOFAILOVER; myself->flags |= nofailover; myself->flags |= CLUSTER_NODE_EXTENSIONS_SUPPORTED | CLUSTER_NODE_LIGHT_HDR_PUBLISH_SUPPORTED | CLUSTER_NODE_LIGHT_HDR_MODULE_SUPPORTED | - CLUSTER_NODE_MULTI_MEET_SUPPORTED; + CLUSTER_NODE_MULTI_MEET_SUPPORTED | + CLUSTER_NODE_FAILOVER_AUTH_NACK_SUPPORTED; if (myself->flags != oldflags) { clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG | CLUSTER_TODO_UPDATE_STATE); @@ -1511,6 +1546,7 @@ void clusterInit(void) { server.cluster->fail_reason = CLUSTER_FAIL_NONE; server.cluster->safe_to_join = 0; server.cluster->size = 0; + server.cluster->size_fail = 0; server.cluster->todo_before_sleep = 0; server.cluster->nodes = dictCreate(&clusterNodesDictType); server.cluster->shards = dictCreate(&clusterSdsToListType); @@ -1519,6 +1555,7 @@ void clusterInit(void) { server.cluster->importing_slots_from = dictCreate(&clusterSlotDictType); server.cluster->failover_auth_time = 0; server.cluster->failover_auth_count = 0; + server.cluster->failover_auth_nack_count = 0; server.cluster->failover_auth_rank = 0; server.cluster->failover_auth_sent = 0; server.cluster->failover_failed_primary_rank = 0; @@ -1554,7 +1591,7 @@ void clusterInit(void) { clusterAddNodeToShard(myself->shard_id, myself); saveconf = 1; } - if (saveconf) clusterSaveConfigOrDie(1); + if (saveconf) clusterSaveConfigOrDie(true); /* Port sanity check II * The other handshake port check is triggered too late to stop @@ -1692,7 +1729,8 @@ void clusterHandleServerShutdown(bool auto_failover) { /* The error logs have been logged in the save function if the save fails. */ serverLog(LL_NOTICE, "Saving the cluster configuration file before exiting."); - clusterSaveConfig(1); + bioDrainWorker(BIO_CLUSTER_SAVE); + clusterSaveConfig(false, true); #if !defined(__sun) /* Unlock the cluster config file before shutdown, see clusterLockConfig. @@ -1731,6 +1769,7 @@ void clusterReset(int hard) { resetManualFailover(); /* Unassign all the slots. */ + hotkeysPurgeAll(); /* Bulk purge before individual clusterDelSlot calls */ for (j = 0; j < CLUSTER_SLOTS; j++) clusterDelSlot(j); /* Recreate shards dict */ @@ -1833,6 +1872,24 @@ clusterLink *createClusterLink(clusterNode *node) { link->send_msg_queue_mem = sizeof(list); link->rcvbuf = zmalloc(link->rcvbuf_alloc = RCVBUF_INIT_LEN); link->rcvbuf_len = 0; + + /* Threaded I/O state */ + link->io_read_state = CLUSTER_LINK_IO_IDLE; + link->io_write_state = CLUSTER_LINK_IO_IDLE; + link->async_close = 0; + link->io_refs = 0; + link->io_result = CLUSTER_IO_OK; + + /* Async write snapshot/result */ + link->io_last_send_block = NULL; + link->io_head_offset = 0; + link->io_nodes_sent = 0; + + link->rcvbuf_alloc_at_dispatch = 0; + link->io_complete_bytes = 0; + link->io_complete_packets = 0; + link->io_read_deferred = 0; + server.stat_cluster_links_memory += link->rcvbuf_alloc + link->send_msg_queue_mem; link->conn = NULL; link->node = node; @@ -1847,22 +1904,23 @@ clusterLink *createClusterLink(clusterNode *node) { /* Free a cluster link, but does not free the associated node of course. * This function will just make sure that the original node associated - * with this link will have the 'link' field set to NULL. */ -void freeClusterLink(clusterLink *link) { + * with this link will have the 'link' field set to NULL. + * + * If I/O jobs are in flight (io_refs > 0), the link is not freed immediately. + * Instead, async_close is set, the link is detached from node fields, and any + * read/write handlers are removed so no new I/O work is scheduled. The actual + * connClose() happens later on the main thread when the last completion + * decrements io_refs to 0, mirroring the client close flow. + * + * Returns 1 if the link was freed immediately, 0 if teardown was deferred. */ +int freeClusterLink(clusterLink *link) { serverAssert(link != NULL); serverLog(LL_DEBUG, "Freeing cluster link for node: %.40s:%s (%s)", clusterLinkGetNodeName(link), link->inbound ? "inbound" : "outbound", clusterLinkGetHumanNodeName(link)); - if (link->conn) { - connClose(link->conn); - link->conn = NULL; - } - server.stat_cluster_links_memory -= sizeof(list) + listLength(link->send_msg_queue) * sizeof(listNode); - listRelease(link->send_msg_queue); - server.stat_cluster_links_memory -= link->rcvbuf_alloc; - zfree(link->rcvbuf); + /* Detach from node regardless of whether we free now or defer. */ if (link->node) { if (link->node->link == link) { serverAssert(!link->inbound); @@ -1872,8 +1930,44 @@ void freeClusterLink(clusterLink *link) { link->node->inbound_link = NULL; link->node->inbound_link_freed_time = mstime(); } + link->node = NULL; + } + + /* If I/O jobs are in flight, defer the actual free. */ + if (link->io_refs > 0) { + serverAssert(link->io_read_state == CLUSTER_LINK_IO_PENDING || + link->io_write_state == CLUSTER_LINK_IO_PENDING); + if (!link->async_close) { + if (link->conn) { + connSetReadHandler(link->conn, NULL); + connSetWriteHandler(link->conn, NULL); + } + link->async_close = 1; + } + return 0; + } + + /* Close the connection now that no I/O jobs are in flight. */ + if (link->conn) { + connClose(link->conn); + link->conn = NULL; } + + /* Immediate free path — both states must be idle. */ + serverAssert(link->io_read_state == CLUSTER_LINK_IO_IDLE); + serverAssert(link->io_write_state == CLUSTER_LINK_IO_IDLE); + serverAssert(link->io_refs == 0); + server.stat_cluster_links_memory -= sizeof(list) + listLength(link->send_msg_queue) * sizeof(listNode); + listRelease(link->send_msg_queue); + + /* Discard any complete packets the worker framed but we never applied. */ + link->io_complete_bytes = 0; + link->io_complete_packets = 0; + + server.stat_cluster_links_memory -= link->rcvbuf_alloc; + zfree(link->rcvbuf); zfree(link); + return 1; } void setClusterNodeToInboundClusterLink(clusterNode *node, clusterLink *link) { @@ -1907,7 +2001,7 @@ void setClusterNodeToInboundClusterLink(clusterNode *node, clusterLink *link) { } } -static void clusterConnAcceptHandler(connection *conn) { +void clusterConnAcceptHandler(connection *conn) { clusterLink *link; if (connGetState(conn) != CONN_STATE_CONNECTED) { @@ -1916,6 +2010,9 @@ static void clusterConnAcceptHandler(connection *conn) { return; } + serverAssert(connGetOwnerKind(conn) == CONN_OWNER_CLUSTER_LINK); + serverAssert(connGetPrivateData(conn) == NULL); + /* Create a link object we use to handle the connection. * It gets passed to the readable handler when data is available. * Initially the link->node pointer is set to NULL as we don't know @@ -1927,6 +2024,10 @@ static void clusterConnAcceptHandler(connection *conn) { /* Register read handler */ connSetReadHandler(conn, clusterReadHandler); + + /* Count a successfully accepted (inbound) cluster link. This reflects + * how often peers (re)establish connections to us. */ + server.cluster->stat_cluster_links_established_inbound++; } void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { @@ -1951,6 +2052,16 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { } connection *conn = connCreateAccepted(connTypeOfCluster(), cfd, &require_auth); + /* Tag inbound cluster bus link as high-priority so cluster gossip and heartbeats + * are processed via QoS ahead of normal client traffic. */ + connSetPriority(conn, true); + /* Mark as cluster-owned before any TLS accept retries so generic + * accept offload routing can safely avoid client assumptions. */ + connSetOwnerKind(conn, CONN_OWNER_CLUSTER_LINK); + /* Only a TLS accept is worth offloading: it runs the handshake. A + * plain TCP accept just flips the connection state, so offloading it + * would cost a worker round trip and an inbox slot for no work. */ + if (connGetType(conn) == CONN_TYPE_TLS) conn->flags |= CONN_FLAG_ALLOW_ACCEPT_OFFLOAD; /* Make sure connection is not in an error state */ if (connGetState(conn) != CONN_STATE_ACCEPTING) { @@ -1965,9 +2076,18 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { /* Use non-blocking I/O for cluster messages. */ serverLog(LL_VERBOSE, "Accepting cluster node connection from %s:%d", cip, cport); - /* Accept the connection now. connAccept() may call our handler directly - * or schedule it for later depending on connection implementation. - */ + /* Install before offloading: that path skips connAccept() below, and a + * TLS retry that cannot re-offload only ever invokes conn_handler. */ + conn->conn_handler = clusterConnAcceptHandler; + + /* Try to offload the TLS accept handshake to an I/O thread. + * If offload succeeds, the completion handler will create the + * clusterLink and install the read handler. */ + if (trySendClusterAcceptToIOThreads(conn) == C_OK) continue; + + /* Synchronous fallback: accept inline. connAccept() may call our + * handler directly or schedule it for later depending on + * connection implementation. */ if (connAccept(conn, clusterConnAcceptHandler) == C_ERR) { if (connGetState(conn) == CONN_STATE_ERROR) serverLog(LL_VERBOSE, "Error accepting cluster node connection: %s", connGetLastError(conn)); @@ -2708,7 +2828,7 @@ void clearNodeFailureIfNeeded(clusterNode *node) { serverLog(LL_NOTICE, "Clear FAIL state for node %.40s (%s): %s is reachable again.", node->name, humanNodename(node), nodeIsReplica(node) ? "replica" : "primary without slots"); node->flags &= ~CLUSTER_NODE_FAIL; - if (nodeIsReplica(myself) && myself->replicaof == node) node->flags &= ~CLUSTER_NODE_MY_PRIMARY_FAIL; + if (nodeIsReplica(myself) && myself->replicaof == node) myself->flags &= ~CLUSTER_NODE_MY_PRIMARY_FAIL; clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE | CLUSTER_TODO_SAVE_CONFIG); } @@ -2723,7 +2843,7 @@ void clearNodeFailureIfNeeded(clusterNode *node) { "Clear FAIL state for node %.40s (%s): is reachable again and nobody is serving its slots after some time.", node->name, humanNodename(node)); node->flags &= ~CLUSTER_NODE_FAIL; - if (nodeIsReplica(myself) && myself->replicaof == node) node->flags &= ~CLUSTER_NODE_MY_PRIMARY_FAIL; + if (nodeIsReplica(myself) && myself->replicaof == node) myself->flags &= ~CLUSTER_NODE_MY_PRIMARY_FAIL; clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE | CLUSTER_TODO_SAVE_CONFIG); } } @@ -3086,6 +3206,7 @@ void clusterSetNodeAsPrimary(clusterNode *n) { n->replicaof = NULL; if (n == myself) { + myself->flags &= ~CLUSTER_NODE_MY_PRIMARY_FAIL; replicationUnsetPrimary(); } @@ -3929,6 +4050,9 @@ int clusterIsValidPacket(clusterLink *link) { } else if (type == CLUSTERMSG_TYPE_FAILOVER_AUTH_REQUEST || type == CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK || type == CLUSTERMSG_TYPE_MFSTART) { explen = sizeof(clusterMsg) - sizeof(union clusterMsgData); + } else if (type == CLUSTERMSG_TYPE_FAILOVER_AUTH_NACK) { + explen = sizeof(clusterMsg) - sizeof(union clusterMsgData); + explen += sizeof(clusterMsgDataFailoverNack); } else if (type == CLUSTERMSG_TYPE_UPDATE) { explen = sizeof(clusterMsg) - sizeof(union clusterMsgData); explen += sizeof(clusterMsgDataUpdate); @@ -4070,6 +4194,13 @@ int clusterProcessPacket(clusterLink *link) { } else { sender->flags &= ~CLUSTER_NODE_MY_PRIMARY_FAIL; } + + /* Check if the node understands FAILOVER_AUTH_NACK packets. */ + if (flags & CLUSTER_NODE_FAILOVER_AUTH_NACK_SUPPORTED) { + sender->flags |= CLUSTER_NODE_FAILOVER_AUTH_NACK_SUPPORTED; + } else { + sender->flags &= ~CLUSTER_NODE_FAILOVER_AUTH_NACK_SUPPORTED; + } } /* Update the last time we saw any data from this node. We @@ -4618,10 +4749,25 @@ int clusterProcessPacket(clusterLink *link) { * equal to epoch where this node started the election. */ if (clusterNodeIsVotingPrimary(sender) && sender_claimed_current_epoch >= server.cluster->failover_auth_epoch) { server.cluster->failover_auth_count++; + serverLog(LL_NOTICE, "Failover auth ACK from %.40s (%s) for epoch %llu (ACKs %d, quorum %d)", + sender->name, humanNodename(sender), (unsigned long long)server.cluster->failover_auth_epoch, + server.cluster->failover_auth_count, (server.cluster->size / 2) + 1); /* Maybe we reached a quorum here, set a flag to make sure * we check ASAP. */ clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_FAILOVER); } + } else if (type == CLUSTERMSG_TYPE_FAILOVER_AUTH_NACK) { + if (!sender) return 1; /* We don't know that node. */ + + /* We consider this nack only if the sender is a primary serving + * a non-zero number of slots, and its currentEpoch is greater or + * equal to epoch where this node started the election. */ + if (server.cluster->failover_auth_time && + server.cluster->failover_auth_sent && + clusterNodeIsVotingPrimary(sender) && + sender_claimed_current_epoch >= server.cluster->failover_auth_epoch) { + clusterProcessFailoverAuthNack(sender, msg); + } } else if (type == CLUSTERMSG_TYPE_MFSTART) { /* This message is acceptable only if I'm a primary and the sender * is one of my replicas. */ @@ -4682,6 +4828,62 @@ int clusterProcessPacket(clusterLink *link) { return 1; } +/* Drain complete packets queued at the start of rcvbuf. + * + * io_complete_bytes marks the bytes the I/O thread determined contain + * only complete packets. clusterProcessPacket() reads from the front of + * rcvbuf, so each packet is slid down to offset 0 in turn and the unparsed + * tail is compacted once at the end. + * + * Returns 1 if the link is still valid after all packets were applied, or + * 0 if packet processing freed the link. */ +static int clusterDrainCompletePackets(clusterLink *link) { + size_t buf_len = link->rcvbuf_len; + size_t consumed = 0; + + while (link->io_complete_bytes > 0) { + clusterMsgHeader *hdr = (clusterMsgHeader *)(link->rcvbuf + consumed); + uint32_t totlen = ntohl(hdr->totlen); + + serverAssert(link->io_complete_bytes >= totlen); + serverAssert(link->io_complete_packets > 0); + + link->io_complete_bytes -= totlen; + link->io_complete_packets--; + + /* Copy just this packet, not the whole remaining tail, which would make + * the drain quadratic. Safe because the copy writes [0, totlen) while + * the bytes not yet consumed start at consumed + totlen. */ + if (consumed > 0) memmove(link->rcvbuf, link->rcvbuf + consumed, totlen); + consumed += totlen; + + link->rcvbuf_len = totlen; + if (!clusterProcessPacket(link)) { + return 0; + } + } + + link->rcvbuf_len = buf_len - consumed; + if (consumed > 0 && link->rcvbuf_len > 0) { + memmove(link->rcvbuf, link->rcvbuf + consumed, link->rcvbuf_len); + } + + return 1; +} + +static void clusterShrinkRcvbuf(clusterLink *link) { + /* Shrink around any leftover partial packet, plus headroom. Requiring an + * empty buffer would pin a busy link at its high-water mark. */ + size_t target = link->rcvbuf_len + RCVBUF_INIT_LEN; + if (target < RCVBUF_INIT_LEN) target = RCVBUF_INIT_LEN; + if (link->rcvbuf_alloc <= target) return; + + size_t prev_rcvbuf_alloc = link->rcvbuf_alloc; + link->rcvbuf = zrealloc(link->rcvbuf, target); + link->rcvbuf_alloc = target; + server.stat_cluster_links_memory += link->rcvbuf_alloc - prev_rcvbuf_alloc; +} + /* This function is called when we detect the link with this node is lost. We set the node as no longer connected. The Cluster Cron will detect this connection and will try to get it connected again. @@ -4698,6 +4900,13 @@ void clusterWriteHandler(connection *conn) { ssize_t nwritten; size_t totwritten = 0; + if (listLength(link->send_msg_queue) == 0) { + connSetWriteHandler(link->conn, NULL); + return; + } + + if (trySendClusterWriteToIOThreads(link) == C_OK) return; + while (totwritten < NET_MAX_WRITES_PER_EVENT && listLength(link->send_msg_queue) > 0) { listNode *head = listFirst(link->send_msg_queue); clusterMsgSendBlock *msgblock = (clusterMsgSendBlock *)head->value; @@ -4732,6 +4941,8 @@ void clusterWriteHandler(connection *conn) { totwritten += nwritten; } + /* Unregister the write handler when the queue is empty to avoid + * burning CPU on spurious writable events. */ if (listLength(link->send_msg_queue) == 0) connSetWriteHandler(link->conn, NULL); } @@ -4753,6 +4964,10 @@ void clusterLinkConnectHandler(connection *conn) { /* Register a read handler from now on */ connSetReadHandler(conn, clusterReadHandler); + /* Count a successfully connected (outbound) cluster link. This reflects + * how often we (re)connect to peers. */ + server.cluster->stat_cluster_links_established_outbound++; + /* Queue a PING in the new connection ASAP: this is crucial * to avoid false positives in failure detection. * @@ -4792,6 +5007,62 @@ static inline int isClusterMsgSignatureAndLengthValid(clusterMsgHeader *hdr) { return 1; } +/* Find the maximal prefix of rcvbuf that contains only complete packets. + * + * Scans the buffer by validating the signature ("RCmb"), minimum header + * length, and total length field. complete_bytes is the number of bytes at the + * start of rcvbuf that contain complete packets, and complete_packets is the + * number of packets in that prefix. + * + * Thread-safe: reads only from the provided buffer and writes only to the + * output parameters. Does not touch clusterNode, clusterState, or any + * main-thread structure. */ +void clusterFindCompletePackets(char *rcvbuf, + size_t rcvbuf_len, + size_t *complete_bytes, + size_t *complete_packets, + clusterIOResult *result) { + size_t offset = 0; + + *complete_bytes = 0; + *complete_packets = 0; + *result = CLUSTER_IO_OK; + + while (offset < rcvbuf_len) { + size_t remaining = rcvbuf_len - offset; + + /* Need at least the header to determine message length. */ + if (remaining < RCVBUF_MIN_READ_LEN) break; + + clusterMsgHeader *hdr = (clusterMsgHeader *)(rcvbuf + offset); + + /* Validate signature and minimum length. */ + if (memcmp(hdr->sig, "RCmb", 4) != 0) { + *complete_bytes = offset; /* preserve any valid prefix already scanned */ + *result = CLUSTER_IO_BAD_HEADER; + return; + } + + uint32_t totlen = ntohl(hdr->totlen); + uint16_t type = ntohs(hdr->type); + uint32_t minlen = IS_LIGHT_MESSAGE(type) ? CLUSTERMSG_LIGHT_MIN_LEN : CLUSTERMSG_MIN_LEN; + + if (totlen < minlen) { + *complete_bytes = offset; /* preserve any valid prefix already scanned */ + *result = CLUSTER_IO_BAD_LENGTH; + return; + } + + /* Wait for the full message to arrive. */ + if (remaining < totlen) break; + + offset += totlen; + (*complete_packets)++; + } + + *complete_bytes = offset; +} + /* Read data. Try to read the first field of the header first to check the * full length of the packet. When a whole packet is in memory this function * will call the function to process the packet. And so forth. */ @@ -4802,6 +5073,18 @@ void clusterReadHandler(connection *conn) { clusterLink *link = connGetPrivateData(conn); unsigned int readlen, rcvbuflen; + /* A worker read job is still in flight or its completion hasn't been + * consumed yet. Do not touch the framed packets or rcvbuf from the main + * thread until clusterHandleReadCompletion() transitions the link back + * to idle. */ + if (link->io_read_state != CLUSTER_LINK_IO_IDLE) return; + + if (!clusterDrainCompletePackets(link)) return; + + /* Try to offload the read first. If offload is unavailable (pool inactive, + * queue full), fall back to the synchronous path below. */ + if (trySendClusterReadToIOThreads(link) == C_OK) return; + while (1) { /* Read as long as there is data to read. */ rcvbuflen = link->rcvbuf_len; if (rcvbuflen < RCVBUF_MIN_READ_LEN) { @@ -4866,13 +5149,8 @@ void clusterReadHandler(connection *conn) { /* Total length obtained? Process this packet. */ if (rcvbuflen >= RCVBUF_MIN_READ_LEN && rcvbuflen == ntohl(hdr->totlen)) { if (clusterProcessPacket(link)) { - if (link->rcvbuf_alloc > RCVBUF_INIT_LEN) { - size_t prev_rcvbuf_alloc = link->rcvbuf_alloc; - zfree(link->rcvbuf); - link->rcvbuf = zmalloc(link->rcvbuf_alloc = RCVBUF_INIT_LEN); - server.stat_cluster_links_memory += link->rcvbuf_alloc - prev_rcvbuf_alloc; - } link->rcvbuf_len = 0; + clusterShrinkRcvbuf(link); } else { return; /* Link no longer valid. */ } @@ -5485,6 +5763,34 @@ void clusterSendFailoverAuth(clusterNode *node) { clusterMsgSendBlockDecrRefCount(msgblock); } +static const char *clusterNackReasonString(uint8_t reason) { + switch (reason) { + case CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_NOT_SAFE: return "not-safe"; + case CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_REQ_EPOCH_OLD: return "req-epoch-old"; + case CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_ALREADY_VOTED: return "already-voted"; + case CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_REQ_IS_PRIMARY: return "req-is-primary"; + case CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_NO_PRIMARY: return "no-primary"; + case CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_PRIMARY_UP: return "primary-up"; + case CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_STALE_CONFIG: return "stale-config"; + default: return "unknown"; + } +} + +/* Send a FAILOVER_AUTH_NACK message to the specified node. */ +void clusterSendFailoverNack(clusterNode *node, uint8_t reason) { + if (!node->link) return; + if (!nodeSupportsFailoverAuthNack(node)) return; + + uint32_t msglen = sizeof(clusterMsg) - sizeof(union clusterMsgData) + sizeof(clusterMsgDataFailoverNack); + clusterMsgSendBlock *msgblock = createClusterMsgSendBlock(CLUSTERMSG_TYPE_FAILOVER_AUTH_NACK, msglen); + + clusterMsg *hdr = getMessageFromSendBlock(msgblock); + memcpy(&hdr->data.failover_nack.nack.reason, &reason, sizeof(reason)); + + clusterSendMessage(node->link, msgblock); + clusterMsgSendBlockDecrRefCount(msgblock); +} + /* Send a MFSTART message to the specified node. */ void clusterSendMFStart(clusterNode *node) { if (!node->link) return; @@ -5515,6 +5821,7 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { if (!server.cluster->safe_to_join) { serverLog(LL_WARNING, "Failover auth denied to %.40s (%s): it is not safe to vote in this moment)", node->name, humanNodename(node)); + clusterSendFailoverNack(node, CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_NOT_SAFE); return; } @@ -5526,6 +5833,7 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { serverLog(LL_WARNING, "Failover auth denied to %.40s (%s): reqEpoch (%llu) < curEpoch(%llu)", node->name, humanNodename(node), (unsigned long long)requestCurrentEpoch, (unsigned long long)server.cluster->currentEpoch); + clusterSendFailoverNack(node, CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_REQ_EPOCH_OLD); return; } @@ -5533,6 +5841,7 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { if (server.cluster->lastVoteEpoch == server.cluster->currentEpoch) { serverLog(LL_WARNING, "Failover auth denied to %.40s (%s): already voted for epoch %llu", node->name, humanNodename(node), (unsigned long long)server.cluster->currentEpoch); + clusterSendFailoverNack(node, CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_ALREADY_VOTED); return; } @@ -5543,12 +5852,15 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { if (clusterNodeIsPrimary(node)) { serverLog(LL_WARNING, "Failover auth denied to %.40s (%s) for epoch %llu: it is a primary node", node->name, humanNodename(node), (unsigned long long)requestCurrentEpoch); + clusterSendFailoverNack(node, CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_REQ_IS_PRIMARY); } else if (primary == NULL) { serverLog(LL_WARNING, "Failover auth denied to %.40s (%s) for epoch %llu: I don't know its primary", node->name, humanNodename(node), (unsigned long long)requestCurrentEpoch); + clusterSendFailoverNack(node, CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_NO_PRIMARY); } else if (!nodeFailed(primary)) { serverLog(LL_WARNING, "Failover auth denied to %.40s (%s) for epoch %llu: its primary is up", node->name, humanNodename(node), (unsigned long long)requestCurrentEpoch); + clusterSendFailoverNack(node, CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_PRIMARY_UP); } return; } @@ -5583,7 +5895,13 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { "Node %.40s (%s) has old slots configuration, sending " "an UPDATE message about %.40s (%s)", node->name, humanNodename(node), slot_owner->name, humanNodename(slot_owner)); + /* Send UPDATE first so the replica can fix its slot config; the NACK + * that follows then triggers fast-fail, letting the replica retry + * with the freshly-updated configEpoch right away instead of waiting + * for auth_timeout. TCP ordering on the same link guarantees the + * UPDATE arrives before the NACK. */ clusterSendUpdate(node->link, slot_owner); + clusterSendFailoverNack(node, CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_STALE_CONFIG); return; } } @@ -5596,6 +5914,41 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { (unsigned long long)server.cluster->currentEpoch); } +/* Handle a FAILOVER_AUTH_NACK from a voter. */ +void clusterProcessFailoverAuthNack(clusterNode *sender, clusterMsg *request) { + /* Ignore NACKs from FAIL nodes to avoid double-counting: FAIL nodes are + * already accounted for in size_fail, and they will never ACK, so including + * their NACK would undercount achievable votes. */ + if (nodeFailed(sender)) { + return; + } + + server.cluster->failover_auth_nack_count++; + + /* A voter that NACKed us in this epoch will not change its mind, so the + * upper bound on the votes we can still collect is the voters that have + * not NACKed, minus FAIL voters that will never reply (they count towards + * size but neither ACK nor NACK). Fast-fail once that bound drops below + * the quorum we need to win.. */ + int needed_quorum = (server.cluster->size / 2) + 1; + int max_possible_acks = server.cluster->size - server.cluster->size_fail - server.cluster->failover_auth_nack_count; + serverLog(LL_NOTICE, "Failover auth NACK [%s] from %.40s (%s) for epoch %llu (NACKs %d, quorum %d)", + clusterNackReasonString(request->data.failover_nack.nack.reason), sender->name, + humanNodename(sender), (unsigned long long)server.cluster->failover_auth_epoch, + server.cluster->failover_auth_nack_count, needed_quorum); + if (max_possible_acks < needed_quorum) { + serverLog(LL_NOTICE, + "Failover election for epoch %llu cannot reach quorum %d (NACKs %d, dead voters %d). " + "Resetting the election since we cannot win an election without quorum.", + (unsigned long long)server.cluster->failover_auth_epoch, needed_quorum, + server.cluster->failover_auth_nack_count, server.cluster->size_fail); + server.cluster->failover_auth_time = 0; + /* Maybe we could start a new election, set a flag here to make sure + * we check as soon as possible, instead of waiting for a cron. */ + clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_FAILOVER); + } +} + /* This function returns the "rank" of this instance, a replica, in the context * of its primary-replicas ring. The rank of the replica is given by the number of * other replicas for the same primary that have a better replication offset @@ -5757,6 +6110,10 @@ void clusterLogCantFailover(int reason) { case CLUSTER_CANT_FAILOVER_WAITING_DELAY: msg = "Waiting the delay before I can start a new failover."; break; case CLUSTER_CANT_FAILOVER_EXPIRED: msg = "Failover attempt expired."; break; case CLUSTER_CANT_FAILOVER_WAITING_VOTES: msg = "Waiting for votes, but majority still not reached."; break; + case CLUSTER_CANT_FAILOVER_NO_DATA: + msg = "Replication offset is 0 and no data has been received from the primary. " + "Please check the 'cluster-replica-no-failover' configuration option."; + break; default: serverPanic("Unknown cant failover reason code."); } lastlog_time = time(NULL); @@ -5852,6 +6209,7 @@ void clusterHandleReplicaFailover(void) { /* Use a failover delay relative to node timeout: 500 for the default node * timeout of 15000, less for lower node timeout, but not more. */ long long delay = min(server.cluster_node_timeout / 30, 500); + if (server.debug_cluster_failover_delay >= 0) delay = server.debug_cluster_failover_delay; /* Pre conditions to run the function, that must be met both in case * of an automatic or manual failover: @@ -5861,7 +6219,7 @@ void clusterHandleReplicaFailover(void) { * not a manual failover. */ if (clusterNodeIsPrimary(myself) || myself->replicaof == NULL || (!nodeFailed(myself->replicaof) && !manual_failover) || - (server.cluster_replica_no_failover && !manual_failover)) { + (server.cluster_replica_no_failover == CLUSTER_REPLICA_NO_FAILOVER_YES && !manual_failover)) { /* There are no reasons to failover, so we set the reason why we * are returning without failing over to NONE. */ server.cluster->cant_failover_reason = CLUSTER_CANT_FAILOVER_NONE; @@ -5894,13 +6252,32 @@ void clusterHandleReplicaFailover(void) { } } + /* Refuse to start an automatic failover while we are still empty, when + * configured to do so. An empty replica has never received any data from + * its primary (e.g. it was just added and hasn't finished the initial + * sync, so its replication offset is 0), so promoting it would make an + * empty dataset the new primary and lose all the data of the shard. + * + * Note that "empty" refers to the data received from the primary, not to + * the number of keys: a replica fully synced with an empty primary has a + * non-zero offset and is therefore not considered empty. + * + * Check bypassed for manual failovers. */ + if (server.cluster_replica_no_failover == CLUSTER_REPLICA_NO_FAILOVER_IF_EMPTY && + !manual_failover && replicationGetReplicaOffset() == 0) { + clusterLogCantFailover(CLUSTER_CANT_FAILOVER_NO_DATA); + return; + } + /* If the previous failover attempt timeout and the retry time has * elapsed, we can set up a new one. */ if (auth_age > auth_retry_time) { server.cluster->failover_auth_time = now + delay + /* Fixed delay to let FAIL msg propagate. */ (delay ? random() % delay : 0); /* Random delay between 0 and the fixed delay. */ + if (server.debug_cluster_failover_delay >= 0) server.cluster->failover_auth_time = now + delay; server.cluster->failover_auth_count = 0; + server.cluster->failover_auth_nack_count = 0; server.cluster->failover_auth_sent = 0; server.cluster->failover_auth_rank = clusterGetReplicaRank(); /* We add another delay that is proportional to the replica rank. @@ -6011,7 +6388,17 @@ void clusterHandleReplicaFailover(void) { /* Ask for votes if needed. */ if (server.cluster->failover_auth_sent == 0) { - server.cluster->currentEpoch++; + if (server.debug_cluster_failover_epoch >= 0) { + /* Testing only: force this election to run in a specific epoch so + * that several replicas can be made to contend in the very same + * epoch, deterministically reproducing a split vote. Consumed + * once; subsequent retries fall back to the normal currentEpoch++ + * so the replicas can eventually win in distinct epochs. */ + server.cluster->currentEpoch = server.debug_cluster_failover_epoch; + server.debug_cluster_failover_epoch = -1; + } else { + server.cluster->currentEpoch++; + } server.cluster->failover_auth_epoch = server.cluster->currentEpoch; serverLog(LL_NOTICE, "Starting a failover election for epoch %llu, node config epoch is %llu", (unsigned long long)server.cluster->currentEpoch, (unsigned long long)nodeEpoch(myself)); @@ -6314,7 +6701,11 @@ static int clusterNodeCronHandleReconnect(clusterNode *node, mstime_t now, long (*cluster_conn_attempts)--; clusterLink *link = createClusterLink(node); link->conn = connCreate(connTypeOfCluster()); + /* Tag outbound cluster bus link as high-priority so node reconnects, gossip ping/pong, + * and failure detection heartbeats operate with QoS priority. */ + connSetPriority(link->conn, true); connSetPrivateData(link->conn, link); + connSetOwnerKind(link->conn, CONN_OWNER_CLUSTER_LINK); if (connConnect(link->conn, node->ip, node->cport, server.bind_source_addr, 0, clusterLinkConnectHandler) == C_ERR) { /* We got a synchronous error from connect before @@ -6351,7 +6742,12 @@ static void freeClusterLinkOnBufferLimitReached(clusterLink *link) { } } -/* Free outbound link to a node if its send buffer size exceeded limit. */ +/* ========================== Wrapper Functions for Testing ========================== */ +void testOnlyFreeClusterLinkOnBufferLimitReached(clusterLink *link) { + freeClusterLinkOnBufferLimitReached(link); +} + +/* Free a link to a node if its buffer size exceeded limit. */ static void clusterNodeCronFreeLinkOnBufferLimitReached(clusterNode *node) { freeClusterLinkOnBufferLimitReached(node->link); freeClusterLinkOnBufferLimitReached(node->inbound_link); @@ -6480,6 +6876,18 @@ void clusterCron(void) { freeClusterLink(node->link); } + /* In some situations the check above cannot disconnect the link, + * because data_received keeps being refreshed by the peer's own PINGs + * even though our PING was lost. If our PING stays outstanding for a + * full node timeout without a PONG, force a reconnect so a fresh PING + * is sent and the stale state clears. */ + if (node->link && + now - node->link->ctime > server.cluster_node_timeout && + node->ping_sent && ping_delay > server.cluster_node_timeout) { + /* Disconnect the link, it will be reconnected automatically. */ + freeClusterLink(node->link); + } + /* If we have currently no active ping in this instance, and the * received PONG is older than half the cluster timeout, send * a new ping now, to ensure all the nodes are pinged without @@ -6594,13 +7002,15 @@ void clusterBeforeSleep(void) { /* Save the config, possibly using fsync. */ if (flags & CLUSTER_TODO_SAVE_CONFIG) { - int fsync = flags & CLUSTER_TODO_FSYNC_CONFIG; + bool fsync = flags & CLUSTER_TODO_FSYNC_CONFIG; if (server.cluster_configfile_save_behavior == CLUSTER_CONFIGFILE_SAVE_BEHAVIOR_SYNC) { /* Sync mode: exit the process if saving fails. */ + bioDrainWorker(BIO_CLUSTER_SAVE); clusterSaveConfigOrDie(fsync); } else if (server.cluster_configfile_save_behavior == CLUSTER_CONFIGFILE_SAVE_BEHAVIOR_BEST_EFFORT) { - /* Best-effort mode: log (don't exit) if saving fails and wait for the next retry. */ - clusterSaveConfigOrLog(fsync); + /* Best-effort mode: save asynchronously via BIO thread; failures are logged (not fatal) + * and the save will be retried on the next config change. */ + clusterSaveConfigBackground(fsync); } } @@ -6726,6 +7136,7 @@ int clusterDelSlot(int slot) { /* Make owner_not_claiming_slot flag consistent with slot ownership information. */ bitmapClearBit(server.cluster->owner_not_claiming_slot, slot); clusterSlotStatReset(slot); + hotkeysPurgeSlot(slot); return C_OK; } @@ -6877,12 +7288,14 @@ void clusterUpdateState(void) { dictEntry *de; server.cluster->size = 0; + server.cluster->size_fail = 0; di = dictGetSafeIterator(server.cluster->nodes); while ((de = dictNext(di)) != NULL) { clusterNode *node = dictGetVal(de); if (clusterNodeIsVotingPrimary(node)) { server.cluster->size++; + if (node->flags & CLUSTER_NODE_FAIL) server.cluster->size_fail++; if ((node->flags & (CLUSTER_NODE_FAIL | CLUSTER_NODE_PFAIL)) == 0) reachable_primaries++; } } @@ -7008,7 +7421,10 @@ int verifyClusterConfigWithData(void) { delKeysInSlot(j, server.lazyfree_lazy_server_del, true, false); } } - if (update_config) clusterSaveConfigOrDie(1); + if (update_config) { + bioDrainWorker(BIO_CLUSTER_SAVE); + clusterSaveConfigOrDie(true); + } return C_OK; } @@ -7041,6 +7457,10 @@ static void clusterSetPrimary(clusterNode *n, int closeSlots, int full_sync_requ } if (closeSlots) clusterCloseAllSlots(); myself->replicaof = n; + if (nodeFailed(n)) + myself->flags |= CLUSTER_NODE_MY_PRIMARY_FAIL; + else + myself->flags &= ~CLUSTER_NODE_MY_PRIMARY_FAIL; updateShardId(myself, n->shard_id); clusterNodeAddReplica(n, myself); replicationSetPrimary(n->ip, getNodeDefaultReplicationPort(n), full_sync_required, true); @@ -7365,6 +7785,7 @@ const char *clusterGetMessageTypeString(int type) { case CLUSTERMSG_TYPE_PUBLISHSHARD: return "publishshard"; case CLUSTERMSG_TYPE_FAILOVER_AUTH_REQUEST: return "auth-req"; case CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK: return "auth-ack"; + case CLUSTERMSG_TYPE_FAILOVER_AUTH_NACK: return "auth-nack"; case CLUSTERMSG_TYPE_UPDATE: return "update"; case CLUSTERMSG_TYPE_MFSTART: return "mfstart"; case CLUSTERMSG_TYPE_MODULE: return "module"; @@ -7583,6 +8004,9 @@ sds genClusterInfoString(sds info) { } dictReleaseIterator(di); + int config_save_status = atomic_load_explicit(&server.cluster_config_save_status, memory_order_relaxed); + time_t config_last_save_time = atomic_load_explicit(&server.cluster_config_last_save_time, memory_order_relaxed); + info = sdscatfmt(info, "cluster_state:%s\r\n" "cluster_slots_assigned:%i\r\n" @@ -7596,11 +8020,15 @@ sds genClusterInfoString(sds info) { "cluster_known_nodes:%U\r\n" "cluster_size:%i\r\n" "cluster_current_epoch:%U\r\n" - "cluster_my_epoch:%U\r\n", + "cluster_my_epoch:%U\r\n" + "cluster_config_save_status:%s\r\n" + "cluster_config_last_save_time:%I\r\n", statestr[server.cluster->state], slots_assigned, slots_ok, slots_pfail, slots_fail, nodes_pfail, nodes_fail, voting_nodes_pfail, voting_nodes_fail, (unsigned long long)dictSize(server.cluster->nodes), server.cluster->size, - (unsigned long long)server.cluster->currentEpoch, (unsigned long long)my_epoch); + (unsigned long long)server.cluster->currentEpoch, (unsigned long long)my_epoch, + (config_save_status == C_OK) ? "ok" : "err", + (long long)config_last_save_time); /* Show stats about messages sent and received. */ long long tot_msg_sent = 0; @@ -7635,8 +8063,23 @@ sds genClusterInfoString(sds info) { (unsigned long long)server.cluster->stats_bus_module_bytes_sent, (unsigned long long)server.cluster->stats_bus_module_bytes_received); - info = sdscatfmt(info, "total_cluster_links_buffer_limit_exceeded:%U\r\n", - (unsigned long long)server.cluster->stat_cluster_links_buffer_limit_exceeded); + info = sdscatfmt(info, + "total_cluster_links_buffer_limit_exceeded:%U\r\n" + "total_cluster_links_established_inbound:%U\r\n" + "total_cluster_links_established_outbound:%U\r\n", + (unsigned long long)server.cluster->stat_cluster_links_buffer_limit_exceeded, + (unsigned long long)server.cluster->stat_cluster_links_established_inbound, + (unsigned long long)server.cluster->stat_cluster_links_established_outbound); + + info = sdscatfmt(info, + "cluster_io_threaded_reads_processed:%I\r\n" + "cluster_io_threaded_writes_processed:%I\r\n" + "cluster_io_threaded_accepts_processed:%I\r\n" + "cluster_io_main_thread_fallbacks:%I\r\n", + (long long)server.stat_cluster_threaded_reads_processed, + (long long)server.stat_cluster_threaded_writes_processed, + (long long)server.stat_cluster_threaded_accepts_processed, + (long long)server.stat_cluster_io_main_thread_fallbacks); return info; } @@ -7696,6 +8139,10 @@ unsigned int delKeysInSlot(unsigned int hashslot, int lazy, bool propagate_del, kvstoreReleaseHashtableIterator(kvs_di); } + /* The slot's keys have been removed locally (flushed or migrated away), so + * drop their hot-key state too. Sampling was suppressed during the loop via + * server_del_keys_in_slot (see hotkeysShouldRecord). */ + hotkeysPurgeSlot(hashslot); server.server_del_keys_in_slot = 0; serverAssert(server.execution_nesting == before_execution_nesting); return j; @@ -8293,7 +8740,8 @@ int clusterCommandSpecial(client *c) { (unsigned long long)myself->configEpoch); addReplySds(c, reply); } else if (!strcasecmp(objectGetVal(c->argv[1]), "saveconfig") && c->argc == 2) { - int retval = clusterSaveConfig(1); + bioDrainWorker(BIO_CLUSTER_SAVE); + int retval = clusterSaveConfig(false, true); if (retval == C_OK) addReply(c, shared.ok); @@ -8600,7 +9048,7 @@ const char **clusterCommandExtendedHelp(void) { "LINKS", " Return information about all network links between this node and its peers.", " Output format is an array where each array element is a map containing attributes of a link", - "MIGRATESLOTS SLOTSRANGE start-slot end-slot [start-slot end-slot ...] NODE node-id [SLOTSRANGE start-slot end-slot [start-slot end-slot ...] NODE node-id ...]", + "MIGRATESLOTS SLOTSRANGE start-slot end-slot [start-slot end-slot ...] NODE node-id [AUTH username password] [SLOTSRANGE start-slot end-slot [start-slot end-slot ...] NODE node-id [AUTH username password] ...]", " Migrate the specified slot ranges from this node to the specified node.", "CANCELSLOTMIGRATIONS ALL", " Cancel all migrations.", @@ -8764,3 +9212,337 @@ bool isAnySlotInManualImportingState(void) { bool isAnySlotInManualMigratingState(void) { return dictSize(server.cluster->migrating_slots_to) > 0; } + +/* ===================== Cluster I/O Thread Worker Functions ================== + * These run on I/O threads. They must NOT touch clusterNode, clusterState, + * server.stat_cluster_links_memory, or any main-thread-only structure. + * + * Read and write jobs are mutually exclusive per link, so the shared + * io_result field still has only one writer at a time. */ + +/* I/O thread worker: read bytes from a cluster link's connection, grow the + * receive buffer as needed, frame packets, and post a completion. + * + * Buffer growth follows the same logic as clusterReadHandler: + * - If < 1 MB, grow to twice the required size. + * - If >= 1 MB, grow by 1 MB. + * stat_cluster_links_memory adjustment is deferred to the main-thread + * completion handler. */ +void clusterReadJob(clusterLink *link) { + connection *conn = link->conn; + clusterIOResult result = CLUSTER_IO_OK; + ssize_t total_read = 0; + + /* I/O thread invariant: we must be in PENDING state. */ + serverAssert(link->io_read_state == CLUSTER_LINK_IO_PENDING); + serverAssert(link->io_write_state == CLUSTER_LINK_IO_IDLE); + + /* The link holds a connection for as long as an I/O job can be in flight: + * link->conn is only cleared on the immediate free path, which is + * unreachable while io_refs > 0, and the async-close path keeps the + * connection alive until the last completion. */ + serverAssert(conn != NULL); + + /* Bounded so one job cannot balloon rcvbuf or hold a worker; the socket stays + * readable and the next event continues. */ + while (total_read < (ssize_t)RCVBUF_MAX_PREALLOC) { + /* Ensure at least some space in rcvbuf. */ + size_t rcvbuf_len = link->rcvbuf_len; + if (rcvbuf_len == link->rcvbuf_alloc) { + size_t required = link->rcvbuf_alloc + 1; + link->rcvbuf_alloc = required < RCVBUF_MAX_PREALLOC ? required * 2 : required + RCVBUF_MAX_PREALLOC; + link->rcvbuf = zrealloc(link->rcvbuf, link->rcvbuf_alloc); + } + + size_t avail = link->rcvbuf_alloc - rcvbuf_len; + ssize_t nread = connRead(conn, link->rcvbuf + rcvbuf_len, avail); + + if (nread > 0) { + link->rcvbuf_len = rcvbuf_len + nread; + total_read += nread; + continue; + } + + if (nread == 0) { + /* EOF */ + result = CLUSTER_IO_EOF; + break; + } + + /* nread == -1 */ + if (connGetState(conn) == CONN_STATE_CONNECTED) { + /* EAGAIN — no more data right now, that's fine. */ + break; + } + /* Real read error. */ + result = CLUSTER_IO_READ_ERROR; + break; + } + + /* If we read something, frame the complete packet prefix and update + * the read timestamp. */ + if (total_read > 0) { + size_t complete_bytes = 0; + size_t complete_packets = 0; + clusterIOResult frame_result; + serverAssert(link->io_complete_bytes == 0); + serverAssert(link->io_complete_packets == 0); + clusterFindCompletePackets(link->rcvbuf, link->rcvbuf_len, + &complete_bytes, &complete_packets, &frame_result); + link->io_complete_bytes = complete_bytes; + link->io_complete_packets = complete_packets; + + /* If framing found a protocol error, that takes priority. */ + if (frame_result != CLUSTER_IO_OK) { + result = frame_result; + } + } + + /* Post result and completion to the main thread. */ + link->io_result = result; + sendToMainThread(link, JOB_RES_CLUSTER_READ); +} + +/* I/O thread worker: write bytes from the canonical send queue to the + * connection, starting at io_head_offset and stopping once it reaches + * io_last_send_block. + * + * The worker does NOT pop nodes or decrement refcounts — clusterMsgSendBlock + * refcounts are non-atomic and blocks can be shared across links. The + * worker records how many head nodes were fully sent (io_nodes_sent) and + * the byte offset into the next partially-sent node (io_head_offset). The + * main-thread completion handler uses these to pop nodes and update memory + * accounting. */ +void clusterWriteJob(clusterLink *link) { + connection *conn = link->conn; + clusterIOResult result = CLUSTER_IO_OK; + int nodes_sent = 0; + listNode *node = listFirst(link->send_msg_queue); + size_t head_offset = link->io_head_offset; + size_t totwritten = 0; + + /* I/O thread invariant: we must be in PENDING state. */ + serverAssert(link->io_write_state == CLUSTER_LINK_IO_PENDING); + serverAssert(link->io_read_state == CLUSTER_LINK_IO_IDLE); + + /* See clusterReadJob(): link->conn outlives any in-flight I/O job. */ + serverAssert(conn != NULL); + + /* Bounded like the synchronous path. A worker is shared, so a link with a + * large backlog must not hold it while other jobs wait; whatever is left + * goes out on the next writable event. */ + while (node && totwritten < NET_MAX_WRITES_PER_EVENT) { + clusterMsgSendBlock *msgblock = (clusterMsgSendBlock *)node->value; + clusterMsg *msg = &msgblock->data[0].msg; + size_t msg_len = ntohl(msg->totlen); + size_t msg_offset = head_offset; + + ssize_t nwritten = connWrite(conn, (char *)msg + msg_offset, msg_len - msg_offset); + if (nwritten <= 0) { + if (nwritten == -1 && connGetState(conn) == CONN_STATE_CONNECTED) { + break; /* EAGAIN */ + } + result = CLUSTER_IO_WRITE_ERROR; + break; + } + + head_offset += nwritten; + totwritten += nwritten; + if (head_offset < msg_len) { + break; /* Partial write */ + } + + /* Fully sent this message — advance to next. */ + head_offset = 0; + nodes_sent++; + if (node == link->io_last_send_block) break; + node = listNextNode(node); + } + + link->io_nodes_sent = nodes_sent; + link->io_head_offset = head_offset; + link->io_result = result; + sendToMainThread(link, JOB_RES_CLUSTER_WRITE); +} + +/* I/O thread worker: perform TLS accept handshake on a cluster connection. + * No clusterLink exists yet — it is created by the main thread on success. */ +void clusterAcceptJob(connection *conn) { + /* The dispatcher holds a reference on the connection for the whole job, so + * it cannot be NULL here. Returning early instead would skip + * sendToMainThread() and leak a pending response forever. */ + serverAssert(conn != NULL); + connAccept(conn, NULL); + sendToMainThread(conn, JOB_RES_CLUSTER_ACCEPT); +} + +/* ===================== Cluster I/O Completion Handlers ===================== + * These handlers are called from processIOThreadsResponses() when cluster + * I/O completions are dequeued from the response queue. The tagged pointer + * is the clusterLink* (read/write) or connection* (accept) directly. */ + +void clusterHandleReadCompletion(clusterLink *link) { + connection *conn = link->conn; + + /* Apply deferred connection state transitions. Even if freeClusterLink() + * was called while the job was in flight, link->conn remains valid until + * the final async_close teardown runs after the last completion. */ + if (conn) { + connSetPostponeUpdateState(conn, 0); + connUpdateState(conn); + } + + /* Transition back to idle and release the I/O ref. */ + serverAssert(link->io_read_state == CLUSTER_LINK_IO_PENDING); + serverAssert(link->io_write_state == CLUSTER_LINK_IO_IDLE); + serverAssert(link->io_refs > 0); + link->io_read_state = CLUSTER_LINK_IO_IDLE; + link->io_refs--; + + /* Update stat_cluster_links_memory for rcvbuf growth that occurred on + * the I/O thread (the I/O thread grows rcvbuf_alloc but does not touch + * the global stat). */ + if (link->rcvbuf_alloc > link->rcvbuf_alloc_at_dispatch) { + server.stat_cluster_links_memory += link->rcvbuf_alloc - link->rcvbuf_alloc_at_dispatch; + } + + /* If the link was already marked for async close, check if we can + * perform the final free now that io_refs has been decremented. */ + if (link->async_close) { + if (link->io_refs == 0) { + freeClusterLink(link); + } + return; + } + + clusterIOResult result = link->io_result; + + /* Handle error results: log and tear down the link. */ + if (result == CLUSTER_IO_BAD_HEADER || result == CLUSTER_IO_BAD_LENGTH) { + /* Drain any valid packets that preceded the bad header/length before + * closing, so we don't silently drop already-complete messages. */ + if (link->io_complete_bytes > 0) { + if (!clusterDrainCompletePackets(link)) return; + } + serverLog(LL_WARNING, "Bad cluster packet header/length from node %.40s:%s (%s)", + clusterLinkGetNodeName(link), + link->inbound ? "inbound" : "outbound", + clusterLinkGetHumanNodeName(link)); + freeClusterLink(link); + return; + } + + if (result == CLUSTER_IO_READ_ERROR || result == CLUSTER_IO_EOF) { + serverLog(LL_DEBUG, "I/O error reading from node link (%.40s:%s) (%s): %s", + clusterLinkGetNodeName(link), + link->inbound ? "inbound" : "outbound", + clusterLinkGetHumanNodeName(link), + (result == CLUSTER_IO_EOF) ? "connection closed" : "read error"); + } + + if (!clusterDrainCompletePackets(link)) return; + + clusterShrinkRcvbuf(link); + + if (result == CLUSTER_IO_READ_ERROR || result == CLUSTER_IO_EOF) { + freeClusterLink(link); + } +} + +void clusterHandleWriteCompletion(clusterLink *link) { + connection *conn = link->conn; + + /* Apply deferred connection state transitions. */ + if (conn) { + connSetPostponeUpdateState(conn, 0); + connUpdateState(conn); + } + + /* Transition back to idle and release the I/O ref. */ + serverAssert(link->io_write_state == CLUSTER_LINK_IO_PENDING); + serverAssert(link->io_read_state == CLUSTER_LINK_IO_IDLE); + serverAssert(link->io_refs > 0); + link->io_write_state = CLUSTER_LINK_IO_IDLE; + link->io_refs--; + + /* Pop fully-sent nodes from the canonical send queue. The I/O thread + * recorded how many nodes it fully sent (io_nodes_sent) without + * modifying the list. We pop them here on the main thread where + * refcount decrements and memory accounting are safe. */ + size_t prev_head_offset = link->head_msg_send_offset; + for (int i = 0; i < link->io_nodes_sent; i++) { + listNode *head = listFirst(link->send_msg_queue); + serverAssert(head != NULL); + clusterMsgSendBlock *msgblock = (clusterMsgSendBlock *)head->value; + clusterMsg *msg = getMessageFromSendBlock(msgblock); + uint32_t msg_len = ntohl(msg->totlen); + size_t start = (i == 0) ? prev_head_offset : 0; + clusterBusAddNetworkBytesByType(ntohs(msg->type) & ~CLUSTERMSG_MODIFIER_MASK, msg_len - start, 1); + uint32_t blocklen = msgblock->totlen; + listDelNode(link->send_msg_queue, head); + link->send_msg_queue_mem -= sizeof(listNode) + blocklen; + server.stat_cluster_links_memory -= sizeof(listNode); + } + + /* Account for bytes written into a partially-sent head node. */ + if (link->io_head_offset > 0) { + listNode *head = listFirst(link->send_msg_queue); + if (head) { + clusterMsgSendBlock *msgblock = (clusterMsgSendBlock *)head->value; + clusterMsg *msg = getMessageFromSendBlock(msgblock); + size_t start = (link->io_nodes_sent == 0) ? prev_head_offset : 0; + size_t partial_bytes = link->io_head_offset - start; + if (partial_bytes > 0) { + clusterBusAddNetworkBytesByType(ntohs(msg->type) & ~CLUSTERMSG_MODIFIER_MASK, partial_bytes, 1); + } + } + } + + link->head_msg_send_offset = listLength(link->send_msg_queue) > 0 ? link->io_head_offset : 0; + link->io_last_send_block = NULL; + link->io_head_offset = 0; + link->io_nodes_sent = 0; + + /* If the link was already marked for async close, check if we can + * perform the final free now that io_refs has been decremented. */ + if (link->async_close) { + if (link->io_refs == 0) { + freeClusterLink(link); + } + return; + } + + clusterIOResult result = link->io_result; + + /* Handle write error: log and tear down the link. */ + if (result == CLUSTER_IO_WRITE_ERROR) { + serverLog(LL_DEBUG, "I/O error writing to node link (%.40s:%s) (%s)", + clusterLinkGetNodeName(link), + link->inbound ? "inbound" : "outbound", + clusterLinkGetHumanNodeName(link)); + freeClusterLink(link); + return; + } + + /* If data remains, wait for the next writable event before attempting + * another offload. This avoids a tight completion -> offload loop when + * the transport reports EAGAIN with no write progress. */ + if (listLength(link->send_msg_queue) > 0) { + if (link->conn) { + connSetWriteHandlerWithBarrier(link->conn, clusterWriteHandler, 1); + } + } else if (link->conn && connHasWriteHandler(link->conn)) { + connSetWriteHandler(link->conn, NULL); + } +} + +void clusterHandleAcceptCompletion(connection *conn) { + conn->flags &= ~CONN_FLAG_ACCEPT_OFFLOAD_PENDING; + /* Runs conn_handler if the handshake finished, re-arms the TLS event if not. */ + connSetPostponeUpdateState(conn, 0); + connUpdateState(conn); + connDecrRefs(conn); + if ((conn->flags & CONN_FLAG_CLOSE_SCHEDULED) && !connHasRefs(conn)) { + connClose(conn); + } +} diff --git a/src/cluster_legacy.h b/src/cluster_legacy.h index acd3eb1de..176521adc 100644 --- a/src/cluster_legacy.h +++ b/src/cluster_legacy.h @@ -2,8 +2,13 @@ #define CLUSTER_LEGACY_H #include + #define CLUSTER_PORT_INCR 10000 /* Cluster port = baseport + PORT_INCR */ +/* Receive buffer sizing for a cluster link. */ +#define RCVBUF_INIT_LEN 1024 +#define RCVBUF_MAX_PREALLOC (1 << 20) /* 1MB */ + /* The following defines are amount of time, sometimes expressed as * multipliers of the node timeout value (when ending with MULT). */ #define CLUSTER_FAIL_REPORT_VALIDITY_MULT 2 /* Fail report validity. */ @@ -17,6 +22,7 @@ #define CLUSTER_CANT_FAILOVER_WAITING_DELAY 2 #define CLUSTER_CANT_FAILOVER_EXPIRED 3 #define CLUSTER_CANT_FAILOVER_WAITING_VOTES 4 +#define CLUSTER_CANT_FAILOVER_NO_DATA 5 #define CLUSTER_CANT_FAILOVER_RELOG_PERIOD 1 /* seconds. */ /* clusterState todo_before_sleep flags. */ @@ -28,6 +34,22 @@ #define CLUSTER_TODO_BROADCAST_ALL (1 << 5) #define CLUSTER_TODO_HANDLE_SLOT_MIGRATION (1 << 6) +/* I/O state for threaded cluster bus offload. */ +typedef enum { + CLUSTER_LINK_IO_IDLE = 0, + CLUSTER_LINK_IO_PENDING, +} clusterLinkIOState; + +/* Result codes for cluster I/O jobs. */ +typedef enum { + CLUSTER_IO_OK = 0, + CLUSTER_IO_BAD_HEADER, + CLUSTER_IO_BAD_LENGTH, + CLUSTER_IO_READ_ERROR, + CLUSTER_IO_EOF, + CLUSTER_IO_WRITE_ERROR, +} clusterIOResult; + /* clusterLink encapsulates everything needed to talk with a remote node. */ typedef struct clusterLink { mstime_t ctime; /* Link creation time */ @@ -41,6 +63,29 @@ typedef struct clusterLink { clusterNode *node; /* Node related to this link. Initialized to NULL when unknown */ int inbound; /* 1 if this link is an inbound link accepted from the related node */ int flags; /* CLUSTER_LINK_... */ + + /* Threaded I/O state (main-thread owned, except where noted) */ + int io_read_state; /* clusterLinkIOState: read job state */ + int io_write_state; /* clusterLinkIOState: write job state */ + int async_close; /* 1 if teardown requested while jobs in flight */ + int io_refs; /* Count of in-flight I/O jobs */ + clusterIOResult io_result; /* Result code from last I/O job (written by I/O thread). + * Shared by read/write jobs because they are mutually exclusive per link. */ + + /* Async write snapshot/result */ + listNode *io_last_send_block; /* Last queue node visible to current write job */ + size_t io_head_offset; /* Snapshot/result offset into queue head */ + int io_nodes_sent; /* Number of fully-sent head nodes (set by I/O thread) */ + + /* Pre-dispatch rcvbuf_alloc for memory accounting on completion */ + size_t rcvbuf_alloc_at_dispatch; /* rcvbuf_alloc when read job was dispatched */ + + /* Async read framing/result */ + size_t io_complete_bytes; /* Bytes at the start of rcvbuf framed as complete packets */ + size_t io_complete_packets; /* Number of complete packets in io_complete_bytes */ + + /* Read/write fairness */ + int io_read_deferred; /* Read skipped while a write was in flight; next write dispatch yields */ } clusterLink; /* Cluster link flags and macros. */ @@ -69,7 +114,8 @@ typedef struct clusterLink { * myself will gossip this flag to other replica in the \ * shard so that the replicas can make a better ranking \ * decisions to help with the failover. */ -#define CLUSTER_NODE_MAX CLUSTER_NODE_MY_PRIMARY_FAIL /* Max bit for CLUSTER_NODE_* flag, update while adding a new flag. */ +#define CLUSTER_NODE_FAILOVER_AUTH_NACK_SUPPORTED (1 << 14) /* This node understands FAILOVER_AUTH_NACK messages. */ +#define CLUSTER_NODE_MAX CLUSTER_NODE_FAILOVER_AUTH_NACK_SUPPORTED /* Max bit for CLUSTER_NODE_* flag, update while adding a new flag. */ /* Ensure cluster node flags never silently grew beyond 16 bits. * The flags in clusterMsg and clusterMsgDataGossip are uint16_t. */ @@ -91,6 +137,7 @@ static_assert(CLUSTER_NODE_MAX <= UINT16_MAX, "cluster node flags must fit in 16 #define nodeSupportsMultiMeet(n) ((n)->flags & CLUSTER_NODE_MULTI_MEET_SUPPORTED) #define nodeInNormalState(n) (!((n)->flags & (CLUSTER_NODE_HANDSHAKE | CLUSTER_NODE_MEET | CLUSTER_NODE_PFAIL | CLUSTER_NODE_FAIL))) #define nodePrimaryIsFail(n) ((n)->flags & CLUSTER_NODE_MY_PRIMARY_FAIL) +#define nodeSupportsFailoverAuthNack(n) ((n)->flags & CLUSTER_NODE_FAILOVER_AUTH_NACK_SUPPORTED) /* Cluster messages header */ @@ -111,7 +158,8 @@ static_assert(CLUSTER_NODE_MAX <= UINT16_MAX, "cluster node flags must fit in 16 #define CLUSTERMSG_TYPE_MFSTART 8 /* Pause clients for manual failover */ #define CLUSTERMSG_TYPE_MODULE 9 /* Module cluster API message. */ #define CLUSTERMSG_TYPE_PUBLISHSHARD 10 /* Pub/Sub Publish shard propagation */ -#define CLUSTERMSG_TYPE_COUNT 11 /* Total number of message types. */ +#define CLUSTERMSG_TYPE_FAILOVER_AUTH_NACK 11 /* No, you don't have my vote. */ +#define CLUSTERMSG_TYPE_COUNT 12 /* Total number of message types. */ #define CLUSTERMSG_LIGHT 0x8000 /* Modifier bit for message types that support light header */ @@ -146,6 +194,10 @@ typedef struct { char nodename[CLUSTER_NAMELEN]; } clusterMsgDataFail; +typedef struct { + uint8_t reason; +} clusterMsgDataFailoverNack; + typedef struct { uint32_t channel_len; uint32_t message_len; @@ -275,6 +327,11 @@ union clusterMsgData { struct { clusterMsgModule msg; } module; + + /* FAILOVER_AUTH_NACK */ + struct { + clusterMsgDataFailoverNack nack; + } failover_nack; }; #define CLUSTER_PROTO_VER 1 /* Cluster bus protocol version. */ @@ -347,6 +404,15 @@ static_assert(offsetof(clusterMsg, data) == 2256, "unexpected field offset"); primary is up. */ #define CLUSTERMSG_FLAG0_EXT_DATA (1 << 2) /* Message contains extension data */ +/* Reason values carried in clusterMsgDataFailoverNack.reason. */ +#define CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_NOT_SAFE 1 /* Voter is not safe to vote yet. */ +#define CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_REQ_EPOCH_OLD 2 /* Request epoch < voter's currentEpoch. */ +#define CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_ALREADY_VOTED 3 /* Voter already voted in this epoch. */ +#define CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_REQ_IS_PRIMARY 4 /* Requester is a primary itself. */ +#define CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_NO_PRIMARY 5 /* Voter doesn't know it's primary. */ +#define CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_PRIMARY_UP 6 /* Voter still sees it's primary up. */ +#define CLUSTERMSG_FAILOVER_AUTH_NACK_REASON_STALE_CONFIG 7 /* Replica's slot config is stale. */ + typedef struct { char sig[4]; /* Signature "RCmb" (Cluster message bus). */ uint32_t totlen; /* Total length of this message */ @@ -448,6 +514,7 @@ struct clusterState { int fail_reason; /* Why the cluster state changes to fail. */ int safe_to_join; /* Can the restarted node safely join the cluster? */ int size; /* Num of primary nodes with at least one slot */ + int size_fail; /* Num of voting primaries currently in FAIL state (subset of size). */ dict *nodes; /* Hash table of name -> clusterNode structures */ dict *shards; /* Hash table of shard_id -> list (of nodes) structures */ dict *nodes_black_list; /* Nodes we don't re-add for a few seconds. */ @@ -460,6 +527,7 @@ struct clusterState { /* The following fields are used to take the replica state on elections. */ mstime_t failover_auth_time; /* Time of previous or next election. */ int failover_auth_count; /* Number of votes received so far. */ + int failover_auth_nack_count; /* Number of rejected votes received so far. */ int failover_auth_sent; /* True if we already asked for votes. */ int failover_auth_rank; /* This replica rank for current auth request. */ int failover_failed_primary_rank; /* The rank of this instance in the context of all failed primary list. */ @@ -493,6 +561,10 @@ struct clusterState { excluding nodes without address. */ unsigned long long stat_cluster_links_buffer_limit_exceeded; /* Total number of cluster links freed due to exceeding buffer limit */ + unsigned long long stat_cluster_links_established_inbound; /* Total number of inbound cluster links + successfully established via accept. */ + unsigned long long stat_cluster_links_established_outbound; /* Total number of outbound cluster links + successfully established via connect. */ /* Bit map for slots that are no longer claimed by the owner in cluster PING * messages. During slot migration, the owner will stop claiming the slot after @@ -504,4 +576,18 @@ struct clusterState { slotStat slot_stats[CLUSTER_SLOTS]; }; +/* Cluster I/O completion handlers called from processIOThreadsResponses(). + * For read/write, the tagged pointer is the clusterLink* itself. + * For accept, the tagged pointer is the connection* (no clusterLink exists yet). + * Implemented in cluster_legacy.c. */ +void clusterHandleReadCompletion(clusterLink *link); +void clusterHandleWriteCompletion(clusterLink *link); +void clusterHandleAcceptCompletion(connection *conn); +void clusterConnAcceptHandler(connection *conn); +void clusterReadJob(clusterLink *link); +void clusterWriteJob(clusterLink *link); + +void testOnlyFreeClusterLinkOnBufferLimitReached(clusterLink *link); +void clusterAcceptJob(connection *conn); + #endif // CLUSTER_LEGACY_H diff --git a/src/cluster_migrateslots.c b/src/cluster_migrateslots.c index c5c554eea..4311982f8 100644 --- a/src/cluster_migrateslots.c +++ b/src/cluster_migrateslots.c @@ -9,6 +9,8 @@ #include "io_threads.h" #include "module.h" #include "functions.h" +#include "sds.h" +#include "server.h" #include #include @@ -89,6 +91,8 @@ typedef struct slotMigrationJob { /* State needed during client establishment */ connection *conn; /* Connection to slot import source node. */ sds response_buf; + sds auth_user; /* User used for AUTH of the export job. */ + sds auth_password; /* Password used for AUTH of the export job */ } slotMigrationJob; static bool isSlotMigrationJobFinished(slotMigrationJob *job); @@ -104,8 +108,11 @@ static void updateSlotMigrationJobState(slotMigrationJob *job, static void sendSyncSlotsMessage(slotMigrationJob *job, const char *subcommand); static void proceedWithSlotMigration(slotMigrationJob *job); static slotMigrationJob *createSlotExportJob(clusterNode *target_node, - list *slot_ranges); + list *slot_ranges, + sds auth_user, + sds auth_password); static bool isSlotExportPauseTimedOut(slotMigrationJob *job); +static void freeSlotMigrationJobAuth(slotMigrationJob *job); static void resetSlotMigrationJob(slotMigrationJob *job); static void finishSlotMigrationJob(slotMigrationJob *job, slotMigrationJobState state, @@ -548,6 +555,12 @@ void clusterCommandSyncSlotsEstablish(client *c) { char *source_node_name = NULL; list *slot_ranges = NULL; + if (c->slot_migration_job) { + addReplyError(c, "CLUSTER SYNCSLOTS ESTABLISH is not allowed on a " + "client that is already a slot migration client"); + return; + } + if (!mustObeyClient(c) && validateSlotMigrationCanStartOrReply(c) == C_ERR) { return; } @@ -731,6 +744,15 @@ void clusterCommandSyncSlotsFailoverGranted(client *c) { /* Sent by a target primary to a replica in its shard to inform that an ongoing * slot import is now finished. */ void clusterCommandSyncSlotsFinish(client *c) { + /* FINISH is sent within the slot migration replication stream, so it must + * originate from a slot migration client (primary/AOF). Reject any other + * client to prevent driving the import state machine to a terminal state. */ + if (!mustObeyClient(c)) { + addReplyError(c, "CLUSTER SYNCSLOTS FINISH should only be used " + "by slot migration clients"); + return; + } + char *name = NULL; char *state = NULL; char *message = NULL; @@ -835,6 +857,12 @@ slotMigrationJob *createSlotImportJob(client *c, job->state = SLOT_IMPORT_WAIT_ACK; job->client = c; job->client->slot_migration_job = job; + if (c && c->conn) { + /* Upgrade connection to high priority */ + if (connSetPriority(c->conn, true) == C_ERR) { + serverLog(LL_WARNING, "Failed to upgrade priority for slot migration connection %d", c->conn->fd); + } + } /* We treat slot imports like primaries. Primaries are expected to have a * dedicated query buffer and allocated replication data. @@ -1161,12 +1189,24 @@ bool clusterSlotFailoverGranted(int slot) { * source will attempt to migrate the slot ranges to the specified target * node. */ void clusterCommandMigrateSlots(client *c) { + /* Redact credentials before validation so errors cannot expose them + * in the command log. */ + for (int i = 2; i < c->argc; i++) { + if (!strcasecmp(objectGetVal(c->argv[i]), "auth")) { + if (i + 1 < c->argc) redactClientCommandArgument(c, i + 1); + if (i + 2 < c->argc) redactClientCommandArgument(c, i + 2); + i += 2; + } + } + if (validateSlotMigrationCanStartOrReply(c) == C_ERR) return; int curr_index = 2; list *new_slot_migrations = listCreate(); listSetFreeMethod(new_slot_migrations, freeSlotMigrationJob); list *slot_ranges = NULL; + sds auth_user = NULL; + sds auth_pass = NULL; while (curr_index < c->argc) { if (strcasecmp(objectGetVal(c->argv[curr_index]), "slotsrange")) { @@ -1233,9 +1273,24 @@ void clusterCommandMigrateSlots(client *c) { } curr_index++; - slotMigrationJob *job = createSlotExportJob(target_node, slot_ranges); + if (curr_index < c->argc) { + sds token = objectGetVal(c->argv[curr_index]); + if (!strcasecmp(token, "auth")) { + if (curr_index + 2 >= c->argc) { + addReplyErrorObject(c, shared.syntaxerr); + goto cleanup; + } + auth_user = sdsdup(objectGetVal(c->argv[curr_index + 1])); + auth_pass = sdsdup(objectGetVal(c->argv[curr_index + 2])); + curr_index += 3; + } + } + + slotMigrationJob *job = createSlotExportJob(target_node, slot_ranges, auth_user, auth_pass); listAddNodeHead(new_slot_migrations, job); slot_ranges = NULL; + auth_user = NULL; + auth_pass = NULL; } /* If we reach here, we have successfully parsed all arguments */ @@ -1263,6 +1318,11 @@ void clusterCommandMigrateSlots(client *c) { cleanup: if (slot_ranges) listRelease(slot_ranges); listRelease(new_slot_migrations); + sdsfree(auth_user); + if (auth_pass) { + memset(auth_pass, 0, sdslen(auth_pass)); + sdsfree(auth_pass); + } } slotMigrationJob *clusterLookupMigrationJob(sds name) { @@ -1331,6 +1391,8 @@ int connectSlotExportJob(slotMigrationJob *job) { port); job->conn = connCreate(connTypeOfReplication()); + /* Set connection to high priority */ + connSetPriority(job->conn, true); if (connConnect(job->conn, n->ip, port, server.bind_source_addr, 0, slotExportConnectHandler) == C_ERR) { return C_ERR; @@ -1388,9 +1450,24 @@ void slotMigrationJobReadAuthResponse(connection *conn) { * job's connection. */ void slotMigrationJobSendAuth(slotMigrationJob *job) { serverAssert(job->type == SLOT_MIGRATION_EXPORT); - serverAssert(server.primary_auth); + serverAssert((job->auth_user == NULL) == (job->auth_password == NULL)); + const char *user = NULL; + size_t user_len = 0; + sds pass; + if (job->auth_user) { + user = job->auth_user; + user_len = sdslen(job->auth_user); + pass = job->auth_password; + } else { + user = server.primary_user; + user_len = user ? strlen(server.primary_user) : 0; + pass = server.primary_auth; + } + serverAssert(pass); - sds err = replicationSendAuth(job->conn); + sds err = replicationSendAuth(job->conn, user, user_len, pass, sdslen(pass)); + /* AUTH is never retried, so the job no longer needs its credentials. */ + freeSlotMigrationJobAuth(job); if (err) { sds status_msg = sdscatfmt(sdsempty(), "Failed to send AUTH command to target node: %s", err); finishSlotMigrationJob(job, SLOT_MIGRATION_JOB_FAILED, status_msg); @@ -1887,7 +1964,9 @@ size_t clusterGetTotalSlotExportBufferMemory(void) { /* Create a slot export job with the given target and slot ranges. */ slotMigrationJob *createSlotExportJob(clusterNode *target_node, - list *slot_ranges) { + list *slot_ranges, + sds auth_user, + sds auth_password) { slotMigrationJob *job = zcalloc(sizeof(slotMigrationJob)); job->ctime = server.unixtime; @@ -1901,6 +1980,8 @@ slotMigrationJob *createSlotExportJob(clusterNode *target_node, memcpy(job->target_node_name, target_node->name, CLUSTER_NAMELEN); memcpy(job->source_node_name, server.cluster->myself->name, CLUSTER_NAMELEN); job->description = generateSlotMigrationJobDescription(job, target_node); + job->auth_user = auth_user; + job->auth_password = auth_password; return job; } @@ -2046,7 +2127,7 @@ void proceedWithSlotMigration(slotMigrationJob *job) { if (!completed) return; serverLog(LL_NOTICE, "Slot migration %s connection established.", job->description); - if (server.primary_auth) { + if (job->auth_password || server.primary_auth) { updateSlotMigrationJobState(job, SLOT_EXPORT_SEND_AUTH); } else { updateSlotMigrationJobState(job, SLOT_EXPORT_SEND_ESTABLISH); @@ -2182,7 +2263,18 @@ void proceedWithSlotMigration(slotMigrationJob *job) { } } -/* Reset the client and connection information associated with the job, leaving +/* Release job-owned credentials without retaining the password in migration history. */ +static void freeSlotMigrationJobAuth(slotMigrationJob *job) { + sdsfree(job->auth_user); + job->auth_user = NULL; + if (job->auth_password) { + memset(job->auth_password, 0, sdslen(job->auth_password)); + sdsfree(job->auth_password); + job->auth_password = NULL; + } +} + +/* Reset the client, connection and authentication information associated with the job, leaving * the migration related metadata. */ void resetSlotMigrationJob(slotMigrationJob *job) { /* Only one of client or conn should be set. */ @@ -2198,6 +2290,7 @@ void resetSlotMigrationJob(slotMigrationJob *job) { sdsfree(job->response_buf); job->response_buf = NULL; + freeSlotMigrationJobAuth(job); } void freeSlotMigrationJob(void *o) { diff --git a/src/commands.def b/src/commands.def index 88b25745f..66dfc061b 100644 --- a/src/commands.def +++ b/src/commands.def @@ -773,17 +773,24 @@ struct COMMAND_ARG CLUSTER_MIGRATESLOTS_migration_group_range_Subargs[] = { {MAKE_ARG("end-slot",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, }; +/* CLUSTER MIGRATESLOTS migration_group auth argument table */ +struct COMMAND_ARG CLUSTER_MIGRATESLOTS_migration_group_auth_Subargs[] = { +{MAKE_ARG("username",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("password",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + /* CLUSTER MIGRATESLOTS migration_group argument table */ struct COMMAND_ARG CLUSTER_MIGRATESLOTS_migration_group_Subargs[] = { {MAKE_ARG("slotsrange-token",ARG_TYPE_PURE_TOKEN,-1,"SLOTSRANGE",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("range",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=CLUSTER_MIGRATESLOTS_migration_group_range_Subargs}, {MAKE_ARG("node-token",ARG_TYPE_PURE_TOKEN,-1,"NODE",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("node-id",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("auth",ARG_TYPE_BLOCK,-1,"AUTH",NULL,"9.2.0",CMD_ARG_OPTIONAL,2,NULL),.subargs=CLUSTER_MIGRATESLOTS_migration_group_auth_Subargs}, }; /* CLUSTER MIGRATESLOTS argument table */ struct COMMAND_ARG CLUSTER_MIGRATESLOTS_Args[] = { -{MAKE_ARG("migration-group",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,4,NULL),.subargs=CLUSTER_MIGRATESLOTS_migration_group_Subargs}, +{MAKE_ARG("migration-group",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,5,NULL),.subargs=CLUSTER_MIGRATESLOTS_migration_group_Subargs}, }; /********** CLUSTER MYID ********************/ @@ -1463,6 +1470,7 @@ commandHistory CLIENT_KILL_History[] = { {"8.0.0","Replaced `master` `TYPE` with `primary`. `master` still supported for backward compatibility."}, {"8.1.0","`ID` option accepts multiple IDs."}, {"9.0.0","Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP."}, +{"9.2.0","Added `H` flag to indicate high priority clients."}, }; #endif @@ -1558,6 +1566,7 @@ commandHistory CLIENT_LIST_History[] = { {"8.0.0","Replaced `master` `TYPE` with `primary`. `master` still supported for backward compatibility."}, {"8.1.0","Added filters USER, ADDR, LADDR, SKIPME, and MAXAGE."}, {"9.0.0","Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP."}, +{"9.2.0","Added `H` flag to indicate high priority clients."}, }; #endif @@ -1905,8 +1914,8 @@ struct COMMAND_STRUCT CLIENT_Subcommands[] = { {MAKE_CMD("id","Returns the unique client ID of the connection.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_ID_History,0,CLIENT_ID_Tips,0,clientIDCommand,2,CMD_ALLOW_BUSY|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,CLIENT_ID_Keyspecs,0,NULL,0)}, {MAKE_CMD("import-source","Marks this client as an import source when the server is in import mode.","O(1)","8.1.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_IMPORT_SOURCE_History,0,CLIENT_IMPORT_SOURCE_Tips,0,clientImportSourceCommand,3,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,CLIENT_IMPORT_SOURCE_Keyspecs,0,NULL,1),.args=CLIENT_IMPORT_SOURCE_Args}, {MAKE_CMD("info","Returns information about the connection.","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_INFO_History,0,CLIENT_INFO_Tips,1,clientInfoCommand,2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,CLIENT_INFO_Keyspecs,0,NULL,0)}, -{MAKE_CMD("kill","Terminates open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_KILL_History,9,CLIENT_KILL_Tips,0,clientKillCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_KILL_Keyspecs,0,NULL,1),.args=CLIENT_KILL_Args}, -{MAKE_CMD("list","Lists open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_LIST_History,9,CLIENT_LIST_Tips,1,clientListCommand,-2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_LIST_Keyspecs,0,NULL,27),.args=CLIENT_LIST_Args}, +{MAKE_CMD("kill","Terminates open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_KILL_History,10,CLIENT_KILL_Tips,0,clientKillCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_KILL_Keyspecs,0,NULL,1),.args=CLIENT_KILL_Args}, +{MAKE_CMD("list","Lists open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_LIST_History,10,CLIENT_LIST_Tips,1,clientListCommand,-2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_LIST_Keyspecs,0,NULL,27),.args=CLIENT_LIST_Args}, {MAKE_CMD("no-evict","Sets the client eviction mode of the connection.","O(1)","7.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_NO_EVICT_History,0,CLIENT_NO_EVICT_Tips,0,clientNoEvictCommand,3,CMD_ALLOW_BUSY|CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_NO_EVICT_Keyspecs,0,NULL,1),.args=CLIENT_NO_EVICT_Args}, {MAKE_CMD("no-touch","Controls whether commands sent by the client affect the LRU/LFU of accessed keys.","O(1)","7.2.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_NO_TOUCH_History,0,CLIENT_NO_TOUCH_Tips,0,clientNoTouchCommand,3,CMD_ALLOW_BUSY|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,CLIENT_NO_TOUCH_Keyspecs,0,NULL,1),.args=CLIENT_NO_TOUCH_Args}, {MAKE_CMD("pause","Suspends commands processing.","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_PAUSE_History,1,CLIENT_PAUSE_Tips,0,clientPauseCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_PAUSE_Keyspecs,0,NULL,2),.args=CLIENT_PAUSE_Args}, @@ -3905,7 +3914,7 @@ struct COMMAND_ARG HGETDEL_Args[] = { #ifndef SKIP_CMD_KEY_SPECS_TABLE /* HGETEX key specs */ keySpec HGETEX_Keyspecs[1] = { -{NULL,CMD_KEY_RW|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} }; #endif @@ -6195,7 +6204,7 @@ struct COMMAND_STRUCT SCRIPT_Subcommands[] = { {MAKE_CMD("flush","Removes all server-side Lua scripts from the script cache.","O(N) with N being the number of scripts in cache","2.6.0",CMD_DOC_NONE,NULL,NULL,"scripting",COMMAND_GROUP_SCRIPTING,SCRIPT_FLUSH_History,1,SCRIPT_FLUSH_Tips,2,scriptCommand,-2,CMD_NOSCRIPT|CMD_STALE,ACL_CATEGORY_SCRIPTING|ACL_CATEGORY_SLOW,NULL,SCRIPT_FLUSH_Keyspecs,0,NULL,1),.args=SCRIPT_FLUSH_Args}, {MAKE_CMD("help","Returns helpful text about the different subcommands.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,"scripting",COMMAND_GROUP_SCRIPTING,SCRIPT_HELP_History,0,SCRIPT_HELP_Tips,0,scriptCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_SCRIPTING|ACL_CATEGORY_SLOW,NULL,SCRIPT_HELP_Keyspecs,0,NULL,0)}, {MAKE_CMD("kill","Terminates a server-side Lua script during execution.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,"scripting",COMMAND_GROUP_SCRIPTING,SCRIPT_KILL_History,0,SCRIPT_KILL_Tips,2,scriptCommand,2,CMD_NOSCRIPT|CMD_ALLOW_BUSY,ACL_CATEGORY_SCRIPTING|ACL_CATEGORY_SLOW,NULL,SCRIPT_KILL_Keyspecs,0,NULL,0)}, -{MAKE_CMD("load","Loads a server-side Lua script to the script cache.","O(N) with N being the length in bytes of the script body.","2.6.0",CMD_DOC_NONE,NULL,NULL,"scripting",COMMAND_GROUP_SCRIPTING,SCRIPT_LOAD_History,0,SCRIPT_LOAD_Tips,2,scriptCommand,3,CMD_NOSCRIPT|CMD_STALE,ACL_CATEGORY_SCRIPTING|ACL_CATEGORY_SLOW,NULL,SCRIPT_LOAD_Keyspecs,0,NULL,1),.args=SCRIPT_LOAD_Args}, +{MAKE_CMD("load","Loads a server-side Lua script to the script cache.","O(N) with N being the length in bytes of the script body.","2.6.0",CMD_DOC_NONE,NULL,NULL,"scripting",COMMAND_GROUP_SCRIPTING,SCRIPT_LOAD_History,0,SCRIPT_LOAD_Tips,2,scriptCommand,3,CMD_DENYOOM|CMD_NOSCRIPT|CMD_STALE,ACL_CATEGORY_SCRIPTING|ACL_CATEGORY_SLOW,NULL,SCRIPT_LOAD_Keyspecs,0,NULL,1),.args=SCRIPT_LOAD_Args}, {MAKE_CMD("show","Show server-side Lua script in the script cache.","O(1).","8.0.0",CMD_DOC_NONE,NULL,NULL,"scripting",COMMAND_GROUP_SCRIPTING,SCRIPT_SHOW_History,0,SCRIPT_SHOW_Tips,0,scriptCommand,3,CMD_NOSCRIPT|CMD_STALE,ACL_CATEGORY_SCRIPTING|ACL_CATEGORY_SLOW,NULL,SCRIPT_SHOW_Keyspecs,0,NULL,1),.args=SCRIPT_SHOW_Args}, {0} }; @@ -6852,6 +6861,31 @@ struct COMMAND_ARG ACL_CAT_Args[] = { {MAKE_ARG("category",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,0,NULL)}, }; +/********** ACL DELROLE ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* ACL DELROLE history */ +#define ACL_DELROLE_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* ACL DELROLE tips */ +const char *ACL_DELROLE_Tips[] = { +"request_policy:all_nodes", +"response_policy:all_succeeded", +}; +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* ACL DELROLE key specs */ +#define ACL_DELROLE_Keyspecs NULL +#endif + +/* ACL DELROLE argument table */ +struct COMMAND_ARG ACL_DELROLE_Args[] = { +{MAKE_ARG("rolename",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + /********** ACL DELUSER ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -6940,6 +6974,28 @@ struct COMMAND_ARG ACL_GENPASS_Args[] = { {MAKE_ARG("bits",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,0,NULL)}, }; +/********** ACL GETROLE ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* ACL GETROLE history */ +#define ACL_GETROLE_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* ACL GETROLE tips */ +#define ACL_GETROLE_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* ACL GETROLE key specs */ +#define ACL_GETROLE_Keyspecs NULL +#endif + +/* ACL GETROLE argument table */ +struct COMMAND_ARG ACL_GETROLE_Args[] = { +{MAKE_ARG("rolename",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + /********** ACL GETUSER ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -6948,6 +7004,7 @@ commandHistory ACL_GETUSER_History[] = { {"6.2.0","Added Pub/Sub channel patterns."}, {"7.0.0","Added selectors and changed the format of key and channel patterns from a list to their rule representation."}, {"9.1.0","Added database permission rules."}, +{"9.2.0","Added roles."}, }; #endif @@ -7047,6 +7104,23 @@ struct COMMAND_ARG ACL_LOG_Args[] = { {MAKE_ARG("operation",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=ACL_LOG_operation_Subargs}, }; +/********** ACL ROLES ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* ACL ROLES history */ +#define ACL_ROLES_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* ACL ROLES tips */ +#define ACL_ROLES_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* ACL ROLES key specs */ +#define ACL_ROLES_Keyspecs NULL +#endif + /********** ACL SAVE ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -7067,6 +7141,32 @@ const char *ACL_SAVE_Tips[] = { #define ACL_SAVE_Keyspecs NULL #endif +/********** ACL SETROLE ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* ACL SETROLE history */ +#define ACL_SETROLE_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* ACL SETROLE tips */ +const char *ACL_SETROLE_Tips[] = { +"request_policy:all_nodes", +"response_policy:all_succeeded", +}; +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* ACL SETROLE key specs */ +#define ACL_SETROLE_Keyspecs NULL +#endif + +/* ACL SETROLE argument table */ +struct COMMAND_ARG ACL_SETROLE_Args[] = { +{MAKE_ARG("rolename",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("rule",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE,0,NULL)}, +}; + /********** ACL SETUSER ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -7075,6 +7175,7 @@ commandHistory ACL_SETUSER_History[] = { {"6.2.0","Added Pub/Sub channel patterns."}, {"7.0.0","Added selectors and key based permissions."}, {"9.1.0","Added database permission rules."}, +{"9.2.0","Added the `role=` rule for assigning roles to a user."}, }; #endif @@ -7134,17 +7235,21 @@ struct COMMAND_ARG ACL_SETUSER_Args[] = { /* ACL command table */ struct COMMAND_STRUCT ACL_Subcommands[] = { {MAKE_CMD("cat","Lists the ACL categories, or the commands inside a category.","O(1) since the categories and commands are a fixed set.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_CAT_History,0,ACL_CAT_Tips,0,aclCommand,-2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_SLOW,NULL,ACL_CAT_Keyspecs,0,NULL,1),.args=ACL_CAT_Args}, +{MAKE_CMD("delrole","Deletes one or more ACL roles. Fails if any role is assigned to a user.","O(N). Where N is the number of roles to delete.","9.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_DELROLE_History,0,ACL_DELROLE_Tips,2,aclCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_DELROLE_Keyspecs,0,NULL,1),.args=ACL_DELROLE_Args}, {MAKE_CMD("deluser","Deletes ACL users, and terminates their connections.","O(1) amortized time considering the typical user.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_DELUSER_History,0,ACL_DELUSER_Tips,2,aclCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_DELUSER_Keyspecs,0,NULL,1),.args=ACL_DELUSER_Args}, {MAKE_CMD("digest","Returns a fingerprint of the ACL rules currently in effect.","O(N). Where N is the number of configured users.","9.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_DIGEST_History,0,ACL_DIGEST_Tips,0,aclCommand,2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_DIGEST_Keyspecs,0,NULL,0)}, {MAKE_CMD("dryrun","Simulates the execution of a command by a user, without executing the command.","O(1).","7.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_DRYRUN_History,0,ACL_DRYRUN_Tips,0,aclCommand,-4,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_DRYRUN_Keyspecs,0,NULL,3),.args=ACL_DRYRUN_Args}, {MAKE_CMD("genpass","Generates a pseudorandom, secure password that can be used to identify ACL users.","O(1)","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_GENPASS_History,0,ACL_GENPASS_Tips,0,aclCommand,-2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_SLOW,NULL,ACL_GENPASS_Keyspecs,0,NULL,1),.args=ACL_GENPASS_Args}, -{MAKE_CMD("getuser","Lists the ACL rules of a user.","O(N). Where N is the number of password, command and pattern rules that the user has.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_GETUSER_History,3,ACL_GETUSER_Tips,0,aclCommand,3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_GETUSER_Keyspecs,0,NULL,1),.args=ACL_GETUSER_Args}, +{MAKE_CMD("getrole","Returns the ACL rules of an ACL role.","O(N). Where N is the number of rules defined for the role.","9.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_GETROLE_History,0,ACL_GETROLE_Tips,0,aclCommand,3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_GETROLE_Keyspecs,0,NULL,1),.args=ACL_GETROLE_Args}, +{MAKE_CMD("getuser","Lists the ACL rules of a user.","O(N). Where N is the number of password, command and pattern rules that the user has.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_GETUSER_History,4,ACL_GETUSER_Tips,0,aclCommand,3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_GETUSER_Keyspecs,0,NULL,1),.args=ACL_GETUSER_Args}, {MAKE_CMD("help","Returns helpful text about the different subcommands.","O(1)","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_HELP_History,0,ACL_HELP_Tips,0,aclCommand,2,CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_SLOW,NULL,ACL_HELP_Keyspecs,0,NULL,0)}, {MAKE_CMD("list","Dumps the effective rules in ACL file format.","O(N). Where N is the number of configured users.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_LIST_History,0,ACL_LIST_Tips,0,aclCommand,2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_LIST_Keyspecs,0,NULL,0)}, {MAKE_CMD("load","Reloads the rules from the configured ACL file.","O(N). Where N is the number of configured users.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_LOAD_History,0,ACL_LOAD_Tips,0,aclCommand,2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_LOAD_Keyspecs,0,NULL,0)}, {MAKE_CMD("log","Lists recent security events generated due to ACL rules.","O(N) with N being the number of entries shown.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_LOG_History,1,ACL_LOG_Tips,0,aclCommand,-2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_LOG_Keyspecs,0,NULL,1),.args=ACL_LOG_Args}, +{MAKE_CMD("roles","Lists all ACL roles.","O(N). Where N is the number of configured roles.","9.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_ROLES_History,0,ACL_ROLES_Tips,0,aclCommand,2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_ROLES_Keyspecs,0,NULL,0)}, {MAKE_CMD("save","Saves the effective ACL rules in the configured ACL file.","O(N). Where N is the number of configured users.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_SAVE_History,0,ACL_SAVE_Tips,2,aclCommand,2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_SAVE_Keyspecs,0,NULL,0)}, -{MAKE_CMD("setuser","Creates and modifies an ACL user and its rules.","O(N). Where N is the number of rules provided.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_SETUSER_History,3,ACL_SETUSER_Tips,2,aclCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_SETUSER_Keyspecs,0,NULL,2),.args=ACL_SETUSER_Args}, +{MAKE_CMD("setrole","Creates and modifies an ACL role and its rules.","O(N+M*C). Where N is the number of rules provided, M the number of users in the role and C the number of connected clients.","9.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_SETROLE_History,0,ACL_SETROLE_Tips,2,aclCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_SETROLE_Keyspecs,0,NULL,2),.args=ACL_SETROLE_Args}, +{MAKE_CMD("setuser","Creates and modifies an ACL user and its rules.","O(N). Where N is the number of rules provided.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_SETUSER_History,4,ACL_SETUSER_Tips,2,aclCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_SETUSER_Keyspecs,0,NULL,2),.args=ACL_SETUSER_Args}, {MAKE_CMD("users","Lists all ACL users.","O(N). Where N is the number of configured users.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_USERS_History,0,ACL_USERS_Tips,0,aclCommand,2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,ACL_USERS_Keyspecs,0,NULL,0)}, {MAKE_CMD("whoami","Returns the authenticated username of the current connection.","O(1)","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_WHOAMI_History,0,ACL_WHOAMI_Tips,0,aclCommand,2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_SLOW,NULL,ACL_WHOAMI_Keyspecs,0,NULL,0)}, {0} @@ -7848,6 +7953,82 @@ struct COMMAND_ARG FLUSHDB_Args[] = { {MAKE_ARG("flush-type",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=FLUSHDB_flush_type_Subargs}, }; +/********** HOTKEYS GET ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HOTKEYS GET history */ +#define HOTKEYS_GET_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HOTKEYS GET tips */ +#define HOTKEYS_GET_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HOTKEYS GET key specs */ +#define HOTKEYS_GET_Keyspecs NULL +#endif + +/********** HOTKEYS HELP ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HOTKEYS HELP history */ +#define HOTKEYS_HELP_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HOTKEYS HELP tips */ +#define HOTKEYS_HELP_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HOTKEYS HELP key specs */ +#define HOTKEYS_HELP_Keyspecs NULL +#endif + +/********** HOTKEYS RESET ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HOTKEYS RESET history */ +#define HOTKEYS_RESET_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HOTKEYS RESET tips */ +#define HOTKEYS_RESET_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HOTKEYS RESET key specs */ +#define HOTKEYS_RESET_Keyspecs NULL +#endif + +/* HOTKEYS command table */ +struct COMMAND_STRUCT HOTKEYS_Subcommands[] = { +{MAKE_CMD("get","Get the hottest keys from the last completed window, ordered by estimated QPS descending","O(N) where N is the number of hot keys in the history","9.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,HOTKEYS_GET_History,0,HOTKEYS_GET_Tips,0,hotkeysGetCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,HOTKEYS_GET_Keyspecs,0,NULL,0)}, +{MAKE_CMD("help","Shows helpful text about the different subcommands","O(1)","9.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,HOTKEYS_HELP_History,0,HOTKEYS_HELP_Tips,0,hotkeysHelpCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_SLOW,NULL,HOTKEYS_HELP_Keyspecs,0,NULL,0)}, +{MAKE_CMD("reset","Reset all hot key statistics and history","O(N) where N is the number of keys to remove from history and statistics","9.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,HOTKEYS_RESET_History,0,HOTKEYS_RESET_Tips,0,hotkeysResetCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,HOTKEYS_RESET_Keyspecs,0,NULL,0)}, +{0} +}; + +/********** HOTKEYS ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* HOTKEYS history */ +#define HOTKEYS_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* HOTKEYS tips */ +#define HOTKEYS_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* HOTKEYS key specs */ +#define HOTKEYS_Keyspecs NULL +#endif + /********** INFO ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -9003,7 +9184,9 @@ struct COMMAND_ARG SINTERSTORE_Args[] = { #ifndef SKIP_CMD_HISTORY_TABLE /* SISMEMBER history */ -#define SISMEMBER_History NULL +commandHistory SISMEMBER_History[] = { +{"9.1.0","Added the `XX` options."}, +}; #endif #ifndef SKIP_CMD_TIPS_TABLE @@ -9022,6 +9205,7 @@ keySpec SISMEMBER_Keyspecs[1] = { struct COMMAND_ARG SISMEMBER_Args[] = { {MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("member",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,"9.1.0",CMD_ARG_OPTIONAL,0,NULL)}, }; /********** SMEMBERS ********************/ @@ -10376,6 +10560,42 @@ struct COMMAND_ARG XACK_Args[] = { {MAKE_ARG("id",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, }; +/********** XACKDEL ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* XACKDEL history */ +#define XACKDEL_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* XACKDEL tips */ +#define XACKDEL_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* XACKDEL key specs */ +keySpec XACKDEL_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* XACKDEL mode argument table */ +struct COMMAND_ARG XACKDEL_mode_Subargs[] = { +{MAKE_ARG("keepref",ARG_TYPE_PURE_TOKEN,-1,"KEEPREF",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("delref",ARG_TYPE_PURE_TOKEN,-1,"DELREF",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("acked",ARG_TYPE_PURE_TOKEN,-1,"ACKED",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* XACKDEL argument table */ +struct COMMAND_ARG XACKDEL_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("group",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("mode",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,3,NULL),.subargs=XACKDEL_mode_Subargs}, +{MAKE_ARG("ids",ARG_TYPE_PURE_TOKEN,-1,"IDS",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("numids",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("id",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + /********** XADD ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -10536,6 +10756,41 @@ struct COMMAND_ARG XDEL_Args[] = { {MAKE_ARG("id",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, }; +/********** XDELEX ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* XDELEX history */ +#define XDELEX_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* XDELEX tips */ +#define XDELEX_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* XDELEX key specs */ +keySpec XDELEX_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* XDELEX mode argument table */ +struct COMMAND_ARG XDELEX_mode_Subargs[] = { +{MAKE_ARG("keepref",ARG_TYPE_PURE_TOKEN,-1,"KEEPREF",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("delref",ARG_TYPE_PURE_TOKEN,-1,"DELREF",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("acked",ARG_TYPE_PURE_TOKEN,-1,"ACKED",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* XDELEX argument table */ +struct COMMAND_ARG XDELEX_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("mode",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,3,NULL),.subargs=XDELEX_mode_Subargs}, +{MAKE_ARG("ids",ARG_TYPE_PURE_TOKEN,-1,"IDS",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("numids",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("id",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)}, +}; + /********** XGROUP CREATE ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -11439,6 +11694,53 @@ struct COMMAND_ARG INCRBYFLOAT_Args[] = { {MAKE_ARG("increment",ARG_TYPE_DOUBLE,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, }; +/********** INCREX ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* INCREX history */ +#define INCREX_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* INCREX tips */ +#define INCREX_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* INCREX key specs */ +keySpec INCREX_Keyspecs[1] = { +{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}} +}; +#endif + +/* INCREX condition argument table */ +struct COMMAND_ARG INCREX_condition_Subargs[] = { +{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* INCREX expiration argument table */ +struct COMMAND_ARG INCREX_expiration_Subargs[] = { +{MAKE_ARG("ex",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("px",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* INCREX increment argument table */ +struct COMMAND_ARG INCREX_increment_Subargs[] = { +{MAKE_ARG("integer",ARG_TYPE_INTEGER,-1,"BYINT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("float",ARG_TYPE_DOUBLE,-1,"BYFLOAT",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* INCREX argument table */ +struct COMMAND_ARG INCREX_Args[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=INCREX_condition_Subargs}, +{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=INCREX_expiration_Subargs}, +{MAKE_ARG("increment",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=INCREX_increment_Subargs}, +}; + /********** LCS ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -11644,6 +11946,7 @@ commandHistory SET_History[] = { {"6.2.0","Added the `GET`, `EXAT` and `PXAT` option."}, {"7.0.0","Allowed the `NX` and `GET` options to be used together."}, {"8.1.0","Added the `IFEQ` option."}, +{"9.2.0","Added the `IFNE` option."}, }; #endif @@ -11664,6 +11967,7 @@ struct COMMAND_ARG SET_condition_Subargs[] = { {MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,"2.6.12",CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,"2.6.12",CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("comparison-value",ARG_TYPE_STRING,-1,"IFEQ","Sets the key's value only if the current value matches the specified comparison value.","8.1.0",CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("comparison-not-equal",ARG_TYPE_STRING,-1,"IFNE","Sets the key's value only if the current value does not match the specified comparison value.","9.2.0",CMD_ARG_NONE,0,NULL)}, }; /* SET expiration argument table */ @@ -11679,7 +11983,7 @@ struct COMMAND_ARG SET_expiration_Subargs[] = { struct COMMAND_ARG SET_Args[] = { {MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("value",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, -{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,3,NULL),.subargs=SET_condition_Subargs}, +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=SET_condition_Subargs}, {MAKE_ARG("get",ARG_TYPE_PURE_TOKEN,-1,"GET",NULL,"6.2.0",CMD_ARG_OPTIONAL,0,NULL)}, {MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=SET_expiration_Subargs}, }; @@ -11832,7 +12136,9 @@ struct COMMAND_ARG SUBSTR_Args[] = { #ifndef SKIP_CMD_HISTORY_TABLE /* EXEC history */ -#define EXEC_History NULL +commandHistory EXEC_History[] = { +{"9.2.0","Added the `IFEQ`, `IFNE`, `NX`, and `XX` options."}, +}; #endif #ifndef SKIP_CMD_TIPS_TABLE @@ -11842,9 +12148,36 @@ struct COMMAND_ARG SUBSTR_Args[] = { #ifndef SKIP_CMD_KEY_SPECS_TABLE /* EXEC key specs */ -#define EXEC_Keyspecs NULL +keySpec EXEC_Keyspecs[1] = { +{"Condition keys are determined by EXEC condition tokens.",CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_UNKNOWN,{{0}},KSPEC_FK_UNKNOWN,{{0}}} +}; #endif +/* EXEC condition ifeq argument table */ +struct COMMAND_ARG EXEC_condition_ifeq_Subargs[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("value",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* EXEC condition ifne argument table */ +struct COMMAND_ARG EXEC_condition_ifne_Subargs[] = { +{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("value",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* EXEC condition argument table */ +struct COMMAND_ARG EXEC_condition_Subargs[] = { +{MAKE_ARG("ifeq",ARG_TYPE_BLOCK,-1,"IFEQ",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=EXEC_condition_ifeq_Subargs}, +{MAKE_ARG("ifne",ARG_TYPE_BLOCK,-1,"IFNE",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=EXEC_condition_ifne_Subargs}, +{MAKE_ARG("nonexisting-key",ARG_TYPE_KEY,0,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("existing-key",ARG_TYPE_KEY,0,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + +/* EXEC argument table */ +struct COMMAND_ARG EXEC_Args[] = { +{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,"9.2.0",CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE,4,NULL),.subargs=EXEC_condition_Subargs}, +}; + /********** MULTI ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -11960,43 +12293,43 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("waitaof","Blocks until all of the preceding write commands sent by the connection are written to the append-only file of the primary and/or replicas.","O(1)","7.2.0",CMD_DOC_NONE,NULL,NULL,"generic",COMMAND_GROUP_GENERIC,WAITAOF_History,0,WAITAOF_Tips,2,waitaofCommand,4,CMD_BLOCKING,ACL_CATEGORY_BLOCKING|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,WAITAOF_Keyspecs,0,NULL,3),.args=WAITAOF_Args}, /* geo */ {MAKE_CMD("geoadd","Adds one or more members to a geospatial index. The key is created if it doesn't exist.","O(log(N)) for each item added, where N is the number of elements in the sorted set.","3.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEOADD_History,1,GEOADD_Tips,0,geoaddCommand,-5,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_GEO|ACL_CATEGORY_SLOW|ACL_CATEGORY_WRITE,NULL,GEOADD_Keyspecs,1,NULL,4),.args=GEOADD_Args}, -{MAKE_CMD("geodist","Returns the distance between two members of a geospatial index.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEODIST_History,0,GEODIST_Tips,0,geodistCommand,-4,CMD_READONLY,ACL_CATEGORY_GEO|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,GEODIST_Keyspecs,1,NULL,4),.args=GEODIST_Args}, -{MAKE_CMD("geohash","Returns members from a geospatial index as geohash strings.","O(1) for each member requested.","3.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEOHASH_History,0,GEOHASH_Tips,0,geohashCommand,-2,CMD_READONLY,ACL_CATEGORY_GEO|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,GEOHASH_Keyspecs,1,NULL,2),.args=GEOHASH_Args}, -{MAKE_CMD("geopos","Returns the longitude and latitude of members from a geospatial index.","O(1) for each member requested.","3.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEOPOS_History,0,GEOPOS_Tips,0,geoposCommand,-2,CMD_READONLY,ACL_CATEGORY_GEO|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,GEOPOS_Keyspecs,1,NULL,2),.args=GEOPOS_Args}, +{MAKE_CMD("geodist","Returns the distance between two members of a geospatial index.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEODIST_History,0,GEODIST_Tips,0,geodistCommand,-4,CMD_READONLY,ACL_CATEGORY_GEO|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,GEODIST_Keyspecs,1,NULL,4),.args=GEODIST_Args,.member_arg_index=2}, +{MAKE_CMD("geohash","Returns members from a geospatial index as geohash strings.","O(1) for each member requested.","3.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEOHASH_History,0,GEOHASH_Tips,0,geohashCommand,-2,CMD_READONLY,ACL_CATEGORY_GEO|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,GEOHASH_Keyspecs,1,NULL,2),.args=GEOHASH_Args,.member_arg_index=2}, +{MAKE_CMD("geopos","Returns the longitude and latitude of members from a geospatial index.","O(1) for each member requested.","3.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEOPOS_History,0,GEOPOS_Tips,0,geoposCommand,-2,CMD_READONLY,ACL_CATEGORY_GEO|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,GEOPOS_Keyspecs,1,NULL,2),.args=GEOPOS_Args,.member_arg_index=2}, {MAKE_CMD("georadius","Queries a geospatial index for members within a distance from a coordinate, optionally stores the result.","O(N+log(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.","3.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEORADIUS_History,2,GEORADIUS_Tips,0,georadiusCommand,-6,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_GEO|ACL_CATEGORY_SLOW|ACL_CATEGORY_WRITE,NULL,GEORADIUS_Keyspecs,3,georadiusGetKeys,11),.args=GEORADIUS_Args}, -{MAKE_CMD("georadiusbymember","Queries a geospatial index for members within a distance from a member, optionally stores the result.","O(N+log(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.","3.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEORADIUSBYMEMBER_History,2,GEORADIUSBYMEMBER_Tips,0,georadiusbymemberCommand,-5,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_GEO|ACL_CATEGORY_SLOW|ACL_CATEGORY_WRITE,NULL,GEORADIUSBYMEMBER_Keyspecs,3,georadiusGetKeys,10),.args=GEORADIUSBYMEMBER_Args}, -{MAKE_CMD("georadiusbymember_ro","Returns members from a geospatial index that are within a distance from a member.","O(N+log(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.","3.2.10",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEORADIUSBYMEMBER_RO_History,2,GEORADIUSBYMEMBER_RO_Tips,0,georadiusbymemberroCommand,-5,CMD_READONLY,ACL_CATEGORY_GEO|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,GEORADIUSBYMEMBER_RO_Keyspecs,1,NULL,9),.args=GEORADIUSBYMEMBER_RO_Args}, +{MAKE_CMD("georadiusbymember","Queries a geospatial index for members within a distance from a member, optionally stores the result.","O(N+log(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.","3.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEORADIUSBYMEMBER_History,2,GEORADIUSBYMEMBER_Tips,0,georadiusbymemberCommand,-5,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_GEO|ACL_CATEGORY_SLOW|ACL_CATEGORY_WRITE,NULL,GEORADIUSBYMEMBER_Keyspecs,3,georadiusGetKeys,10),.args=GEORADIUSBYMEMBER_Args,.member_arg_index=2}, +{MAKE_CMD("georadiusbymember_ro","Returns members from a geospatial index that are within a distance from a member.","O(N+log(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.","3.2.10",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEORADIUSBYMEMBER_RO_History,2,GEORADIUSBYMEMBER_RO_Tips,0,georadiusbymemberroCommand,-5,CMD_READONLY,ACL_CATEGORY_GEO|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,GEORADIUSBYMEMBER_RO_Keyspecs,1,NULL,9),.args=GEORADIUSBYMEMBER_RO_Args,.member_arg_index=2}, {MAKE_CMD("georadius_ro","Returns members from a geospatial index that are within a distance from a coordinate.","O(N+log(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.","3.2.10",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEORADIUS_RO_History,2,GEORADIUS_RO_Tips,0,georadiusroCommand,-6,CMD_READONLY,ACL_CATEGORY_GEO|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,GEORADIUS_RO_Keyspecs,1,NULL,10),.args=GEORADIUS_RO_Args}, {MAKE_CMD("geosearch","Queries a geospatial index for members inside an area of a box, circle, or a polygon.","O(N+log(M)) where N is the number of elements in the grid-aligned bounding box area around the shape provided as the filter and M is the number of items inside the shape","6.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEOSEARCH_History,2,GEOSEARCH_Tips,0,geosearchCommand,-7,CMD_READONLY,ACL_CATEGORY_GEO|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,GEOSEARCH_Keyspecs,1,NULL,8),.args=GEOSEARCH_Args}, {MAKE_CMD("geosearchstore","Queries a geospatial index for members inside an area of a box, a circle, or a polygon, optionally stores the result.","O(N+log(M)) where N is the number of elements in the grid-aligned bounding box area around the shape provided as the filter and M is the number of items inside the shape","6.2.0",CMD_DOC_NONE,NULL,NULL,"geo",COMMAND_GROUP_GEO,GEOSEARCHSTORE_History,2,GEOSEARCHSTORE_Tips,0,geosearchstoreCommand,-8,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_GEO|ACL_CATEGORY_SLOW|ACL_CATEGORY_WRITE,NULL,GEOSEARCHSTORE_Keyspecs,2,NULL,7),.args=GEOSEARCHSTORE_Args}, /* hash */ -{MAKE_CMD("hdel","Deletes one or more fields and their values from a hash. Deletes the hash if no fields remain.","O(N) where N is the number of fields to be removed.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HDEL_History,1,HDEL_Tips,0,hdelCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HDEL_Keyspecs,1,NULL,2),.args=HDEL_Args}, -{MAKE_CMD("hexists","Determines whether a field exists in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXISTS_History,0,HEXISTS_Tips,0,hexistsCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HEXISTS_Keyspecs,1,NULL,2),.args=HEXISTS_Args}, +{MAKE_CMD("hdel","Deletes one or more fields and their values from a hash. Deletes the hash if no fields remain.","O(N) where N is the number of fields to be removed.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HDEL_History,1,HDEL_Tips,0,hdelCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HDEL_Keyspecs,1,NULL,2),.args=HDEL_Args,.member_arg_index=2}, +{MAKE_CMD("hexists","Determines whether a field exists in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXISTS_History,0,HEXISTS_Tips,0,hexistsCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HEXISTS_Keyspecs,1,NULL,2),.args=HEXISTS_Args,.member_arg_index=2}, {MAKE_CMD("hexpire","Sets expiry time on hash fields.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRE_History,0,HEXPIRE_Tips,0,hexpireCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HEXPIRE_Keyspecs,1,NULL,4),.args=HEXPIRE_Args}, {MAKE_CMD("hexpireat","Sets expiry time on hash fields.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIREAT_History,0,HEXPIREAT_Tips,0,hexpireatCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HEXPIREAT_Keyspecs,1,NULL,4),.args=HEXPIREAT_Args}, -{MAKE_CMD("hexpiretime","Returns Unix timestamps in seconds since the epoch at which the given key's field(s) will expire.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRETIME_History,0,HEXPIRETIME_Tips,0,hexpiretimeCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HEXPIRETIME_Keyspecs,1,NULL,2),.args=HEXPIRETIME_Args}, -{MAKE_CMD("hget","Returns the value of a field in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGET_History,0,HGET_Tips,0,hgetCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HGET_Keyspecs,1,NULL,2),.args=HGET_Args}, +{MAKE_CMD("hexpiretime","Returns Unix timestamps in seconds since the epoch at which the given key's field(s) will expire.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRETIME_History,0,HEXPIRETIME_Tips,0,hexpiretimeCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HEXPIRETIME_Keyspecs,1,NULL,2),.args=HEXPIRETIME_Args,.member_arg_index=4}, +{MAKE_CMD("hget","Returns the value of a field in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGET_History,0,HGET_Tips,0,hgetCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HGET_Keyspecs,1,NULL,2),.args=HGET_Args,.member_arg_index=2}, {MAKE_CMD("hgetall","Returns all fields and values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETALL_History,0,HGETALL_Tips,1,hgetallCommand,2,CMD_READONLY,ACL_CATEGORY_HASH|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,HGETALL_Keyspecs,1,NULL,1),.args=HGETALL_Args}, -{MAKE_CMD("hgetdel","Returns the values of one or more fields and deletes them from a hash.","O(N) where N is the number of fields to be retrieved and deleted.","9.1.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETDEL_History,0,HGETDEL_Tips,0,hgetdelCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HGETDEL_Keyspecs,1,NULL,2),.args=HGETDEL_Args}, +{MAKE_CMD("hgetdel","Returns the values of one or more fields and deletes them from a hash.","O(N) where N is the number of fields to be retrieved and deleted.","9.1.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETDEL_History,0,HGETDEL_Tips,0,hgetdelCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HGETDEL_Keyspecs,1,NULL,2),.args=HGETDEL_Args,.member_arg_index=4}, {MAKE_CMD("hgetex","Gets the value of one or more fields of a given hash key, and optionally sets their expiration time or time-to-live (TTL).","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETEX_History,0,HGETEX_Tips,0,hgetexCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HGETEX_Keyspecs,1,NULL,3),.args=HGETEX_Args}, -{MAKE_CMD("hincrby","Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBY_History,0,HINCRBY_Tips,0,hincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HINCRBY_Keyspecs,1,NULL,3),.args=HINCRBY_Args}, -{MAKE_CMD("hincrbyfloat","Increments the floating point value of a field by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBYFLOAT_History,0,HINCRBYFLOAT_Tips,0,hincrbyfloatCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HINCRBYFLOAT_Keyspecs,1,NULL,3),.args=HINCRBYFLOAT_Args}, +{MAKE_CMD("hincrby","Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBY_History,0,HINCRBY_Tips,0,hincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HINCRBY_Keyspecs,1,NULL,3),.args=HINCRBY_Args,.member_arg_index=2}, +{MAKE_CMD("hincrbyfloat","Increments the floating point value of a field by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBYFLOAT_History,0,HINCRBYFLOAT_Tips,0,hincrbyfloatCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HINCRBYFLOAT_Keyspecs,1,NULL,3),.args=HINCRBYFLOAT_Args,.member_arg_index=2}, {MAKE_CMD("hkeys","Returns all fields in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HKEYS_History,0,HKEYS_Tips,1,hkeysCommand,2,CMD_READONLY,ACL_CATEGORY_HASH|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,HKEYS_Keyspecs,1,NULL,1),.args=HKEYS_Args}, {MAKE_CMD("hlen","Returns the number of fields in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HLEN_History,0,HLEN_Tips,0,hlenCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HLEN_Keyspecs,1,NULL,1),.args=HLEN_Args}, -{MAKE_CMD("hmget","Returns the values of all fields in a hash.","O(N) where N is the number of fields being requested.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HMGET_History,0,HMGET_Tips,0,hmgetCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HMGET_Keyspecs,1,NULL,2),.args=HMGET_Args}, -{MAKE_CMD("hmset","Sets the values of multiple fields.","O(N) where N is the number of fields being set.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HMSET_History,0,HMSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HMSET_Keyspecs,1,NULL,2),.args=HMSET_Args}, -{MAKE_CMD("hpersist","Remove the existing expiration on a hash key's field(s).","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPERSIST_History,0,HPERSIST_Tips,0,hpersistCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HPERSIST_Keyspecs,1,NULL,2),.args=HPERSIST_Args}, +{MAKE_CMD("hmget","Returns the values of all fields in a hash.","O(N) where N is the number of fields being requested.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HMGET_History,0,HMGET_Tips,0,hmgetCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HMGET_Keyspecs,1,NULL,2),.args=HMGET_Args,.member_arg_index=2}, +{MAKE_CMD("hmset","Sets the values of multiple fields.","O(N) where N is the number of fields being set.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HMSET_History,0,HMSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HMSET_Keyspecs,1,NULL,2),.args=HMSET_Args,.member_arg_index=2}, +{MAKE_CMD("hpersist","Remove the existing expiration on a hash key's field(s).","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPERSIST_History,0,HPERSIST_Tips,0,hpersistCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HPERSIST_Keyspecs,1,NULL,2),.args=HPERSIST_Args,.member_arg_index=4}, {MAKE_CMD("hpexpire","Sets expiry time on hash object.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRE_History,0,HPEXPIRE_Tips,0,hpexpireCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HPEXPIRE_Keyspecs,1,NULL,4),.args=HPEXPIRE_Args}, {MAKE_CMD("hpexpireat","Sets expiration time on hash field.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIREAT_History,0,HPEXPIREAT_Tips,0,hpexpireatCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HPEXPIREAT_Keyspecs,1,NULL,4),.args=HPEXPIREAT_Args}, -{MAKE_CMD("hpexpiretime","Returns the Unix timestamp in milliseconds since Unix epoch at which the given key's field(s) will expire.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRETIME_History,0,HPEXPIRETIME_Tips,0,hpexpiretimeCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HPEXPIRETIME_Keyspecs,1,NULL,2),.args=HPEXPIRETIME_Args}, -{MAKE_CMD("hpttl","Returns the remaining time to live in milliseconds of a hash key's field(s) that have an associated expiration.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPTTL_History,0,HPTTL_Tips,0,hpttlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HPTTL_Keyspecs,1,NULL,2),.args=HPTTL_Args}, +{MAKE_CMD("hpexpiretime","Returns the Unix timestamp in milliseconds since Unix epoch at which the given key's field(s) will expire.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRETIME_History,0,HPEXPIRETIME_Tips,0,hpexpiretimeCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HPEXPIRETIME_Keyspecs,1,NULL,2),.args=HPEXPIRETIME_Args,.member_arg_index=4}, +{MAKE_CMD("hpttl","Returns the remaining time to live in milliseconds of a hash key's field(s) that have an associated expiration.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPTTL_History,0,HPTTL_Tips,0,hpttlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HPTTL_Keyspecs,1,NULL,2),.args=HPTTL_Args,.member_arg_index=4}, {MAKE_CMD("hrandfield","Returns one or more random fields from a hash.","O(N) where N is the number of fields returned","6.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HRANDFIELD_History,0,HRANDFIELD_Tips,1,hrandfieldCommand,-2,CMD_READONLY,ACL_CATEGORY_HASH|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,HRANDFIELD_Keyspecs,1,NULL,2),.args=HRANDFIELD_Args}, {MAKE_CMD("hscan","Iterates over fields and values of a hash.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSCAN_History,0,HSCAN_Tips,1,hscanCommand,-3,CMD_READONLY,ACL_CATEGORY_HASH|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,HSCAN_Keyspecs,1,NULL,5),.args=HSCAN_Args}, -{MAKE_CMD("hset","Creates or modifies the value of a field in a hash.","O(1) for each field/value pair added, so O(N) to add N field/value pairs when the command is called with multiple field/value pairs.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSET_History,1,HSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HSET_Keyspecs,1,NULL,2),.args=HSET_Args}, +{MAKE_CMD("hset","Creates or modifies the value of a field in a hash.","O(1) for each field/value pair added, so O(N) to add N field/value pairs when the command is called with multiple field/value pairs.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSET_History,1,HSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HSET_Keyspecs,1,NULL,2),.args=HSET_Args,.member_arg_index=2}, {MAKE_CMD("hsetex","Sets the value of one or more fields of a given hash key, and optionally sets their expiration time.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETEX_History,0,HSETEX_Tips,0,hsetexCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HSETEX_Keyspecs,1,NULL,5),.args=HSETEX_Args}, -{MAKE_CMD("hsetnx","Sets the value of a field in a hash only when the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETNX_History,0,HSETNX_Tips,0,hsetnxCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HSETNX_Keyspecs,1,NULL,3),.args=HSETNX_Args}, -{MAKE_CMD("hstrlen","Returns the length of the value of a field.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSTRLEN_History,0,HSTRLEN_Tips,0,hstrlenCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HSTRLEN_Keyspecs,1,NULL,2),.args=HSTRLEN_Args}, -{MAKE_CMD("httl","Returns the remaining time to live (in seconds) of a hash key's field(s) that have an associated expiration.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HTTL_History,0,HTTL_Tips,0,httlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HTTL_Keyspecs,1,NULL,2),.args=HTTL_Args}, +{MAKE_CMD("hsetnx","Sets the value of a field in a hash only when the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETNX_History,0,HSETNX_Tips,0,hsetnxCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_WRITE,NULL,HSETNX_Keyspecs,1,NULL,3),.args=HSETNX_Args,.member_arg_index=2}, +{MAKE_CMD("hstrlen","Returns the length of the value of a field.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSTRLEN_History,0,HSTRLEN_Tips,0,hstrlenCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HSTRLEN_Keyspecs,1,NULL,2),.args=HSTRLEN_Args,.member_arg_index=2}, +{MAKE_CMD("httl","Returns the remaining time to live (in seconds) of a hash key's field(s) that have an associated expiration.","O(N) where N is the number of specified fields.","9.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HTTL_History,0,HTTL_Tips,0,httlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HASH|ACL_CATEGORY_READ,NULL,HTTL_Keyspecs,1,NULL,2),.args=HTTL_Args,.member_arg_index=4}, {MAKE_CMD("hvals","Returns all values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HVALS_History,0,HVALS_Tips,1,hvalsCommand,2,CMD_READONLY,ACL_CATEGORY_HASH|ACL_CATEGORY_READ|ACL_CATEGORY_SLOW,NULL,HVALS_Keyspecs,1,NULL,1),.args=HVALS_Args}, /* hyperloglog */ {MAKE_CMD("pfadd","Adds elements to a HyperLogLog key. Creates the key if it doesn't exist.","O(1) to add every element.","2.8.9",CMD_DOC_NONE,NULL,NULL,"hyperloglog",COMMAND_GROUP_HYPERLOGLOG,PFADD_History,0,PFADD_Tips,0,pfaddCommand,-2,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_HYPERLOGLOG|ACL_CATEGORY_WRITE,NULL,PFADD_Keyspecs,1,NULL,2),.args=PFADD_Args}, @@ -12060,6 +12393,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("failover","Starts a coordinated failover from a server to one of its replicas.","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FAILOVER_History,0,FAILOVER_Tips,0,failoverCommand,-1,CMD_ADMIN|CMD_NOSCRIPT|CMD_STALE,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,FAILOVER_Keyspecs,0,NULL,3),.args=FAILOVER_Args}, {MAKE_CMD("flushall","Removes all keys from all databases.","O(N) where N is the total number of keys in all databases","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FLUSHALL_History,2,FLUSHALL_Tips,2,flushallCommand,-1,CMD_WRITE|CMD_ALL_DBS,ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_KEYSPACE|ACL_CATEGORY_SLOW|ACL_CATEGORY_WRITE,NULL,FLUSHALL_Keyspecs,0,NULL,1),.args=FLUSHALL_Args}, {MAKE_CMD("flushdb","Removes all keys from the current database.","O(N) where N is the number of keys in the selected database","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FLUSHDB_History,2,FLUSHDB_Tips,2,flushdbCommand,-1,CMD_WRITE,ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_KEYSPACE|ACL_CATEGORY_SLOW|ACL_CATEGORY_WRITE,NULL,FLUSHDB_Keyspecs,0,NULL,1),.args=FLUSHDB_Args}, +{MAKE_CMD("hotkeys","A container for hot key commands","Depends on subcommand.","9.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,HOTKEYS_History,0,HOTKEYS_Tips,0,NULL,-2,0,ACL_CATEGORY_SLOW,NULL,HOTKEYS_Keyspecs,0,NULL,0),.subcommands=HOTKEYS_Subcommands}, {MAKE_CMD("info","Returns information and statistics about the server.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,INFO_History,1,INFO_Tips,3,infoCommand,-1,CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,INFO_Keyspecs,0,NULL,1),.args=INFO_Args}, {MAKE_CMD("lastsave","Returns the Unix timestamp of the last successful save to disk.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,LASTSAVE_History,0,LASTSAVE_Tips,1,lastsaveCommand,1,CMD_LOADING|CMD_STALE|CMD_FAST,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_FAST,NULL,LASTSAVE_Keyspecs,0,NULL,0)}, {MAKE_CMD("latency","A container for latency diagnostics commands.","Depends on subcommand.","2.8.13",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,LATENCY_History,0,LATENCY_Tips,0,NULL,-2,0,ACL_CATEGORY_SLOW,NULL,LATENCY_Keyspecs,0,NULL,0),.subcommands=LATENCY_Subcommands}, @@ -12087,13 +12421,13 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("sinter","Returns the intersect of multiple sets.","O(N*M) worst case where N is the cardinality of the smallest set and M is the number of sets.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SINTER_History,0,SINTER_Tips,1,sinterCommand,-2,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SET|ACL_CATEGORY_SLOW,NULL,SINTER_Keyspecs,1,NULL,1),.args=SINTER_Args}, {MAKE_CMD("sintercard","Returns the number of members of the intersect of multiple sets.","O(N*M) worst case where N is the cardinality of the smallest set and M is the number of sets.","7.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SINTERCARD_History,0,SINTERCARD_Tips,0,sinterCardCommand,-3,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SET|ACL_CATEGORY_SLOW,NULL,SINTERCARD_Keyspecs,1,sintercardGetKeys,3),.args=SINTERCARD_Args}, {MAKE_CMD("sinterstore","Stores the intersect of multiple sets in a key.","O(N*M) worst case where N is the cardinality of the smallest set and M is the number of sets.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SINTERSTORE_History,0,SINTERSTORE_Tips,0,sinterstoreCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SET|ACL_CATEGORY_SLOW|ACL_CATEGORY_WRITE,NULL,SINTERSTORE_Keyspecs,2,NULL,2),.args=SINTERSTORE_Args}, -{MAKE_CMD("sismember","Determines whether a member belongs to a set.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SISMEMBER_History,0,SISMEMBER_Tips,0,sismemberCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SET,NULL,SISMEMBER_Keyspecs,1,NULL,2),.args=SISMEMBER_Args}, +{MAKE_CMD("sismember","Determines whether a member belongs to a set.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SISMEMBER_History,1,SISMEMBER_Tips,0,sismemberCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SET,NULL,SISMEMBER_Keyspecs,1,NULL,3),.args=SISMEMBER_Args,.member_arg_index=2}, {MAKE_CMD("smembers","Returns all members of a set.","O(N) where N is the set cardinality.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SMEMBERS_History,0,SMEMBERS_Tips,1,sinterCommand,2,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SET|ACL_CATEGORY_SLOW,NULL,SMEMBERS_Keyspecs,1,NULL,1),.args=SMEMBERS_Args}, -{MAKE_CMD("smismember","Determines whether multiple members belong to a set.","O(N) where N is the number of elements being checked for membership","6.2.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SMISMEMBER_History,0,SMISMEMBER_Tips,0,smismemberCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SET,NULL,SMISMEMBER_Keyspecs,1,NULL,2),.args=SMISMEMBER_Args}, +{MAKE_CMD("smismember","Determines whether multiple members belong to a set.","O(N) where N is the number of elements being checked for membership","6.2.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SMISMEMBER_History,0,SMISMEMBER_Tips,0,smismemberCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SET,NULL,SMISMEMBER_Keyspecs,1,NULL,2),.args=SMISMEMBER_Args,.member_arg_index=2}, {MAKE_CMD("smove","Moves a member from one set to another.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SMOVE_History,0,SMOVE_Tips,0,smoveCommand,4,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_SET|ACL_CATEGORY_WRITE,NULL,SMOVE_Keyspecs,2,NULL,3),.args=SMOVE_Args}, {MAKE_CMD("spop","Returns one or more random members from a set after removing them. Deletes the set if the last member was popped.","Without the count argument O(1), otherwise O(N) where N is the value of the passed count.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SPOP_History,1,SPOP_Tips,1,spopCommand,-2,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_SET|ACL_CATEGORY_WRITE,NULL,SPOP_Keyspecs,1,NULL,2),.args=SPOP_Args}, {MAKE_CMD("srandmember","Gets one or multiple random members from a set.","Without the count argument O(1), otherwise O(N) where N is the absolute value of the passed count.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SRANDMEMBER_History,1,SRANDMEMBER_Tips,1,srandmemberCommand,-2,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SET|ACL_CATEGORY_SLOW,NULL,SRANDMEMBER_Keyspecs,1,NULL,2),.args=SRANDMEMBER_Args}, -{MAKE_CMD("srem","Removes one or more members from a set. Deletes the set if the last member was removed.","O(N) where N is the number of members to be removed.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SREM_History,1,SREM_Tips,0,sremCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_SET|ACL_CATEGORY_WRITE,NULL,SREM_Keyspecs,1,NULL,2),.args=SREM_Args}, +{MAKE_CMD("srem","Removes one or more members from a set. Deletes the set if the last member was removed.","O(N) where N is the number of members to be removed.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SREM_History,1,SREM_Tips,0,sremCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_SET|ACL_CATEGORY_WRITE,NULL,SREM_Keyspecs,1,NULL,2),.args=SREM_Args,.member_arg_index=2}, {MAKE_CMD("sscan","Iterates over members of a set.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SSCAN_History,0,SSCAN_Tips,1,sscanCommand,-3,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SET|ACL_CATEGORY_SLOW,NULL,SSCAN_Keyspecs,1,NULL,4),.args=SSCAN_Args}, {MAKE_CMD("sunion","Returns the union of multiple sets.","O(N) where N is the total number of elements in all given sets.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SUNION_History,0,SUNION_Tips,1,sunionCommand,-2,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SET|ACL_CATEGORY_SLOW,NULL,SUNION_Keyspecs,1,NULL,1),.args=SUNION_Args}, {MAKE_CMD("sunionstore","Stores the union of multiple sets in a key.","O(N) where N is the total number of elements in all given sets.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SUNIONSTORE_History,0,SUNIONSTORE_Tips,0,sunionstoreCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SET|ACL_CATEGORY_SLOW|ACL_CATEGORY_WRITE,NULL,SUNIONSTORE_Keyspecs,2,NULL,2),.args=SUNIONSTORE_Args}, @@ -12106,13 +12440,13 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("zcount","Returns the count of members in a sorted set that have scores within a range.","O(log(N)) with N being the number of elements in the sorted set.","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZCOUNT_History,0,ZCOUNT_Tips,0,zcountCommand,4,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SORTEDSET,NULL,ZCOUNT_Keyspecs,1,NULL,3),.args=ZCOUNT_Args}, {MAKE_CMD("zdiff","Returns the difference between multiple sorted sets.","O(L + (N-K)log(N)) worst case where L is the total number of elements in all the sets, N is the size of the first set, and K is the size of the result set.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZDIFF_History,0,ZDIFF_Tips,0,zdiffCommand,-3,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZDIFF_Keyspecs,1,zunionInterDiffGetKeys,3),.args=ZDIFF_Args}, {MAKE_CMD("zdiffstore","Stores the difference of multiple sorted sets in a key.","O(L + (N-K)log(N)) worst case where L is the total number of elements in all the sets, N is the size of the first set, and K is the size of the result set.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZDIFFSTORE_History,0,ZDIFFSTORE_Tips,0,zdiffstoreCommand,-4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZDIFFSTORE_Keyspecs,2,zunionInterDiffStoreGetKeys,3),.args=ZDIFFSTORE_Args}, -{MAKE_CMD("zincrby","Increments the score of a member in a sorted set.","O(log(N)) where N is the number of elements in the sorted set.","1.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINCRBY_History,0,ZINCRBY_Tips,0,zincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZINCRBY_Keyspecs,1,NULL,3),.args=ZINCRBY_Args}, +{MAKE_CMD("zincrby","Increments the score of a member in a sorted set.","O(log(N)) where N is the number of elements in the sorted set.","1.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINCRBY_History,0,ZINCRBY_Tips,0,zincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZINCRBY_Keyspecs,1,NULL,3),.args=ZINCRBY_Args,.member_arg_index=3}, {MAKE_CMD("zinter","Returns the intersect of multiple sorted sets.","O(N*K)+O(M*log(M)) worst case with N being the smallest input sorted set, K being the number of input sorted sets and M being the number of elements in the resulting sorted set.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINTER_History,0,ZINTER_Tips,0,zinterCommand,-3,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZINTER_Keyspecs,1,zunionInterDiffGetKeys,5),.args=ZINTER_Args}, {MAKE_CMD("zintercard","Returns the number of members of the intersect of multiple sorted sets.","O(N*K) worst case with N being the smallest input sorted set, K being the number of input sorted sets.","7.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINTERCARD_History,0,ZINTERCARD_Tips,0,zinterCardCommand,-3,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZINTERCARD_Keyspecs,1,zunionInterDiffGetKeys,3),.args=ZINTERCARD_Args}, {MAKE_CMD("zinterstore","Stores the intersect of multiple sorted sets in a key.","O(N*K)+O(M*log(M)) worst case with N being the smallest input sorted set, K being the number of input sorted sets and M being the number of elements in the resulting sorted set.","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZINTERSTORE_History,0,ZINTERSTORE_Tips,0,zinterstoreCommand,-4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZINTERSTORE_Keyspecs,2,zunionInterDiffStoreGetKeys,5),.args=ZINTERSTORE_Args}, {MAKE_CMD("zlexcount","Returns the number of members in a sorted set within a lexicographical range.","O(log(N)) with N being the number of elements in the sorted set.","2.8.9",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZLEXCOUNT_History,0,ZLEXCOUNT_Tips,0,zlexcountCommand,4,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SORTEDSET,NULL,ZLEXCOUNT_Keyspecs,1,NULL,3),.args=ZLEXCOUNT_Args}, {MAKE_CMD("zmpop","Returns the highest- or lowest-scoring members from one or more sorted sets after removing them. Deletes the sorted set if the last member was popped.","O(K) + O(M*log(N)) where K is the number of provided keys, N being the number of elements in the sorted set, and M being the number of elements popped.","7.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZMPOP_History,0,ZMPOP_Tips,0,zmpopCommand,-4,CMD_WRITE,ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZMPOP_Keyspecs,1,zmpopGetKeys,4),.args=ZMPOP_Args}, -{MAKE_CMD("zmscore","Returns the score of one or more members in a sorted set.","O(N) where N is the number of members being requested.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZMSCORE_History,0,ZMSCORE_Tips,0,zmscoreCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SORTEDSET,NULL,ZMSCORE_Keyspecs,1,NULL,2),.args=ZMSCORE_Args}, +{MAKE_CMD("zmscore","Returns the score of one or more members in a sorted set.","O(N) where N is the number of members being requested.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZMSCORE_History,0,ZMSCORE_Tips,0,zmscoreCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SORTEDSET,NULL,ZMSCORE_Keyspecs,1,NULL,2),.args=ZMSCORE_Args,.member_arg_index=2}, {MAKE_CMD("zpopmax","Returns the highest-scoring members from a sorted set after removing them. Deletes the sorted set if the last member was popped.","O(log(N)*M) with N being the number of elements in the sorted set, and M being the number of elements popped.","5.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZPOPMAX_History,0,ZPOPMAX_Tips,0,zpopmaxCommand,-2,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZPOPMAX_Keyspecs,1,NULL,2),.args=ZPOPMAX_Args}, {MAKE_CMD("zpopmin","Returns the lowest-scoring members from a sorted set after removing them. Deletes the sorted set if the last member was popped.","O(log(N)*M) with N being the number of elements in the sorted set, and M being the number of elements popped.","5.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZPOPMIN_History,0,ZPOPMIN_Tips,0,zpopminCommand,-2,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZPOPMIN_Keyspecs,1,NULL,2),.args=ZPOPMIN_Args}, {MAKE_CMD("zrandmember","Returns one or more random members from a sorted set.","O(N) where N is the number of members returned","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZRANDMEMBER_History,0,ZRANDMEMBER_Tips,1,zrandmemberCommand,-2,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZRANDMEMBER_Keyspecs,1,NULL,2),.args=ZRANDMEMBER_Args}, @@ -12120,25 +12454,27 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("zrangebylex","Returns members in a sorted set within a lexicographical range.","O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).","2.8.9",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZRANGEBYLEX_History,1,ZRANGEBYLEX_Tips,0,zrangebylexCommand,-4,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZRANGEBYLEX_Keyspecs,1,NULL,5),.args=ZRANGEBYLEX_Args}, {MAKE_CMD("zrangebyscore","Returns members in a sorted set within a range of scores.","O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).","1.0.5",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZRANGEBYSCORE_History,2,ZRANGEBYSCORE_Tips,0,zrangebyscoreCommand,-4,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZRANGEBYSCORE_Keyspecs,1,NULL,6),.args=ZRANGEBYSCORE_Args}, {MAKE_CMD("zrangestore","Stores a range of members from sorted set in a key.","O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements stored into the destination key.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZRANGESTORE_History,0,ZRANGESTORE_Tips,0,zrangestoreCommand,-5,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZRANGESTORE_Keyspecs,2,NULL,7),.args=ZRANGESTORE_Args}, -{MAKE_CMD("zrank","Returns the index of a member in a sorted set ordered by ascending scores.","O(log(N))","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZRANK_History,1,ZRANK_Tips,0,zrankCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SORTEDSET,NULL,ZRANK_Keyspecs,1,NULL,3),.args=ZRANK_Args}, -{MAKE_CMD("zrem","Removes one or more members from a sorted set. Deletes the sorted set if all members were removed.","O(M*log(N)) with N being the number of elements in the sorted set and M the number of elements to be removed.","1.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREM_History,1,ZREM_Tips,0,zremCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZREM_Keyspecs,1,NULL,2),.args=ZREM_Args}, +{MAKE_CMD("zrank","Returns the index of a member in a sorted set ordered by ascending scores.","O(log(N))","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZRANK_History,1,ZRANK_Tips,0,zrankCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SORTEDSET,NULL,ZRANK_Keyspecs,1,NULL,3),.args=ZRANK_Args,.member_arg_index=2}, +{MAKE_CMD("zrem","Removes one or more members from a sorted set. Deletes the sorted set if all members were removed.","O(M*log(N)) with N being the number of elements in the sorted set and M the number of elements to be removed.","1.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREM_History,1,ZREM_Tips,0,zremCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZREM_Keyspecs,1,NULL,2),.args=ZREM_Args,.member_arg_index=2}, {MAKE_CMD("zremrangebylex","Removes members in a sorted set within a lexicographical range. Deletes the sorted set if all members were removed.","O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements removed by the operation.","2.8.9",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREMRANGEBYLEX_History,0,ZREMRANGEBYLEX_Tips,0,zremrangebylexCommand,4,CMD_WRITE,ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZREMRANGEBYLEX_Keyspecs,1,NULL,3),.args=ZREMRANGEBYLEX_Args}, {MAKE_CMD("zremrangebyrank","Removes members in a sorted set within a range of indexes. Deletes the sorted set if all members were removed.","O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements removed by the operation.","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREMRANGEBYRANK_History,0,ZREMRANGEBYRANK_Tips,0,zremrangebyrankCommand,4,CMD_WRITE,ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZREMRANGEBYRANK_Keyspecs,1,NULL,3),.args=ZREMRANGEBYRANK_Args}, {MAKE_CMD("zremrangebyscore","Removes members in a sorted set within a range of scores. Deletes the sorted set if all members were removed.","O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements removed by the operation.","1.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREMRANGEBYSCORE_History,0,ZREMRANGEBYSCORE_Tips,0,zremrangebyscoreCommand,4,CMD_WRITE,ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZREMRANGEBYSCORE_Keyspecs,1,NULL,3),.args=ZREMRANGEBYSCORE_Args}, {MAKE_CMD("zrevrange","Returns members in a sorted set within a range of indexes in reverse order.","O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements returned.","1.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREVRANGE_History,1,ZREVRANGE_Tips,0,zrevrangeCommand,-4,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZREVRANGE_Keyspecs,1,NULL,5),.args=ZREVRANGE_Args}, {MAKE_CMD("zrevrangebylex","Returns members in a sorted set within a lexicographical range in reverse order.","O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).","2.8.9",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREVRANGEBYLEX_History,1,ZREVRANGEBYLEX_Tips,0,zrevrangebylexCommand,-4,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZREVRANGEBYLEX_Keyspecs,1,NULL,5),.args=ZREVRANGEBYLEX_Args}, {MAKE_CMD("zrevrangebyscore","Returns members in a sorted set within a range of scores in reverse order.","O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).","2.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREVRANGEBYSCORE_History,2,ZREVRANGEBYSCORE_Tips,0,zrevrangebyscoreCommand,-4,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZREVRANGEBYSCORE_Keyspecs,1,NULL,6),.args=ZREVRANGEBYSCORE_Args}, -{MAKE_CMD("zrevrank","Returns the index of a member in a sorted set ordered by descending scores.","O(log(N))","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREVRANK_History,1,ZREVRANK_Tips,0,zrevrankCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SORTEDSET,NULL,ZREVRANK_Keyspecs,1,NULL,3),.args=ZREVRANK_Args}, +{MAKE_CMD("zrevrank","Returns the index of a member in a sorted set ordered by descending scores.","O(log(N))","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZREVRANK_History,1,ZREVRANK_Tips,0,zrevrankCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SORTEDSET,NULL,ZREVRANK_Keyspecs,1,NULL,3),.args=ZREVRANK_Args,.member_arg_index=2}, {MAKE_CMD("zscan","Iterates over members and scores of a sorted set.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZSCAN_History,1,ZSCAN_Tips,1,zscanCommand,-3,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZSCAN_Keyspecs,1,NULL,5),.args=ZSCAN_Args}, -{MAKE_CMD("zscore","Returns the score of a member in a sorted set.","O(1)","1.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZSCORE_History,0,ZSCORE_Tips,0,zscoreCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SORTEDSET,NULL,ZSCORE_Keyspecs,1,NULL,2),.args=ZSCORE_Args}, +{MAKE_CMD("zscore","Returns the score of a member in a sorted set.","O(1)","1.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZSCORE_History,0,ZSCORE_Tips,0,zscoreCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_SORTEDSET,NULL,ZSCORE_Keyspecs,1,NULL,2),.args=ZSCORE_Args,.member_arg_index=2}, {MAKE_CMD("zunion","Returns the union of multiple sorted sets.","O(N)+O(M*log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set.","6.2.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZUNION_History,0,ZUNION_Tips,0,zunionCommand,-3,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET,NULL,ZUNION_Keyspecs,1,zunionInterDiffGetKeys,5),.args=ZUNION_Args}, {MAKE_CMD("zunionstore","Stores the union of multiple sorted sets in a key.","O(N)+O(M log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set.","2.0.0",CMD_DOC_NONE,NULL,NULL,"sorted_set",COMMAND_GROUP_SORTED_SET,ZUNIONSTORE_History,0,ZUNIONSTORE_Tips,0,zunionstoreCommand,-4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_SORTEDSET|ACL_CATEGORY_WRITE,NULL,ZUNIONSTORE_Keyspecs,2,zunionInterDiffStoreGetKeys,5),.args=ZUNIONSTORE_Args}, /* stream */ {MAKE_CMD("xack","Returns the number of messages that were successfully acknowledged by the consumer group member of a stream.","O(1) for each message ID processed.","5.0.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XACK_History,0,XACK_Tips,0,xackCommand,-4,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_STREAM|ACL_CATEGORY_WRITE,NULL,XACK_Keyspecs,1,NULL,3),.args=XACK_Args}, +{MAKE_CMD("xackdel","Acknowledge and (if possible) delete stream message(s).","O(1) for each single item to delete in the stream, regardless of the stream size","9.2.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XACKDEL_History,0,XACKDEL_Tips,0,xackdelCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_WRITE|ACL_CATEGORY_STREAM,NULL,XACKDEL_Keyspecs,1,NULL,6),.args=XACKDEL_Args}, {MAKE_CMD("xadd","Appends a new message to a stream. Creates the key if it doesn't exist.","O(1) when adding a new entry, O(N) when trimming where N being the number of entries evicted.","5.0.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XADD_History,2,XADD_Tips,1,xaddCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_STREAM|ACL_CATEGORY_WRITE,NULL,XADD_Keyspecs,1,NULL,5),.args=XADD_Args}, {MAKE_CMD("xautoclaim","Changes, or acquires, ownership of messages in a consumer group, as if the messages were delivered to a consumer group member.","O(1) if COUNT is small.","6.2.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XAUTOCLAIM_History,1,XAUTOCLAIM_Tips,1,xautoclaimCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_STREAM|ACL_CATEGORY_WRITE,NULL,XAUTOCLAIM_Keyspecs,1,NULL,7),.args=XAUTOCLAIM_Args}, {MAKE_CMD("xclaim","Changes, or acquires, ownership of a message in a consumer group, as if the message was delivered to a consumer group member.","O(log N) with N being the number of messages in the PEL of the consumer group.","5.0.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XCLAIM_History,0,XCLAIM_Tips,1,xclaimCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_STREAM|ACL_CATEGORY_WRITE,NULL,XCLAIM_Keyspecs,1,NULL,11),.args=XCLAIM_Args}, {MAKE_CMD("xdel","Returns the number of messages after removing them from a stream.","O(1) for each single item to delete in the stream, regardless of the stream size.","5.0.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XDEL_History,0,XDEL_Tips,0,xdelCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_STREAM|ACL_CATEGORY_WRITE,NULL,XDEL_Keyspecs,1,NULL,2),.args=XDEL_Args}, +{MAKE_CMD("xdelex","Delete stream message(s) with extended options","O(1) for each single item to delete in the stream, regardless of the stream size","9.2.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XDELEX_History,0,XDELEX_Tips,0,xdelexCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_WRITE|ACL_CATEGORY_STREAM,NULL,XDELEX_Keyspecs,1,NULL,5),.args=XDELEX_Args}, {MAKE_CMD("xgroup","A container for consumer groups commands.","Depends on subcommand.","5.0.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XGROUP_History,0,XGROUP_Tips,0,NULL,-2,0,ACL_CATEGORY_SLOW,NULL,XGROUP_Keyspecs,0,NULL,0),.subcommands=XGROUP_Subcommands}, {MAKE_CMD("xinfo","A container for stream introspection commands.","Depends on subcommand.","5.0.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XINFO_History,0,XINFO_Tips,0,NULL,-2,0,ACL_CATEGORY_SLOW,NULL,XINFO_Keyspecs,0,NULL,0),.subcommands=XINFO_Subcommands}, {MAKE_CMD("xlen","Returns the number of messages in a stream.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,"stream",COMMAND_GROUP_STREAM,XLEN_History,0,XLEN_Tips,0,xlenCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_STREAM,NULL,XLEN_Keyspecs,1,NULL,1),.args=XLEN_Args}, @@ -12162,13 +12498,14 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("incr","Increments the integer value of a key by one. Uses 0 as initial value if the key doesn't exist.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,INCR_History,0,INCR_Tips,0,incrCommand,2,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,INCR_Keyspecs,1,NULL,1),.args=INCR_Args}, {MAKE_CMD("incrby","Increments the integer value of a key by a number. Uses 0 as initial value if the key doesn't exist.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,INCRBY_History,0,INCRBY_Tips,0,incrbyCommand,3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,INCRBY_Keyspecs,1,NULL,2),.args=INCRBY_Args}, {MAKE_CMD("incrbyfloat","Increments the floating point value of a key by a number. Uses 0 as initial value if the key doesn't exist.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,INCRBYFLOAT_History,0,INCRBYFLOAT_Tips,0,incrbyfloatCommand,3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,INCRBYFLOAT_Keyspecs,1,NULL,2),.args=INCRBYFLOAT_Args}, +{MAKE_CMD("increx","Increments the numeric value of a key with an option to expire. Uses 0 as initial value if the key doesn't exist.","O(1)","9.2.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,INCREX_History,0,INCREX_Tips,0,increxCommand,-2,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,INCREX_Keyspecs,1,NULL,4),.args=INCREX_Args}, {MAKE_CMD("lcs","Finds the longest common substring.","O(N*M) where N and M are the lengths of s1 and s2, respectively","7.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,LCS_History,0,LCS_Tips,0,lcsCommand,-3,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_STRING,NULL,LCS_Keyspecs,1,NULL,6),.args=LCS_Args}, {MAKE_CMD("mget","Atomically returns the string values of one or more keys.","O(N) where N is the number of keys to retrieve.","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,MGET_History,0,MGET_Tips,1,mgetCommand,-2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_READ|ACL_CATEGORY_STRING,NULL,MGET_Keyspecs,1,NULL,1),.args=MGET_Args}, {MAKE_CMD("mset","Atomically creates or modifies the string values of one or more keys.","O(N) where N is the number of keys to set.","1.0.1",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,MSET_History,0,MSET_Tips,2,msetCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,MSET_Keyspecs,1,NULL,1),.args=MSET_Args}, {MAKE_CMD("msetex","Atomically creates or modifies the string values of one or more keys, and optionally set their expiration.","O(N) where N is the number of keys to set.","9.1.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,MSETEX_History,0,MSETEX_Tips,2,msetexCommand,-4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE|ACL_CATEGORY_SLOW,NULL,MSETEX_Keyspecs,1,NULL,4),.args=MSETEX_Args}, {MAKE_CMD("msetnx","Atomically modifies the string values of one or more keys only when all keys don't exist.","O(N) where N is the number of keys to set.","1.0.1",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,MSETNX_History,0,MSETNX_Tips,0,msetnxCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,MSETNX_Keyspecs,1,NULL,1),.args=MSETNX_Args}, {MAKE_CMD("psetex","Sets both string value and expiration time in milliseconds of a key. The key is created if it doesn't exist.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,PSETEX_History,0,PSETEX_Tips,0,psetexCommand,4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,PSETEX_Keyspecs,1,NULL,3),.args=PSETEX_Args}, -{MAKE_CMD("set","Sets the string value of a key, ignoring its type. The key is created if it doesn't exist.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SET_History,5,SET_Tips,0,setCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,SET_Keyspecs,1,setGetKeys,5),.args=SET_Args}, +{MAKE_CMD("set","Sets the string value of a key, ignoring its type. The key is created if it doesn't exist.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SET_History,6,SET_Tips,0,setCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,SET_Keyspecs,1,setGetKeys,5),.args=SET_Args}, {MAKE_CMD("setex","Sets the string value and expiration time of a key. Creates the key if it doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SETEX_History,0,SETEX_Tips,0,setexCommand,4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,SETEX_Keyspecs,1,NULL,3),.args=SETEX_Args}, {MAKE_CMD("setnx","Sets the string value of a key only when the key doesn't exist.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SETNX_History,0,SETNX_Tips,0,setnxCommand,3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_FAST|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,SETNX_Keyspecs,1,NULL,2),.args=SETNX_Args}, {MAKE_CMD("setrange","Overwrites a part of a string value with another by an offset. Creates the key if it doesn't exist.","O(1), not counting the time taken to copy the new string in place. Usually, this string is very small so the amortized complexity is O(1). Otherwise, complexity is O(M) with M being the length of the value argument.","2.2.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SETRANGE_History,0,SETRANGE_Tips,0,setrangeCommand,4,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SLOW|ACL_CATEGORY_STRING|ACL_CATEGORY_WRITE,NULL,SETRANGE_Keyspecs,1,NULL,3),.args=SETRANGE_Args}, @@ -12176,7 +12513,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("substr","Returns a substring from a string value.","O(N) where N is the length of the returned string. The complexity is ultimately determined by the returned length, but because creating a substring from an existing string is very cheap, it can be considered O(1) for small strings.","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SUBSTR_History,0,SUBSTR_Tips,0,getrangeCommand,4,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_STRING,NULL,SUBSTR_Keyspecs,1,NULL,3),.args=SUBSTR_Args}, /* transactions */ {MAKE_CMD("discard","Discards a transaction.","O(N), when N is the number of queued commands","2.0.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,DISCARD_History,0,DISCARD_Tips,0,discardCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_FAST|ACL_CATEGORY_TRANSACTION,NULL,DISCARD_Keyspecs,0,NULL,0)}, -{MAKE_CMD("exec","Executes all commands in a transaction.","Depends on commands in the transaction","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,EXEC_History,0,EXEC_Tips,0,execCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_SLOW|ACL_CATEGORY_TRANSACTION,NULL,EXEC_Keyspecs,0,NULL,0)}, +{MAKE_CMD("exec","Executes all commands in a transaction if conditions match.","O(N) for N conditions, plus the complexity of commands in the transaction","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,EXEC_History,1,EXEC_Tips,0,execCommand,-1,CMD_NOSCRIPT|CMD_LOADING|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SLOW|ACL_CATEGORY_TRANSACTION,NULL,EXEC_Keyspecs,1,execGetKeys,1),.args=EXEC_Args}, {MAKE_CMD("multi","Starts a transaction.","O(1)","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,MULTI_History,0,MULTI_Tips,0,multiCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_NO_MULTI|CMD_ALLOW_BUSY,ACL_CATEGORY_FAST|ACL_CATEGORY_TRANSACTION,NULL,MULTI_Keyspecs,0,NULL,0)}, {MAKE_CMD("unwatch","Forgets about watched keys of a transaction.","O(1)","2.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,UNWATCH_History,0,UNWATCH_Tips,0,unwatchCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_FAST|ACL_CATEGORY_TRANSACTION,NULL,UNWATCH_Keyspecs,0,NULL,0)}, {MAKE_CMD("watch","Monitors changes to keys to determine the execution of a transaction.","O(1) for every key.","2.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,WATCH_History,0,WATCH_Tips,0,watchCommand,-2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_NO_MULTI|CMD_ALLOW_BUSY,ACL_CATEGORY_FAST|ACL_CATEGORY_TRANSACTION,NULL,WATCH_Keyspecs,1,NULL,1),.args=WATCH_Args}, diff --git a/src/commands.h b/src/commands.h index eef77e6c5..808bdf658 100644 --- a/src/commands.h +++ b/src/commands.h @@ -24,6 +24,7 @@ typedef enum { #define COMMAND_HGET 2 #define COMMAND_HSET 3 #define COMMAND_MSET 4 +#define COMMAND_INCREX 5 /* Command flags. Please check the definition of struct serverCommand in this file * for more information about the meaning of every flag. */ @@ -57,6 +58,7 @@ typedef enum { #define CMD_MODULE_GETCHANNELS (1ULL << 27) /* Use the modules getchannels interface. */ #define CMD_TOUCHES_ARBITRARY_KEYS (1ULL << 28) #define CMD_ALL_DBS (1ULL << 29) +#define CMD_WRITE_FIRSTKEY_ONLY (1ULL << 30) /* Command flags. Please don't forget to add command flag documentation in struct * serverCommand in server.h file. */ diff --git a/src/commands/acl-delrole.json b/src/commands/acl-delrole.json new file mode 100644 index 000000000..fe0e124f7 --- /dev/null +++ b/src/commands/acl-delrole.json @@ -0,0 +1,38 @@ +{ + "DELROLE": { + "summary": "Deletes one or more ACL roles. Fails if any role is assigned to a user.", + "complexity": "O(N). Where N is the number of roles to delete.", + "group": "server", + "since": "9.2.0", + "arity": -3, + "container": "ACL", + "function": "aclCommand", + "command_flags": [ + "ADMIN", + "NOSCRIPT", + "LOADING", + "STALE", + "SENTINEL" + ], + "command_tips": [ + "REQUEST_POLICY:ALL_NODES", + "RESPONSE_POLICY:ALL_SUCCEEDED" + ], + "reply_schema": { + "type": "integer", + "description": "The number of roles deleted." + }, + "arguments": [ + { + "name": "rolename", + "type": "string", + "multiple": true + } + ], + "acl_categories": [ + "ADMIN", + "DANGEROUS", + "SLOW" + ] + } +} diff --git a/src/commands/acl-getrole.json b/src/commands/acl-getrole.json new file mode 100644 index 000000000..ccf36ce72 --- /dev/null +++ b/src/commands/acl-getrole.json @@ -0,0 +1,88 @@ +{ + "GETROLE": { + "summary": "Returns the ACL rules of an ACL role.", + "complexity": "O(N). Where N is the number of rules defined for the role.", + "group": "server", + "since": "9.2.0", + "arity": 3, + "container": "ACL", + "function": "aclCommand", + "command_flags": [ + "ADMIN", + "NOSCRIPT", + "LOADING", + "STALE", + "SENTINEL" + ], + "reply_schema": { + "oneOf": [ + { + "description": "A set of ACL rule definitions for the role.", + "type": "object", + "additionalProperties": false, + "properties": { + "commands": { + "description": "Root selector's commands.", + "type": "string" + }, + "keys": { + "description": "Root selector's keys.", + "type": "string" + }, + "channels": { + "description": "Root selector's channels.", + "type": "string" + }, + "databases": { + "description": "Root selector's databases.", + "type": "string" + }, + "selectors": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "commands": { + "type": "string" + }, + "keys": { + "type": "string" + }, + "channels": { + "type": "string" + }, + "databases": { + "type": "string" + } + } + } + }, + "users": { + "description": "List of usernames assigned to this role.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "description": "Role not found.", + "type": "null" + } + ] + }, + "arguments": [ + { + "name": "rolename", + "type": "string" + } + ], + "acl_categories": [ + "ADMIN", + "DANGEROUS", + "SLOW" + ] + } +} diff --git a/src/commands/acl-getuser.json b/src/commands/acl-getuser.json index c235d4172..44e3a1f1b 100644 --- a/src/commands/acl-getuser.json +++ b/src/commands/acl-getuser.json @@ -19,6 +19,10 @@ [ "9.1.0", "Added database permission rules." + ], + [ + "9.2.0", + "Added roles." ] ], "command_flags": [ @@ -89,6 +93,13 @@ } } } + }, + "roles": { + "description": "List of role names assigned to this user.", + "type": "array", + "items": { + "type": "string" + } } } }, diff --git a/src/commands/acl-roles.json b/src/commands/acl-roles.json new file mode 100644 index 000000000..40d14e215 --- /dev/null +++ b/src/commands/acl-roles.json @@ -0,0 +1,30 @@ +{ + "ROLES": { + "summary": "Lists all ACL roles.", + "complexity": "O(N). Where N is the number of configured roles.", + "group": "server", + "since": "9.2.0", + "arity": 2, + "container": "ACL", + "function": "aclCommand", + "command_flags": [ + "ADMIN", + "NOSCRIPT", + "LOADING", + "STALE", + "SENTINEL" + ], + "reply_schema": { + "type": "array", + "description": "List of existing ACL roles.", + "items": { + "type": "string" + } + }, + "acl_categories": [ + "ADMIN", + "DANGEROUS", + "SLOW" + ] + } +} diff --git a/src/commands/acl-setrole.json b/src/commands/acl-setrole.json new file mode 100644 index 000000000..deec03389 --- /dev/null +++ b/src/commands/acl-setrole.json @@ -0,0 +1,42 @@ +{ + "SETROLE": { + "summary": "Creates and modifies an ACL role and its rules.", + "complexity": "O(N+M*C). Where N is the number of rules provided, M the number of users in the role and C the number of connected clients.", + "group": "server", + "since": "9.2.0", + "arity": -3, + "container": "ACL", + "function": "aclCommand", + "command_flags": [ + "ADMIN", + "NOSCRIPT", + "LOADING", + "STALE", + "SENTINEL" + ], + "command_tips": [ + "REQUEST_POLICY:ALL_NODES", + "RESPONSE_POLICY:ALL_SUCCEEDED" + ], + "reply_schema": { + "const": "OK" + }, + "arguments": [ + { + "name": "rolename", + "type": "string" + }, + { + "name": "rule", + "type": "string", + "optional": true, + "multiple": true + } + ], + "acl_categories": [ + "ADMIN", + "DANGEROUS", + "SLOW" + ] + } +} diff --git a/src/commands/acl-setuser.json b/src/commands/acl-setuser.json index bd0dce481..df1610f71 100644 --- a/src/commands/acl-setuser.json +++ b/src/commands/acl-setuser.json @@ -19,6 +19,10 @@ [ "9.1.0", "Added database permission rules." + ], + [ + "9.2.0", + "Added the `role=` rule for assigning roles to a user." ] ], "command_flags": [ diff --git a/src/commands/client-kill.json b/src/commands/client-kill.json index e34ed463f..529de31a5 100644 --- a/src/commands/client-kill.json +++ b/src/commands/client-kill.json @@ -43,6 +43,10 @@ [ "9.0.0", "Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP." + ], + [ + "9.2.0", + "Added `H` flag to indicate high priority clients." ] ], "command_flags": [ diff --git a/src/commands/client-list.json b/src/commands/client-list.json index 1d8b0a932..c934935e4 100644 --- a/src/commands/client-list.json +++ b/src/commands/client-list.json @@ -43,6 +43,10 @@ [ "9.0.0", "Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP." + ], + [ + "9.2.0", + "Added `H` flag to indicate high priority clients." ] ], "command_flags": [ diff --git a/src/commands/cluster-migrateslots.json b/src/commands/cluster-migrateslots.json index a43bb9815..e30094494 100644 --- a/src/commands/cluster-migrateslots.json +++ b/src/commands/cluster-migrateslots.json @@ -51,6 +51,23 @@ "type": "string", "pattern": "^[0-9a-fA-F]{40}$", "description": "40 character node name of the node to migrate to" + }, + { + "token": "AUTH", + "name": "auth", + "type": "block", + "optional": true, + "since": "9.2.0", + "arguments": [ + { + "name": "username", + "type": "string" + }, + { + "name": "password", + "type": "string" + } + ] } ] } diff --git a/src/commands/exec.json b/src/commands/exec.json index 8c4aba48a..f2168dfce 100644 --- a/src/commands/exec.json +++ b/src/commands/exec.json @@ -1,20 +1,98 @@ { "EXEC": { - "summary": "Executes all commands in a transaction.", - "complexity": "Depends on commands in the transaction", + "summary": "Executes all commands in a transaction if conditions match.", + "complexity": "O(N) for N conditions, plus the complexity of commands in the transaction", "group": "transactions", "since": "1.2.0", - "arity": 1, + "arity": -1, "function": "execCommand", + "get_keys_function": "execGetKeys", + "history": [ + [ + "9.2.0", + "Added the `IFEQ`, `IFNE`, `NX`, and `XX` options." + ] + ], "command_flags": [ "NOSCRIPT", "LOADING", + "NO_MANDATORY_KEYS", "STALE" ], "acl_categories": [ "SLOW", "TRANSACTION" ], + "key_specs": [ + { + "notes": "Condition keys are determined by EXEC condition tokens.", + "flags": [ + "RO", + "ACCESS" + ], + "begin_search": { + "unknown": null + }, + "find_keys": { + "unknown": null + } + } + ], + "arguments": [ + { + "name": "condition", + "type": "oneof", + "since": "9.2.0", + "optional": true, + "multiple": true, + "arguments": [ + { + "token": "IFEQ", + "name": "ifeq", + "type": "block", + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "value", + "type": "string" + } + ] + }, + { + "token": "IFNE", + "name": "ifne", + "type": "block", + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "value", + "type": "string" + } + ] + }, + { + "token": "NX", + "name": "nonexisting-key", + "type": "key", + "key_spec_index": 0 + }, + { + "token": "XX", + "name": "existing-key", + "type": "key", + "key_spec_index": 0 + } + ] + } + ], "reply_schema": { "oneOf": [ { @@ -22,7 +100,7 @@ "type": "array" }, { - "description": "The transaction was aborted because a `WATCH`ed key was touched", + "description": "The transaction was aborted because a `WATCH`ed key was touched or an EXEC condition did not match", "type": "null" } ] diff --git a/src/commands/geodist.json b/src/commands/geodist.json index 6f10c2d2e..ea4d01f87 100644 --- a/src/commands/geodist.json +++ b/src/commands/geodist.json @@ -42,7 +42,7 @@ }, { "name": "member1", - "type": "string" + "type": "member" }, { "name": "member2", diff --git a/src/commands/geohash.json b/src/commands/geohash.json index fcf5212f2..b1b8654f3 100644 --- a/src/commands/geohash.json +++ b/src/commands/geohash.json @@ -42,7 +42,7 @@ }, { "name": "member", - "type": "string", + "type": "member", "multiple": true, "optional": true } diff --git a/src/commands/geopos.json b/src/commands/geopos.json index 8089920bd..95b67c4a9 100644 --- a/src/commands/geopos.json +++ b/src/commands/geopos.json @@ -42,7 +42,7 @@ }, { "name": "member", - "type": "string", + "type": "member", "multiple": true, "optional": true } diff --git a/src/commands/georadiusbymember.json b/src/commands/georadiusbymember.json index 9994a8df4..e87651649 100644 --- a/src/commands/georadiusbymember.json +++ b/src/commands/georadiusbymember.json @@ -94,7 +94,7 @@ }, { "name": "member", - "type": "string" + "type": "member" }, { "name": "radius", diff --git a/src/commands/georadiusbymember_ro.json b/src/commands/georadiusbymember_ro.json index 2e6ac22f9..454689f4c 100644 --- a/src/commands/georadiusbymember_ro.json +++ b/src/commands/georadiusbymember_ro.json @@ -52,7 +52,7 @@ }, { "name": "member", - "type": "string" + "type": "member" }, { "name": "radius", diff --git a/src/commands/hdel.json b/src/commands/hdel.json index 39ad2b84c..41e3fab6b 100644 --- a/src/commands/hdel.json +++ b/src/commands/hdel.json @@ -53,7 +53,7 @@ }, { "name": "field", - "type": "string", + "type": "field", "multiple": true } ] diff --git a/src/commands/hexists.json b/src/commands/hexists.json index 0403e6117..ab6b1e369 100644 --- a/src/commands/hexists.json +++ b/src/commands/hexists.json @@ -54,7 +54,7 @@ }, { "name": "field", - "type": "string" + "type": "field" } ] } diff --git a/src/commands/hexpiretime.json b/src/commands/hexpiretime.json index 24c908430..347b54018 100644 --- a/src/commands/hexpiretime.json +++ b/src/commands/hexpiretime.json @@ -77,7 +77,7 @@ }, { "name": "field", - "type": "string", + "type": "field", "multiple": true } ] diff --git a/src/commands/hget.json b/src/commands/hget.json index b81b03a9a..f69e0f51e 100644 --- a/src/commands/hget.json +++ b/src/commands/hget.json @@ -55,7 +55,7 @@ }, { "name": "field", - "type": "string" + "type": "field" } ] } diff --git a/src/commands/hgetdel.json b/src/commands/hgetdel.json index fdec47bb2..4037d570a 100644 --- a/src/commands/hgetdel.json +++ b/src/commands/hgetdel.json @@ -71,7 +71,7 @@ }, { "name": "field", - "type": "string", + "type": "field", "multiple": true } ] diff --git a/src/commands/hgetex.json b/src/commands/hgetex.json index 1e11e551f..41acfb230 100644 --- a/src/commands/hgetex.json +++ b/src/commands/hgetex.json @@ -19,7 +19,8 @@ { "flags": [ "RW", - "ACCESS" + "ACCESS", + "UPDATE" ], "begin_search": { "index": { diff --git a/src/commands/hincrby.json b/src/commands/hincrby.json index ddc42fc30..0e5b1315e 100644 --- a/src/commands/hincrby.json +++ b/src/commands/hincrby.json @@ -49,7 +49,7 @@ }, { "name": "field", - "type": "string" + "type": "field" }, { "name": "increment", diff --git a/src/commands/hincrbyfloat.json b/src/commands/hincrbyfloat.json index 91cf9452f..fc3db7d59 100644 --- a/src/commands/hincrbyfloat.json +++ b/src/commands/hincrbyfloat.json @@ -49,7 +49,7 @@ }, { "name": "field", - "type": "string" + "type": "field" }, { "name": "increment", diff --git a/src/commands/hmget.json b/src/commands/hmget.json index 22bb670df..fb831af71 100644 --- a/src/commands/hmget.json +++ b/src/commands/hmget.json @@ -58,7 +58,7 @@ }, { "name": "field", - "type": "string", + "type": "field", "multiple": true } ] diff --git a/src/commands/hmset.json b/src/commands/hmset.json index dbeabebcb..31dfbbec3 100644 --- a/src/commands/hmset.json +++ b/src/commands/hmset.json @@ -52,7 +52,7 @@ "arguments": [ { "name": "field", - "type": "string" + "type": "field" }, { "name": "value", diff --git a/src/commands/hotkeys-get.json b/src/commands/hotkeys-get.json new file mode 100644 index 000000000..81fd94206 --- /dev/null +++ b/src/commands/hotkeys-get.json @@ -0,0 +1,43 @@ +{ + "GET": { + "summary": "Get the hottest keys from the last completed window, ordered by estimated QPS descending", + "complexity": "O(N) where N is the number of hot keys in the history", + "group": "server", + "since": "9.2.0", + "arity": 2, + "function": "hotkeysGetCommand", + "command_flags": [ + "ADMIN", + "LOADING", + "STALE" + ], + "acl_categories": [ + "ADMIN", + "DANGEROUS", + "SLOW" + ], + "container": "HOTKEYS", + "reply_schema": { + "description": "Hottest keys from the last completed window, ordered by estimated QPS (descending)", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { + "description": "Key name", + "type": "string" + }, + "db": { + "description": "Database id", + "type": "integer" + }, + "qps": { + "description": "Estimated accesses per second over the last completed window", + "type": "integer" + } + } + } + } + } +} diff --git a/src/commands/hotkeys-help.json b/src/commands/hotkeys-help.json new file mode 100644 index 000000000..56974b5cd --- /dev/null +++ b/src/commands/hotkeys-help.json @@ -0,0 +1,25 @@ +{ + "HELP": { + "summary": "Shows helpful text about the different subcommands", + "complexity": "O(1)", + "group": "server", + "since": "9.2.0", + "arity": 2, + "container": "HOTKEYS", + "function": "hotkeysHelpCommand", + "command_flags": [ + "LOADING", + "STALE" + ], + "reply_schema": { + "type": "array", + "description": "Helpful text about subcommands.", + "items": { + "type": "string" + } + }, + "acl_categories": [ + "SLOW" + ] + } +} diff --git a/src/commands/hotkeys-reset.json b/src/commands/hotkeys-reset.json new file mode 100644 index 000000000..62105e3cf --- /dev/null +++ b/src/commands/hotkeys-reset.json @@ -0,0 +1,24 @@ +{ + "RESET": { + "summary": "Reset all hot key statistics and history", + "complexity": "O(N) where N is the number of keys to remove from history and statistics", + "group": "server", + "since": "9.2.0", + "arity": 2, + "function": "hotkeysResetCommand", + "command_flags": [ + "ADMIN", + "LOADING", + "STALE" + ], + "acl_categories": [ + "ADMIN", + "DANGEROUS", + "SLOW" + ], + "container": "HOTKEYS", + "reply_schema": { + "const": "OK" + } + } +} diff --git a/src/commands/hotkeys.json b/src/commands/hotkeys.json new file mode 100644 index 000000000..af96d26d1 --- /dev/null +++ b/src/commands/hotkeys.json @@ -0,0 +1,12 @@ +{ + "HOTKEYS": { + "summary": "A container for hot key commands", + "complexity": "Depends on subcommand.", + "group": "server", + "since": "9.2.0", + "arity": -2, + "acl_categories": [ + "SLOW" + ] + } +} diff --git a/src/commands/hpersist.json b/src/commands/hpersist.json index 59ae09a12..5732f1248 100644 --- a/src/commands/hpersist.json +++ b/src/commands/hpersist.json @@ -76,7 +76,7 @@ }, { "name": "field", - "type": "string", + "type": "field", "multiple": true } ] diff --git a/src/commands/hpexpiretime.json b/src/commands/hpexpiretime.json index 2156b72fd..884b61ac2 100644 --- a/src/commands/hpexpiretime.json +++ b/src/commands/hpexpiretime.json @@ -77,7 +77,7 @@ }, { "name": "field", - "type": "string", + "type": "field", "multiple": true } ] diff --git a/src/commands/hpttl.json b/src/commands/hpttl.json index 9910b77c5..9d4c4df46 100644 --- a/src/commands/hpttl.json +++ b/src/commands/hpttl.json @@ -77,7 +77,7 @@ }, { "name": "field", - "type": "string", + "type": "field", "multiple": true } ] diff --git a/src/commands/hset.json b/src/commands/hset.json index 6b0dbec96..a3149e4b6 100644 --- a/src/commands/hset.json +++ b/src/commands/hset.json @@ -59,7 +59,7 @@ "arguments": [ { "name": "field", - "type": "string" + "type": "field" }, { "name": "value", diff --git a/src/commands/hsetnx.json b/src/commands/hsetnx.json index fcc61afaf..b67ec23e6 100644 --- a/src/commands/hsetnx.json +++ b/src/commands/hsetnx.json @@ -56,7 +56,7 @@ }, { "name": "field", - "type": "string" + "type": "field" }, { "name": "value", diff --git a/src/commands/hstrlen.json b/src/commands/hstrlen.json index 35bf8ae9d..ba1e08764 100644 --- a/src/commands/hstrlen.json +++ b/src/commands/hstrlen.json @@ -47,7 +47,7 @@ }, { "name": "field", - "type": "string" + "type": "field" } ] } diff --git a/src/commands/httl.json b/src/commands/httl.json index e39e15e75..6e3ffb357 100644 --- a/src/commands/httl.json +++ b/src/commands/httl.json @@ -77,7 +77,7 @@ }, { "name": "field", - "type": "string", + "type": "field", "multiple": true } ] diff --git a/src/commands/increx.json b/src/commands/increx.json new file mode 100644 index 000000000..011566494 --- /dev/null +++ b/src/commands/increx.json @@ -0,0 +1,122 @@ +{ + "INCREX": { + "summary": "Increments the numeric value of a key with an option to expire. Uses 0 as initial value if the key doesn't exist.", + "complexity": "O(1)", + "group": "string", + "since": "9.2.0", + "arity": -2, + "function": "increxCommand", + "command_flags": [ + "WRITE", + "DENYOOM", + "FAST" + ], + "acl_categories": [ + "FAST", + "STRING", + "WRITE" + ], + "key_specs": [ + { + "flags": [ + "RW", + "ACCESS", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "condition", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "nx", + "type": "pure-token", + "token": "NX" + }, + { + "name": "xx", + "type": "pure-token", + "token": "XX" + } + ] + }, + { + "name": "expiration", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "ex", + "type": "integer", + "token": "EX" + }, + { + "name": "px", + "type": "integer", + "token": "PX" + }, + { + "name": "unix-time-seconds", + "type": "unix-time", + "token": "EXAT" + }, + { + "name": "unix-time-milliseconds", + "type": "unix-time", + "token": "PXAT" + } + ] + }, + { + "name": "increment", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "integer", + "type": "integer", + "token": "BYINT" + }, + { + "name": "float", + "type": "double", + "token": "BYFLOAT" + } + ] + } + ], + "reply_schema": { + "type": "array", "minItems": 2, "maxItems": 2, + "items": [ + { + "description": "Value after the increment; the current value if the increment was not applied.", + "type": "number" + }, + { + "description": "Increment actually applied; 0 if skipped.", + "type": "number" + } + ] + } + } +} diff --git a/src/commands/script-load.json b/src/commands/script-load.json index 7ab9e20bb..5babe405f 100644 --- a/src/commands/script-load.json +++ b/src/commands/script-load.json @@ -8,6 +8,7 @@ "container": "SCRIPT", "function": "scriptCommand", "command_flags": [ + "DENYOOM", "NOSCRIPT", "STALE" ], diff --git a/src/commands/set.json b/src/commands/set.json index d3f1a9258..6947fdee6 100644 --- a/src/commands/set.json +++ b/src/commands/set.json @@ -27,6 +27,10 @@ [ "8.1.0", "Added the `IFEQ` option." + ], + [ + "9.2.0", + "Added the `IFNE` option." ] ], "command_flags": [ @@ -114,6 +118,13 @@ "token": "IFEQ", "since": "8.1.0", "summary": "Sets the key's value only if the current value matches the specified comparison value." + }, + { + "name": "comparison-not-equal", + "type": "string", + "token": "IFNE", + "since": "9.2.0", + "summary": "Sets the key's value only if the current value does not match the specified comparison value." } ] }, diff --git a/src/commands/sismember.json b/src/commands/sismember.json index 2315a6ad4..4d9235bc6 100644 --- a/src/commands/sismember.json +++ b/src/commands/sismember.json @@ -4,8 +4,14 @@ "complexity": "O(1)", "group": "set", "since": "1.0.0", - "arity": 3, + "arity": -3, "function": "sismemberCommand", + "history": [ + [ + "9.1.0", + "Added the `XX` options." + ] + ], "command_flags": [ "READONLY", "FAST" @@ -38,11 +44,15 @@ "oneOf": [ { "const": 0, - "description": "The element is not a member of the set, or the key does not exist." + "description": "The element is not a member of the set, or the key does not exist (when `XX` option is not specified)." }, { "const": 1, "description": "The element is a member of the set." + }, + { + "const": -1, + "description": "The key does not exist (when `XX` option is specified)." } ] }, @@ -54,7 +64,14 @@ }, { "name": "member", - "type": "string" + "type": "member" + }, + { + "name": "xx", + "token": "XX", + "type": "pure-token", + "since": "9.1.0", + "optional": true } ] } diff --git a/src/commands/smismember.json b/src/commands/smismember.json index 26e23b000..10bdb35cb 100644 --- a/src/commands/smismember.json +++ b/src/commands/smismember.json @@ -60,7 +60,7 @@ }, { "name": "member", - "type": "string", + "type": "member", "multiple": true } ] diff --git a/src/commands/srem.json b/src/commands/srem.json index 77b582930..475284620 100644 --- a/src/commands/srem.json +++ b/src/commands/srem.json @@ -54,7 +54,7 @@ }, { "name": "member", - "type": "string", + "type": "member", "multiple": true } ] diff --git a/src/commands/xackdel.json b/src/commands/xackdel.json new file mode 100644 index 000000000..62d0e7ab3 --- /dev/null +++ b/src/commands/xackdel.json @@ -0,0 +1,97 @@ +{ + "XACKDEL": { + "summary": "Acknowledge and (if possible) delete stream message(s).", + "complexity": "O(1) for each single item to delete in the stream, regardless of the stream size", + "group": "stream", + "since": "9.2.0", + "arity": -6, + "function": "xackdelCommand", + "command_flags": [ + "WRITE", + "FAST" + ], + "acl_categories": [ + "FAST", + "WRITE", + "STREAM" + ], + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "group", + "type": "string" + }, + { + "name": "mode", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "keepref", + "type": "pure-token", + "token": "KEEPREF" + }, + { + "name": "delref", + "type": "pure-token", + "token": "DELREF" + }, + { + "name": "acked", + "type": "pure-token", + "token": "ACKED" + } + ] + }, + { + "name": "ids", + "type": "pure-token", + "token": "IDS" + }, + { + "name": "numids", + "type": "integer" + }, + { + "name": "id", + "type": "string", + "multiple": true + } + ], + "reply_schema": { + "description": "The command returns an integer for each stream message: -1=message not found, 1=acked and deleted, 2=acked but not deleted.", + "type": "array", + "minItems": 1, + "items": { + "description": "Status of the stream message, -1=message not found, 1=acked and deleted, 2=acked but not deleted.", + "type": "integer", + "minimum": -1, + "maximum": 2 + } + } + } +} diff --git a/src/commands/xdelex.json b/src/commands/xdelex.json new file mode 100644 index 000000000..11fa5a18a --- /dev/null +++ b/src/commands/xdelex.json @@ -0,0 +1,93 @@ +{ + "XDELEX": { + "summary": "Delete stream message(s) with extended options", + "complexity": "O(1) for each single item to delete in the stream, regardless of the stream size", + "group": "stream", + "since": "9.2.0", + "arity": -5, + "function": "xdelexCommand", + "command_flags": [ + "WRITE", + "FAST" + ], + "acl_categories": [ + "FAST", + "WRITE", + "STREAM" + ], + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ], + "arguments": [ + { + "name": "key", + "type": "key", + "key_spec_index": 0 + }, + { + "name": "mode", + "type": "oneof", + "optional": true, + "arguments": [ + { + "name": "keepref", + "type": "pure-token", + "token": "KEEPREF" + }, + { + "name": "delref", + "type": "pure-token", + "token": "DELREF" + }, + { + "name": "acked", + "type": "pure-token", + "token": "ACKED" + } + ] + }, + { + "name": "ids", + "token": "IDS", + "type": "pure-token" + }, + { + "name": "numids", + "type": "integer" + }, + { + "name": "id", + "type": "string", + "multiple": true + } + ], + "reply_schema": { + "description": "The command returns an integer for each stream message: -1=message not found, 1=message was deleted, 2=message was not deleted due to existing references (ACKED mode).", + "type": "array", + "minItems": 0, + "items": { + "description": "Status of the stream message, -1=message not found, 1=message was deleted, 2=message was not deleted due to existing references (ACKED mode).", + "type": "integer", + "minimum": -1, + "maximum": 2 + } + } + } +} diff --git a/src/commands/xpending.json b/src/commands/xpending.json index 8e599c6f2..6aef2f4ae 100644 --- a/src/commands/xpending.json +++ b/src/commands/xpending.json @@ -113,6 +113,30 @@ } } ] + }, + { + "description": "Summary form when there are no pending messages.", + "type": "array", + "minItems": 4, + "maxItems": 4, + "items": [ + { + "description": "Total number of pending messages", + "const": 0 + }, + { + "description": "Minimal pending entry ID", + "type": "null" + }, + { + "description": "Maximal pending entry ID", + "type": "null" + }, + { + "description": "Consumers with pending messages", + "type": "null" + } + ] } ] }, diff --git a/src/commands/zincrby.json b/src/commands/zincrby.json index 453e18bc9..29ef4bb5c 100644 --- a/src/commands/zincrby.json +++ b/src/commands/zincrby.json @@ -53,7 +53,7 @@ }, { "name": "member", - "type": "string" + "type": "member" } ] } diff --git a/src/commands/zmscore.json b/src/commands/zmscore.json index c08f699c3..e561ef482 100644 --- a/src/commands/zmscore.json +++ b/src/commands/zmscore.json @@ -59,7 +59,7 @@ }, { "name": "member", - "type": "string", + "type": "member", "multiple": true } ] diff --git a/src/commands/zrank.json b/src/commands/zrank.json index 34491feca..eb07fbfc6 100644 --- a/src/commands/zrank.json +++ b/src/commands/zrank.json @@ -75,7 +75,7 @@ }, { "name": "member", - "type": "string" + "type": "member" }, { "name": "withscore", diff --git a/src/commands/zrem.json b/src/commands/zrem.json index c372f4868..a64de58af 100644 --- a/src/commands/zrem.json +++ b/src/commands/zrem.json @@ -54,7 +54,7 @@ }, { "name": "member", - "type": "string", + "type": "member", "multiple": true } ] diff --git a/src/commands/zrevrank.json b/src/commands/zrevrank.json index 411a4c169..52500527a 100644 --- a/src/commands/zrevrank.json +++ b/src/commands/zrevrank.json @@ -75,7 +75,7 @@ }, { "name": "member", - "type": "string" + "type": "member" }, { "name": "withscore", diff --git a/src/commands/zscore.json b/src/commands/zscore.json index 987c69997..1f8b58528 100644 --- a/src/commands/zscore.json +++ b/src/commands/zscore.json @@ -55,7 +55,7 @@ }, { "name": "member", - "type": "string" + "type": "member" } ] } diff --git a/src/compression.c b/src/compression.c index b024b2c81..722a7b0b0 100644 --- a/src/compression.c +++ b/src/compression.c @@ -10,6 +10,7 @@ #include "serverassert.h" #include +/* Returns a static algorithm name for logs and config output. */ const char *compressionAlgoName(compressionAlgo algo) { switch (algo) { case ALGO_NONE: @@ -25,14 +26,18 @@ const char *compressionAlgoName(compressionAlgo algo) { /* ===== Compressor ===== */ +/* Compressor lifecycle. Codec dispatch used by streamWriter and by the + * replication write path; callers own sticky error state while these + * functions manage only codec state. checksum_flags is a bitwise combination + * of STREAM_CHECKSUM_* values. */ int streamCompressorInit(streamCompressor *compressor, compressionAlgo algo, int level, - bool codec_checksum) { + uint8_t checksum_flags) { memset(compressor, 0, sizeof(*compressor)); compressor->algo = algo; compressor->level = level; - compressor->codec_checksum = codec_checksum; + compressor->checksum_flags = checksum_flags; switch (algo) { case ALGO_LZ4: @@ -51,6 +56,12 @@ size_t streamCompressorOutputBound(const streamCompressor *compressor, size_t in } } +/* Feeds raw input into the compressor and writes compressed bytes to output. + * Called repeatedly to build a complete frame: COMPRESS_FLUSH_CONTINUE keeps + * buffering, COMPRESS_FLUSH_SYNC drains buffered bytes but leaves the frame + * open, and COMPRESS_FLUSH_END closes it. output must be at least + * streamCompressorOutputBound(compressor, input_len) bytes. Returns bytes + * written, or -1 on error. */ ssize_t streamCompressorFeed(streamCompressor *compressor, uint8_t *output, size_t output_capacity, @@ -77,6 +88,7 @@ void streamCompressorFree(streamCompressor *compressor) { /* ===== Decompressor ===== */ +/* Codec dispatch shared by the pull and push stream readers. */ int streamDecompressorInit(streamDecompressor *decompressor, compressionAlgo algo, bool skip_codec_checksum_validation) { diff --git a/src/compression.h b/src/compression.h index 5c432a14c..ba47e8b7b 100644 --- a/src/compression.h +++ b/src/compression.h @@ -22,30 +22,26 @@ typedef enum { typedef enum { COMPRESS_FLUSH_CONTINUE = 0, /* Buffer internally. */ COMPRESS_FLUSH_END = 1, /* Finalize frame. */ + COMPRESS_FLUSH_SYNC = 2, /* Drain buffered bytes, keep frame open. */ } compressFlushMode; -/* Returns a static algorithm name for logs and config output. */ const char *compressionAlgoName(compressionAlgo algo); /* ===== Compressor ===== */ +#define STREAM_CHECKSUM_BLOCK (1u << 0) +#define STREAM_CHECKSUM_CONTENT (1u << 1) + typedef struct { compressionAlgo algo; int level; /* 0 selects the codec default. */ void *ctx; bool stream_started; - bool codec_checksum; + uint8_t checksum_flags; } streamCompressor; -/* Compressor lifecycle. Codec dispatch used by streamWriter; the writer owns - * sticky error state while these functions manage only codec state. */ -int streamCompressorInit(streamCompressor *compressor, compressionAlgo algo, int level, bool codec_checksum); +int streamCompressorInit(streamCompressor *compressor, compressionAlgo algo, int level, uint8_t checksum_flags); size_t streamCompressorOutputBound(const streamCompressor *compressor, size_t input_len); -/* Feeds raw input into the compressor and writes compressed bytes to output. - * Called repeatedly to build a complete frame: COMPRESS_FLUSH_CONTINUE keeps - * buffering and COMPRESS_FLUSH_END closes the frame. output must be at least - * streamCompressorOutputBound(compressor, input_len) bytes. Returns bytes written, - * or -1 on error. */ ssize_t streamCompressorFeed(streamCompressor *compressor, uint8_t *output, size_t output_capacity, @@ -64,8 +60,6 @@ typedef struct { size_t input_hint; /* Preferred compressed bytes for next feed, 0 if unknown. */ } streamDecompressor; -/* Decompressor lifecycle. Codec dispatch used by streamReader; the reader owns - * buffering and sticky error state. */ int streamDecompressorInit(streamDecompressor *decompressor, compressionAlgo algo, bool skip_codec_checksum_validation); diff --git a/src/compression_lz4.c b/src/compression_lz4.c index 97b8e1e01..6f593f88d 100644 --- a/src/compression_lz4.c +++ b/src/compression_lz4.c @@ -72,10 +72,10 @@ ssize_t compressionLz4CompressFeed(streamCompressor *compressor, if (!compressor->stream_started) { LZ4F_preferences_t prefs = lz4f_prefs; prefs.compressionLevel = compressor->level; - prefs.frameInfo.blockChecksumFlag = compressor->codec_checksum + prefs.frameInfo.blockChecksumFlag = compressor->checksum_flags & STREAM_CHECKSUM_BLOCK ? LZ4F_blockChecksumEnabled : LZ4F_noBlockChecksum; - prefs.frameInfo.contentChecksumFlag = compressor->codec_checksum + prefs.frameInfo.contentChecksumFlag = compressor->checksum_flags & STREAM_CHECKSUM_CONTENT ? LZ4F_contentChecksumEnabled : LZ4F_noContentChecksum; size_t r = LZ4F_compressBegin(cctx, output, output_capacity, &prefs); @@ -94,6 +94,14 @@ ssize_t compressionLz4CompressFeed(streamCompressor *compressor, switch (flush_mode) { case COMPRESS_FLUSH_CONTINUE: break; + case COMPRESS_FLUSH_SYNC: { + /* Emit buffered bytes without ending the frame. */ + if (offset >= output_capacity) return -1; + size_t r = LZ4F_flush(cctx, output + offset, output_capacity - offset, NULL); + if (LZ4F_isError(r)) return -1; + offset += r; + break; + } case COMPRESS_FLUSH_END: { if (offset >= output_capacity) return -1; size_t r = LZ4F_compressEnd(cctx, output + offset, output_capacity - offset, NULL); diff --git a/src/compression_stream.c b/src/compression_stream.c index 0dd621c4f..6dedda6ad 100644 --- a/src/compression_stream.c +++ b/src/compression_stream.c @@ -25,9 +25,7 @@ static bool vcsHasMagicPrefix(const uint8_t *buf, size_t len) { return memcmp(buf, VCS_MAGIC, n) == 0; } -static int writeVcsEnvelope(streamWriterWriteFn write_cb, - void *ctx, - compressionAlgo algo) { +int vcsBuildEnvelope(uint8_t *buf, compressionAlgo algo, uint8_t stream_kind) { uint8_t codec; switch (algo) { case ALGO_LZ4: @@ -44,14 +42,21 @@ static int writeVcsEnvelope(streamWriterWriteFn write_cb, [VCS_OFFSET_VERSION] = VCS_VERSION, [VCS_OFFSET_CODEC] = codec, [VCS_OFFSET_RESERVED] = 0, - [VCS_OFFSET_STREAM_KIND] = VCS_STREAM_RDB, + [VCS_OFFSET_STREAM_KIND] = stream_kind, }; + memcpy(buf, envelope, VCS_ENVELOPE_SIZE); + return C_OK; +} + +static int writeVcsEnvelope(streamWriterWriteFn write_cb, void *ctx, compressionAlgo algo, uint8_t stream_kind) { + uint8_t envelope[VCS_ENVELOPE_SIZE]; + if (vcsBuildEnvelope(envelope, algo, stream_kind) == C_ERR) return C_ERR; return write_cb(ctx, envelope, VCS_ENVELOPE_SIZE); } /* Reject a nonzero reserved byte so a future envelope extension fails loudly * rather than being silently misinterpreted. */ -static int readVcsEnvelope(const uint8_t *buf, compressionAlgo *algo) { +static int readVcsEnvelope(const uint8_t *buf, uint8_t expected_stream_kind, compressionAlgo *algo) { if (buf[VCS_OFFSET_VERSION] != VCS_VERSION) return C_ERR; switch (buf[VCS_OFFSET_CODEC]) { @@ -62,7 +67,7 @@ static int readVcsEnvelope(const uint8_t *buf, compressionAlgo *algo) { return C_ERR; } if (buf[VCS_OFFSET_RESERVED] != 0) return C_ERR; - if (buf[VCS_OFFSET_STREAM_KIND] != VCS_STREAM_RDB) return C_ERR; + if (buf[VCS_OFFSET_STREAM_KIND] != expected_stream_kind) return C_ERR; return C_OK; } @@ -75,7 +80,8 @@ int streamWriterInit(streamWriter *writer, compressionAlgo algo, bool codec_chec writer->write_cb = write_cb; writer->write_ctx = write_ctx; - if (streamCompressorInit(&writer->compressor, algo, 0, codec_checksum) == C_ERR) { + uint8_t checksum_flags = codec_checksum ? STREAM_CHECKSUM_BLOCK | STREAM_CHECKSUM_CONTENT : 0; + if (streamCompressorInit(&writer->compressor, algo, 0, checksum_flags) == C_ERR) { writer->state = STREAM_WRITER_STATE_ERROR; return C_ERR; } @@ -83,11 +89,11 @@ int streamWriterInit(streamWriter *writer, compressionAlgo algo, bool codec_chec } /* Envelope is emitted lazily so a writer that's created but never written - * doesn't leave a stub envelope on the sink. */ + * doesn't leave a stub envelope on the output. */ static int streamWriterEnsureEnvelope(streamWriter *writer) { if (writer->state == STREAM_WRITER_STATE_ACTIVE) return C_OK; if (writer->state != STREAM_WRITER_STATE_INITIAL) return C_ERR; - if (writeVcsEnvelope(writer->write_cb, writer->write_ctx, writer->compressor.algo) == C_ERR) { + if (writeVcsEnvelope(writer->write_cb, writer->write_ctx, writer->compressor.algo, VCS_STREAM_RDB) == C_ERR) { writer->state = STREAM_WRITER_STATE_ERROR; return C_ERR; } @@ -171,6 +177,7 @@ int streamReaderInit(streamReader *reader, const streamReaderConfig *cfg, stream reader->buffer_size = cfg->buffer_size < STREAM_READER_BUFFER_SIZE_MIN ? STREAM_READER_BUFFER_SIZE_MIN : cfg->buffer_size; + reader->eof_mid_frame_is_truncation = cfg->eof_mid_frame_is_truncation; compressionAlgo algo = ALGO_NONE; while (true) { size_t need = reader->probe.header_len < VCS_MAGIC_SIZE @@ -197,7 +204,7 @@ int streamReaderInit(streamReader *reader, const streamReaderConfig *cfg, stream } if (reader->probe.header_len == VCS_ENVELOPE_SIZE) { - if (readVcsEnvelope(reader->probe.header, &algo) == C_ERR) { + if (readVcsEnvelope(reader->probe.header, VCS_STREAM_RDB, &algo) == C_ERR) { streamReaderSetError(reader, STREAM_READER_ERROR_INCOMPATIBLE); return C_ERR; } @@ -359,7 +366,11 @@ ssize_t streamReaderRead(streamReader *reader, void *buf, size_t len) { available = reader->decompressed_buf_len; if (available == 0 && fill_result == C_ERR) return total > 0 ? (ssize_t)total : -1; if (available == 0 && !reader->decompressor.frame_done) { - streamReaderSetError(reader, STREAM_READER_ERROR_CORRUPT); + /* A codec failure would already have latched an error above, so + * this is a short read: recoverable only if more bytes can arrive. */ + streamReaderSetError(reader, reader->eof_mid_frame_is_truncation + ? STREAM_READER_ERROR_TRUNCATED + : STREAM_READER_ERROR_CORRUPT); return total > 0 ? (ssize_t)total : -1; } if (available == 0) break; @@ -420,3 +431,173 @@ void streamReaderFree(streamReader *reader) { reader->decompressed_buf_len = 0; reader->decompressed_buf_pos = 0; } + +/* ===== Push reader ===== */ + +/* Decoded-output room offered to the codec per feed iteration: bounds how + * much the caller's sds over-allocates per iteration while the drain loop + * empties the codec's buffered output. */ +#define STREAM_PUSH_READER_OUTPUT_CHUNK_SIZE (16 * 1024) + +void streamPushReaderInit(streamPushReader *reader, uint8_t expected_stream_kind) { + memset(reader, 0, sizeof(*reader)); + reader->expected_stream_kind = expected_stream_kind; +} + +void streamPushReaderFree(streamPushReader *reader) { + if (reader->state == STREAM_PUSH_READER_COMPRESSED) streamDecompressorFree(&reader->decompressor); + sdsfree(reader->pending_input); + reader->pending_input = NULL; + reader->pending_input_pos = 0; + reader->codec_needs_drain = false; + reader->state = STREAM_PUSH_READER_PROBE; + reader->envelope_len = 0; +} + +/* Drain compressed bytes [in, in+len) through the codec, appending decoded + * output to *out within the remaining scheduling budget. */ +static streamPushReaderResult +streamPushReaderFeedCodec(streamPushReader *reader, const uint8_t *in, size_t len, size_t *input_consumed, sds *out, size_t *budget) { + size_t off = 0; + size_t room = 0; + ssize_t produced = 0; + do { + if (*budget == 0) { + *input_consumed = off; + return STREAM_PUSH_READER_NEED_OUTPUT; + } + room = STREAM_PUSH_READER_OUTPUT_CHUNK_SIZE; + if (room > *budget) room = *budget; + size_t used = sdslen(*out); + *out = sdsMakeRoomFor(*out, room); + size_t consumed = 0; + const uint8_t *feed_input = in ? in + off : NULL; + produced = streamDecompressorFeed(&reader->decompressor, (uint8_t *)*out + used, room, feed_input, len - off, + &consumed); + if (produced < 0) { + *input_consumed = off; + return STREAM_PUSH_READER_ERR; + } + serverAssert((size_t)produced <= room); + serverAssert(consumed <= len - off); + if (produced > 0) { + sdsIncrLen(*out, (size_t)produced); + *budget -= (size_t)produced; + } + off += consumed; + /* Report the frame end; for a long-lived stream this means the + * source ended it unexpectedly. */ + if (reader->decompressor.frame_done) { + *input_consumed = off; + return STREAM_PUSH_READER_FRAME_DONE; + } + /* The codec always makes progress given input and output room; no + * progress with input still pending is a stuck state. Fail rather + * than let the caller drop the unconsumed tail. Gated on pending + * input: empty-input drain iterations legitimately produce 0. */ + if (off < len && consumed == 0 && produced == 0) { + *input_consumed = off; + return STREAM_PUSH_READER_ERR; + } + /* Keep draining with empty input while the codec may hold buffered + * output, which is only the case when it filled the entire room. */ + } while (off < len || (size_t)produced == room); + *input_consumed = off; + return STREAM_PUSH_READER_OK; +} + +static streamPushReaderResult streamPushReaderFeedInput(streamPushReader *reader, const void *src, size_t len, size_t *input_consumed, sds *out, size_t output_budget) { + const uint8_t *in = src; + size_t off = 0; + size_t budget = output_budget; + *input_consumed = 0; + + /* Probe phase: classify the stream from its leading bytes. The magic may + * arrive split across feeds, so bytes accumulate until the prefix matches + * or rules out the VCS magic. */ + if (reader->state == STREAM_PUSH_READER_PROBE) { + while (reader->envelope_len < VCS_MAGIC_SIZE && off < len) { + reader->envelope[reader->envelope_len++] = in[off++]; + if (!vcsHasMagicPrefix(reader->envelope, reader->envelope_len)) { + reader->state = STREAM_PUSH_READER_PASSTHROUGH; + break; + } + } + + if (reader->state == STREAM_PUSH_READER_PROBE) { + /* Magic matches so far; gather the rest of the envelope. */ + while (reader->envelope_len < VCS_ENVELOPE_SIZE && off < len) + reader->envelope[reader->envelope_len++] = in[off++]; + if (reader->envelope_len < VCS_ENVELOPE_SIZE) { + *input_consumed = off; + return STREAM_PUSH_READER_OK; /* Need more header. */ + } + + compressionAlgo algo = ALGO_NONE; + if (readVcsEnvelope(reader->envelope, reader->expected_stream_kind, &algo) != C_OK) { + *input_consumed = off; + return STREAM_PUSH_READER_ERR; + } + if (streamDecompressorInit(&reader->decompressor, algo, false) != C_OK) { + *input_consumed = off; + return STREAM_PUSH_READER_ERR; + } + reader->state = STREAM_PUSH_READER_COMPRESSED; + } + } + + if (reader->state == STREAM_PUSH_READER_PASSTHROUGH) { + /* Replay any buffered magic-prefix bytes once, then forward the rest. */ + *out = sdscatlen(*out, reader->envelope, reader->envelope_len); + reader->envelope_len = 0; + if (off < len) *out = sdscatlen(*out, in + off, len - off); + *input_consumed = len; + return STREAM_PUSH_READER_OK; + } + if (off < len || len == 0) { + size_t codec_consumed = 0; + const uint8_t *codec_input = in ? in + off : NULL; + streamPushReaderResult result = streamPushReaderFeedCodec(reader, codec_input, len - off, &codec_consumed, + out, &budget); + *input_consumed = off + codec_consumed; + return result; + } + *input_consumed = off; + return STREAM_PUSH_READER_OK; +} + +bool streamPushReaderHasPendingDecode(const streamPushReader *reader) { + return reader->codec_needs_drain || reader->pending_input != NULL; +} + +streamPushReaderResult streamPushReaderFeed(streamPushReader *reader, const void *src, size_t len, sds *out, size_t output_budget) { + bool resuming = streamPushReaderHasPendingDecode(reader); + serverAssert(len == 0 || !resuming); + + const uint8_t *input = src; + size_t input_len = len; + if (reader->pending_input) { + input = (const uint8_t *)reader->pending_input + reader->pending_input_pos; + input_len = sdslen(reader->pending_input) - reader->pending_input_pos; + } else if (reader->codec_needs_drain) { + input = NULL; + input_len = 0; + } + + size_t consumed = 0; + streamPushReaderResult result = streamPushReaderFeedInput(reader, input, input_len, &consumed, out, output_budget); + serverAssert(consumed <= input_len); + + if (reader->pending_input) { + reader->pending_input_pos += consumed; + if (reader->pending_input_pos == sdslen(reader->pending_input)) { + sdsfree(reader->pending_input); + reader->pending_input = NULL; + reader->pending_input_pos = 0; + } + } else if (result == STREAM_PUSH_READER_NEED_OUTPUT && consumed < input_len) { + reader->pending_input = sdsnewlen(input + consumed, input_len - consumed); + } + reader->codec_needs_drain = result == STREAM_PUSH_READER_NEED_OUTPUT && consumed == input_len; + return result; +} diff --git a/src/compression_stream.h b/src/compression_stream.h index 1a6b6f626..73a3c4cae 100644 --- a/src/compression_stream.h +++ b/src/compression_stream.h @@ -8,6 +8,7 @@ #define COMPRESSION_STREAM_H #include "compression.h" +#include "sds.h" /* VCS envelope: * [0..2] magic "VCS" @@ -37,6 +38,8 @@ /* Identifies an RDB payload in the envelope. */ #define VCS_STREAM_RDB 0x01 +/* Identifies a replication stream payload in the envelope. */ +#define VCS_STREAM_REPL 0x02 typedef int (*streamWriterWriteFn)(void *ctx, const uint8_t *data, size_t len); /* Returns >0 bytes read, 0 on EOF, -1 on error. Partial reads allowed. */ @@ -71,10 +74,14 @@ int streamWriterFinish(streamWriter *writer); /* Releases resources without implicitly finalizing the frame. */ void streamWriterFree(streamWriter *writer); +/* Build a 7-byte VCS envelope in a producer-owned output buffer. + * Returns C_ERR when algo has no wire codec id. */ +int vcsBuildEnvelope(uint8_t *buf, compressionAlgo algo, uint8_t stream_kind); + /* ===== Reader ===== */ /* Default decompressed-output buffer size. Tiny caller values are clamped up - * so the decoder can always make forward progress without growing internal + * so the reader can always make forward progress without growing internal * state. The compressed-input buffer only needs to hold one LZ4 block. */ #define STREAM_READER_BUFFER_SIZE_DEFAULT (1024 * 1024) #define STREAM_READER_BUFFER_SIZE_MIN (128 * 1024) @@ -86,6 +93,7 @@ typedef struct { bool allow_passthrough; bool skip_codec_checksum_validation; size_t buffer_size; + bool eof_mid_frame_is_truncation; /* Set for sources that can deliver the rest later. */ } streamReaderConfig; typedef enum { @@ -94,6 +102,7 @@ typedef enum { STREAM_READER_ERROR_INCOMPATIBLE = 2, STREAM_READER_ERROR_CORRUPT = 3, STREAM_READER_ERROR_INTERNAL = 4, + STREAM_READER_ERROR_TRUNCATED = 5, /* Clean EOF before frame end: recoverable short read, not corruption. */ } streamReaderErrorKind; typedef enum { @@ -111,6 +120,7 @@ typedef struct streamReader { } probe; size_t probe_replay_pos; /* Passthrough bytes left to replay from probe. */ size_t buffer_size; + bool eof_mid_frame_is_truncation; streamReaderErrorKind error_kind; streamReaderState state; @@ -141,4 +151,46 @@ ssize_t streamReaderRead(streamReader *reader, void *buf, size_t len); int streamReaderFinish(streamReader *reader); void streamReaderFree(streamReader *reader); +/* ===== Push reader ===== */ + +/* Push-mode counterpart of streamReader for callers that receive bytes from a + * non-blocking source (event loop) instead of pulling through a read callback. + * The stream's leading bytes classify it: a VCS envelope of the expected + * stream kind activates the codec; anything else switches to passthrough and + * bytes are forwarded verbatim. Output is appended to a caller-provided sds. */ + +/* OK and ERR intentionally match C_OK and C_ERR. The additional results + * distinguish a closed live frame from resumable output backpressure. */ +typedef enum { + STREAM_PUSH_READER_OK = 0, /* Input consumed; output (possibly 0 bytes) appended. */ + STREAM_PUSH_READER_ERR = -1, /* Envelope or codec error. */ + STREAM_PUSH_READER_FRAME_DONE = -2, /* The compressed frame ended. */ + STREAM_PUSH_READER_NEED_OUTPUT = 1, /* Decode budget reached; resume with an empty feed. */ +} streamPushReaderResult; + +typedef enum { + STREAM_PUSH_READER_PROBE = 0, /* Still classifying the leading bytes. */ + STREAM_PUSH_READER_COMPRESSED, /* VCS envelope seen; codec active. */ + STREAM_PUSH_READER_PASSTHROUGH, /* Non-VCS stream; bytes forwarded as-is. */ +} streamPushReaderState; + +typedef struct streamPushReader { + streamDecompressor decompressor; /* Valid once state == COMPRESSED. */ + streamPushReaderState state; + uint8_t expected_stream_kind; /* Required VCS stream kind. */ + uint8_t envelope[VCS_ENVELOPE_SIZE]; /* Leading bytes gathered during probe. */ + size_t envelope_len; + sds pending_input; /* Wire bytes retained when the output budget is exhausted. */ + size_t pending_input_pos; + bool codec_needs_drain; /* Codec may have output left after consuming all input. */ +} streamPushReader; + +/* Feed appends decoded or passthrough bytes to *out. output_budget limits + * decoded output per call. On STREAM_PUSH_READER_NEED_OUTPUT, call Feed again + * with no input before reading more source bytes. */ +void streamPushReaderInit(streamPushReader *reader, uint8_t expected_stream_kind); +void streamPushReaderFree(streamPushReader *reader); +bool streamPushReaderHasPendingDecode(const streamPushReader *reader); +streamPushReaderResult streamPushReaderFeed(streamPushReader *reader, const void *src, size_t len, sds *out, size_t output_budget); + #endif /* COMPRESSION_STREAM_H */ diff --git a/src/config.c b/src/config.c index ddf571571..6ceae2b69 100644 --- a/src/config.c +++ b/src/config.c @@ -31,6 +31,7 @@ #include "io_threads.h" #include "sds.h" #include "server.h" +#include "hotkeys.h" #include "cluster.h" #include "connection.h" #include "bio.h" @@ -38,6 +39,7 @@ #include "cluster_migrateslots.h" #include "eval.h" #include "lrulfu.h" +#include "throttle_repl.h" #include #include @@ -182,6 +184,20 @@ configEnum rdb_compression_enum[] = {{"no", RDB_COMPRESSION_NO}, {"lz4", RDB_COMPRESSION_LZ4}, {NULL, 0}}; +configEnum bgsave_method_enum[] = {{"fork", RDB_BGSAVE_TYPE_FORK}, + {"forkless", RDB_BGSAVE_TYPE_FORKLESS}, + {NULL, 0}}; + +configEnum cluster_replica_no_failover_enum[] = {{"no", CLUSTER_REPLICA_NO_FAILOVER_NO}, + {"yes", CLUSTER_REPLICA_NO_FAILOVER_YES}, + {"if-empty", CLUSTER_REPLICA_NO_FAILOVER_IF_EMPTY}, + {NULL, 0}}; + +configEnum repl_compression_enum[] = {{"no", REPL_COMPRESSION_NO}, + {"yes", REPL_COMPRESSION_YES}, + {"lz4", REPL_COMPRESSION_LZ4}, + {NULL, 0}}; + /* Output buffer limits presets. */ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {0, 0, 0}, /* normal */ @@ -593,11 +609,19 @@ void loadServerConfigFromString(sds config) { } else if (!strcasecmp(argv[0], "user") && argc >= 2) { int argc_err; if (ACLAppendUserForLoading(argv, argc, &argc_err) == C_ERR) { - const char *errmsg = ACLSetUserStringError(); + const char *errmsg = ACLSetStringError(); snprintf(buf, sizeof(buf), "Error in user declaration '%s': %s", argv[argc_err], errmsg); err = buf; goto loaderr; } + } else if (!strcasecmp(argv[0], "role") && argc >= 2) { + int argc_err; + if (ACLAppendRoleForLoading(argv, argc, &argc_err) == C_ERR) { + const char *errmsg = ACLSetStringError(); + snprintf(buf, sizeof(buf), "Error in role declaration '%s': %s", argv[argc_err], errmsg); + err = buf; + goto loaderr; + } } else if (!strcasecmp(argv[0], "loadmodule") && argc >= 2) { moduleEnqueueLoadModule(argv[1], &argv[2], argc - 2); } else if (strchr(argv[0], '.')) { @@ -645,6 +669,11 @@ void loadServerConfigFromString(sds config) { err = "replicaof directive not allowed in cluster mode"; goto loaderr; } + if (server.bgsave_default_method == RDB_BGSAVE_TYPE_FORKLESS && !server.forkless_infrastructure_enabled) { + err = "'bgsave-default-method forkless' can only be selected when the server was started with " + "'forkless-infrastructure-enabled yes'"; + goto loaderr; + } /* To ensure backward compatibility and work while hz is out of range */ if (server.hz < CONFIG_MIN_HZ) server.hz = CONFIG_MIN_HZ; @@ -1238,7 +1267,7 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { /* The following is a list of config features that are only supported in * config file parsing and are not recognized by lookupConfig */ strcasecmp(argv[0], "include") && strcasecmp(argv[0], "rename-command") && strcasecmp(argv[0], "user") && - strcasecmp(argv[0], "loadmodule") && strcasecmp(argv[0], "sentinel"))) { + strcasecmp(argv[0], "role") && strcasecmp(argv[0], "loadmodule") && strcasecmp(argv[0], "sentinel"))) { /* The line is either unparsable for some reason, for * instance it may have unbalanced quotes, may contain a * config that doesn't exist anymore, for instance a module that got @@ -1503,36 +1532,36 @@ void rewriteConfigSaveOption(standardConfig *config, const char *name, struct re rewriteConfigMarkAsProcessed(state, name); } -/* Rewrite the user option. */ -void rewriteConfigUserOption(struct rewriteConfigState *state) { +/* Rewrite the user or role option. */ +static void rewriteConfigAclOption(struct rewriteConfigState *state, const char *directive, rax *table) { /* If there is a user file defined we just mark this configuration * directive as processed, so that all the lines containing users * inside the config file gets discarded. */ if (server.acl_filename[0] != '\0') { - rewriteConfigMarkAsProcessed(state, "user"); + rewriteConfigMarkAsProcessed(state, directive); return; } - /* Otherwise, scan the list of users and rewrite every line. Note that - * in case the list here is empty, the effect will just be to comment - * all the users directive inside the config file. */ + /* Otherwise, scan the table and rewrite every line. Note that in case the + * table here is empty, the effect will just be to comment all the matching + * directives inside the config file. */ raxIterator ri; - raxStart(&ri, Users); + raxStart(&ri, table); raxSeek(&ri, "^", NULL, 0); while (raxNext(&ri)) { user *u = ri.data; - sds line = sdsnew("user "); + sds line = sdscatfmt(sdsempty(), "%s ", directive); line = sdscatsds(line, u->name); line = sdscatlen(line, " ", 1); robj *descr = ACLDescribeUser(u); line = sdscatsds(line, objectGetVal(descr)); decrRefCount(descr); - rewriteConfigRewriteLine(state, "user", line, 1); + rewriteConfigRewriteLine(state, directive, line, 1); } raxStop(&ri); - /* Mark "user" as processed in case there are no defined users. */ - rewriteConfigMarkAsProcessed(state, "user"); + /* Mark the directive as processed in case the table is empty. */ + rewriteConfigMarkAsProcessed(state, directive); } /* Rewrite the dir option, always using absolute paths.*/ @@ -1862,7 +1891,8 @@ int rewriteConfig(char *path, int force_write) { } dictReleaseIterator(di); - rewriteConfigUserOption(state); + rewriteConfigAclOption(state, "role", Roles); + rewriteConfigAclOption(state, "user", Users); rewriteConfigLoadmoduleOption(state); /* Rewrite Sentinel config if in Sentinel mode. */ @@ -2446,6 +2476,19 @@ static void numericConfigRewrite(standardConfig *config, const char *name, struc {.type = SPECIAL_CONFIG, \ embedCommonConfig(name, alias, modifiable) embedConfigInterface(NULL, setfn, getfn, rewritefn, applyfn)} +static int isValidBgsaveDefaultMethod(int val, const char **err) { + /* During startup config parsing the directives are applied one by one, so + * forkless-infrastructure-enabled may not have been read yet when this + * value is set. We will check it when loading the config string */ + if (reading_config_file) return 1; + if (val == RDB_BGSAVE_TYPE_FORKLESS && !server.forkless_infrastructure_enabled) { + *err = "'forkless' can only be selected when the server was started with " + "'forkless-infrastructure-enabled yes'"; + return 0; + } + return 1; +} + static int isValidActiveDefrag(int val, const char **err) { #ifndef HAVE_DEFRAG if (val) { @@ -2555,6 +2598,15 @@ static int isValidAnnouncedIp(char *val, const char **err) { return 1; } +static int isValidPrioritySubnets(char *val, const char **err) { + return validatePrioritySubnets(val, err) == C_OK; +} + +static int updatePrioritySubnetsConfig(const char **err) { + UNUSED(err); + return updatePrioritySubnets(server.priority_subnets) == C_OK; +} + static int isValidAnnouncedHostname(char *val, const char **err) { if (strlen(val) >= NET_HOST_STR_LEN) { *err = "Hostnames must be less than " STRINGIFY(NET_HOST_STR_LEN) " characters"; @@ -2653,6 +2705,16 @@ static int updateDefragConfiguration(const char **err) { return 1; } +/* Dynamic configuration apply callback for priority-preemptive-poll-interval-us. + * Updates the preemption threshold on the active event loop. */ +static int updatePriorityPreemptivePollInterval(const char **err) { + UNUSED(err); + if (server.el) { + aeSetQoSPreemptCheckInterval(server.el, server.priority_preemptive_poll_interval_us); + } + return 1; +} + static int updateJemallocBgThread(const char **err) { UNUSED(err); set_jemalloc_bg_thread(server.jemalloc_bg_thread); @@ -2678,6 +2740,9 @@ static int updateMaxmemory(const char **err) { } startEvictionTimeProc(); } + /* maxmemory-scripts can be a percentage of maxmemory, in that case the + * scripts eviction limit changed together with maxmemory. */ + if (server.maxmemory_scripts < 0) startScriptsEvictionTimeProc(); return 1; } @@ -3339,6 +3404,12 @@ static int applyClientMaxMemoryUsage(const char **err) { return 1; } +static int updateMaxmemoryScripts(const char **err) { + UNUSED(err); + startScriptsEvictionTimeProc(); + return 1; +} + #define HASH_SEED_MAX_LEN 256 static int isValidDbHashSeed(sds val, const char **err) { if (sdslen(val) > HASH_SEED_MAX_LEN) { @@ -3357,6 +3428,7 @@ standardConfig static_configs[] = { createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, server.rdb_del_sync_files, 0, NULL, NULL), createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL), createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL), + createEnumConfig("bgsave-default-method", NULL, MODIFIABLE_CONFIG, bgsave_method_enum, server.bgsave_default_method, RDB_BGSAVE_TYPE_FORK, isValidBgsaveDefaultMethod, NULL), createBoolConfig("set-proc-title", NULL, IMMUTABLE_CONFIG, server.set_proc_title, 1, NULL, NULL), /* Should setproctitle be used? */ createBoolConfig("lazyfree-lazy-eviction", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 1, NULL, NULL), createBoolConfig("lazyfree-lazy-expire", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 1, NULL, NULL), @@ -3367,6 +3439,7 @@ standardConfig static_configs[] = { createBoolConfig("repl-mptcp", NULL, IMMUTABLE_CONFIG, server.repl_mptcp, 0, isValidMptcp, NULL), createBoolConfig("repl-diskless-sync", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.repl_diskless_sync, 1, NULL, NULL), createBoolConfig("dual-channel-replication-enabled", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.dual_channel_replication, 0, NULL, NULL), + createBoolConfig("repl-throttling-enabled", NULL, MODIFIABLE_CONFIG, throttleRepl_config.repl_throttling_enabled, 0, NULL, NULL), createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, 1, NULL, NULL), createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, 0, NULL, NULL), createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, 1, NULL, updateClusterState), @@ -3374,13 +3447,13 @@ standardConfig static_configs[] = { createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, 1, NULL, NULL), createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, 1, NULL, NULL), createBoolConfig("aof-timestamp-enabled", NULL, MODIFIABLE_CONFIG, server.aof_timestamp_enabled, 0, NULL, NULL), - createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_replica_no_failover, 0, NULL, updateClusterFlags), /* Failover by default. */ createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_replica_lazy_flush, 1, NULL, NULL), createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, 1, NULL, NULL), createBoolConfig("replica-read-only", "slave-read-only", DEBUG_CONFIG | MODIFIABLE_CONFIG, server.repl_replica_ro, 1, NULL, NULL), createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_replica_ignore_maxmemory, 1, NULL, NULL), createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, updateJemallocBgThread), createBoolConfig("activedefrag", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.active_defrag_enabled, CONFIG_ACTIVE_DEFRAG_DEFAULT, isValidActiveDefrag, NULL), + createBoolConfig("forkless-infrastructure-enabled", NULL, IMMUTABLE_CONFIG, server.forkless_infrastructure_enabled, 0, NULL, NULL), createBoolConfig("syslog-enabled", NULL, IMMUTABLE_CONFIG, server.syslog_enabled, 0, NULL, NULL), createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, server.cluster_enabled, 0, NULL, NULL), createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG | DENY_LOADING_CONFIG, server.aof_enabled, 0, NULL, updateAppendOnly), @@ -3428,6 +3501,8 @@ standardConfig static_configs[] = { createStringConfig("proc-title-template", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.proc_title_template, CONFIG_DEFAULT_PROC_TITLE_TEMPLATE, isValidProcTitleTemplate, updateProcTitleTemplate), createStringConfig("bind-source-addr", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bind_source_addr, NULL, NULL, NULL), createStringConfig("logfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.logfile, "", NULL, NULL), + createStringConfig("priority-subnets", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.priority_subnets, NULL, isValidPrioritySubnets, updatePrioritySubnetsConfig), + #ifdef LOG_REQ_RES createStringConfig("req-res-logfile", NULL, IMMUTABLE_CONFIG | HIDDEN_CONFIG, EMPTY_STRING_IS_NULL, server.req_res_logfile, NULL, NULL, NULL), #endif @@ -3461,6 +3536,8 @@ standardConfig static_configs[] = { createEnumConfig("log-timestamp-format", NULL, MODIFIABLE_CONFIG, log_timestamp_format_enum, server.log_timestamp_format, LOG_TIMESTAMP_LEGACY, NULL, NULL), createEnumConfig("rdb-version-check", NULL, MODIFIABLE_CONFIG, rdb_version_check_enum, server.rdb_version_check, RDB_VERSION_CHECK_STRICT, NULL, NULL), createEnumConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, rdb_compression_enum, server.rdb_compression, RDB_COMPRESSION_YES, NULL, NULL), + createEnumConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, cluster_replica_no_failover_enum, server.cluster_replica_no_failover, CLUSTER_REPLICA_NO_FAILOVER_NO, NULL, updateClusterFlags), /* Failover by default. */ + createEnumConfig("repl-compression", NULL, MODIFIABLE_CONFIG, repl_compression_enum, server.repl_compression, REPL_COMPRESSION_NO, NULL, NULL), /* Integer configs */ createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.config_databases, 16, INTEGER_CONFIG, NULL, NULL), @@ -3481,6 +3558,7 @@ standardConfig static_configs[] = { createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, 10, INTEGER_CONFIG, NULL, NULL), /* Default: don't defrag when fragmentation is below 10% */ createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, 100, INTEGER_CONFIG, NULL, updateDefragConfiguration), /* Default: maximum defrag force at 100% fragmentation */ createIntConfig("active-defrag-cycle-us", NULL, MODIFIABLE_CONFIG, 0, 100000, server.active_defrag_cycle_us, 500, INTEGER_CONFIG, NULL, updateDefragConfiguration), + createIntConfig("priority-preemptive-poll-interval-us", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.priority_preemptive_poll_interval_us, 2000, INTEGER_CONFIG, NULL, updatePriorityPreemptivePollInterval), createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, lfu_config_log_factor, 10, INTEGER_CONFIG, NULL, NULL), createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, lfu_config_decay_time, 1, INTEGER_CONFIG, NULL, NULL), createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.replica_priority, 100, INTEGER_CONFIG, NULL, NULL), @@ -3513,9 +3591,13 @@ standardConfig static_configs[] = { createIntConfig("rdma-rx-size", NULL, IMMUTABLE_CONFIG, 64 * 1024, 16 * 1024 * 1024, server.rdma_ctx_config.rx_size, 1024 * 1024, INTEGER_CONFIG, NULL, NULL), createIntConfig("rdma-completion-vector", NULL, IMMUTABLE_CONFIG, -1, 1024, server.rdma_ctx_config.completion_vector, -1, INTEGER_CONFIG, NULL, NULL), createIntConfig("cluster-message-gossip-perc", NULL, MODIFIABLE_CONFIG | HIDDEN_CONFIG, 1, 100, server.cluster_message_gossip_perc, 10, INTEGER_CONFIG, NULL, NULL), + createIntConfig("hotkeys-sampling-percentage", NULL, MODIFIABLE_CONFIG, 1, 100, server.hotkeys_sampling_percentage, 1, INTEGER_CONFIG, NULL, hotkeysSamplingCallback), + createIntConfig("hotkeys-top-k", NULL, MODIFIABLE_CONFIG, 0, 1000, server.hotkeys_top_k, 0, INTEGER_CONFIG, NULL, hotkeysTopKCallback), + createIntConfig("hotkeys-window-seconds", NULL, MODIFIABLE_CONFIG, 1, 300, server.hotkeys_window_seconds, 1, INTEGER_CONFIG, NULL, hotkeysWindowCallback), /* Unsigned int configs */ createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, server.maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients), + createUIntConfig("maxclients-reserved", NULL, MODIFIABLE_CONFIG, 0, UINT_MAX, server.maxclients_reserved, 0, INTEGER_CONFIG, NULL, NULL), createUIntConfig("unixsocketperm", NULL, IMMUTABLE_CONFIG, 0, 0777, server.unix_ctx_config.perm, 0, OCTAL_CONFIG, NULL, NULL), createUIntConfig("socket-mark-id", NULL, IMMUTABLE_CONFIG, 0, UINT_MAX, server.socket_mark_id, 0, INTEGER_CONFIG, NULL, NULL), createUIntConfig("max-new-connections-per-cycle", NULL, MODIFIABLE_CONFIG, 1, 1000, server.max_new_conns_per_cycle, 10, INTEGER_CONFIG, NULL, NULL), @@ -3566,6 +3648,7 @@ standardConfig static_configs[] = { createSizeTConfig("tracking-table-max-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 1 million keys max. */ createSizeTConfig("client-query-buffer-limit", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, 1024 * 1024, LONG_MAX, server.client_max_querybuf_len, 1024 * 1024 * 1024, MEMORY_CONFIG, NULL, NULL), /* Default: 1GB max query buffer. */ createSSizeTConfig("maxmemory-clients", NULL, MODIFIABLE_CONFIG, -100, SSIZE_MAX, server.maxmemory_clients, 0, MEMORY_CONFIG | PERCENT_CONFIG, NULL, applyClientMaxMemoryUsage), + createSSizeTConfig("maxmemory-scripts", NULL, MODIFIABLE_CONFIG, -100, SSIZE_MAX, server.maxmemory_scripts, 0, MEMORY_CONFIG | PERCENT_CONFIG, NULL, updateMaxmemoryScripts), createSSizeTConfig("slot-migration-max-failover-repl-bytes", NULL, MODIFIABLE_CONFIG, -1, SSIZE_MAX, server.slot_migration_max_failover_repl_bytes, 0, MEMORY_CONFIG | SIGNED_MEMORY_CONFIG, NULL, NULL), /* Other configs */ @@ -3590,6 +3673,9 @@ standardConfig static_configs[] = { createStringConfig("tls-client-cert-file", NULL, VOLATILE_CONFIG | MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_cert_file, NULL, NULL, applyTlsCfg), createStringConfig("tls-client-key-file", NULL, VOLATILE_CONFIG | MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_key_file, NULL, NULL, applyTlsCfg), createStringConfig("tls-client-key-file-pass", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_key_file_pass, NULL, NULL, applyTlsCfg), + createStringConfig("tls-alt-cert-file", NULL, VOLATILE_CONFIG | MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.alt_cert_file, NULL, NULL, applyTlsCfg), + createStringConfig("tls-alt-key-file", NULL, VOLATILE_CONFIG | MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.alt_key_file, NULL, NULL, applyTlsCfg), + createStringConfig("tls-alt-key-file-pass", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.alt_key_file_pass, NULL, NULL, applyTlsCfg), createStringConfig("tls-dh-params-file", NULL, VOLATILE_CONFIG | MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.dh_params_file, NULL, NULL, applyTlsCfg), createStringConfig("tls-ca-cert-file", NULL, VOLATILE_CONFIG | MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_file, NULL, NULL, applyTlsCfg), createStringConfig("tls-ca-cert-dir", NULL, VOLATILE_CONFIG | MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_dir, NULL, NULL, applyTlsCfg), diff --git a/src/connection.c b/src/connection.c index 819cdeb87..2c714f144 100644 --- a/src/connection.c +++ b/src/connection.c @@ -168,3 +168,51 @@ sds getListensInfoString(sds info) { return info; } +/* Set connection priority. If the connection already has active events + * registered in the event loop, migrate them to the new priority level. + * Handles postponed state safely if the socket is offloaded to IO threads, + * and preserves AE_BARRIER ordering flags. + * Returns C_OK on success, or C_ERR if event migration fails. */ +int connSetPriority(connection *conn, bool is_priority) { + serverAssert(conn != NULL); + if (conn->is_priority == is_priority) return C_OK; + + /* Fast path: if no socket exists yet, update priority field directly */ + if (conn->fd == -1) { + conn->is_priority = is_priority; + return C_OK; + } + + int mask = aeGetFileEvents(server.el, conn->fd); + if (mask == AE_NONE) { + conn->is_priority = is_priority; + return C_OK; + } + + /* If socket state update is postponed by IO threads, update priority field only; + * connUpdateState() will register with the new priority upon IO completion. */ + if (conn->flags & CONN_FLAG_POSTPONE_UPDATE_STATE) { + conn->is_priority = is_priority; + return C_OK; + } + + /* Dynamic migration: active events exist on this socket */ + bool old_priority = conn->is_priority; + conn->is_priority = is_priority; + + /* If transport has custom state updater (e.g. TLS), delegate to it */ + if (conn->type && conn->type->update_state) { + conn->type->update_state(conn); + } else { + mask = (mask & ~AE_HIGH_PRIORITY); /* Strip off old priority flag */ + if (conn->is_priority) mask |= AE_HIGH_PRIORITY; /* Add new priority flag */ + + if (aeCreateFileEvent(server.el, conn->fd, mask, conn->type->ae_handler, conn) == AE_ERR) { + return C_ERR; + } + } + + serverLog(LL_DEBUG, "Connection fd %d priority updated from %s to %s", + conn->fd, old_priority ? "prioritized" : "normal", is_priority ? "prioritized" : "normal"); + return C_OK; +} diff --git a/src/connection.h b/src/connection.h index 5527ee2a7..7fe6de8a4 100644 --- a/src/connection.h +++ b/src/connection.h @@ -32,9 +32,10 @@ #define VALKEY_CONNECTION_H #include +#include #include -#include #include +#include #include #include "ae.h" @@ -61,9 +62,20 @@ typedef enum { CONN_STATE_ERROR } ConnectionState; -#define CONN_FLAG_CLOSE_SCHEDULED (1 << 0) /* Closed scheduled by a handler */ -#define CONN_FLAG_WRITE_BARRIER (1 << 1) /* Write barrier requested */ -#define CONN_FLAG_ALLOW_ACCEPT_OFFLOAD (1 << 2) /* Connection accept can be offloaded to IO threads. */ +/* Identifies the type of owner stored in conn->private_data. + * Used by connection-layer safety assertions to avoid unsafe casts. */ +typedef enum { + CONN_OWNER_CLIENT = 0, /* private_data points to a client (default) */ + CONN_OWNER_CLUSTER_LINK, /* private_data points to a clusterLink */ +} ConnectionOwnerKind; + +#define CONN_FLAG_CLOSE_SCHEDULED (1 << 0) /* Closed scheduled by a handler */ +#define CONN_FLAG_WRITE_BARRIER (1 << 1) /* Write barrier requested */ +#define CONN_FLAG_ALLOW_ACCEPT_OFFLOAD (1 << 2) /* Connection accept can be offloaded to IO threads. */ +#define CONN_FLAG_ACCEPT_OFFLOAD_PENDING (1 << 3) /* Accept offload job is currently in flight. */ +#define CONN_FLAG_POSTPONE_UPDATE_STATE (1 << 4) /* Connection update state is postponed by IO threads \ + * to prevent main thread event loop races while worker \ + * threads access the socket buffers. */ #define CONN_POSTPONE_READ (1 << 0) #define CONN_POSTPONE_WRITE (1 << 1) @@ -160,7 +172,8 @@ typedef struct ConnectionType { struct user *(*get_peer_user)(connection *conn, sds *cert_username); /* Miscellaneous */ - int (*connIntegrityChecked)(void); // return 1 if connection type has built-in integrity checks + int (*connIntegrityChecked)(void); // return 1 if connection type has built-in integrity checks + int (*is_closing)(connection *conn); // return 1 if connection is closed } ConnectionType; struct connection { @@ -171,6 +184,8 @@ struct connection { short int flags; short int refs; unsigned short int iovcnt; + bool is_priority; /* true if connection is prioritized for QoS */ + ConnectionOwnerKind owner_kind; void *private_data; ConnectionCallbackFunc conn_handler; ConnectionCallbackFunc write_handler; @@ -272,8 +287,7 @@ static inline int connWritev(connection *conn, const struct iovec *iov, int iovc * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. */ static inline int connRead(connection *conn, void *buf, size_t buf_len) { - int ret = conn->type->read(conn, buf, buf_len); - return ret; + return conn->type->read(conn, buf, buf_len); } /* Register a write handler, to be called when the connection is writable. @@ -397,6 +411,15 @@ static inline int connHasReadHandler(connection *conn) { return conn->read_handler != NULL; } +/* Check if the remote side has closed the connection. */ +static inline int connIsClosing(connection *conn) { + if (!conn->type->is_closing) return 0; + return conn->type->is_closing(conn); +} + +/* Shared is_closing implementation for TCP socket-based connections. */ +int connTcpSocketIsClosing(connection *conn); + /* Associate a private data pointer with the connection */ static inline void connSetPrivateData(connection *conn, void *data) { conn->private_data = data; @@ -407,6 +430,16 @@ static inline void *connGetPrivateData(connection *conn) { return conn->private_data; } +/* Set the owner kind for the connection */ +static inline void connSetOwnerKind(connection *conn, ConnectionOwnerKind kind) { + conn->owner_kind = kind; +} + +/* Get the owner kind for the connection */ +static inline ConnectionOwnerKind connGetOwnerKind(connection *conn) { + return conn->owner_kind; +} + /* Return a text that describes the connection, suitable for inclusion * in CLIENT LIST and similar outputs. * @@ -508,6 +541,16 @@ static inline aeFileProc *connAcceptHandler(ConnectionType *ct) { /* Get Listeners information, note that caller should free the non-empty string */ sds getListensInfoString(sds info); +/* Connection QoS / Priority. */ +int connSetPriority(connection *conn, bool is_priority); +static inline bool connIsPriority(const connection *conn) { + return conn && conn->is_priority; +} + +/* Get AE priority flag for a connection. */ +static inline int connGetAEPriorityFlag(const connection *conn) { + return (conn && conn->is_priority) ? AE_HIGH_PRIORITY : AE_NONE; +} int RedisRegisterConnectionTypeSocket(void); int RedisRegisterConnectionTypeUnix(void); int RedisRegisterConnectionTypeTLS(void); @@ -525,8 +568,14 @@ static inline void connUpdateState(connection *conn) { } static inline void connSetPostponeUpdateState(connection *conn, int postpone_mask) { - if (conn && conn->type && conn->type->postpone_update_state) { - conn->type->postpone_update_state(conn, postpone_mask); + if (conn) { + if (postpone_mask) + conn->flags |= CONN_FLAG_POSTPONE_UPDATE_STATE; + else + conn->flags &= ~CONN_FLAG_POSTPONE_UPDATE_STATE; + if (conn->type && conn->type->postpone_update_state) { + conn->type->postpone_update_state(conn, postpone_mask); + } } } diff --git a/src/db.c b/src/db.c index 4c5c428c6..8860340f4 100644 --- a/src/db.c +++ b/src/db.c @@ -28,6 +28,8 @@ */ #include "server.h" +#include "listpack.h" +#include "hotkeys.h" #include "ordered_index.h" #include "cluster.h" #include "cluster_migrateslots.h" @@ -38,7 +40,10 @@ #include "module.h" #include "vector.h" #include "expire.h" +#include "bgiteration.h" +#include "forkless.h" #include "crc16_slottable.h" +#include "bgiteration.h" /*----------------------------------------------------------------------------- * C-level DB API @@ -123,6 +128,10 @@ robj *lookupKey(serverDb *db, robj *key, int flags) { /* TODO: Use separate misses stats and notify event for WRITE */ } + /* Charge this lookup to hot-key detection. All the policy (whether detection + * is on, which lookups count, and sampling) lives in hotkeys.c. */ + hotkeysRecordLookup(key, db->id, flags); + return val; } @@ -369,6 +378,7 @@ static void dbSetValue(serverDb *db, robj *key, robj **valref, int overwrite, vo objectSetLRU(val, objectGetLRU(old)); long long expire = objectGetExpire(old); new = objectSetKeyAndExpire(val, objectGetVal(key), expire); + bgIteration_updateDbEntryPtr(old, new); *oldref = new; /* Replace the old value at its location in the expire space. */ if (expire >= 0) { @@ -438,6 +448,7 @@ void setKey(client *c, serverDb *db, robj *key, robj **valref, int flags) { } else { dbSetValue(db, key, valref, 1, NULL); } + bgIteration_dbEntryModified(*valref); if (!(flags & SETKEY_KEEPTTL)) removeExpire(db, key); if (!(flags & SETKEY_NO_SIGNAL)) signalModifiedKey(c, db, key); } @@ -483,6 +494,9 @@ int dbGenericDeleteWithDictIndex(serverDb *db, robj *key, int async, int flags, hashtablePosition pos; void **ref = kvstoreHashtableTwoPhasePopFindRef(db->keys, dict_index, objectGetVal(key), &pos); if (ref != NULL) { + bgIteration_keyDelete(db->id, (sds)objectGetVal(key)); + hotkeysRecordDelete(key, db->id, flags); + robj *val = *ref; /* VM_StringDMA may call dbUnshareStringValue which may free val, so we * need to incr to retain val */ @@ -672,6 +686,9 @@ long long emptyData(int dbnum, int flags, void(callback)(hashtable *)) { return -1; } + /* bgIteration must be notified for flushall. */ + if (dbnum == -1) bgIteration_flushall(); + /* Fire the flushdb modules event. */ moduleFireServerEvent(VALKEYMODULE_EVENT_FLUSHDB, VALKEYMODULE_SUBEVENT_FLUSHDB_START, &fi); @@ -691,6 +708,13 @@ long long emptyData(int dbnum, int flags, void(callback)(hashtable *)) { /* Empty the database structure. */ removed = emptyDbStructure(server.db, dbnum, async, callback); + if (hotkeysEnabled()) { + if (dbnum == -1) + hotkeysPurgeAll(); + else + hotkeysPurgeDb(dbnum); + } + if (dbnum == -1) flushReplicaKeysWithExpireList(async); if (with_functions) { @@ -762,6 +786,7 @@ long long dbTotalServerKeyCount(void) { void signalModifiedKey(client *c, serverDb *db, robj *key) { touchWatchedKey(db, key); trackingInvalidateKey(c, key, 1); + bgIteration_keyModified(db->id, objectGetVal(key)); } void signalFlushedDb(int dbid, int async) { @@ -797,9 +822,8 @@ void signalFlushedDb(int dbid, int async) { * async: flushes the database in an async manner. * no option: determine sync or async according to the value of lazyfree-lazy-user-flush. * - * On success C_OK is returned and the flags are stored in *flags, otherwise - * C_ERR is returned and the function sends an error to the client. */ -int getFlushCommandFlags(client *c, int *flags) { + * On success the C_OK is returned, otherwise C_ERR is returned. */ +int parseFlushCommandFlags(client *c, int *flags) { /* Parse the optional ASYNC option. */ if (c->argc == 2 && !strcasecmp(objectGetVal(c->argv[1]), "sync")) { *flags = EMPTYDB_NO_FLAGS; @@ -808,16 +832,23 @@ int getFlushCommandFlags(client *c, int *flags) { } else if (c->argc == 1) { *flags = server.lazyfree_lazy_user_flush ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS; } else { - addReplyErrorObject(c, shared.syntaxerr); return C_ERR; } return C_OK; } +/* Parses the flush command flags and returns an error to the client on failure */ +int parseFlushCommandFlagsOrReply(client *c, int *flags) { + int result = parseFlushCommandFlags(c, flags); + if (result == C_ERR) addReplyErrorObject(c, shared.syntaxerr); + return result; +} + /* Flushes the whole server data set. */ void flushAllDataAndResetRDB(int flags) { server.dirty += emptyData(-1, flags, NULL); - if (server.child_type == CHILD_TYPE_RDB) killRDBChild(); + if (isForkBgsaveInProgress()) killRDBChild(); + if (isForklessSaveInProgress()) forklessSaveCancel(); if (server.child_type == CHILD_TYPE_SLOT_MIGRATION) killSlotMigrationChild(); if (server.saveparamslen > 0) { rdbSaveInfo rsi, *rsiptr; @@ -839,7 +870,7 @@ void flushAllDataAndResetRDB(int flags) { void flushdbCommand(client *c) { int flags; - if (getFlushCommandFlags(c, &flags) == C_ERR) return; + if (parseFlushCommandFlagsOrReply(c, &flags) == C_ERR) return; /* flushdb should not flush the functions */ server.dirty += emptyData(c->db->id, flags | EMPTYDB_NOFUNCTIONS, NULL); @@ -863,7 +894,7 @@ void flushdbCommand(client *c) { * Flushes the whole server data set. */ void flushallCommand(client *c) { int flags; - if (getFlushCommandFlags(c, &flags) == C_ERR) return; + if (parseFlushCommandFlagsOrReply(c, &flags) == C_ERR) return; /* flushall should not flush the functions */ flushAllDataAndResetRDB(flags | EMPTYDB_NOFUNCTIONS); @@ -1344,7 +1375,8 @@ void scanGenericCommandWithOptions(client *c, robj *o, unsigned long long cursor setTypeReleaseIterator(si); cursor = 0; } else if ((objectGetType(o) == OBJ_HASH || o->type == OBJ_ZSET) && o->encoding == OBJ_ENCODING_LISTPACK) { - unsigned char *p = lpFirst(objectGetVal(o)); + unsigned char *zl = objectGetVal(o); + unsigned char *p = lpFirst(zl); unsigned char *str; int64_t len; unsigned char intbuf[LP_INTBUF_SIZE]; @@ -1353,9 +1385,13 @@ void scanGenericCommandWithOptions(client *c, robj *o, unsigned long long cursor str = lpGet(p, &len, intbuf); /* point to the value */ p = lpNext(objectGetVal(o), p); + unsigned char *vptr = p; + /* Skip fields not visible in the current context */ + long long expiry = hashTypeListpackGetExpiry(zl, vptr); + int is_valid = hashTypeListpackFieldIsValid(expiry); + p = lpNext(zl, vptr); + if (!is_valid) continue; if (opts->use_pattern && !stringmatchlen(opts->pat, opts->patlen, (char *)str, len, 0)) { - /* jump to the next key/val pair */ - p = lpNext(objectGetVal(o), p); continue; } /* add key object */ @@ -1363,11 +1399,10 @@ void scanGenericCommandWithOptions(client *c, robj *o, unsigned long long cursor addScanDataItem(&result, (const char *)item, sdslen(item)); /* add value object */ if (!opts->only_keys) { - str = lpGet(p, &len, intbuf); + str = lpGet(vptr, &len, intbuf); item = sdsnewlen(str, len); addScanDataItem(&result, (const char *)item, sdslen(item)); } - p = lpNext(objectGetVal(o), p); } cursor = 0; } else { @@ -1495,6 +1530,8 @@ void shutdownCommand(client *c) { return; } + /* Clear pending_command to avoid re-execution. */ + c->flag.pending_command = 0; blockClientShutdown(c); if (prepareForShutdown(c, flags) == C_OK) exit(0); /* If we're here, then shutdown is ongoing (the client is still blocked) or @@ -2301,7 +2338,7 @@ robj *dbFindExpires(serverDb *db, sds key) { } unsigned long long dbSize(serverDb *db) { - return kvstoreSize(db->keys); + return (db->keys) ? kvstoreSize(db->keys) : 0; } unsigned long long dbScan(serverDb *db, unsigned long long cursor, kvstoreScanFunction scan_cb, void *privdata) { @@ -2720,16 +2757,21 @@ int genericGetKeys(int storeKeyOfs, keys = getKeysPrepareResult(result, numkeys); result->numkeys = numkeys; + int keyIdx = 0; + /* If there's a destination key, put this first */ + if (storeKeyOfs) { + keys[keyIdx].pos = storeKeyOfs; + keys[keyIdx].flags = 0; + keyIdx++; + } + /* Add all key positions for argv[firstKeyOfs...n] to keys[] */ for (i = 0; i < num; i++) { - keys[i].pos = firstKeyOfs + (i * keyStep); - keys[i].flags = 0; + keys[keyIdx].pos = firstKeyOfs + (i * keyStep); + keys[keyIdx].flags = 0; + keyIdx++; } - if (storeKeyOfs) { - keys[num].pos = storeKeyOfs; - keys[num].flags = 0; - } return result->numkeys; } @@ -2846,6 +2888,10 @@ int sortGetKeys(struct serverCommand *cmd, robj **argv, int argc, getKeysResult found_store = 1; keys[num].pos = i + 1; /* */ keys[num].flags = CMD_KEY_OW | CMD_KEY_UPDATE; + /* Skip the destination. It is a key name, so it must never be + * examined as an option: a key that spells one would hide the + * later STORE clause that SORT actually writes to. */ + i++; break; } } diff --git a/src/debug.c b/src/debug.c index a71b91fd3..25d328fe2 100644 --- a/src/debug.c +++ b/src/debug.c @@ -455,6 +455,14 @@ void debugCommand(client *c) { " Disable sending cluster ping to a random node every second.", "DISABLE-CLUSTER-RECONNECTION <0|1>", " Disable cluster reconnection of cluster nodes.", + "CLUSTER-FAILOVER-DELAY ", + " Override the failover delay. -1 is the default value, meaning don't", + " override, values >= 0 will be used for the failover delay.", + "CLUSTER-FAILOVER-EPOCH ", + " Force the next failover election started by this replica to run in", + " the given epoch instead of currentEpoch+1. -1 (default) disables the", + " override. It is consumed once: subsequent retries use currentEpoch+1", + " again. Useful to make several replicas contend in the same epoch.", "OOM", " Crash the server simulating an out-of-memory error.", "PANIC", @@ -544,6 +552,8 @@ void debugCommand(client *c) { " Protect a client from being freed, forcing deferred close.", "FORCE-TLS-WRITE-ERROR <0|1>", " Force TLS write error for testing.", + "BIO-DRAIN ", + " Wait for the specified bio job queue to become empty.", NULL}; addExtendedReplyHelp(c, help, clusterDebugCommandExtendedHelp()); } else if (!strcasecmp(objectGetVal(c->argv[1]), "segfault")) { @@ -643,21 +653,41 @@ void debugCommand(client *c) { long packet_type; if (getLongFromObjectOrReply(c, c->argv[2], &packet_type, NULL) != C_OK) return; server.cluster_drop_packet_filter = packet_type; + serverLog(LL_NOTICE, "Setting drop-cluster-packet-filter to %ld", packet_type); addReply(c, shared.ok); } else if (!strcasecmp(objectGetVal(c->argv[1]), "close-cluster-link-on-packet-drop") && c->argc == 3) { - server.debug_cluster_close_link_on_packet_drop = atoi(objectGetVal(c->argv[2])); + server.debug_cluster_close_link_on_packet_drop = (atoi(objectGetVal(c->argv[2])) != 0); + serverLog(LL_NOTICE, "Setting close-cluster-link-on-packet-drop to %d", (atoi(objectGetVal(c->argv[2])) != 0)); addReply(c, shared.ok); } else if (!strcasecmp(objectGetVal(c->argv[1]), "disable-cluster-random-ping") && c->argc == 3) { - server.debug_cluster_disable_random_ping = atoi(objectGetVal(c->argv[2])); + server.debug_cluster_disable_random_ping = (atoi(objectGetVal(c->argv[2])) != 0); addReply(c, shared.ok); } else if (!strcasecmp(objectGetVal(c->argv[1]), "disable-cluster-reconnection") && c->argc == 3) { - server.debug_cluster_disable_reconnection = atoi(objectGetVal(c->argv[2])); + server.debug_cluster_disable_reconnection = (atoi(objectGetVal(c->argv[2])) != 0); + addReply(c, shared.ok); + } else if (!strcasecmp(objectGetVal(c->argv[1]), "cluster-failover-delay") && c->argc == 3) { + int delay_ms; + if (getIntFromObjectOrReply(c, c->argv[2], &delay_ms, NULL) != C_OK) return; + if (delay_ms < -1) { + addReplyError(c, "delay-ms must be -1 (default) or a non-negative value in ms"); + return; + } + server.debug_cluster_failover_delay = delay_ms; + addReply(c, shared.ok); + } else if (!strcasecmp(objectGetVal(c->argv[1]), "cluster-failover-epoch") && c->argc == 3) { + long long epoch; + if (getLongLongFromObjectOrReply(c, c->argv[2], &epoch, NULL) != C_OK) return; + if (epoch < -1) { + addReplyError(c, "epoch must be -1 (default) or a non-negative value"); + return; + } + server.debug_cluster_failover_epoch = epoch; addReply(c, shared.ok); } else if (!strcasecmp(objectGetVal(c->argv[1]), "slotmigration")) { if (!strcasecmp(objectGetVal(c->argv[2]), "prevent-pause")) { - server.debug_slot_migration_prevent_pause = atoi(objectGetVal(c->argv[3])); + server.debug_slot_migration_prevent_pause = (atoi(objectGetVal(c->argv[3])) != 0); } else if (!strcasecmp(objectGetVal(c->argv[2]), "prevent-failover")) { - server.debug_slot_migration_prevent_failover = atoi(objectGetVal(c->argv[3])); + server.debug_slot_migration_prevent_failover = (atoi(objectGetVal(c->argv[3])) != 0); } else { addReplySubcommandSyntaxError(c); return; @@ -1119,6 +1149,17 @@ void debugCommand(client *c) { #else addReplyError(c, "TLS is not enabled"); #endif + } else if (!strcasecmp(objectGetVal(c->argv[1]), "bio-drain") && c->argc == 3) { + int type; + const char *name = objectGetVal(c->argv[2]); + if (!strcasecmp(name, "BIO_CLUSTER_SAVE")) { + type = BIO_CLUSTER_SAVE; + } else { + addReplySubcommandSyntaxError(c); + return; + } + bioDrainWorker(type); + addReply(c, shared.ok); } else if (!handleDebugClusterCommand(c)) { addReplySubcommandSyntaxError(c); return; diff --git a/src/defrag.c b/src/defrag.c index 4debda161..65b966392 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -44,6 +44,7 @@ #include "eval.h" #include "script.h" #include "module.h" +#include "bgiteration.h" #include #include @@ -658,9 +659,12 @@ static void defragKey(defragKeysCtx *ctx, robj **elemref) { unsigned char *newzl; ob = *elemref; + if (bgIteration_isEntryInuse(ob)) return; + /* Try to defrag robj and/or string value. */ if ((newob = activeDefragStringOb(ob))) { *elemref = newob; + bgIteration_updateDbEntryPtr(ob, newob); if (objectGetExpire(newob) >= 0) { /* Replace the pointer in the expire table without accessing the old * pointer. */ @@ -765,6 +769,11 @@ static void defragPubsubScanCallback(void *privdata, void *elemref) { * and 1 if time is up and more work is needed. */ static int defragLaterItem(robj *ob, unsigned long *cursor, monotime endtime, int dbid) { if (ob) { + if (bgIteration_isEntryInuse(ob)) { + *cursor = 0; + return 0; + } + if (ob->type == OBJ_LIST && ob->encoding == OBJ_ENCODING_QUICKLIST) { return scanLaterList(ob, cursor, endtime); } else if (ob->type == OBJ_SET && ob->encoding == OBJ_ENCODING_HASHTABLE) { @@ -959,7 +968,7 @@ static doneStatus defragLuaScripts(monotime endtime, void *target, void *privdat /* In case we are in the process of eval some script we do not want to replace the script being run * so we just bail out without really defragging here. */ if (scriptIsRunning()) return DEFRAG_DONE; - activeDefragSdsDict(evalScriptsDict(), DEFRAG_SDS_DICT_VAL_LUA_SCRIPT); + activeDefragSdsDict(evalCtxScriptsDict(), DEFRAG_SDS_DICT_VAL_LUA_SCRIPT); return DEFRAG_DONE; } @@ -967,9 +976,7 @@ static doneStatus defragLuaScripts(monotime endtime, void *target, void *privdat static doneStatus defragModuleGlobals(monotime endtime, void *target, void *privdata) { UNUSED(target); UNUSED(privdata); - if (endtime == 0) return DEFRAG_NOT_DONE; // required initialization - moduleDefragGlobals(); - return DEFRAG_DONE; + return moduleDefragGlobals(endtime) ? DEFRAG_NOT_DONE : DEFRAG_DONE; } diff --git a/src/eval.c b/src/eval.c index 1ec2affe4..1525f5381 100644 --- a/src/eval.c +++ b/src/eval.c @@ -54,6 +54,7 @@ void evalGenericCommandWithDebugging(client *c, int evalsha); +static void evalCtxDeleteScript(sds sha); typedef struct evalScript { compiledFunction *script; @@ -89,9 +90,10 @@ dictType shaScriptObjectDictType = { /* Eval context */ struct evalCtx { - dict *scripts; /* A dictionary of SHA1 -> evalScript */ - list *scripts_lru_list; /* A list of SHA1, first in first out LRU eviction. */ - unsigned long long scripts_mem; /* Cached scripts' memory + oh */ + dict *scripts; /* A dictionary of SHA1 -> evalScript */ + list *scripts_lru_list; /* A list of SHA1, first in first out LRU eviction. */ + unsigned long long scripts_mem; /* Cached scripts' memory + oh */ + unsigned long long eval_scripts_mem; /* Cached eval scripts' memory + oh */ } evalCtx; /* Initialize the scripting environment. @@ -102,13 +104,14 @@ struct evalCtx { void evalInit(void) { /* Initialize a dictionary we use to map SHAs to scripts. * - * Initialize a list we use for script evictions. + * Initialize a list we use for script LRU evictions. * Note that we duplicate the sha when adding to the lru list due to defrag, * and we need to free them respectively. */ evalCtx.scripts = dictCreate(&shaScriptObjectDictType); evalCtx.scripts_lru_list = listCreate(); listSetFreeMethod(evalCtx.scripts_lru_list, sdsfreeVoid); evalCtx.scripts_mem = 0; + evalCtx.eval_scripts_mem = 0; } /* --------------------------------------------------------------------------- @@ -158,6 +161,24 @@ void freeEvalScripts(dict *scripts, list *scripts_lru_list, list *engine_callbac } } +static void scriptsMemoryAdd(size_t memory, int is_eval) { + evalCtx.scripts_mem += memory; + if (is_eval) evalCtx.eval_scripts_mem += memory; +} + +static void scriptsMemorySubtract(size_t memory, int is_eval) { + evalCtx.scripts_mem -= memory; + if (is_eval) evalCtx.eval_scripts_mem -= memory; +} + +/* Remove an LRU node and account for its duplicated SHA. Both counters include + * this list-owned allocation; dictionary-owned memory is handled separately. */ +static void scriptsLRUDeleteNode(listNode *node) { + sds sha = listNodeValue(node); + scriptsMemorySubtract(sdsAllocSize(sha), 1); + listDelNode(evalCtx.scripts_lru_list, node); +} + static void resetEngineEvalEnvCallback(scriptingEngine *engine, void *context) { int async = context != NULL; callableLazyEnvReset *callback = scriptingEngineCallResetEnvFunc(engine, VMSE_EVAL, async); @@ -181,7 +202,7 @@ void evalRelease(int async) { } } -/* Remove all cached eval scripts associated with the given scripting engine. +/* Remove all cached scripts associated with the given scripting engine. * Called when a scripting engine is unregistered to avoid dangling engine * pointers in the eval script cache. */ void evalRemoveScriptsFromEngine(scriptingEngine *engine) { @@ -189,14 +210,7 @@ void evalRemoveScriptsFromEngine(scriptingEngine *engine) { dictEntry *entry; while ((entry = dictNext(iter))) { evalScript *es = dictGetVal(entry); - if (es->engine == engine) { - sds sha = dictGetKey(entry); - evalCtx.scripts_mem -= sdsAllocSize(sha) + getStringObjectSdsUsedMemory(es->body); - if (es->node) { - listDelNode(evalCtx.scripts_lru_list, es->node); - } - dictDelete(evalCtx.scripts, sha); - } + if (es->engine == engine) evalCtxDeleteScript(dictGetKey(entry)); } dictReleaseIterator(iter); } @@ -325,46 +339,173 @@ uint64_t evalGetCommandFlags(client *c, uint64_t cmd_flags) { return scriptFlagsToCmdFlags(cmd_flags, script_flags); } -/* Delete an eval script with the specified sha. +/* Delete a cached script with the specified sha. * - * This will delete the script from the scripting engine and delete the script - * from server. */ -static void evalDeleteScript(client *c, sds sha) { - /* Delete the script from server. */ + * This removes the script from the scripting engine, the script dictionary, and + * the EVAL LRU list when it has one. */ +static void evalCtxDeleteScript(sds sha) { dictEntry *de = dictUnlink(evalCtx.scripts, sha); - serverAssertWithInfo(c, NULL, de); + serverAssert(de); + sds dict_sha = dictGetKey(de); evalScript *es = dictGetVal(de); - evalCtx.scripts_mem -= sdsAllocSize(sha) + getStringObjectSdsUsedMemory(es->body); + scriptsMemorySubtract(sdsAllocSize(dict_sha) + getStringObjectSdsUsedMemory(es->body), es->node != NULL); + if (es->node) { + scriptsLRUDeleteNode(es->node); + es->node = NULL; + } dictFreeUnlinkedEntry(evalCtx.scripts, de); } -/* Users who abuse EVAL will generate a new lua script on each call, which can +/* Add a script to the LRU eviction list, evicting oldest scripts if necessary. + * + * Users who abuse EVAL will generate a new lua script on each call, which can * consume large amounts of memory over time. Since EVAL is mostly the one that * abuses the lua cache, and these won't have pipeline issues (scripts won't - * disappear when EVALSHA needs it and cause failure), we implement script eviction + * disappear when EVALSHA needs it and cause failure), we implement script LRU eviction * only for these (not for one loaded with SCRIPT LOAD). Considering that we don't * have many scripts, then unlike keys, we don't need to worry about the memory * usage of keeping a true sorted LRU linked list. * - * Returns the corresponding node added, which is used to save it in scriptHolder + * This function enforces a maximum count limit (LRU_LIST_LENGTH = 500) on cached + * scripts loaded via EVAL. When the limit is reached, the oldest (least recently + * used) scripts are evicted to make room for new ones. + * + * Note: Scripts loaded via SCRIPT LOAD are not added to this LRU list and are + * exempt from count-based eviction. maxmemory-scripts only applies to scripts + * loaded via EVAL. + * + * Returns the corresponding node added, which is used to save it in evalScript * and use it for quick removal and re-insertion into an LRU list each time the * script is used. */ #define LRU_LIST_LENGTH 500 -static listNode *scriptsLRUAdd(client *c, sds sha) { +static listNode *scriptsLRUAdd(sds sha) { /* Evict oldest. */ while (listLength(evalCtx.scripts_lru_list) >= LRU_LIST_LENGTH) { - listNode *ln = listFirst(evalCtx.scripts_lru_list); - sds oldest = listNodeValue(ln); - evalDeleteScript(c, oldest); - listDelNode(evalCtx.scripts_lru_list, ln); + sds oldest = listNodeValue(listFirst(evalCtx.scripts_lru_list)); + evalCtxDeleteScript(oldest); server.stat_evictedscripts++; } /* Add current. */ - listAddNodeTail(evalCtx.scripts_lru_list, sdsdup(sha)); + sds lru_sha = sdsdup(sha); + listAddNodeTail(evalCtx.scripts_lru_list, lru_sha); + scriptsMemoryAdd(sdsAllocSize(lru_sha), 1); return listLast(evalCtx.scripts_lru_list); } +/* Returns the actual scripts eviction limit based on current configuration or + * 0 if no limit. */ +size_t getScriptsMemoryLimit(void) { + size_t maxmemory_scripts_actual = SIZE_MAX; + + if (server.maxmemory_scripts < 0 && server.maxmemory > 0) { + /* Handle percentage of maxmemory (negative value represents percentage). */ + unsigned long long maxmemory_scripts_bytes = + (unsigned long long)((double)server.maxmemory * -(double)server.maxmemory_scripts / 100); + if (maxmemory_scripts_bytes <= SIZE_MAX) maxmemory_scripts_actual = maxmemory_scripts_bytes; + } else if (server.maxmemory_scripts > 0) { + /* Absolute value specified. */ + maxmemory_scripts_actual = server.maxmemory_scripts; + } else { + /* maxmemory-scripts is 0, no memory limit enforced. */ + return 0; + } + + /* Don't allow a too small maxmemory-scripts to avoid cases where we can't + * cache any scripts at all due to bad configuration. Minimum is 128KB. */ + if (maxmemory_scripts_actual < 1024 * 128) maxmemory_scripts_actual = 1024 * 128; + + return maxmemory_scripts_actual; +} + +static int isScriptsEvictionProcRunning = 0; + +/* Script eviction return codes. */ +#define SCRIPTS_EVICT_OK 0 /* Memory is OK or eviction completed successfully. */ +#define SCRIPTS_EVICT_RUNNING 1 /* Memory still over limit, time limit reached, need async continuation. */ + +/* Perform memory-based EVAL script evictions when maxmemory-scripts limit is exceeded. + * + * SCRIPT LOAD scripts are not considered by this limit or eviction mechanism. + * Their memory is accounted for by the global maxmemory limit instead. Since + * SCRIPT LOAD has CMD_DENYOOM, processCommand() may evict keys or reject the + * command with OOM when the instance cannot accept it. + * + * Eviction strategy: + * - EVAL scripts are selected from their LRU list. + * - Eviction is skipped while a long-running command has yielded to the event + * loop, so the currently executing script cannot be freed underneath it. + * - To avoid blocking the server, eviction is time-limited. If the time limit is + * reached while memory is still over the limit, a time proc continues eviction. + * + * Returns: + * - SCRIPTS_EVICT_OK: Memory is within limits or no limit configured. + * - SCRIPTS_EVICT_RUNNING: Eviction still needed, async proc scheduled. */ +static int performScriptsEvictions(void) { + /* Do not evict while a long-running command has yielded to the event loop; + * the next eligible trigger will retry the eviction. */ + if (isInsideYieldingLongCommand()) return SCRIPTS_EVICT_OK; + + /* Nothing to evict if no scripts cached. */ + if (dictSize(evalCtx.scripts) == 0) return SCRIPTS_EVICT_OK; + + /* Check if memory-based eviction is enabled. */ + size_t script_eviction_limit = getScriptsMemoryLimit(); + if (script_eviction_limit == 0) return SCRIPTS_EVICT_OK; + + int scripts_evicted = 0; + unsigned long scripts_eviction_time_limit_us = 500; /* 500 microseconds max per call */ + monotime scripts_eviction_timer; + elapsedStart(&scripts_eviction_timer); + + /* Evict EVAL scripts until their memory usage is under the limit. */ + while (evalScriptsMemoryOverhead() > script_eviction_limit) { + listNode *node = listFirst(evalCtx.scripts_lru_list); + if (node == NULL) return SCRIPTS_EVICT_OK; + + sds sha = listNodeValue(node); + evalCtxDeleteScript(sha); + server.stat_evictedscripts++; + scripts_evicted++; + + if (scripts_evicted % 16 == 0) { + /* After some time, exit the loop early. We don't want to spend too much + * time here and block the server. */ + if (elapsedUs(scripts_eviction_timer) > scripts_eviction_time_limit_us) { + if (evalScriptsMemoryOverhead() > script_eviction_limit) { + /* Still need to evict scripts, start the eviction timer proc. */ + startScriptsEvictionTimeProc(); + return SCRIPTS_EVICT_RUNNING; + } else { + return SCRIPTS_EVICT_OK; + } + } + } + } + + return SCRIPTS_EVICT_OK; +} + +/* Time event proc for script eviction. */ +static long long scriptsEvictionTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData) { + UNUSED(eventLoop); + UNUSED(id); + UNUSED(clientData); + + if (performScriptsEvictions() == SCRIPTS_EVICT_RUNNING) return 0; /* Keep evicting */ + + isScriptsEvictionProcRunning = 0; + return AE_NOMORE; +} + +/* Start the scripts eviction time proc if not already running. */ +void startScriptsEvictionTimeProc(void) { + if (!isScriptsEvictionProcRunning) { + isScriptsEvictionProcRunning = 1; + aeCreateTimeEvent(server.el, 0, scriptsEvictionTimeProc, NULL, NULL); + } +} + static int evalRegisterNewScript(client *c, robj *body, char **sha) { serverAssert(sha != NULL); @@ -382,7 +523,14 @@ static int evalRegisterNewScript(client *c, robj *body, char **sha) { if (entry != NULL) { evalScript *es = dictGetVal(entry); if (es->node) { - listDelNode(evalCtx.scripts_lru_list, es->node); + sds dict_sha = dictGetKey(entry); + size_t sha_mem = sdsAllocSize(dict_sha); + size_t body_mem = getStringObjectSdsUsedMemory(es->body); + /* The dictionary entry and body remain cached, but no longer + * belong to the EVAL-only accounting. */ + scriptsMemorySubtract(sha_mem + body_mem, 1); + scriptsMemoryAdd(sha_mem + body_mem, 0); + scriptsLRUDeleteNode(es->node); es->node = NULL; } @@ -449,6 +597,9 @@ static int evalRegisterNewScript(client *c, robj *body, char **sha) { serverAssert(num_compiled_functions == 1); + /* Try evict EVAL scripts before actually adding the script. */ + if (!is_script_load) performScriptsEvictions(); + /* We also save a SHA1 -> Original script map in a dictionary * so that we can replicate / write in the AOF all the * EVALSHA commands as EVAL using the original script. */ @@ -458,13 +609,14 @@ static int evalRegisterNewScript(client *c, robj *body, char **sha) { es->flags = script_flags; sds _sha = sdsnew(*sha); if (!is_script_load) { - /* Script eviction only applies to EVAL, not SCRIPT LOAD. */ - es->node = scriptsLRUAdd(c, _sha); + /* Script LRU eviction only applies to EVAL, not SCRIPT LOAD. */ + es->node = scriptsLRUAdd(_sha); } es->body = body; + int retval = dictAdd(evalCtx.scripts, _sha, es); serverAssert(retval == DICT_OK); - evalCtx.scripts_mem += sdsAllocSize(_sha) + getStringObjectSdsUsedMemory(body); + scriptsMemoryAdd(sdsAllocSize(_sha) + getStringObjectSdsUsedMemory(body), !is_script_load); incrRefCount(body); zfree(functions); @@ -697,17 +849,24 @@ unsigned long evalMemory(void) { return memory; } -dict *evalScriptsDict(void) { +dict *evalCtxScriptsDict(void) { return evalCtx.scripts; } -unsigned long evalScriptsMemory(void) { +/* Return the memory overhead used by cached scripts. */ +unsigned long scriptsMemoryOverhead(void) { return evalCtx.scripts_mem + dictMemUsage(evalCtx.scripts) + dictSize(evalCtx.scripts) * sizeof(evalScript) + listLength(evalCtx.scripts_lru_list) * sizeof(listNode); } +/* Return the memory overhead used by EVAL scripts. */ +unsigned long evalScriptsMemoryOverhead(void) { + return evalCtx.eval_scripts_mem + + listLength(evalCtx.scripts_lru_list) * (sizeof(evalScript) + sizeof(listNode)); +} + /* Wrapper for EVAL / EVALSHA that enables debugging, and makes sure * that when EVAL returns, whatever happened, the session is ended. */ void evalGenericCommandWithDebugging(client *c, int evalsha) { diff --git a/src/expire.c b/src/expire.c index a70044a8b..0ff99d175 100644 --- a/src/expire.c +++ b/src/expire.c @@ -39,6 +39,7 @@ #include "cluster.h" #include "cluster_migrateslots.h" #include "util.h" +#include "bgiteration.h" /*----------------------------------------------------------------------------- * Incremental collection of expired keys. @@ -168,13 +169,18 @@ void fieldExpireScanCallback(void *privdata, void *volaKey, int didx) { robj *o = volaKey; serverAssert(o); serverAssert(hashTypeHasVolatileFields(o)); + + data->has_more_expired_entries = false; + data->sampled++; + + if (bgIteration_isEntryInuse(o)) return; + mstime_t now = server.mstime; size_t expired_fields = dbReclaimExpiredFields(o, data->db, now, data->max_entries, didx); if (expired_fields) { data->has_more_expired_entries = (expired_fields == data->max_entries); data->expired++; } - data->sampled++; } static int expireShouldSkipTableForSamplingCb(hashtable *ht) { diff --git a/src/expire.h b/src/expire.h index cb6103f01..8c3cec184 100644 --- a/src/expire.h +++ b/src/expire.h @@ -3,6 +3,7 @@ /* Include feature-test macros early for unit tests that include expire.h * before server.h. */ +#include "dict.h" #include "fmacros.h" #include diff --git a/src/fbtree.c b/src/fbtree.c index a06432adc..392f4ae81 100644 --- a/src/fbtree.c +++ b/src/fbtree.c @@ -226,7 +226,21 @@ static inline void innerNodeMoveChildren(innerNode *node, int dst_idx, int src_i } static bool updateCommonPrefix(innerNode *inner) { - if (inner->header.num_items < 2) return false; + if (inner->header.num_items < 2) { + /* A node reduced to one child (by a range delete removing all its + * siblings) can still be holding the prefix it derived when it had + * >= 2 anchors. The surviving child is child 0, whose key range + * extends BELOW its own high key, so when the range delete truncates + * it innerNodeRefreshChildMeta() installs a smaller anchor that need + * not start with the retained prefix. Drop compression here: + * prefix_len 0 is trivially a prefix of every anchor. */ + if (inner->prefix_len != 0) { + innerNodeSetPrefix(inner, "", 0); + recomputeFeatures(inner); + return true; + } + return false; + } /* Compute the common prefix of this node's first and last anchors. * Anchors are high keys of children, so this is the common prefix among @@ -1524,7 +1538,7 @@ typedef struct { * Returns the leaf node reached. * * When called with a compile-time-constant function pointer (e.g., - * findChildByScoreWrapper), the compiler can inline both this helper and the + * findChildByValueWrapper), the compiler can inline both this helper and the * callback at -O2, producing specialized code with no indirect calls. */ typedef int (*findChildFn)(innerNode *inner, const void *key); @@ -1560,10 +1574,6 @@ static int findChildByValueWrapper(innerNode *inner, const void *key) { * Returns <0, 0, or >0 like memcmp/sdscmp. */ typedef int (*leafCmpFn)(const_sds element, const void *key); -static int leafCmpByScore(const_sds element, const void *key) { - return memcmp(element, key, SCORE_SIZE); -} - static int leafCmpByValue(const_sds element, const void *key) { return sdscmp(element, (const_sds)key); } @@ -1800,14 +1810,25 @@ static unsigned long deleteRangeSameLeaf(fbtreeIndex *fbt, * If callback is non-NULL, it is invoked for each deleted item before sdsfree. * Returns the number of elements deleted. */ static unsigned long deleteRangeCore(fbtreeIndex *fbt, BoundaryPaths *bp, fbtreeItemCallback callback, void *callback_ctx) { - /* Empty range check: score/value path builders can produce boundary - * indices that fall outside the leaf (no matching elements in that leaf). - * Rank paths never hit this since ranks are pre-validated. */ - if (bp->start_idx >= bp->start_leaf->header.num_items && bp->end_idx < 0) return 0; + /* No whole-range empty short-circuit is needed here for the + * leaf-untouched-on-both-sides case: unlike deleteRangeSameLeaf (where + * an out-of-range boundary index unambiguously means nothing to delete + * within that single leaf), a diverged-path range can still be + * non-empty even when both boundary leaves are untouched, because + * whole subtrees strictly between the two descent paths -- at the split + * node's own children, or at any level below either boundary's descent + * path -- may lie wholly inside [min_key, max_key]. Checking only the + * split node's immediate children (as an adjacency test) misses middle + * subtrees that appear one or more levels further down, so no local + * check here can be both cheap and correct. The splice and fixup below + * handle the truly-empty case naturally: when there is really nothing + * in range, every per-level loop below finds no boundary-untouched + * work and no middle children to free, and the function falls through + * having deleted (and returned) zero elements. */ + int split_depth = bp->shared_depth - 1; if (bp->end_idx < 0 && bp->start_leaf == bp->end_leaf) return 0; /* --- Phase 1: Boundaries diverged. Process the split. --- */ - int split_depth = bp->shared_depth - 1; innerNode *split_node = (innerNode *)bp->shared_path[split_depth]; int li = bp->shared_left_idx[split_depth]; int ri = bp->shared_right_idx[split_depth]; @@ -2110,6 +2131,141 @@ unsigned long fbtreeDeleteRangeByRank(fbtreeIndex *fbt, return deleteRangeSameLeaf(fbt, &bp, callback, callback_ctx); } +/* Lexicographically next 8-byte score prefix. Returns false when `in` is all + * 0xFF, i.e. no score prefix sorts above it. Incrementing the big-endian + * 8-byte value yields the next possible prefix in memcmp order, so an + * exclusive score bound can be rewritten as an inclusive bound on the next + * prefix. This is pure byte-string reasoning; it does not assume anything + * about how scores are normalized. */ +static bool scorePrefixNext(const char in[static SCORE_SIZE], char out[static SCORE_SIZE]) { + memcpy(out, in, SCORE_SIZE); + for (int i = SCORE_SIZE - 1; i >= 0; i--) { + unsigned char c = (unsigned char)out[i]; + if (c != 0xFF) { + out[i] = (char)(c + 1); + return true; + } + out[i] = 0; + } + return false; +} + +/* Rank of the first element whose score prefix is >= `score`, or the tree + * length when no such element exists. This is a whole-tree lower bound: it + * stays correct when a run of elements sharing `score` spans several leaves. + * Used for the rare unbounded-upper edge (the upper bound reaches past the + * largest representable score prefix) where a shared two-boundary descent has + * no finite hi key. */ +static unsigned long lowerBoundRankByScore(fbtreeIndex *fbt, const char *score) { + fbtreeIterator iterator; + fbtreeInitIterator(&iterator, fbt); + long rank = fbtreeSeekToScore(score, &iterator); + /* fbtreeSeekToScore returns the count of elements before the position, so + * it is never negative (0 for an empty tree). */ + assert(rank >= 0); + return (unsigned long)rank; +} + +/* Turn a score range with inclusive/exclusive bounds into the half-open + * score-prefix window [lo, hi): every element with lo <= score(e) < hi is in + * range. Returns false when the window is provably empty. When the upper edge + * would exceed the largest representable prefix, *hi_unbounded is set (the + * window runs to the tree end) and hi is left unset. + * + * Both edges are expressed as lower-bound prefixes (the first prefix at or + * above a value), which is what makes the range duplicate-safe: a run of + * elements sharing a score is included or excluded as a unit even when it spans + * several leaves. */ +static bool scoreRangeBounds(const char *min_score, + const char *max_score, + int min_ex, + int max_ex, + char lo[SCORE_SIZE], + char hi[SCORE_SIZE], + bool *hi_unbounded) { + *hi_unbounded = false; + + if (min_ex) { + if (!scorePrefixNext(min_score, lo)) return false; /* nothing sorts above min */ + } else { + memcpy(lo, min_score, SCORE_SIZE); + } + + if (max_ex) { + memcpy(hi, max_score, SCORE_SIZE); + } else if (!scorePrefixNext(max_score, hi)) { + *hi_unbounded = true; /* max is the largest possible prefix: no upper cut */ + } + + if (!*hi_unbounded && memcmp(lo, hi, SCORE_SIZE) >= 0) return false; + return true; +} + +/* Two independent lower-bound searches (first element with score prefix >= key) + * over two DIFFERENT leaves, stepped in lockstep so both leaves' pointer-chase + * cache misses are outstanding at once (memory-level parallelism). Each + * values[mid] is a separately allocated sds, so every probe is a miss; + * interleaving the two searches and prefetching both payloads per step lets the + * out-of-order core overlap them instead of serializing 2*log2(leaf) misses. + * Mirrors resolveBothIdxPrefetch but resolves pure lower bounds (both edges are + * "first >= key"), which is the half-open form the score range needs. */ +static void lowerBoundBothByScore(const leafNode *lo_leaf, const char *lo, const leafNode *hi_leaf, const char *hi, int *out_lo_idx, int *out_hi_idx) { + int llo = 0, lhi = lo_leaf->header.num_items; + int hlo = 0, hhi = hi_leaf->header.num_items; + while (llo < lhi || hlo < hhi) { + int lmid = (llo + lhi) / 2; + int hmid = (hlo + hhi) / 2; + if (llo < lhi) __builtin_prefetch(lo_leaf->values[lmid]); + if (hlo < hhi) __builtin_prefetch(hi_leaf->values[hmid]); + if (llo < lhi) { + if (memcmp(lo_leaf->values[lmid], lo, SCORE_SIZE) < 0) + llo = lmid + 1; + else + lhi = lmid; + } + if (hlo < hhi) { + if (memcmp(hi_leaf->values[hmid], hi, SCORE_SIZE) < 0) + hlo = hmid + 1; + else + hhi = hmid; + } + } + *out_lo_idx = llo; + *out_hi_idx = hlo; +} + +/* Resolve a finite half-open score window [lo, hi) with ONE shared tree descent + * (see buildBoundaryPaths). Fills `bp` with both boundary leaves and their + * sub-paths, records the lower-bound leaf indices (bp->start_idx = first element + * >= lo; the hi lower bound is returned via *out_hi_idx), and reports the global + * ranks of both edges. Returns whether both edges fall in the same leaf. + * + * Both leaf-local edges use lower-bound ("first element with score prefix >= + * key") resolution; the child_sizes accumulation in rankFromBoundaryPath turns + * each leaf-local index into a global rank, all from a single root->leaf + * descent. Because both edges are lower bounds the window is duplicate-score + * safe: hi marks the first element excluded from the range, so an equal-score + * run is kept or dropped as a unit even when it continues into the next leaf. */ +static bool scoreRangeSharedDescent(fbtreeIndex *fbt, const char *lo, const char *hi, BoundaryPaths *bp, int *out_hi_idx, unsigned long *out_start_rank, unsigned long *out_end_rank) { + bool same_leaf = buildBoundaryPaths(fbt, bp, lo, hi, findChildByScoreWrapper); + + int lo_idx, hi_idx; + if (same_leaf) { + /* One leaf: nothing independent to overlap, resolve directly. */ + lo_idx = leafNodeBinarySearchByScore(bp->start_leaf, lo); + hi_idx = leafNodeBinarySearchByScore(bp->end_leaf, hi); + } else { + /* Two leaves: software-pipeline the searches so the misses overlap. */ + lowerBoundBothByScore(bp->start_leaf, lo, bp->end_leaf, hi, &lo_idx, &hi_idx); + } + + bp->start_idx = lo_idx; + *out_hi_idx = hi_idx; + *out_start_rank = rankFromBoundaryPath(bp, false, lo_idx); + *out_end_rank = rankFromBoundaryPath(bp, true, hi_idx); + return same_leaf; +} + /* Delete elements with score prefix in [min_score, max_score]. * min_ex/max_ex: if true, the corresponding bound is exclusive. * Score is an 8-byte big-endian normalized prefix (as stored in the tree). @@ -2124,9 +2280,11 @@ unsigned long fbtreeDeleteRangeByScore(fbtreeIndex *fbt, void *callback_ctx) { if (!fbt->root) return 0; - /* Delete-all short-circuit */ - sds first = leafNodeLowKey(fbt->leftmost_leaf); - sds last = leafNodeHighKey(fbt->rightmost_leaf); + /* Delete-all short-circuit: when both bounds fall outside the stored score + * range every element qualifies, so free the whole tree in one pass rather + * than descending for boundaries. */ + const_sds first = leafNodeLowKey(fbt->leftmost_leaf); + const_sds last = leafNodeHighKey(fbt->rightmost_leaf); int min_covers = min_ex ? memcmp(min_score, first, SCORE_SIZE) < 0 : memcmp(min_score, first, SCORE_SIZE) <= 0; int max_covers = max_ex ? memcmp(max_score, last, SCORE_SIZE) > 0 : memcmp(max_score, last, SCORE_SIZE) >= 0; if (min_covers && max_covers) { @@ -2135,15 +2293,34 @@ unsigned long fbtreeDeleteRangeByScore(fbtreeIndex *fbt, return count; } - /* Quick check: empty range */ - int range_cmp = memcmp(min_score, max_score, SCORE_SIZE); - if (range_cmp > 0 || (range_cmp == 0 && (min_ex || max_ex))) return 0; - - /* Descend once to build the boundary paths, then delete. */ + char lo[SCORE_SIZE], hi[SCORE_SIZE]; + bool hi_unbounded; + if (!scoreRangeBounds(min_score, max_score, min_ex, max_ex, lo, hi, &hi_unbounded)) return 0; + + if (hi_unbounded) { + /* Range runs to the tree end: the upper bound reaches past the largest + * representable score prefix, so there is no finite hi key to anchor the + * right boundary of a shared descent. min is finite here (the fully + * covered range was handled by the delete-all short-circuit above). One + * lower-bound seek plus the rank-based delete. */ + unsigned long start = lowerBoundRankByScore(fbt, lo); + unsigned long length = fbtreeLength(fbt); + if (length <= start) return 0; + return fbtreeDeleteRangeByRank(fbt, start, length - 1, callback, callback_ctx); + } + + /* Single shared descent locates both boundary leaves and records the paths + * that deleteRangeCore / deleteRangeSameLeaf consume. The hi lower bound is + * the first element NOT in range, so the last element to delete sits one + * position before it (end_idx = hi_idx - 1). When hi_idx == 0 the end leaf + * holds no in-range element and end_idx becomes -1, the "end leaf untouched" + * case the delete engine handles for the value path as well. */ BoundaryPaths bp; - bool same_leaf = buildBoundaryPaths(fbt, &bp, min_score, max_score, findChildByScoreWrapper); - bp.start_idx = resolveStartIdx(bp.start_leaf, min_score, min_ex, leafCmpByScore); - bp.end_idx = resolveEndIdx(bp.end_leaf, max_score, max_ex, leafCmpByScore); + int hi_idx; + unsigned long start_rank, end_rank; + bool same_leaf = scoreRangeSharedDescent(fbt, lo, hi, &bp, &hi_idx, &start_rank, &end_rank); + if (end_rank <= start_rank) return 0; /* empty range */ + bp.end_idx = hi_idx - 1; return same_leaf ? deleteRangeSameLeaf(fbt, &bp, callback, callback_ctx) : deleteRangeCore(fbt, &bp, callback, callback_ctx); } @@ -2196,33 +2373,29 @@ unsigned long fbtreeCountRangeByScore(fbtreeIndex *fbt, int max_ex) { if (!fbt->root) return 0; - /* Whole-tree short-circuit */ - sds first = leafNodeLowKey(fbt->leftmost_leaf); - sds last = leafNodeHighKey(fbt->rightmost_leaf); + /* Whole-tree short-circuit: when both bounds fall outside the stored score + * range every element qualifies, so answer with the tree length rather than + * paying for the two boundary descents. `ZCOUNT key -inf +inf` is the common + * case this serves. */ + const_sds first = leafNodeLowKey(fbt->leftmost_leaf); + const_sds last = leafNodeHighKey(fbt->rightmost_leaf); int min_covers = min_ex ? memcmp(min_score, first, SCORE_SIZE) < 0 : memcmp(min_score, first, SCORE_SIZE) <= 0; int max_covers = max_ex ? memcmp(max_score, last, SCORE_SIZE) > 0 : memcmp(max_score, last, SCORE_SIZE) >= 0; if (min_covers && max_covers) return fbtreeLength(fbt); - /* Empty range */ - int range_cmp = memcmp(min_score, max_score, SCORE_SIZE); - if (range_cmp > 0 || (range_cmp == 0 && (min_ex || max_ex))) return 0; + char lo[SCORE_SIZE], hi[SCORE_SIZE]; + bool hi_unbounded; + if (!scoreRangeBounds(min_score, max_score, min_ex, max_ex, lo, hi, &hi_unbounded)) return 0; - BoundaryPaths bp; - bool same_leaf = buildBoundaryPaths(fbt, &bp, min_score, max_score, findChildByScoreWrapper); - if (same_leaf) { - /* One leaf: nothing to overlap, resolve directly. */ - bp.start_idx = resolveStartIdx(bp.start_leaf, min_score, min_ex, leafCmpByScore); - bp.end_idx = resolveEndIdx(bp.end_leaf, max_score, max_ex, leafCmpByScore); - } else { - /* Two leaves: software-pipeline the searches so both misses overlap. */ - resolveBothIdxPrefetch(bp.start_leaf, min_score, min_ex, bp.end_leaf, max_score, max_ex, - leafCmpByScore, &bp.start_idx, &bp.end_idx); - } + /* Unbounded upper edge (upper bound reaches past the largest representable + * score prefix): one lower-bound seek; the rest of the tree is in range. */ + if (hi_unbounded) return fbtreeLength(fbt) - lowerBoundRankByScore(fbt, lo); - /* start_idx is the first in-range element; end_idx is the last in-range - * element (inclusive), so its one-past position is end_idx + 1. */ - unsigned long start_rank = rankFromBoundaryPath(&bp, false, bp.start_idx); - unsigned long end_rank = rankFromBoundaryPath(&bp, true, bp.end_idx + 1); + /* Finite window: one shared descent yields both boundary ranks. */ + BoundaryPaths bp; + int hi_idx; + unsigned long start_rank, end_rank; + scoreRangeSharedDescent(fbt, lo, hi, &bp, &hi_idx, &start_rank, &end_rank); return end_rank > start_rank ? end_rank - start_rank : 0; } diff --git a/src/forkless.c b/src/forkless.c new file mode 100644 index 000000000..843a109cd --- /dev/null +++ b/src/forkless.c @@ -0,0 +1,437 @@ +#include "forkless.h" +#include "server.h" +#include "bgiteration.h" +#include "mutexqueue.h" +#include "rdb.h" +#include "bio.h" + +static const void *PROCESS_COMPLETE_ITEM = (void *)-1; +static const int SNAPSHOT_FILE_CLOSE_MONITOR_INTERVAL_MS = 200; + +typedef struct { + rio save_rio; /* Must be 1st to permit cast from rio back to forklessSaveInfo */ + int cur_db; /* Last selectDb issued */ + bgIterator *iterator; + uint64_t bytes_written; + int err_code; + mutexQueue *foreground_queue; + bool terminated; + sds temp_file; + sds final_file; +} forklessSaveInfo; + +/* Keep a global indicator of the current iterator (for cancellation purposes). */ +static forklessSaveInfo *currentForklessSave = NULL; + +/* rio check_abort_between_writes callback: checks if the forkless save iterator is being terminated. */ +static int forklessSaveShouldAbort(rio *r) { + static_assert(offsetof(forklessSaveInfo, save_rio) == 0, "rio must be castable to forklessSaveInfo"); + forklessSaveInfo *saveInfo = (forklessSaveInfo *)r; + return saveInfo->iterator && bgIteratorIsTerminating(saveInfo->iterator); +} + +static int writeSelectDb(forklessSaveInfo *saveInfo, int new_db) { + if (new_db == saveInfo->cur_db) return C_OK; + + if (rdbSaveType(&saveInfo->save_rio, RDB_OPCODE_SELECTDB) == -1) { + serverLog(LL_WARNING, "forkless-save: error while writing OPCODE_SELECTDB"); + return C_ERR; + } + if (rdbSaveLen(&saveInfo->save_rio, new_db) == -1) { + serverLog(LL_WARNING, "forkless-save: error while writing selectDb value"); + return C_ERR; + } + saveInfo->cur_db = new_db; + return C_OK; +} + +static int writeDbSizeHints(forklessSaveInfo *saveInfo) { + for (int dbid = 0; dbid < server.dbnum; dbid++) { + serverDb *db = server.db[dbid]; + if (db == NULL || dbSize(db) == 0) continue; + if (writeSelectDb(saveInfo, dbid) != C_OK) return C_ERR; + if (rdbSaveDbSizeHints(&saveInfo->save_rio, db, 0) < 0) return C_ERR; + } + return C_OK; +} + +/* Entry point for background thread. + * Upon entering: + * - The RDB header has been written (magic, aux fields, functions) + * - The DB size hints have been written + * This function is responsible for writing all of the dictionary entries. */ +static void *forklessSaveProcessor(void *arg) { + serverAssert(!onServerMainThread()); + forklessSaveInfo *saveInfo = arg; + + serverLog(LL_NOTICE, "forkless-save: background processor started"); + int err = C_OK; + + saveInfo->save_rio.check_abort_between_writes = forklessSaveShouldAbort; + + const unsigned statsIntervalMs = 1000; + monotime lastStatsTime; + elapsedStart(&lastStatsTime); + + bool done = false; + bool terminated = false; + long items = 0; + while (!done && err == C_OK) { + bgIteratorItem *item = bgIteratorRead(saveInfo->iterator); + + switch (item->type) { + case BGITERATOR_ITEM_COMPLETE: + done = true; + break; + + case BGITERATOR_ITEM_TERMINATED: + terminated = true; + done = true; + break; + + case BGITERATOR_ITEM_DBENTRY: + if ((err = writeSelectDb(saveInfo, item->dbid)) == C_ERR) break; + items++; + + robj key; + initStaticStringObject(key, objectGetKey(item->u.dbe.de)); + robj *o = item->u.dbe.de; + + long long expire = objectGetExpire(item->u.dbe.de); + if (rdbSaveKeyValuePair(&saveInfo->save_rio, &key, o, expire, item->dbid, RDB_VERSION) == -1) { + serverLog(LL_WARNING, "forkless-save: error writing KV pair"); + err = C_ERR; + } + break; + default: + /* bgIteration may deliver item types that are not necessarily relevant to us. + * New types may also be added in the future. It is the client's responsibility + * to filter out irrelevant types, so we simply ignore them here. */ + break; + } + + if (elapsedMs(lastStatsTime) >= statsIntervalMs) { + elapsedStart(&lastStatsTime); + atomic_store_explicit(&server.stat_current_save_keys_processed, items, memory_order_relaxed); + } + } + + if (err != C_OK && bgIteratorIsTerminating(saveInfo->iterator) && !rioGetWriteError(&saveInfo->save_rio)) { + /* The write returned an error because the abort check stopped it, not + * from a real I/O failure (no RIO write error is set). Treat it as a + * cancel. */ + terminated = true; + err = C_OK; + } + + char *message = ""; + if (terminated) + message = "TERMINATED"; + else if (err != C_OK) + message = "***ERROR***"; + serverLog(LL_NOTICE, "forkless-save: background processor finished. %ld items processed. %s", + items, message); + + saveInfo->err_code = err; + bgIteratorClose(saveInfo->iterator); + return NULL; +} + +static void cleanupSaveInfoAndEmitEndMetrics(forklessSaveInfo *saveInfo) { + bool cancelled = saveInfo->terminated && saveInfo->err_code == C_OK; + bool success = !saveInfo->terminated && saveInfo->err_code == C_OK; + + /* A cancel must not count as a failed save, so skip the metrics that set + * lastbgsave_status (like the fork child's SIGUSR1 whitelist). */ + if (!cancelled) rdbRecordEndMetrics(RDB_BGSAVE_TYPE_FORKLESS, saveInfo->err_code, time(NULL)); + /* startSaving() fired the persistence start event in this process, so a + * terminal event must be emitted even on cancel to balance it. */ + stopSaving(success); + /* Finalize the save state in any case. */ + rdbClearSaveState(time(NULL)); + + if (cancelled) { + serverLog(LL_WARNING, "forkless-save: forkless save cancelled. %lld seconds.", (long long)server.rdb_save_time_last); + } else if (success) { + serverLog(LL_NOTICE, "forkless-save: forkless save complete. %lld seconds.", (long long)server.rdb_save_time_last); + } else { + serverLog(LL_WARNING, "forkless-save: forkless save failed. %lld seconds.", (long long)server.rdb_save_time_last); + } + currentForklessSave = NULL; + atomic_store_explicit(&server.stat_current_save_keys_processed, 0, memory_order_relaxed); + atomic_store_explicit(&server.stat_current_save_keys_total, 0, memory_order_relaxed); + + serverAssert(saveInfo->temp_file == NULL); + zfree(saveInfo); +} + +/* Routine for background thread to close and rename the forkless save snapshot file. + * Closing the file requires synchronously flushing the content to disk, which can + * take some time. */ +static void forklessSaveCloseSnapshotFile(void *args[]) { + serverAssert(!onServerMainThread()); + forklessSaveInfo *saveInfo = (forklessSaveInfo *)args[0]; + /* Error or not, close the file... */ + /* Flush the RIO buffer to the OS before fsync, otherwise any bytes still + * buffered (including the tail written after the last autosync boundary and + * the RDB footer) are not covered by the fsync below. */ + if (rioFlush(&saveInfo->save_rio) == 0) { + serverLog(LL_WARNING, "forkless-save: error flushing temp file [%s]: %s", + saveInfo->temp_file, strerror(errno)); + saveInfo->err_code = C_ERR; + } + if (valkey_fsync(fileno(saveInfo->save_rio.io.file.fp)) != 0) { + serverLog(LL_WARNING, "forkless-save: error fsyncing temp file [%s]: %s", + saveInfo->temp_file, strerror(errno)); + saveInfo->err_code = C_ERR; + } + if (fclose(saveInfo->save_rio.io.file.fp) != 0) { + serverLog(LL_WARNING, "forkless-save: error closing temp file [%s]: %s", + saveInfo->temp_file, strerror(errno)); + saveInfo->err_code = C_ERR; + } + + if (!saveInfo->terminated && saveInfo->err_code == C_OK) { + if (rename(saveInfo->temp_file, saveInfo->final_file) != 0) { + serverLog(LL_WARNING, "forkless-save: error moving temp file [%s] to destination [%s]: %s", + saveInfo->temp_file, saveInfo->final_file, strerror(errno)); + saveInfo->err_code = C_ERR; + } else if (fsyncFileDir(saveInfo->final_file) != 0) { + /* fsync the directory so the rename itself survives a crash. */ + serverLog(LL_WARNING, "forkless-save: error syncing directory for [%s]: %s", + saveInfo->final_file, strerror(errno)); + saveInfo->err_code = C_ERR; + } + } + + if (saveInfo->terminated || saveInfo->err_code != C_OK) { + bg_unlink(saveInfo->temp_file); + } + sdsfree(saveInfo->temp_file); + sdsfree(saveInfo->final_file); + saveInfo->temp_file = NULL; + saveInfo->final_file = NULL; + /* Notify the main thread that I am done closing the file. */ + mutexQueueAdd(saveInfo->foreground_queue, (void *)PROCESS_COMPLETE_ITEM); +} + +/* Timer proc which runs in the main valkey event loop. It monitors to see when the background thread + * completes the action to close and rename the snapshot file at the end of disk based forkless save, + * and performs the final clean-up actions. */ +static long long snapshotEndMonitorTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData) { + UNUSED(eventLoop); + UNUSED(id); + serverAssert(onServerMainThread()); + + forklessSaveInfo *saveInfo = (forklessSaveInfo *)clientData; + + /* I own this mutex queue from the main thread, check to see if the background + job is done or not. Note we only expect a single notification event here. */ + if (mutexQueuePop(saveInfo->foreground_queue, false) != NULL) { + mutexQueueRelease(saveInfo->foreground_queue); + saveInfo->foreground_queue = NULL; + cleanupSaveInfoAndEmitEndMetrics(saveInfo); + return AE_NOMORE; + } + return SNAPSHOT_FILE_CLOSE_MONITOR_INTERVAL_MS; +} + +void forklessSaveComplete(bool terminated, void *privdata) { + serverAssert(onServerMainThread()); + serverLog(LL_NOTICE, "forkless-save: completion proc - %s", (terminated) ? "terminated" : "ok"); + + forklessSaveInfo *saveInfo = privdata; + saveInfo->terminated = terminated; + /* The save iterator should be terminated and freed at this point in time. */ + saveInfo->iterator = NULL; + currentForklessSave = NULL; + /* For file based forkless save, we need to generate the RDB end marker. and complete the save */ + if (!saveInfo->terminated && saveInfo->err_code == C_OK) { + saveInfo->err_code = rdbWriteFooter(&saveInfo->save_rio, REPLICA_REQ_NONE) == C_ERR ? C_ERR : C_OK; + } + + /* Done writing, capture bytes written (regardless of pass/fail) */ + saveInfo->bytes_written = saveInfo->save_rio.processed_bytes; + + /* Start a cron job to check for the background job completion */ + aeCreateTimeEvent(server.el, SNAPSHOT_FILE_CLOSE_MONITOR_INTERVAL_MS, snapshotEndMonitorTimeProc, saveInfo, NULL); + /* Submit a background job to close and rename the snapshot file */ + saveInfo->foreground_queue = mutexQueueCreate(); // The monitor proc will delete this + bioCreateLazyFreeJob(forklessSaveCloseSnapshotFile, 1, saveInfo); + serverLog(LL_NOTICE, "forkless-save: created background thread to perform snapshot file close and rename"); + /* We will now wait for the background closeSnapshotFile job to complete. + * The remainder of the cleanup will be performed in the snapshotEndMonitorTimeProc. */ +} + +static int forklessSaveCommonStart(forklessSaveInfo *saveInfo) { + serverAssert(onServerMainThread()); + + saveInfo->cur_db = -1; + + serverLog(LL_NOTICE, "Using forkless save for next backup"); + rdbRecordStartMetrics(RDB_BGSAVE_TYPE_FORKLESS); + startSaving(RDBFLAGS_FORKLESS_SAVE); + + rdbSaveInfo rsi, *rsiptr = rdbPopulateSaveInfo(&rsi); + if (rdbWriteHeader(&saveInfo->save_rio, REPLICA_REQ_NONE, RDB_VERSION, RDBFLAGS_NONE, rsiptr) == C_ERR) return C_ERR; + + if (writeDbSizeHints(saveInfo) == C_ERR) return C_ERR; + + return C_OK; +} + +static void startBackgroundThread(forklessSaveInfo *saveInfo) { + serverAssert(onServerMainThread()); + + pthread_t thread_id; + pthread_attr_t attr; + int pthread_rc; + serverInitThreadAttribute(&attr); + pthread_rc = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + serverAssert(pthread_rc == 0); + pthread_rc = pthread_create(&thread_id, &attr, &forklessSaveProcessor, saveInfo); + serverAssert(pthread_rc == 0); + pthread_rc = pthread_attr_destroy(&attr); + serverAssert(pthread_rc == 0); +} + +/* Save a point-in-time snapshot to the given filename. + * The filename must be under the server's current working directory. + * Writes to a temp file and renames to the final filename on completion. */ +int forklessSaveToDisk(const char *filename) { + serverAssert(onServerMainThread()); + serverAssert(currentForklessSave == NULL); + serverAssert(!isSaveInProgress()); + serverAssert(filename); + serverLog(LL_NOTICE, "Beginning forklessSaveToDisk"); + + server.stat_rdb_saves++; + + /* Use a forkless-specific name with a unique counter so the temp file can't + * collide with a fork-based rdbSave() (same process) or another forkless + * save. */ + char tmpfile[256]; + snprintf(tmpfile, sizeof(tmpfile), "temp-forkless-%d-%lld.rdb", (int)getpid(), (long long)server.stat_rdb_saves); + + FILE *file = fopen(tmpfile, "wb"); + if (file == NULL) { + serverLog(LL_WARNING, "forkless-save: failed to open temp file [%s] for forkless save: %s", + tmpfile, strerror(errno)); + return C_ERR; + } + + forklessSaveInfo *saveInfo = zcalloc(sizeof(forklessSaveInfo)); + saveInfo->temp_file = sdsnew(tmpfile); + saveInfo->final_file = sdsnew(filename); + + rioInitWithFile(&saveInfo->save_rio, file); + if (server.rdb_save_incremental_fsync) { + rioSetAutoSync(&saveInfo->save_rio, REDIS_AUTOSYNC_BYTES); + rioSetReclaimCache(&saveInfo->save_rio, 1); + } + + int rc = forklessSaveCommonStart(saveInfo); + if (rc != C_OK) goto werr; + + /* Saving to a file indicates a consistent snapshot (a backup at a point in time) */ + saveInfo->iterator = bgIteratorCreateFullScanIter(FORKLESS_SAVE_FILE_ITER_NAME, + BGITERATOR_CONSISTENCY_START, NULL, forklessSaveComplete, saveInfo); + if (saveInfo->iterator == NULL) { + serverLog(LL_WARNING, "forkless-save: error creating iterator"); + goto werr; + } + currentForklessSave = saveInfo; + + atomic_store_explicit(&server.stat_current_save_keys_total, dbTotalServerKeyCount(), memory_order_relaxed); + atomic_store_explicit(&server.stat_current_save_keys_processed, 0, memory_order_relaxed); + + startBackgroundThread(saveInfo); + + /* at this point, background iteration has started (saveInfo will be freed later) */ + return C_OK; + +werr: + saveInfo->err_code = C_ERR; + rdbRecordEndMetrics(RDB_BGSAVE_TYPE_FORKLESS, C_ERR, time(NULL)); + rdbClearSaveState(time(NULL)); + serverLog(LL_WARNING, "forkless-save: forkless save failed. %lld seconds.", (long long)server.rdb_save_time_last); + stopSaving(0); + currentForklessSave = NULL; + + if (file != NULL) { + if (fclose(file) != 0) { + serverLog(LL_WARNING, "forkless-save: Could not close temp file [%s]: %s", + saveInfo->temp_file, strerror(errno)); + } + if (unlink(saveInfo->temp_file) != 0) { + serverLog(LL_WARNING, "forkless-save: Could not delete temp file [%s]: %s", + saveInfo->temp_file, strerror(errno)); + } + } + sdsfree(saveInfo->temp_file); + sdsfree(saveInfo->final_file); + zfree(saveInfo); + return C_ERR; +} + +/* Cancels the currently running forkless save, if one is in progress. */ +void forklessSaveCancel(void) { + serverAssert(onServerMainThread()); + if (currentForklessSave == NULL) return; + bgIteratorTerminate(currentForklessSave->iterator); +} + +int isForklessSaveInProgress(void) { + return server.cur_bgsave_type == RDB_BGSAVE_TYPE_FORKLESS; +} + +/* Appends forkless save INFO metrics to the provided sds string. */ +sds forkless_catInfo(sds info) { + long long estimated_seconds_remaining = -1; + + if (onServerMainThread()) { + bgIterator *iter = bgIteratorFind(FORKLESS_SAVE_FILE_ITER_NAME); + if (iter != NULL) { + bgIteratorStatus status = {0}; + bgIteratorGetStatus(iter, &status); + + if (status.dbentries_processed > 0) { + long long total_keys = + (long long)atomic_load_explicit(&server.stat_current_save_keys_total, memory_order_relaxed); + /* The ETA is best effort. Clamp at 0 since dbentries_processed + * can exceed total_keys (e.g. a full sync may process more than + * the start-time key count). */ + long long remaining = max(total_keys - (long long)status.dbentries_processed, 0); + estimated_seconds_remaining = remaining * status.runtime_ms / status.dbentries_processed / 1000; + } + } + } + + return sdscatprintf(info, "forkless_estimated_seconds_remaining:%lld\r\n", estimated_seconds_remaining); +} + +/* Appends forkless debug metrics to the provided sds string. */ +sds forkless_catDebugInfo(sds info) { + bgIteratorStatus status = {0}; + long long current_item_ms = -1; + + if (onServerMainThread()) { + bgIterator *iter = bgIteratorFind(FORKLESS_SAVE_FILE_ITER_NAME); + if (iter != NULL) { + bgIteratorGetStatus(iter, &status); + current_item_ms = status.current_item_ms; + } + } + + return sdscatprintf(info, + "forkless_current_item_ms:%lld\r\n" + "forkless_current_queue_length:%lu\r\n" + "forkless_queue_length_target:%lu\r\n" + "forkless_dbentries_queued:%lu\r\n" + "forkless_dbentries_processed:%lu\r\n", + current_item_ms, + status.queue_length, + status.queue_length_target, + status.dbentries_queued, + status.dbentries_processed); +} diff --git a/src/forkless.h b/src/forkless.h new file mode 100644 index 000000000..e5393e52f --- /dev/null +++ b/src/forkless.h @@ -0,0 +1,14 @@ +#ifndef __FORKLESS_H__ +#define __FORKLESS_H__ + +#include "server.h" + +#define FORKLESS_SAVE_FILE_ITER_NAME "forkless_save_file" + +int forklessSaveToDisk(const char *filename); +void forklessSaveCancel(void); +int isForklessSaveInProgress(void); +sds forkless_catInfo(sds info); +sds forkless_catDebugInfo(sds info); + +#endif diff --git a/src/hashtable.c b/src/hashtable.c index 89db564ea..c5cd889e1 100644 --- a/src/hashtable.c +++ b/src/hashtable.c @@ -344,7 +344,7 @@ typedef struct { } position; static_assert(sizeof(hashtablePosition) >= sizeof(position), - "Opaque iterator size"); + "Opaque position size"); /* State for incremental find. */ typedef struct { @@ -1406,13 +1406,13 @@ void hashtableResumeAutoShrink(hashtable *ht) { * spaces, "holes", in the bucket chains, which wastes memory. Additionally, we * pause auto shrink when rehashing is paused, meaning the hashtable will not * shrink the bucket count. */ -static void hashtablePauseRehashing(hashtable *ht) { +void hashtablePauseRehashing(hashtable *ht) { ht->pause_rehash++; hashtablePauseAutoShrink(ht); } /* Resumes incremental rehashing, after pausing it. */ -static void hashtableResumeRehashing(hashtable *ht) { +void hashtableResumeRehashing(hashtable *ht) { ht->pause_rehash--; assert(ht->pause_rehash >= 0); hashtableResumeAutoShrink(ht); @@ -1910,7 +1910,7 @@ bool hashtableIncrementalFindStep(hashtableIncrementalFindState *state) { const void *elem_key = entryGetKey(ht, entry); if (compareKeys(ht, data->key, elem_key)) { /* It's a match. */ - data->state = HASHTABLE_FOUND; + data->state = validateElementIfNeeded(ht, entry) ? HASHTABLE_FOUND : HASHTABLE_NOT_FOUND; return false; } /* No match. Look for next candidate entry in the bucket. */ @@ -1990,6 +1990,37 @@ bool hashtableIncrementalFindGetResult(hashtableIncrementalFindState *state, voi } } +/* Provides batch lookup. Compared with serial single-key lookups, it can improve + * performance by parallelizing memory accesses. Each bit in the returned bitmap + * indicates whether the key at the same index was found. */ +uint32_t hashtableFindBatch(hashtable *ht, int numkeys, const void **keys, void **found_entries) { + assert(numkeys >= 0 && numkeys <= HASHTABLE_FIND_BATCH_MAX_SIZE); + if (numkeys == 0) return 0; + + rehashStepOnReadIfNeeded(ht); + + hashtableIncrementalFindState states[numkeys]; + for (int i = 0; i < numkeys; i++) { + hashtableIncrementalFindInit(&states[i], ht, keys[i]); + } + + size_t incomplete; + do { + incomplete = 0; + for (int i = 0; i < numkeys; i++) { + incomplete += hashtableIncrementalFindStep(&states[i]); + } + } while (incomplete != 0); + + uint32_t result = 0; + for (int i = 0; i < numkeys; i++) { + if (hashtableIncrementalFindGetResult(&states[i], &found_entries[i])) { + result |= (uint32_t)1 << i; + } + } + return result; +} + /* --- Scan --- */ /* Scan is a stateless iterator. It works with a cursor that is returned to the @@ -2054,13 +2085,17 @@ size_t hashtableScan(hashtable *ht, size_t cursor, hashtableScanFunction fn, voi * A cursor of 0 means the scan has not started, so no keys have been passed. */ bool hashtableScanHasPassedKey(hashtable *ht, const void *key, size_t cursor) { if (cursor == 0) return false; - size_t mask = expToMask(ht->bucket_exp[0]); - uint64_t hash = hashKey(ht, key); - size_t bucket_idx = hash & mask; - size_t cursor_idx = cursor & mask; - /* In reverse-bit-increment order, a bucket has been visited if its - * reversed index is less than the reversed cursor index. */ - return rev(bucket_idx) < rev(cursor_idx); + if (hashtableSize(ht) == 0) return true; + + /* The scan visits buckets in reverse-binary order based on the smallest + * table. During rehashing, a small-table bucket and its corresponding + * large-table buckets are processed together, so the small-table mask + * determines ordering in both cases. */ + int exp = ht->bucket_exp[0]; + if (hashtableIsRehashing(ht) && ht->bucket_exp[1] < exp) exp = ht->bucket_exp[1]; + size_t mask = expToMask(exp); + size_t bucket_idx = hashKey(ht, key) & mask; + return rev(bucket_idx) < rev(cursor & mask); } /* Like hashtableScan, but additionally reallocates the memory used by the dict diff --git a/src/hashtable.h b/src/hashtable.h index 4af1ad0dc..6fff926a8 100644 --- a/src/hashtable.h +++ b/src/hashtable.h @@ -92,7 +92,8 @@ typedef enum { typedef void (*hashtableScanFunction)(void *privdata, void *entry); /* Constants */ -#define HASHTABLE_BUCKET_SIZE 64 /* bytes, the most common cache line size */ +#define HASHTABLE_BUCKET_SIZE 64 /* bytes, the most common cache line size */ +#define HASHTABLE_FIND_BATCH_MAX_SIZE 32 /* Limited by the result bitmap size. */ /* Scan flags */ #define HASHTABLE_SCAN_EMIT_REF (1 << 0) @@ -129,6 +130,8 @@ size_t hashtableMemUsage(const hashtable *ht); void hashtablePauseAutoShrink(hashtable *ht); void hashtableResumeAutoShrink(hashtable *ht); bool hashtableIsRehashing(hashtable *ht); +void hashtablePauseRehashing(hashtable *ht); +void hashtableResumeRehashing(hashtable *ht); bool hashtableIsRehashingPaused(hashtable *ht); ssize_t hashtableGetRehashingIndex(hashtable *ht); void hashtableRehashingInfo(hashtable *ht, size_t *from_size, size_t *to_size); @@ -144,6 +147,7 @@ void hashtableSetCanAbortShrink(bool can_abort); /* Entries */ bool hashtableFind(hashtable *ht, const void *key, void **found); +uint32_t hashtableFindBatch(hashtable *ht, int numkeys, const void **keys, void **found_entries); void **hashtableFindRef(hashtable *ht, const void *key); bool hashtableAdd(hashtable *ht, void *entry); bool hashtableAddOrFind(hashtable *ht, void *entry, void **existing); diff --git a/src/hotkeys.c b/src/hotkeys.c new file mode 100644 index 000000000..10617408c --- /dev/null +++ b/src/hotkeys.c @@ -0,0 +1,354 @@ +/* + * Copyright Valkey Contributors. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "server.h" +#include "hotkeys.h" +#include "cluster.h" +#include "monotonic.h" +#include "space_saving.h" + +/* --------------------------------------------------------------------------- + * Hot-key detection + * + * A frozen-window Space-Saving manager (spaceSavingManager, see space_saving.h) + * does the heavy lifting: it tracks the top-K (key, db) pairs, keeping a live + * window accumulating the current `hotkeys-window-seconds` and a frozen snapshot + * of the last completed window, which is what HOTKEYS GET reports. This file + * supplies the policy around it: the sampling/enable configuration, the + * invalidation predicates, and the HOTKEYS commands. + * --------------------------------------------------------------------------*/ + +/* Create a frozen-window manager sized and timed from the current config. */ +static spaceSavingManager *hotkeysCreateManager(void) { + uint64_t window_us = (uint64_t)server.hotkeys_window_seconds * 1000000ULL; + spaceSavingManager *m = spaceSavingManagerCreate(server.hotkeys_top_k, window_us, getMonotonicUs()); + if (m) spaceSavingManagerSetLiveSamplingPercentage(m, server.hotkeys_sampling_percentage); + return m; +} + +/* =========================================================================== + * Invalidation helpers + * ==========================================================================*/ + +void hotkeysPurgeAll(void) { + if (!server.hotkeys_manager) return; + /* Reset preserves the live window's sampling percentage, so there is nothing + * to re-establish here. */ + spaceSavingManagerReset(server.hotkeys_manager, getMonotonicUs()); +} + +/* Periodic maintenance from serverCron: close any window that has fully elapsed + * so a completed window is frozen on schedule even when there is no traffic (and + * so any future window-boundary work — history, notifications — has a place to + * hang). Cheap: a subtract and a compare unless a boundary was actually crossed. + * No-op when detection is disabled. */ +void hotkeysCron(void) { + if (server.hotkeys_manager) spaceSavingManagerRotate(server.hotkeys_manager, getMonotonicUs()); +} + +/* The cluster hash slot is not stored per entry — it is derived from the key + * name on demand, only when a slot-scoped purge asks for it. */ +static int hotkeysItemInSlot(sds key, int dbid, void *arg) { + UNUSED(dbid); + return (int)keyHashSlot(key, (int)sdslen(key)) == *(int *)arg; +} + +static int hotkeysItemInDb(sds key, int dbid, void *arg) { + UNUSED(key); + return dbid == *(int *)arg; +} + +/* Drop every entry on `slot` from both windows, so a removed slot's keys + * disappear from reports immediately and do not resurface on rotation. */ +void hotkeysPurgeSlot(int slot) { + if (server.hotkeys_manager) spaceSavingManagerRemoveIf(server.hotkeys_manager, hotkeysItemInSlot, &slot); +} + +/* Drop every entry in database `dbid` from both windows. */ +void hotkeysPurgeDb(int dbid) { + if (server.hotkeys_manager) spaceSavingManagerRemoveIf(server.hotkeys_manager, hotkeysItemInDb, &dbid); +} + +/* Note: RENAME / MOVE / SWAPDB are intentionally NOT re-attributed. An entry is + * keyed by (key name, db), so after one of these commands a tracked entry keeps + * its old identity and may briefly be reported under the pre-command name/db. + * This is accepted for simplicity: the stale entry is harmless and ages out + * with the window — it stops accruing new hits immediately and disappears once + * the window rotates (from the live window on the next rotation, from the + * frozen snapshot one rotation later), so it lingers at most for the reporting + * window. */ + +/* =========================================================================== + * Per-access detection hook + * ==========================================================================*/ + +/* Record one sampled access (read or write) of `key` in database `dbid`. */ +static void hotkeysRecordSample(robj *key, int dbid) { + spaceSavingManager *m = server.hotkeys_manager; + if (!m || !key) return; + sds k = objectGetVal(key); + if (!k) return; + recordSpaceSavingManagerSample(m, k, dbid); +} + +/* True when the current activity is a genuine client executing a command — a + * real client that is actually processing a command, and is not the replication + * link/AOF, and not RDB/AOF loading, and not an administrative bulk slot + * deletion (delKeysInSlot, e.g. CLUSTER FLUSHSLOT / slot migration — that is not + * user key access and must not feed or evict the sampler). Importing traffic is + * user-driven load and is counted. */ +static bool hotkeysShouldRecord(void) { + client *c = server.current_client; + return c != NULL && c->flag.executing_command && !mustObeyClient(c) && !server.loading && + !server.server_del_keys_in_slot; +} + +/* Charge a sampled read/write access of `key` in `dbid`, for a lookup carrying + * `lookup_flags` (LOOKUP_*). + * + * Lookups flagged LOOKUP_NOHOTKEYS are skipped as introspection (OBJECT, DEBUG, + * the cluster redirect lookup). Note this tests that dedicated bit and NOT + * LOOKUP_NOEFFECTS, which is a mask of several flags: a lookup carrying only + * LOOKUP_NOTOUCH (EXISTS/TYPE/TTL, or any hit from a CLIENT NO-TOUCH client) is + * a genuine client access. */ +void hotkeysRecordLookup(robj *key, int dbid, int lookup_flags) { + if (!hotkeysEnabled() || (lookup_flags & LOOKUP_NOHOTKEYS)) return; + if (!hotkeysShouldRecord()) return; + if (!bernoulliSampleHit(server.hotkeys_sampling_percentage)) return; + hotkeysRecordSample(key, dbid); +} + +/* Charge a sampled removal of `key` in `dbid`. `del_flags` are the DB_FLAG_* + * deletion reasons: only a genuine client-issued DEL/UNLINK counts, not passive + * expiry or eviction (DB_FLAG_KEY_EXPIRED / DB_FLAG_KEY_EVICTED). A deletion is + * activity on the key, so it is charged like any other access. */ +void hotkeysRecordDelete(robj *key, int dbid, int del_flags) { + if (!hotkeysEnabled() || !(del_flags & DB_FLAG_KEY_DELETED)) return; + if (!hotkeysShouldRecord()) return; + if (!bernoulliSampleHit(server.hotkeys_sampling_percentage)) return; + hotkeysRecordSample(key, dbid); +} + +/* =========================================================================== + * HOTKEYS commands + * ==========================================================================*/ + +typedef struct { + sds key; + uint64_t qps; + int dbid; +} hotkeysCollected; + +static int hotkeysCollectedCmpDesc(const void *a, const void *b) { + const hotkeysCollected *ea = a; + const hotkeysCollected *eb = b; + if (eb->qps > ea->qps) return 1; + if (eb->qps < ea->qps) return -1; + return 0; +} + +/* Compute (a * b) / c rounded to nearest, without overflowing the intermediate + * product. Uses a 128-bit intermediate where the compiler has one (as + * monotonic.c does); the uint64 fallback is exact for every reachable input, + * since overflowing it would take upwards of 9e10 sampled hits on one key + * inside a single window. */ +static uint64_t hotkeysMulDivRound(uint64_t a, uint64_t b, uint64_t c) { +#ifdef __SIZEOF_INT128__ + __uint128_t num = (__uint128_t)a * b; + return (uint64_t)((num + c / 2) / c); +#else + return (a * b + c / 2) / c; +#endif +} + +/* Recover a per-second rate from a frozen (count, error) pair whose counts were + * Bernoulli-sampled at `sample_percentage` percent over a window that really + * lasted `duration_us` microseconds. Uses the midpoint of the [count-error, + * count] band (the *2 keeps error/2 exact) and scales the sampled count back up + * by 100/sample_percentage. + * + * The denominator is the window's MEASURED duration, not the configured + * `hotkeys-window-seconds`. Rotation is driven by serverCron, so a window is + * closed at or after its nominal boundary and holds the traffic of that whole + * real interval; dividing by the nominal length would over-report by the + * rotation lag (up to ~1/server.hz, i.e. ~10% at the default hz with a 1s + * window) and always in the same direction. Integer arithmetic, rounded to + * nearest; 0 for non-positive inputs. */ +static uint64_t hotkeysEstimateQps(uint64_t count, uint64_t error, int sample_percentage, uint64_t duration_us) { + if (sample_percentage <= 0 || duration_us == 0) return 0; + uint64_t twice_midpoint = 2 * count - error; + uint64_t den = 2ULL * (uint64_t)sample_percentage * duration_us; + return hotkeysMulDivRound(twice_midpoint, 100ULL * 1000000ULL, den); +} + +void hotkeysGetCommand(client *c) { + /* Report an empty result rather than an error when detection is off, as + * SLOWLOG GET and LATENCY HISTORY do: a polling client then has one shape to + * parse and does not have to match on an error string to tell "disabled" + * from "nothing is hot". */ + if (!hotkeysEnabled()) { + addReplyArrayLen(c, 0); + return; + } + /* Detection is enabled, so the manager must already exist (created by + * hotkeysInit / the config callbacks whenever top-k is turned on). */ + spaceSavingManager *m = server.hotkeys_manager; + serverAssert(m != NULL); + + /* Close any window that has fully elapsed so we report the latest + * completed window. */ + spaceSavingManagerRotate(m, getMonotonicUs()); + + int cap = spaceSavingManagerCount(m); + if (cap == 0) { + addReplyArrayLen(c, 0); + return; + } + + hotkeysCollected *arr = zmalloc(cap * sizeof(hotkeysCollected)); + /* Estimate with the sampling percentage that produced the frozen window (the + * current config may have changed since) and the interval it really spanned. */ + int frozen_pct = spaceSavingManagerFrozenSamplingPercentage(m); + uint64_t frozen_duration_us = spaceSavingManagerFrozenDurationUs(m); + for (int i = 0; i < cap; i++) { + uint64_t count, error; + spaceSavingManagerAt(m, i, &arr[i].key, &arr[i].dbid, &count, &error); + arr[i].qps = hotkeysEstimateQps(count, error, frozen_pct, frozen_duration_us); + } + + qsort(arr, cap, sizeof(hotkeysCollected), hotkeysCollectedCmpDesc); + + int limit = cap < server.hotkeys_top_k ? cap : server.hotkeys_top_k; + addReplyArrayLen(c, limit); + for (int j = 0; j < limit; j++) { + addReplyMapLen(c, 3); + addReplyBulkCString(c, "key"); + addReplyBulkCBuffer(c, arr[j].key, sdslen(arr[j].key)); + addReplyBulkCString(c, "db"); + addReplyLongLong(c, arr[j].dbid); + addReplyBulkCString(c, "qps"); + addReplyLongLong(c, arr[j].qps); + } + zfree(arr); +} + +void hotkeysResetCommand(client *c) { + /* Nothing to clear when detection is off; still report success, so callers + * need not special-case the disabled state. */ + if (hotkeysEnabled()) hotkeysPurgeAll(); + addReply(c, shared.ok); +} + +void hotkeysHelpCommand(client *c) { + const char *help[] = { + "GET", + " Return the hottest keys of the last completed window, ordered by", + " estimated accesses per second (descending). Each entry reports the", + " key name, the database it was accessed in, and the estimated QPS.", + "RESET", + " Clear all collected hot key statistics.", + NULL, + }; + addReplyHelp(c, help); +} + +/* =========================================================================== + * Generic hotkey API + * ==========================================================================*/ + +/* Is hot-key detection currently enabled? Tracking zero keys is the same thing + * as not tracking, so `hotkeys-top-k` doubles as the on/off switch: 0 disables + * detection, any positive value enables it and sets the Space-Saving capacity. + * The sampling percentage only sets how much traffic is sampled while enabled. */ +bool hotkeysEnabled(void) { + return server.hotkeys_top_k > 0; +} + +/* Number of sampled observations in the last completed window (N). The + * Space-Saving guarantee is stated relative to N: only keys with frequency + * above N/K are guaranteed tracked, so operators use it to gauge the detection + * floor and how much to trust a given entry. 0 when detection is disabled. */ +static uint64_t hotkeysLastWindowSamples(void) { + return server.hotkeys_manager ? spaceSavingManagerFrozenTotal(server.hotkeys_manager) : 0; +} + +/* Real duration of the last completed window, in microseconds. 0 means there is + * no completed window: detection was just enabled or reset, or the last window + * was dropped for spanning more than twice the configured length — those cases + * are not distinguishable from this value alone. */ +static uint64_t hotkeysLastWindowDurationUs(void) { + return server.hotkeys_manager ? spaceSavingManagerFrozenDurationUs(server.hotkeys_manager) : 0; +} + +/* Append the fields of the INFO "hotkeys" section. The caller emits the section + * header; this owns which fields the section carries. */ +sds genHotkeysInfoString(sds info) { + /* N for the last completed window: only keys above N/K are guaranteed + * tracked, so this gives operators the detection floor of a report. */ + info = sdscatprintf(info, "hotkeys_last_window_samples:%llu\r\n", (unsigned long long)hotkeysLastWindowSamples()); + /* The real span the report was measured over, which is the configured window + * plus the rotation lag — and the QPS denominator. */ + info = sdscatprintf(info, "hotkeys_last_window_duration_ms:%llu\r\n", + (unsigned long long)(hotkeysLastWindowDurationUs() / 1000)); + return info; +} + +/* Reconfigure the manager in place from the current config: the in-progress + * (live) window is reset (its counts were gathered under the old config), but + * the last completed (frozen) window is KEPT along with the config that + * produced it, so an operator's in-flight HOTKEYS GET still sees it. No-op when + * detection is disabled (no manager). Use HOTKEYS RESET to discard everything. */ +static void hotkeysManagerReconfigure(void) { + if (!server.hotkeys_manager) return; + spaceSavingManagerReconfigure(server.hotkeys_manager, server.hotkeys_top_k, + (uint64_t)server.hotkeys_window_seconds * 1000000ULL, getMonotonicUs()); + spaceSavingManagerSetLiveSamplingPercentage(server.hotkeys_manager, server.hotkeys_sampling_percentage); +} + +/* Create or free the manager to match the enabled state. */ +static void hotkeysManagerSetEnabled(int enabled) { + if (enabled && !server.hotkeys_manager) { + server.hotkeys_manager = hotkeysCreateManager(); + } else if (!enabled && server.hotkeys_manager) { + spaceSavingManagerRelease(server.hotkeys_manager); + server.hotkeys_manager = NULL; + } +} + +/* Bring up hot-key detection at server startup (creates the manager if enabled). */ +void hotkeysInit(void) { + hotkeysManagerSetEnabled(hotkeysEnabled()); +} + +/* =========================================================================== + * Config callbacks + * ==========================================================================*/ + +/* Sampling percentage only changes how much traffic is sampled; reconfigure in + * place so a live query still sees the last completed window (no-op if disabled). */ +int hotkeysSamplingCallback(const char **err) { + UNUSED(err); + hotkeysManagerReconfigure(); + return 1; +} + +/* top-k is also the on/off switch (0 disables), so it drives the manager + * lifecycle: crossing 0 creates or frees it, while a change that stays enabled + * reconfigures in place and keeps the last completed window. */ +int hotkeysTopKCallback(const char **err) { + UNUSED(err); + if (hotkeysEnabled() && server.hotkeys_manager) + hotkeysManagerReconfigure(); + else + hotkeysManagerSetEnabled(hotkeysEnabled()); + return 1; +} + +int hotkeysWindowCallback(const char **err) { + UNUSED(err); + hotkeysManagerReconfigure(); + return 1; +} diff --git a/src/hotkeys.h b/src/hotkeys.h new file mode 100644 index 000000000..919f69f8c --- /dev/null +++ b/src/hotkeys.h @@ -0,0 +1,46 @@ +#ifndef HOTKEYS_H +#define HOTKEYS_H + +#include +#include + +#include "sds.h" + +/* + * Server-side hot key detection. The Space-Saving algorithm, its frozen-window + * manager and the tracked (key, db) item live in space_saving.{c,h}; this module + * (hotkeys.c) is the policy layer around it: what counts as a recordable access, + * the sampling and enable configuration, config wiring, and the HOTKEYS + * commands. + */ + +typedef struct serverObject robj; + +/* Config callbacks (wired from config.c). */ +int hotkeysSamplingCallback(const char **err); +int hotkeysTopKCallback(const char **err); +int hotkeysWindowCallback(const char **err); + +/* Is hot-key detection currently enabled (hotkeys-top-k > 0)? */ +bool hotkeysEnabled(void); +/* Append the INFO "hotkeys" section fields (the caller emits the header). */ +sds genHotkeysInfoString(sds info); +/* Create the manager at server startup if detection is enabled. */ +void hotkeysInit(void); +/* Periodic maintenance (call from serverCron): freeze elapsed windows on time. */ +void hotkeysCron(void); + +/* Drop tracked keys: all, or scoped to a cluster slot / database. */ +void hotkeysPurgeAll(void); +void hotkeysPurgeSlot(int slot); +void hotkeysPurgeDb(int dbid); + +/* Charge a sampled access of `key` in database `dbid` to hot-key detection. + * Both apply the whole policy themselves (enabled, which activity counts, + * sampling), so callers in the data path need no hot-key knowledge: + * - Lookup: `lookup_flags` are the LOOKUP_* flags of the lookup. + * - Delete: `del_flags` are the DB_FLAG_* deletion reasons. */ +void hotkeysRecordLookup(robj *key, int dbid, int lookup_flags); +void hotkeysRecordDelete(robj *key, int dbid, int del_flags); + +#endif /* HOTKEYS_H */ diff --git a/src/io_threads.c b/src/io_threads.c index 6e3d91a03..ad3ee24ba 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -5,6 +5,10 @@ */ #include "io_threads.h" +#include "ae.h" +#include "cluster.h" +#include "cluster_legacy.h" +#include "connhelpers.h" #include "cluster_migrateslots.h" #include "connection.h" #include "queues.h" @@ -15,21 +19,37 @@ #define IO_SPMC_QUEUE_SIZE 4096 #define IO_SPSC_QUEUE_SIZE 4096 +/* QoS Swim Lanes for I/O threads + * High priority queues: reserved for critical internal communication such as + * cluster bus messages, slot migration, and replication streams + * Normal priority queues: used for normal client connections + */ +typedef enum { + /* Normal priority jobs - used for normal client connections */ + JOB_PRIORITY_NORMAL = 0, + /* High priority jobs - used for critical internal communication such as + * cluster bus messages, slot migration, and replication streams */ + JOB_PRIORITY_HIGH, + /* Number of priority levels */ + JOB_PRIORITY_COUNT +} jobPriority; + static _Thread_local int thread_id = 0; -static _Thread_local mpscTicket io_thread_ticket = {0}; +static _Thread_local mpscTicket io_thread_ticket[JOB_PRIORITY_COUNT] = {0}; /* Backlog of responses when io_shared_outbox is full. Should be rare. */ -static _Thread_local list *pending_io_responses = NULL; +static _Thread_local list *pending_io_responses[JOB_PRIORITY_COUNT] = {NULL, NULL}; static pthread_t io_threads[IO_THREADS_MAX_NUM] = {0}; static pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM]; static int cur_epoll_thread = 0; // Main -> IO: Shared Queue (Single Producer Multi Consumer) where all IO threads pull jobs from -static spmcQueue io_shared_inbox = {0}; +static spmcQueue io_shared_inbox[JOB_PRIORITY_COUNT] = {0}; // IO -> Main: Response Channel (Multi Producer Single Consumer) used by IO threads to send results back to main-thread -static mpscQueue io_shared_outbox = {0}; +static mpscQueue io_shared_outbox[JOB_PRIORITY_COUNT] = {0}; // Main -> IO (Thread-Specific) for tasks that must run on specific IO thread where IO threads check their private inbox before the shared queue static spscQueue io_private_inbox[IO_THREADS_MAX_NUM] = {0}; static size_t io_jobs_submitted; static _Atomic(size_t) io_jobs_finished; +static size_t cluster_io_pending_responses; static int io_threads_initialized = 0; _Atomic long long used_active_time_io_thread[IO_THREADS_MAX_NUM] = {0}; @@ -39,6 +59,10 @@ _Atomic long long used_active_time_io_thread[IO_THREADS_MAX_NUM] = {0}; #define JOB_TAG_MASK 0x7 #define JOB_PTR_MASK (~(uintptr_t)JOB_TAG_MASK) +static inline jobPriority getJobPriority(const client *c) { + return (c && connIsPriority(c->conn)) ? JOB_PRIORITY_HIGH : JOB_PRIORITY_NORMAL; +} + static inline void *tagJob(void *ptr, int type) { return (void *)((uintptr_t)ptr | type); } @@ -75,8 +99,8 @@ static size_t getPendingIOThreadsJobs(void) { } /* Read/write jobs awaiting response from IO threads. */ -static int getPendingIOResponsesCount(void) { - return server.stat_io_writes_pending + server.stat_io_reads_pending; +static size_t getPendingIOResponsesCount(void) { + return server.stat_io_writes_pending + server.stat_io_reads_pending + cluster_io_pending_responses; } /* Drains the I/O threads queue by waiting for all jobs to be processed. @@ -189,8 +213,8 @@ void IOThreadsAfterSleep(int numevents) { if (now - last_sample_time < IO_SAMPLE_RATE_MS) return; last_sample_time = now; - size_t q_size = spmcSize(&io_shared_inbox); - spmc_size_sum += q_size; + spmc_size_sum += spmcSize(&io_shared_inbox[JOB_PRIORITY_NORMAL]); + spmc_size_sum += spmcSize(&io_shared_inbox[JOB_PRIORITY_HIGH]); sample_count++; trackInstantaneousMetric(STATS_METRIC_IO_WAIT, spmc_size_sum, sample_count, 1); @@ -225,7 +249,7 @@ void IOThreadsAfterSleep(int numevents) { /* Don't suspend if work remains in the specific thread's queue... */ if (!spscIsEmpty(&io_private_inbox[tid])) return; /* ...or if we are dropping to 1 thread but the global queue still has work */ - if (target == 1 && !spmcIsEmpty(&io_shared_inbox)) return; + if (target == 1 && (!spmcIsEmpty(&io_shared_inbox[JOB_PRIORITY_NORMAL]) || !spmcIsEmpty(&io_shared_inbox[JOB_PRIORITY_HIGH]))) return; pthread_mutex_lock(&io_threads_mutex[tid]); server.active_io_threads_num--; @@ -242,11 +266,11 @@ void ioThreadPoll(aeEventLoop *el) { atomic_store_explicit(&server.io_poll_state, AE_IO_STATE_DONE, memory_order_release); } -static void flushPendingIOResponses(int blocking) { - if (!pending_io_responses) return; +static void flushPendingIOResponsesList(list **pending_list, mpscQueue *outbox, mpscTicket *ticket, int blocking) { + if (*pending_list == NULL) return; listIter li; listNode *ln; - listRewind(pending_io_responses, &li); + listRewind(*pending_list, &li); while ((ln = listNext(&li))) { void *job = listNodeValue(ln); @@ -254,21 +278,26 @@ static void flushPendingIOResponses(int blocking) { /* Try to enqueue. If blocking is set, retry until success. */ do { - pushed = mpscEnqueue(&io_shared_outbox, job, &io_thread_ticket); + pushed = mpscEnqueue(outbox, job, ticket); if (pushed || !blocking || server.crashed) break; /* On server crash we kill the IO threads, no point in sending back jobs to the main-thread. */ atomic_thread_fence(memory_order_acquire); } while (true); if (pushed) { - listDelNode(pending_io_responses, ln); + listDelNode(*pending_list, ln); } else { return; } } /* List is fully drained */ - listRelease(pending_io_responses); - pending_io_responses = NULL; + listRelease(*pending_list); + *pending_list = NULL; +} + +static void flushPendingIOResponses(int blocking) { + flushPendingIOResponsesList(&pending_io_responses[JOB_PRIORITY_HIGH], &io_shared_outbox[JOB_PRIORITY_HIGH], &io_thread_ticket[JOB_PRIORITY_HIGH], blocking); + flushPendingIOResponsesList(&pending_io_responses[JOB_PRIORITY_NORMAL], &io_shared_outbox[JOB_PRIORITY_NORMAL], &io_thread_ticket[JOB_PRIORITY_NORMAL], blocking); } /* Define a cleanup function that will clean all thread resources */ @@ -282,6 +311,41 @@ void cleanupThreadResources(void *dummy) { freeSharedQueryBuf(); } +static inline void processTaggedSPMCJob(void *tagged_job) { + void *data; + int type; + untagJob(tagged_job, &data, &type); + + switch (type) { + case JOB_REQ_READ_CLIENT: + ioThreadReadQueryFromClient((client *)data); + break; + case JOB_REQ_WRITE_CLIENT: + ioThreadWriteToClient((client *)data); + break; + case JOB_REQ_FREE_OBJ: + decrRefCount(data); + break; + case JOB_REQ_ACCEPT: + ioThreadAccept((client *)data); + break; + case JOB_REQ_POLL: + ioThreadPoll((aeEventLoop *)data); + break; + case JOB_REQ_CLUSTER_READ: + clusterReadJob((clusterLink *)data); + break; + case JOB_REQ_CLUSTER_WRITE: + clusterWriteJob((clusterLink *)data); + break; + case JOB_REQ_CLUSTER_ACCEPT: + clusterAcceptJob((connection *)data); + break; + default: + serverPanic("Invalid SPMC job type: %d", type); + } +} + static void *IOThreadMain(void *myid) { /* The ID is the thread ID number (from 1 to server.io_threads_num-1). ID 0 is the main thread. */ long id = (long)myid; @@ -317,10 +381,10 @@ static void *IOThreadMain(void *myid) { untagJob(batch_jobs[i], &data, &type); switch (type) { - case JOB_REQ_FREE_ARGV: + case JOB_SPSC_FREE_ARGV: ioThreadFreeArgv((robj **)data); break; - case JOB_REQ_POLL: + case JOB_SPSC_POLL: ioThreadPoll((aeEventLoop *)data); break; default: @@ -332,31 +396,14 @@ static void *IOThreadMain(void *myid) { /* PRIORITY 2: Shared Global Queue (SPMC) * Only checked after SPSC is drained. */ - void *tagged_job = spmcDequeue(&io_shared_inbox); - if (tagged_job) { - void *data; - int type; - untagJob(tagged_job, &data, &type); - - switch (type) { - case JOB_REQ_READ_CLIENT: - ioThreadReadQueryFromClient((client *)data); - break; - case JOB_REQ_WRITE_CLIENT: - ioThreadWriteToClient((client *)data); - break; - case JOB_REQ_FREE_OBJ: - decrRefCount(data); - break; - case JOB_REQ_ACCEPT: - ioThreadAccept((client *)data); - break; - case JOB_REQ_POLL: - ioThreadPoll((aeEventLoop *)data); - break; - default: - serverPanic("Invalid SPMC job type: %d", type); - } + void *tagged_job; + if ((tagged_job = spmcDequeue(&io_shared_inbox[JOB_PRIORITY_HIGH])) != NULL) { + processTaggedSPMCJob(tagged_job); + processed++; + } + + if ((tagged_job = spmcDequeue(&io_shared_inbox[JOB_PRIORITY_NORMAL])) != NULL) { + processTaggedSPMCJob(tagged_job); processed++; } @@ -366,7 +413,11 @@ static void *IOThreadMain(void *myid) { /* If both queues were empty (no processing done), wait for signal. */ if (processed == 0) { - if (unlikely(pending_io_responses)) { + int has_pending = 0; + for (int p = 0; p < JOB_PRIORITY_COUNT; p++) { + if (pending_io_responses[p]) has_pending = 1; + } + if (unlikely(has_pending)) { flushPendingIOResponses(0); } else { /* If it is locked. We should block until main thread unlocks it. */ @@ -453,7 +504,12 @@ int updateIOThreads(const char **err) { * in that state, we will deadlock (Main thread waits for worker, Worker waits for queue space). */ size_t pending = getPendingIOResponsesCount(); - if (pending > io_shared_outbox.queue_size) { + /* Since pending is the sum of all in-flight read/write jobs, in the worst-case scenario where + * 100% of the traffic happens to be on one priority lane, that outbox will receive at most pending + * responses. If pending fits within each queue's capacity, neither queue can ever overflow or cause + * workers to block while draining*/ + if (pending > io_shared_outbox[JOB_PRIORITY_NORMAL].queue_size || + pending > io_shared_outbox[JOB_PRIORITY_HIGH].queue_size) { if (err) *err = "Can't update IO threads under load, try again later"; return 0; } @@ -492,10 +548,13 @@ void initIOThreads(int prev_threads_num) { server.active_io_threads_num = 1; /* We start with threads not active. */ server.io_poll_state = AE_IO_STATE_NONE; server.io_ae_fired_events = 0; - spmcInit(&io_shared_inbox, IO_SPMC_QUEUE_SIZE); - mpscInit(&io_shared_outbox, IO_MPSC_QUEUE_SIZE); + for (int p = 0; p < JOB_PRIORITY_COUNT; p++) { + spmcInit(&io_shared_inbox[p], IO_SPMC_QUEUE_SIZE); + mpscInit(&io_shared_outbox[p], IO_MPSC_QUEUE_SIZE); + } io_jobs_submitted = 0; atomic_init(&io_jobs_finished, 0); + cluster_io_pending_responses = 0; prefetchCommandsBatchInit(); io_threads_initialized = 1; } @@ -506,6 +565,54 @@ void initIOThreads(int prev_threads_num) { } } +void testOnlyInitIOThreadQueues(void) { + for (int p = 0; p < JOB_PRIORITY_COUNT; p++) { + if (io_shared_inbox[p].buffer) spmcFree(&io_shared_inbox[p]); + if (io_shared_outbox[p].buffer) mpscFree(&io_shared_outbox[p]); + if (pending_io_responses[p]) { + listRelease(pending_io_responses[p]); + pending_io_responses[p] = NULL; + } + spmcInit(&io_shared_inbox[p], IO_SPMC_QUEUE_SIZE); + mpscInit(&io_shared_outbox[p], IO_MPSC_QUEUE_SIZE); + io_thread_ticket[p] = (mpscTicket){0}; + } + io_jobs_submitted = 0; + atomic_store_explicit(&io_jobs_finished, 0, memory_order_relaxed); + cluster_io_pending_responses = 0; +} + +void testOnlyFreeIOThreadQueues(void) { + for (int p = 0; p < JOB_PRIORITY_COUNT; p++) { + if (pending_io_responses[p]) { + listRelease(pending_io_responses[p]); + pending_io_responses[p] = NULL; + } + spmcFree(&io_shared_inbox[p]); + mpscFree(&io_shared_outbox[p]); + io_thread_ticket[p] = (mpscTicket){0}; + } + io_jobs_submitted = 0; + atomic_store_explicit(&io_jobs_finished, 0, memory_order_relaxed); + cluster_io_pending_responses = 0; +} + +/* Fill the shared inbox so the next dispatch has to take its enqueue-failure + * path. The queue is file-static, so tests cannot do this themselves. */ +void testOnlyFillIOThreadInbox(void) { + for (int p = 0; p < JOB_PRIORITY_COUNT; p++) { + while (spmcEnqueue(&io_shared_inbox[p], (void *)-1)) { + /* Keep going until the queue rejects the push. */ + } + } +} + +/* Expose the cluster pending-response count so tests can assert that a failed + * or completed dispatch leaves no response outstanding. */ +size_t testOnlyGetClusterIOPendingResponses(void) { + return cluster_io_pending_responses; +} + int trySendReadToIOThreads(client *c) { if (server.active_io_threads_num <= 1) return C_ERR; /* Fake/teardown clients may have no connection; never offload those. */ @@ -518,6 +625,9 @@ int trySendReadToIOThreads(client *c) { if (c->io_write_state == CLIENT_PENDING_IO) return C_OK; /* For simplicity, don't offload replica clients reads as read traffic from replica is negligible */ if (getClientType(c) == CLIENT_TYPE_REPLICA) return C_ERR; + /* A live replication stream reader must run on the main thread; the IO-thread + * read path does not decode. Destroyed once the probe resolves to plaintext. */ + if (c->flag.primary && server.repl_stream_reader) return C_ERR; /* With Lua debug client we may call connWrite directly in the main thread */ if (c->flag.lua_debug) return C_ERR; /* For simplicity let the main-thread handle the blocked clients */ @@ -534,7 +644,8 @@ int trySendReadToIOThreads(client *c) { c->io_read_state = CLIENT_PENDING_IO; connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); - if (unlikely(spmcEnqueue(&io_shared_inbox, tagJob(c, JOB_REQ_READ_CLIENT)) == false)) { + jobPriority qidx = getJobPriority(c); + if (unlikely(spmcEnqueue(&io_shared_inbox[qidx], tagJob(c, JOB_REQ_READ_CLIENT)) == false)) { c->read_flags = 0; c->io_read_state = CLIENT_IDLE; connSetPostponeUpdateState(c->conn, 0); @@ -595,7 +706,9 @@ int trySendWriteToIOThreads(client *c) { c->io_write_state = CLIENT_PENDING_IO; connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); void *job = tagJob(c, JOB_REQ_WRITE_CLIENT); - if (unlikely(spmcEnqueue(&io_shared_inbox, job) == false)) { + + jobPriority qidx = getJobPriority(c); + if (unlikely(spmcEnqueue(&io_shared_inbox[qidx], job) == false)) { c->io_write_state = CLIENT_IDLE; connSetPostponeUpdateState(c->conn, 0); c->write_flags = 0; @@ -622,6 +735,171 @@ int trySendWriteToIOThreads(client *c) { return C_OK; } +/* Try to offload a cluster link read to an I/O thread. + * Enqueues a tagged job onto io_shared_inbox (SPMC queue). + * Returns C_OK if offloaded or if a job is already pending (to prevent + * the caller from falling back to synchronous I/O on a connection + * with an in-flight worker job). + * Returns C_ERR if fallback is needed (pool inactive or spmcEnqueue fails). */ +int trySendClusterReadToIOThreads(struct clusterLink *link) { + /* If any I/O job is already in flight for this link, return C_OK + * so the caller does NOT fall back to synchronous I/O. */ + if (link->io_read_state != CLUSTER_LINK_IO_IDLE) return C_OK; + if (link->io_write_state != CLUSTER_LINK_IO_IDLE) { + link->io_read_deferred = 1; + return C_OK; + } + link->io_read_deferred = 0; + + /* Invariant: io_refs must be 0 when both states are IDLE. */ + serverAssert(link->io_refs == 0); + + /* clusterReadHandler() drains any queued complete packets before + * attempting a new dispatch. */ + serverAssert(link->io_complete_bytes == 0); + serverAssert(link->io_complete_packets == 0); + + /* The connection is not established yet. See the equivalent guard in + * trySendClusterWriteToIOThreads() for why we return C_OK here. */ + if (connGetState(link->conn) != CONN_STATE_CONNECTED) return C_OK; + + /* No I/O thread pool available — synchronous fallback. */ + if (server.active_io_threads_num <= 1) { + server.stat_cluster_io_main_thread_fallbacks++; + return C_ERR; + } + + /* Postpone connection state updates while the I/O thread operates. */ + connSetPostponeUpdateState(link->conn, 1); + + /* Transition link to pending-read state. */ + link->io_read_state = CLUSTER_LINK_IO_PENDING; + link->io_refs++; + link->rcvbuf_alloc_at_dispatch = link->rcvbuf_alloc; + + /* Enqueue the read job. */ + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_HIGH], tagJob(link, JOB_REQ_CLUSTER_READ)) == false)) { + /* Rollback on enqueue failure. */ + link->io_read_state = CLUSTER_LINK_IO_IDLE; + link->io_refs--; + connSetPostponeUpdateState(link->conn, 0); + server.stat_cluster_io_main_thread_fallbacks++; + return C_ERR; + } + + io_jobs_submitted++; + cluster_io_pending_responses++; + return C_OK; +} + +/* Try to offload a cluster link write to an I/O thread. + * Enqueues a tagged job onto io_shared_inbox after snapshotting the current + * head offset and the last queue node visible to the worker. New messages + * appended by clusterSendMessage during the write stay queued on the main + * thread and are picked up by a later dispatch. + * Returns C_OK if offloaded or if a job is already pending (to prevent + * the caller from falling back to synchronous I/O on a connection + * with an in-flight worker job). + * Returns C_ERR if fallback is needed (pool inactive or spmcEnqueue fails). */ +int trySendClusterWriteToIOThreads(struct clusterLink *link) { + listNode *last_send_block; + + /* If any I/O job is already in flight for this link, return C_OK + * so the caller does NOT fall back to synchronous I/O. */ + if (link->io_write_state != CLUSTER_LINK_IO_IDLE) return C_OK; + if (link->io_read_state != CLUSTER_LINK_IO_IDLE) return C_OK; + + /* Invariant: io_refs must be 0 when both states are IDLE. */ + serverAssert(link->io_refs == 0); + + /* Nothing to write. */ + if (listLength(link->send_msg_queue) == 0) return C_OK; + + /* The connection is still being established (TCP connect or TLS handshake + * in progress). Don't dispatch: connWrite() fails with a non-EAGAIN error + * on a connection that isn't connected yet, the worker reports + * CLUSTER_IO_WRITE_ERROR and the completion handler turns that into a link + * teardown, so a slow handshake would kill the link. Nothing is stranded: + * the caller installed the write handler and the connection layer drives it + * once the handshake completes. Return C_OK so the caller neither retries + * synchronously (which fails the same way, see clusterWriteHandler) nor + * records a main-thread fallback. */ + if (connGetState(link->conn) != CONN_STATE_CONNECTED) return C_OK; + + /* No I/O thread pool available — synchronous fallback. */ + if (server.active_io_threads_num <= 1) { + server.stat_cluster_io_main_thread_fallbacks++; + return C_ERR; + } + + /* Yield one dispatch to a read skipped earlier: WRITE_BARRIER fires writable + * first, so a never-empty send queue would re-claim the link and never read. */ + if (link->io_read_deferred) { + link->io_read_deferred = 0; + return C_OK; + } + + last_send_block = listLast(link->send_msg_queue); + serverAssert(last_send_block != NULL); + + /* Postpone connection state updates while the I/O thread operates. */ + connSetPostponeUpdateState(link->conn, 1); + + /* Snapshot the canonical queue for one write job. */ + link->io_last_send_block = last_send_block; + link->io_head_offset = link->head_msg_send_offset; + link->io_nodes_sent = 0; + + /* Transition link to pending-write state. */ + link->io_write_state = CLUSTER_LINK_IO_PENDING; + link->io_refs++; + + /* Enqueue the write job. */ + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_HIGH], tagJob(link, JOB_REQ_CLUSTER_WRITE)) == false)) { + link->io_write_state = CLUSTER_LINK_IO_IDLE; + link->io_refs--; + link->io_last_send_block = NULL; + link->io_head_offset = 0; + link->io_nodes_sent = 0; + connSetPostponeUpdateState(link->conn, 0); + server.stat_cluster_io_main_thread_fallbacks++; + return C_ERR; + } + + io_jobs_submitted++; + cluster_io_pending_responses++; + return C_OK; +} + +/* Try to offload a cluster TLS accept to an I/O thread. + * Called from clusterAcceptHandler BEFORE any clusterLink exists. + * Returns C_OK if offloaded, C_ERR if fallback is needed. */ +int trySendClusterAcceptToIOThreads(connection *conn) { + if (!(conn->flags & CONN_FLAG_ALLOW_ACCEPT_OFFLOAD)) return C_ERR; + /* A cluster accept job is already in flight for this connection. */ + if (conn->flags & CONN_FLAG_ACCEPT_OFFLOAD_PENDING) return C_OK; + if (server.active_io_threads_num <= 1) { + server.stat_cluster_io_main_thread_fallbacks++; + return C_ERR; + } + + conn->flags |= CONN_FLAG_ACCEPT_OFFLOAD_PENDING; + connSetPostponeUpdateState(conn, 1); + connIncrRefs(conn); + + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_HIGH], tagJob(conn, JOB_REQ_CLUSTER_ACCEPT)) == false)) { + connDecrRefs(conn); + connSetPostponeUpdateState(conn, 0); + conn->flags &= ~CONN_FLAG_ACCEPT_OFFLOAD_PENDING; + server.stat_cluster_io_main_thread_fallbacks++; + return C_ERR; + } + + io_jobs_submitted++; + cluster_io_pending_responses++; + return C_OK; +} + /* Internal function to free the client's argv in an IO thread. */ void ioThreadFreeArgv(robj **argv) { int last_arg = 0; @@ -687,7 +965,7 @@ int tryOffloadFreeArgvToIOThreads(client *c, int argc, robj **argv) { * this is the last argument to free. With this approach, we don't need to * send the argc to the IO thread and we can send just the argv ptr. */ argv[last_arg_to_free]->refcount = 0; - void *job = tagJob(argv, JOB_REQ_FREE_ARGV); + void *job = tagJob(argv, JOB_SPSC_FREE_ARGV); /* We pass false to enqueue the job without committing the queue index immediately. * This allows us to batch multiple free jobs together and * commit them in a single operation later in the event loop. This reduces the overhead @@ -712,7 +990,7 @@ int tryOffloadFreeObjToIOThreads(robj *obj) { if (obj->encoding != OBJ_ENCODING_RAW || obj->type != OBJ_STRING) return C_ERR; void *job = tagJob(obj, JOB_REQ_FREE_OBJ); - if (unlikely(spmcEnqueue(&io_shared_inbox, job) == false)) return C_ERR; + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_NORMAL], job) == false)) return C_ERR; io_jobs_submitted++; server.stat_io_freed_objects++; return C_OK; @@ -754,14 +1032,12 @@ void trySendPollJobToIOThreads(void) { return; } - void *job = tagJob(server.el, JOB_REQ_POLL); - server.io_poll_state = AE_IO_STATE_POLL; aeSetPollProtect(server.el, 1); /* Use SPMC to minimize polling overhead. At high thread counts, use private SPSC queues for lower latency. */ if (server.active_io_threads_num <= 9) { - if (unlikely(spmcEnqueue(&io_shared_inbox, job) == false)) { + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_NORMAL], tagJob(server.el, JOB_REQ_POLL)) == false)) { server.io_poll_state = AE_IO_STATE_NONE; aeSetPollProtect(server.el, 0); return; @@ -773,7 +1049,7 @@ void trySendPollJobToIOThreads(void) { aeSetPollProtect(server.el, 0); return; } - spscEnqueue(&io_private_inbox[cur_epoll_thread], job, true); + spscEnqueue(&io_private_inbox[cur_epoll_thread], tagJob(server.el, JOB_SPSC_POLL), true); } aeSetCustomPollProc(server.el, getIOThreadPollResults); @@ -781,16 +1057,22 @@ void trySendPollJobToIOThreads(void) { } void sendToMainThread(void *data, int type) { - if (unlikely(pending_io_responses)) { - flushPendingIOResponses(0); + jobPriority qidx = JOB_PRIORITY_NORMAL; + if (type == JOB_RES_READ_CLIENT || type == JOB_RES_WRITE_CLIENT) { + client *c = (client *)data; + qidx = getJobPriority(c); + } else if (type == JOB_RES_CLUSTER_READ || type == JOB_RES_CLUSTER_WRITE || type == JOB_RES_CLUSTER_ACCEPT) { + qidx = JOB_PRIORITY_HIGH; + } + if (unlikely(pending_io_responses[qidx])) { + flushPendingIOResponsesList(&pending_io_responses[qidx], &io_shared_outbox[qidx], &io_thread_ticket[qidx], 0); } void *job = tagJob(data, type); - if (unlikely(pending_io_responses || !mpscEnqueue(&io_shared_outbox, job, &io_thread_ticket))) { - /* Failed to push new job: initialize list if needed and save job */ - if (pending_io_responses == NULL) { - pending_io_responses = listCreate(); + if (unlikely(pending_io_responses[qidx] || !mpscEnqueue(&io_shared_outbox[qidx], job, &io_thread_ticket[qidx]))) { + if (pending_io_responses[qidx] == NULL) { + pending_io_responses[qidx] = listCreate(); } - listAddNodeTail(pending_io_responses, job); + listAddNodeTail(pending_io_responses[qidx], job); } } @@ -821,7 +1103,14 @@ int trySendAcceptToIOThreads(connection *conn) { return C_ERR; } + /* Cluster TLS accepts have no client private-data yet. Route them to the + * dedicated cluster accept offload path. */ + if (connGetOwnerKind(conn) == CONN_OWNER_CLUSTER_LINK) { + return trySendClusterAcceptToIOThreads(conn); + } + client *c = connGetPrivateData(conn); + serverAssert(c != NULL); if (c->io_read_state != CLIENT_IDLE) { return C_OK; } @@ -835,7 +1124,7 @@ int trySendAcceptToIOThreads(connection *conn) { connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); void *job = tagJob(c, JOB_REQ_ACCEPT); - if (unlikely(spmcEnqueue(&io_shared_inbox, job) == false)) { + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_NORMAL], job) == false)) { c->io_read_state = CLIENT_IDLE; c->flag.pending_read = 0; connSetPostponeUpdateState(c->conn, 0); @@ -873,7 +1162,8 @@ static void handleReadJobs(client **read_jobs, int read_count) { client *c = lookupClientByID(read_client_ids[i]); if (!c || !c->conn) continue; - if (processPendingCommandAndInputBuffer(c) == C_OK) beforeNextClient(c); + if (processPendingCommandAndInputBuffer(c) == C_ERR) continue; + beforeNextClient(c); c = lookupClientByID(read_client_ids[i]); if (!c || !c->conn) continue; @@ -895,55 +1185,83 @@ static void handleWriteJobs(client **write_jobs, int write_count) { } } -int processIOThreadsResponses(void) { - /* We don't check for threads number since some threads may return jobs then deactivate/shut-down */ - - /* Quick check if any pending operations exist */ - if (getPendingIOResponsesCount() == 0) return 0; - - int total_processed = 0; +static int processOutboxBatch(mpscQueue *outbox) { void *jobs[JOB_BATCH_SIZE]; client *read_jobs[JOB_BATCH_SIZE]; client *write_jobs[JOB_BATCH_SIZE]; + int received_responses = 0; + int read_count = 0; + int write_count = 0; - /* Loop until we consume all pending jobs */ - while (1) { - int received_responses = 0; - int dequeued_count = 0; - int read_count = 0; - int write_count = 0; - - /* Try to dequeue JOB_BATCH_SIZE */ - while (received_responses < JOB_BATCH_SIZE) { - dequeued_count = mpscDequeueBatch(&io_shared_outbox, jobs, JOB_BATCH_SIZE - received_responses); + /* Try to dequeue JOB_BATCH_SIZE */ + while (received_responses < JOB_BATCH_SIZE) { + int dequeued_count = mpscDequeueBatch(outbox, jobs, JOB_BATCH_SIZE - received_responses); - /* Stop if we can't get more jobs from the queue. */ - if (dequeued_count == 0) break; + /* Stop if we can't get more jobs from the queue. */ + if (dequeued_count == 0) break; - received_responses += dequeued_count; - total_processed += dequeued_count; + received_responses += dequeued_count; - for (int i = 0; i < dequeued_count; i++) { - void *data; - int job_type; - untagJob(jobs[i], &data, &job_type); + for (int i = 0; i < dequeued_count; i++) { + void *data; + int job_type; + untagJob(jobs[i], &data, &job_type); + if (job_type == JOB_RES_READ_CLIENT) { client *c = (client *)data; - if (job_type == JOB_RES_READ_CLIENT) { - serverAssert(c->io_read_state == CLIENT_COMPLETED_IO); - read_jobs[read_count++] = c; - } else if (job_type == JOB_RES_WRITE_CLIENT) { - serverAssert(c->io_write_state == CLIENT_COMPLETED_IO); - write_jobs[write_count++] = c; - } else { - serverPanic("Unknown job type %d", job_type); - } + serverAssert(c->io_read_state == CLIENT_COMPLETED_IO); + read_jobs[read_count++] = c; + } else if (job_type == JOB_RES_WRITE_CLIENT) { + client *c = (client *)data; + serverAssert(c->io_write_state == CLIENT_COMPLETED_IO); + write_jobs[write_count++] = c; + } else if (job_type == JOB_RES_CLUSTER_READ) { + serverAssert(cluster_io_pending_responses > 0); + cluster_io_pending_responses--; + server.stat_cluster_threaded_reads_processed++; + clusterHandleReadCompletion((struct clusterLink *)data); + } else if (job_type == JOB_RES_CLUSTER_WRITE) { + serverAssert(cluster_io_pending_responses > 0); + cluster_io_pending_responses--; + server.stat_cluster_threaded_writes_processed++; + clusterHandleWriteCompletion((struct clusterLink *)data); + } else if (job_type == JOB_RES_CLUSTER_ACCEPT) { + serverAssert(cluster_io_pending_responses > 0); + cluster_io_pending_responses--; + server.stat_cluster_threaded_accepts_processed++; + clusterHandleAcceptCompletion((connection *)data); + } else { + serverPanic("Unknown job type %d", job_type); } } + } - if (read_count) handleReadJobs(read_jobs, read_count); - if (write_count) handleWriteJobs(write_jobs, write_count); + if (read_count) handleReadJobs(read_jobs, read_count); + if (write_count) handleWriteJobs(write_jobs, write_count); + return received_responses; +} - /* If the queue was empty at the last try - don't try again */ - if (dequeued_count == 0) return total_processed; +/* Process completed IO jobs from worker threads back onto the main thread. + * Drains the high-priority outbox first to guarantee control-plane responsiveness, + * and performs periodic preemptive polling of QoS events while consuming normal jobs. */ +int processIOThreadsResponses(void) { + /* We don't check for threads number since some threads may return jobs then deactivate/shut-down */ + + /* Quick check if any pending operations exist across any priority level */ + if (getPendingIOResponsesCount() == 0) return 0; + + int total_processed = 0; + /* Loop until we consume all pending jobs */ + while (1) { + /* 1. Strict Priority: First, drain high-priority events (cluster bus, replication, and slot migration jobs) */ + int processed = processOutboxBatch(&io_shared_outbox[JOB_PRIORITY_HIGH]); + if (processed == 0) { + /* 2. Preemptive Poll: When high-priority outbox is empty, check if any new + * high-priority events arrived on QoS channels before processing normal traffic. */ + aeProcessQoSEventsPreemptively(server.el); + } + /* 3. Drain normal client events */ + processed += processOutboxBatch(&io_shared_outbox[JOB_PRIORITY_NORMAL]); + total_processed += processed; + if (processed == 0) return total_processed; } } diff --git a/src/io_threads.h b/src/io_threads.h index 4202f6508..06d4cf889 100644 --- a/src/io_threads.h +++ b/src/io_threads.h @@ -3,23 +3,43 @@ #include "server.h" +/* Tag values for tagged pointers on work/response queues. + * Tags must fit in 3 bits (0-7) due to 8-byte alignment from jemalloc + * with --with-lg-quantum=3. SPSC and SPMC tags are separate enums + * because they occupy independent queues and may reuse values. */ + +/* Tags for the SPSC private inbox (main thread → specific I/O thread). */ +typedef enum { + JOB_SPSC_FREE_ARGV = 0, + JOB_SPSC_POLL = 1, +} JobRequestSPSC; + +/* Tags for the SPMC shared inbox (main thread → any I/O thread). */ typedef enum { JOB_REQ_READ_CLIENT = 0, JOB_REQ_WRITE_CLIENT, - JOB_REQ_FREE_ARGV, JOB_REQ_FREE_OBJ, JOB_REQ_POLL, JOB_REQ_ACCEPT, + JOB_REQ_CLUSTER_READ, + JOB_REQ_CLUSTER_WRITE, + JOB_REQ_CLUSTER_ACCEPT, JOB_REQ_COUNT -} JobRequest; -_Static_assert(JOB_REQ_COUNT <= 8, "JOB_REQ_COUNT must not exceed 8 for pointer arithmetic"); +} JobRequestSPMC; +static_assert(JOB_REQ_COUNT <= 8, "JOB_REQ_COUNT must not exceed 8 for pointer arithmetic"); +/* Tags for the MPSC response queue (I/O threads → main thread). */ typedef enum { JOB_RES_READ_CLIENT = 0, JOB_RES_WRITE_CLIENT, + JOB_RES_CLUSTER_READ, + JOB_RES_CLUSTER_WRITE, + JOB_RES_CLUSTER_ACCEPT, JOB_RES_COUNT } JobResult; -_Static_assert(JOB_RES_COUNT <= 8, "JOB_RES_COUNT must not exceed 8 for pointer arithmetic"); +static_assert(JOB_RES_COUNT <= 8, "JOB_RES_COUNT must not exceed 8 for pointer arithmetic"); + +typedef void (*job_handler)(void *); void initIOThreads(int prev_threads_num); void killIOThreads(void); @@ -31,8 +51,16 @@ int tryOffloadFreeArgvToIOThreads(client *c, int argc, robj **argv); void IOThreadsAfterSleep(int numevents); void IOThreadsBeforeSleep(long long current_time); void drainIOThreadsQueue(void); +void testOnlyInitIOThreadQueues(void); +void testOnlyFreeIOThreadQueues(void); +void testOnlyFillIOThreadInbox(void); +size_t testOnlyGetClusterIOPendingResponses(void); void trySendPollJobToIOThreads(void); int trySendAcceptToIOThreads(connection *conn); +struct clusterLink; +int trySendClusterReadToIOThreads(struct clusterLink *link); +int trySendClusterWriteToIOThreads(struct clusterLink *link); +int trySendClusterAcceptToIOThreads(connection *conn); int updateIOThreads(const char **err); long long getIOThreadActiveTimeMicroseconds(int id); int clientHasPendingIO(struct client *c); diff --git a/src/latency.h b/src/latency.h index a1e327bbb..eed2f14f1 100644 --- a/src/latency.h +++ b/src/latency.h @@ -103,11 +103,13 @@ typedef struct durationStats { } durationStats; typedef enum { - EL_DURATION_TYPE_EL = 0, // cumulative time duration metric of the whole eventloop - EL_DURATION_TYPE_CMD, // cumulative time duration metric of executing commands - EL_DURATION_TYPE_AOF, // cumulative time duration metric of flushing AOF in eventloop - EL_DURATION_TYPE_CRON, // cumulative time duration metric of cron (serverCron and beforeSleep, but excluding IO and - // AOF) + EL_DURATION_TYPE_EL = 0, // cumulative time duration metric of the whole eventloop + EL_DURATION_TYPE_CMD, // cumulative time duration metric of executing commands + EL_DURATION_TYPE_AOF, // cumulative time duration metric of flushing AOF in eventloop + EL_DURATION_TYPE_CRON, // cumulative time duration metric of cron (serverCron and beforeSleep, but excluding IO and + // AOF) + EL_DURATION_TYPE_PRIORITY_EL, // cumulative time duration metric of priority eventloop + EL_DURATION_TYPE_PRIORITY_CMD, // cumulative time duration metric of priority commands execution EL_DURATION_TYPE_NUM } DurationType; diff --git a/src/listpack.c b/src/listpack.c index 5668f19a8..b700c5f90 100644 --- a/src/listpack.c +++ b/src/listpack.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "listpack.h" #include "listpack_malloc.h" @@ -47,7 +48,6 @@ #define LP_HDR_SIZE 6 /* 32 bit total len + 16 bit number of elements. */ #define LP_HDR_NUMELE_UNKNOWN UINT16_MAX -#define LP_MAX_INT_ENCODING_LEN 9 #define LP_MAX_BACKLEN_SIZE 5 #define LP_ENCODING_INT 0 #define LP_ENCODING_STRING 1 @@ -94,6 +94,18 @@ #define LP_ENCODING_32BIT_STR_MASK 0xFF #define LP_ENCODING_IS_32BIT_STR(byte) (((byte) & LP_ENCODING_32BIT_STR_MASK) == LP_ENCODING_32BIT_STR) + +/* Tagged (metadata) entry marker (11110101). + * + * The tag is followed by an inner (metadata) element. A tagged element doesn't + * count in the listpack's number of elements and is skipped when traversing the + * listpack. + * + * Tagged Element: [F5][metadata_element][backlen] */ +#define LP_ENCODING_TAGGED 0xF5 +#define LP_ENCODING_TAGGED_MASK 0xFF +#define LP_ENCODING_IS_TAGGED(byte) (((byte) & LP_ENCODING_TAGGED_MASK) == LP_ENCODING_TAGGED) + #define LP_EOF 0xFF #define LP_ENCODING_6BIT_STR_LEN(p) ((p)[0] & 0x3F) @@ -101,9 +113,6 @@ #define LP_ENCODING_32BIT_STR_LEN(p) \ (((uint32_t)(p)[1] << 0) | ((uint32_t)(p)[2] << 8) | ((uint32_t)(p)[3] << 16) | ((uint32_t)(p)[4] << 24)) -#define lpGetTotalBytes(p) \ - (((uint32_t)(p)[0] << 0) | ((uint32_t)(p)[1] << 8) | ((uint32_t)(p)[2] << 16) | ((uint32_t)(p)[3] << 24)) - #define lpGetNumElements(p) (((uint32_t)(p)[4] << 0) | ((uint32_t)(p)[5] << 8)) #define lpSetTotalBytes(p, v) \ do { \ @@ -164,6 +173,15 @@ void lpFree(unsigned char *lp) { lp_free(lp); } +/* Get value stored in the metadata */ +long long lpGetMetadataValue(unsigned char *p) { + unsigned char *inner = p + 1; + unsigned int slen; + long long value = 0; + lpGetValue(inner, &slen, &value); + return value; +} + /* Same as lpFree, but useful for when you are passing the listpack * into a generic free function that expects (void *) */ void lpFreeVoid(void *lp) { @@ -181,7 +199,7 @@ unsigned char *lpShrinkToFit(unsigned char *lp) { } /* Stores the integer encoded representation of 'v' in the 'intenc' buffer. */ -static inline void lpEncodeIntegerGetType(int64_t v, unsigned char *intenc, uint64_t *enclen) { +void lpEncodeIntegerGetType(int64_t v, unsigned char *intenc, uint64_t *enclen) { if (v >= 0 && v <= 127) { /* Single byte 0-127 integer. */ intenc[0] = v; @@ -355,6 +373,12 @@ static inline uint32_t lpCurrentEncodedSizeUnsafe(unsigned char *p) { if (LP_ENCODING_IS_64BIT_INT(p[0])) return 9; if (LP_ENCODING_IS_12BIT_STR(p[0])) return 2 + LP_ENCODING_12BIT_STR_LEN(p); if (LP_ENCODING_IS_32BIT_STR(p[0])) return 5 + LP_ENCODING_32BIT_STR_LEN(p); + if (LP_ENCODING_IS_TAGGED(p[0])) { + unsigned char *inner = p + 1; + uint32_t inner_size = lpCurrentEncodedSizeUnsafe(inner); + return inner_size + 1; /* tagged byte + inner element */ + } + if (p[0] == LP_EOF) return 1; return 0; } @@ -373,6 +397,7 @@ static inline uint32_t lpCurrentEncodedSizeBytes(unsigned char *p) { if (LP_ENCODING_IS_64BIT_INT(p[0])) return 1; if (LP_ENCODING_IS_12BIT_STR(p[0])) return 2; if (LP_ENCODING_IS_32BIT_STR(p[0])) return 5; + if (LP_ENCODING_IS_TAGGED(p[0])) return 1; if (p[0] == LP_EOF) return 1; return 0; } @@ -393,13 +418,20 @@ unsigned char *lpSkip(unsigned char *p) { * already pointed to the last element of the listpack. */ unsigned char *lpNext(unsigned char *lp, unsigned char *p) { assert(p); - p = lpSkip(p); - if (unlikely(p[0] == LP_EOF)) { - size_t bytes = lpBytes(lp); - /* EOF must only appear at the end of a listpack. */ - assert(p + 1 == lp + bytes); - return NULL; - } + do { + p = lpSkip(p); + if (unlikely(p[0] == LP_EOF)) { + size_t bytes = lpBytes(lp); + /* EOF must only appear at the end of a listpack. */ + assert(p + 1 == lp + bytes); + return NULL; + } + /* Metadata (tagged) entries are logically part of the real element + * that precedes them and are invisible to logical iteration: they can + * only be reached through lpGetMetadata(). For listpacks without + * metadata this is a well-predicted not-taken branch on a byte that + * was just read for the EOF check. */ + } while (unlikely(LP_ENCODING_IS_TAGGED(p[0]))); return p; } @@ -408,25 +440,33 @@ unsigned char *lpNext(unsigned char *lp, unsigned char *p) { * already pointed to the first element of the listpack. */ unsigned char *lpPrev(unsigned char *lp, unsigned char *p) { assert(p); - if (p - lp == LP_HDR_SIZE) return NULL; - p--; /* Seek the first backlen byte of the last element. */ - uint64_t prevlen = lpDecodeBacklen(p); - prevlen += lpEncodeBacklen(NULL, prevlen); - p -= prevlen - 1; /* Seek the first byte of the previous entry. */ - return p; + while (p - lp != LP_HDR_SIZE) { + p--; /* Seek the first backlen byte of the last element. */ + uint64_t prevlen = lpDecodeBacklen(p); + prevlen += lpEncodeBacklen(NULL, prevlen); + p -= prevlen - 1; /* Seek the first byte of the previous entry. */ + /* Skip metadata entries, see lpNext(). */ + if (likely(!LP_ENCODING_IS_TAGGED(p[0]))) return p; + } + return NULL; } /* Return a pointer to the first element of the listpack, or NULL if the * listpack has no elements. */ unsigned char *lpFirst(unsigned char *lp) { unsigned char *p = lp + LP_HDR_SIZE; /* Skip the header. */ - if (unlikely(p[0] == LP_EOF)) { - size_t bytes = lpBytes(lp); - /* EOF must only appear at the end of a listpack. */ - assert(p + 1 == lp + bytes); - return NULL; + while (1) { + if (unlikely(p[0] == LP_EOF)) { + size_t bytes = lpBytes(lp); + /* EOF must only appear at the end of a listpack. */ + assert(p + 1 == lp + bytes); + return NULL; + } + /* Skip metadata entries, see lpNext(). A metadata entry should never + * lead a listpack, this is just defensive. */ + if (likely(!LP_ENCODING_IS_TAGGED(p[0]))) return p; + p = lpSkip(p); } - return p; } /* Return a pointer to the last element of the listpack, or NULL if the @@ -460,6 +500,36 @@ unsigned long lpLength(unsigned char *lp) { return count; } +/* Returns 1 if the element at 'p' is a metadata tagged element */ +int lpIsMetadata(unsigned char *p) { + if (p[0] == LP_EOF) return 0; + return LP_ENCODING_IS_TAGGED(p[0]); +} + +/* If the real element pointed by 'p' is trailed by a metadata (tagged) entry, + * return a pointer to it, otherwise NULL. Metadata entries are logically + * coupled to the real element that precedes them and are skipped by the + * logical iterators (lpFirst/lpLast/lpNext/lpPrev/lpSeek); this accessor is + * the only way to reach them. */ +unsigned char *lpGetMetadata(unsigned char *lp, unsigned char *p) { + assert(p); + p = lpSkip(p); + if (unlikely(p[0] == LP_EOF)) { + /* EOF must only appear at the end of a listpack. */ + assert(p + 1 == lp + lpBytes(lp)); + return NULL; + } + return LP_ENCODING_IS_TAGGED(p[0]) ? p : NULL; +} + +/* Pointer to the first physical entry (after the header). May point at a + * real element, a leading metadata entry, or the EOF terminator on an empty + * listpack. Unlike lpFirst() it does not skip metadata; use only with + * EOF/metadata-aware accessors. */ +unsigned char *lpStart(unsigned char *lp) { + return lp + LP_HDR_SIZE; +} + /* Return the listpack element pointed by 'p'. * * The function changes behavior depending on the passed 'intbuf' value. @@ -609,6 +679,18 @@ unsigned char *lpFind(unsigned char *lp, unsigned char *p, unsigned char *s, uin assert(p); while (p) { + /* Check if we are reading a metadata entry if so skip it */ + if (lpIsMetadata(p)) { + p = lpSkip(p); + if (unlikely(p[0] == LP_EOF)) { + /* EOF must only appear at the end of a listpack. */ + assert(p + 1 == lp + lp_bytes); + break; + } + assert(p >= lp + LP_HDR_SIZE && p < lp + lp_bytes); + continue; + } + if (skipcnt == 0) { value = lpGetWithSize(p, &ll, NULL, &entry_size); if (value) { @@ -690,13 +772,22 @@ unsigned char *lpFind(unsigned char *lp, unsigned char *p, unsigned char *s, uin * For deletion operations (both 'elestr' and 'eleint' set to NULL) 'newp' is * set to the next element, on the right of the deleted one, or to NULL if the * deleted element was the last one. */ -unsigned char *lpInsert(unsigned char *lp, - unsigned char *elestr, - unsigned char *eleint, - uint32_t size, - unsigned char *p, - int where, - unsigned char **newp) { +/* What kind of entry an insertion produces. LP_ENTRY_METADATA entries carry + * the LP_ENCODING_TAGGED marker, are skipped by the logical iterators and are + * not counted in the header numele field. */ +typedef enum { + LP_ENTRY_DATA = 0, + LP_ENTRY_METADATA +} lpEntryType; + +static unsigned char *lpInsertImpl(unsigned char *lp, + unsigned char *elestr, + unsigned char *eleint, + uint32_t size, + unsigned char *p, + int where, + unsigned char **newp, + lpEntryType type) { unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; unsigned char backlen[LP_MAX_BACKLEN_SIZE]; @@ -708,6 +799,11 @@ unsigned char *lpInsert(unsigned char *lp, * it to LP_REPLACE. */ if (del_ele) where = LP_REPLACE; + /* Metadata (tagged) entries are not counted in the header numele field, + * which only tracks real elements. Determine before any memory movement + * whether this operation adds/removes a metadata entry. */ + int ele_is_meta = del_ele ? LP_ENCODING_IS_TAGGED(p[0]) : (type == LP_ENTRY_METADATA); + /* If we need to insert after the current element, we just jump to the * next element (that could be the EOF one) and handle the case of * inserting before. So the function will actually deal with just two @@ -723,7 +819,10 @@ unsigned char *lpInsert(unsigned char *lp, unsigned long poff = p - lp; int enctype; - if (elestr) { + if (type == LP_ENTRY_METADATA) { + enctype = LP_ENCODING_TAGGED; + enclen = size + 1; /* size is the encoded value +1 for the prefixed tag */ + } else if (elestr) { /* Calling lpEncodeGetType() results into the encoded version of the * element to be stored into 'intenc' in case it is representable as * an integer: in that case, the function returns LP_ENCODING_INT. @@ -795,6 +894,13 @@ unsigned char *lpInsert(unsigned char *lp, if (!del_ele) { if (enctype == LP_ENCODING_INT) { memcpy(dst, eleint, enclen); + } else if (enctype == LP_ENCODING_TAGGED) { + dst[0] = LP_ENCODING_TAGGED; + if (eleint) { + memcpy(dst + 1, eleint, size); + } else if (elestr) { + memcpy(dst + 1, elestr, size); + } } else if (elestr) { lpEncodeString(dst, elestr, size); } else { @@ -805,8 +911,8 @@ unsigned char *lpInsert(unsigned char *lp, dst += backlen_size; } - /* Update header. */ - if (where != LP_REPLACE || del_ele) { + /* Update header. Metadata entries are invisible to numele. */ + if ((where != LP_REPLACE || del_ele) && !ele_is_meta) { uint32_t num_elements = lpGetNumElements(lp); if (num_elements != LP_HDR_NUMELE_UNKNOWN) { if (!del_ele) @@ -839,6 +945,28 @@ unsigned char *lpInsert(unsigned char *lp, return lp; } +/* Public lpInsert(), inserting a regular (data) element. See lpInsertImpl() + * for the full contract. */ +unsigned char *lpInsert(unsigned char *lp, + unsigned char *elestr, + unsigned char *eleint, + uint32_t size, + unsigned char *p, + int where, + unsigned char **newp) { + return lpInsertImpl(lp, elestr, eleint, size, p, where, newp, LP_ENTRY_DATA); +} + +/* Insert a metadata (tagged) entry holding the integer payload 'eleint' of + * length 'size' (as produced by lpEncodeIntegerGetType()). Metadata payloads + * are integer-only: the accessor lpGetMetadataValue() has no way to return a + * string. */ +unsigned char * +lpInsertMetadata(unsigned char *lp, unsigned char *eleint, uint32_t size, unsigned char *p, int where, unsigned char **newp) { + assert(eleint != NULL); + return lpInsertImpl(lp, NULL, eleint, size, p, where, newp, LP_ENTRY_METADATA); +} + /* This is just a wrapper for lpInsert() to directly use a string. */ unsigned char * lpInsertString(unsigned char *lp, unsigned char *s, uint32_t slen, unsigned char *p, int where, unsigned char **newp) { @@ -908,6 +1036,14 @@ unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **new return lpInsert(lp, NULL, NULL, 0, p, LP_REPLACE, newp); } +/* This is just a wrapper around lpDelete to remove tagged metadata element + * from listpack. 'metadata_ptr' MUST be non-null and should point at a + * tagged entry */ +unsigned char *lpRemoveMetadata(unsigned char *lp, unsigned char *metadata_ptr) { + assert(lpIsMetadata(metadata_ptr)); + return lpDelete(lp, metadata_ptr, NULL); +} + /* Delete a range of entries from the listpack start with the element pointed by 'p'. */ unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsigned long num) { size_t bytes = lpBytes(lp); @@ -919,11 +1055,16 @@ unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsi if (num == 0) return lp; /* Nothing to delete, return ASAP. */ /* Find the next entry to the last entry that needs to be deleted. + * 'num' counts real elements; metadata (tagged) entries trailing a real + * element are logically coupled to it and are deleted along with it + * (they are not counted in 'deleted', which tracks numele adjustments). * lpLength may be unreliable due to corrupt data, so we cannot * treat 'num' as the number of elements to be deleted. */ while (num--) { deleted++; tail = lpSkip(tail); + /* Consume metadata entries trailing the deleted element. */ + while (tail[0] != LP_EOF && LP_ENCODING_IS_TAGGED(tail[0])) tail = lpSkip(tail); if (unlikely(tail[0] == LP_EOF)) { /* EOF must only appear at the end of a listpack. */ assert(tail + 1 == lp + bytes); @@ -1219,6 +1360,19 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) { /* make sure the encoded entry length doesn't reach outside the edge of the listpack */ if (OUT_OF_RANGE(p + lenbytes)) return 0; + /* For tagged (metadata) entries the size header lives in the inner + * element: lenbytes above only covers the tag byte, so before + * lpCurrentEncodedSizeUnsafe() recurses into the inner header we must + * validate that header's bytes are in range as well. Nested tags are + * invalid. */ + if (LP_ENCODING_IS_TAGGED(p[0])) { + unsigned char *inner = p + 1; + if (LP_ENCODING_IS_TAGGED(inner[0])) return 0; + uint32_t inner_lenbytes = lpCurrentEncodedSizeBytes(inner); + if (!inner_lenbytes) return 0; + if (OUT_OF_RANGE(inner + inner_lenbytes)) return 0; + } + /* get the entry length and encoded backlen. */ unsigned long entrylen = lpCurrentEncodedSizeUnsafe(p); unsigned long encodedBacklen = lpEncodeBacklen(NULL, entrylen); @@ -1241,7 +1395,7 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) { /* Validate the integrity of the data structure. * Validates the header and scans all entries one by one. */ -int lpValidateIntegrity(unsigned char *lp, size_t size, listpackValidateEntryCB entry_cb, void *cb_userdata) { +int lpValidateIntegrity(unsigned char *lp, size_t size, listpackValidateEntryCB entry_cb, void *cb_userdata, int allow_metadata) { /* Check that we can actually read the header. (and EOF) */ if (size < LP_HDR_SIZE + 1) return 0; @@ -1254,6 +1408,7 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, listpackValidateEntryCB /* Validate the individual entries. */ uint32_t count = 0; + int seen_real = 0; uint32_t numele = lpGetNumElements(lp); unsigned char *p = lp + LP_HDR_SIZE; while (p && p[0] != LP_EOF) { @@ -1263,10 +1418,20 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, listpackValidateEntryCB * to avoid callback crash due to corrupt listpack. */ if (!lpValidateNext(lp, &p, bytes)) return 0; + /* Metadata (tagged) entries are only legal where the caller allows + * them (hash listpacks) and are not counted in numele. They must + * trail a real element, with one exception: a single tagged entry may + * lead the listpack as the owning type's aggregate header. */ + if (lpIsMetadata(prev)) { + if (!allow_metadata) return 0; + if (!seen_real && !(prev == lpStart(lp))) return 0; + } else { + count++; + seen_real = 1; + } + /* Optionally let the caller validate the entry too. */ if (entry_cb && !entry_cb(prev, numele, cb_userdata)) return 0; - - count++; } /* Make sure 'p' really does point to the end of the listpack. */ diff --git a/src/listpack.h b/src/listpack.h index 8f9f50170..9be003d41 100644 --- a/src/listpack.h +++ b/src/listpack.h @@ -39,6 +39,14 @@ #include #define LP_INTBUF_SIZE 21 /* 20 digits of -2^63 + 1 null term = 21. */ +#define LP_MAX_INT_ENCODING_LEN 9 +/* Worst-case on-wire size of a tagged metadata entry: tag byte + widest + * integer encoding + 1-byte backlen. Callers sizing an addition that + * includes metadata (e.g. via lpSafeToAdd) must account for this. */ +#define LP_METADATA_MAX_ENTRY_BYTES (1 + LP_MAX_INT_ENCODING_LEN + 1) + +#define lpGetTotalBytes(p) \ + (((uint32_t)(p)[0] << 0) | ((uint32_t)(p)[1] << 8) | ((uint32_t)(p)[2] << 16) | ((uint32_t)(p)[3] << 24)) /* lpInsert() where argument possible values: */ #define LP_BEFORE 0 @@ -61,13 +69,18 @@ unsigned char *lpShrinkToFit(unsigned char *lp); unsigned char * lpInsertString(unsigned char *lp, unsigned char *s, uint32_t slen, unsigned char *p, int where, unsigned char **newp); unsigned char *lpInsertInteger(unsigned char *lp, long long lval, unsigned char *p, int where, unsigned char **newp); +void lpEncodeIntegerGetType(int64_t v, unsigned char *intenc, uint64_t *enclen); unsigned char *lpPrepend(unsigned char *lp, unsigned char *s, uint32_t slen); unsigned char *lpPrependInteger(unsigned char *lp, long long lval); unsigned char *lpAppend(unsigned char *lp, unsigned char *s, uint32_t slen); unsigned char *lpAppendInteger(unsigned char *lp, long long lval); +long long lpGetMetadataValue(unsigned char *p); +unsigned char * +lpInsertMetadata(unsigned char *lp, unsigned char *eleint, uint32_t size, unsigned char *p, int where, unsigned char **newp); unsigned char *lpReplace(unsigned char *lp, unsigned char **p, unsigned char *s, uint32_t slen); unsigned char *lpReplaceInteger(unsigned char *lp, unsigned char **p, long long lval); unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **newp); +unsigned char *lpRemoveMetadata(unsigned char *lp, unsigned char *metadata_ptr); unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsigned long num); unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num); unsigned char *lpBatchDelete(unsigned char *lp, unsigned char **ps, unsigned long count); @@ -85,7 +98,10 @@ size_t lpBytes(unsigned char *lp); size_t lpEstimateBytesRepeatedInteger(long long lval, unsigned long rep); unsigned char *lpSeek(unsigned char *lp, long index); typedef int (*listpackValidateEntryCB)(unsigned char *p, unsigned int head_count, void *userdata); -int lpValidateIntegrity(unsigned char *lp, size_t size, listpackValidateEntryCB entry_cb, void *cb_userdata); +int lpIsMetadata(unsigned char *p); +unsigned char *lpGetMetadata(unsigned char *lp, unsigned char *p); +unsigned char *lpStart(unsigned char *lp); +int lpValidateIntegrity(unsigned char *lp, size_t size, listpackValidateEntryCB entry_cb, void *cb_userdata, int allow_metadata); unsigned char *lpValidateFirst(unsigned char *lp); int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes); unsigned int lpCompare(unsigned char *p, unsigned char *s, uint32_t slen); diff --git a/src/memory_prefetch.c b/src/memory_prefetch.c index 892ccdfc7..aa74d4a7c 100644 --- a/src/memory_prefetch.c +++ b/src/memory_prefetch.c @@ -13,14 +13,26 @@ #include "io_threads.h" typedef enum { - PREFETCH_ENTRY, /* Initial state, prefetch entries associated with the given key's hash */ - PREFETCH_VALUE, /* prefetch the value object of the entry found in the previous step */ - PREFETCH_DONE /* Indicates that prefetching for this key is complete */ + PREFETCH_ENTRY, /* Initial state, prefetch entries associated with the given key's hash */ + PREFETCH_VALUE, /* prefetch the value object of the entry found in the previous step */ + PREFETCH_VALUE_NESTED, /* nested prefetch of inner hashtable for hash/zset types */ + PREFETCH_DONE /* Indicates that prefetching for this key is complete */ } PrefetchState; +typedef enum { + NESTED_PREFETCH_INIT, /* Init incremental find on inner hashtable */ + NESTED_PREFETCH_STEP, /* Step through incremental find */ + NESTED_PREFETCH_VALUE, /* Prefetch the found entry's value (non-embedded only) */ +} NestedPrefetchPhase; + typedef struct KeyPrefetchInfo { PrefetchState state; /* Current state of the prefetch operation */ hashtableIncrementalFindState hashtab_state; + /* Fields for nested prefetching of inner hashtables (hash/zset) */ + robj *member; /* field/member to look up in the inner table, NULL if none */ + int inner_is_zset; /* inner table is a zset index: lookup keys need marking */ + NestedPrefetchPhase nested_phase; + hashtableIncrementalFindState inner_hashtab_state; } KeyPrefetchInfo; /* PrefetchCommandsBatch structure holds the state of the current batch of client commands being processed. */ @@ -34,6 +46,7 @@ typedef struct PrefetchCommandsBatch { int *slots; /* Array of slots for each key */ void **keys; /* Array of keys to prefetch in the current batch */ client **clients; /* Array of clients in the current batch */ + robj **key_members; /* Member to prefetch for each key (NULL = no nested prefetch) */ hashtable **keys_tables; /* Main table for each key */ KeyPrefetchInfo *prefetch_info; /* Prefetch info for each key */ } PrefetchCommandsBatch; @@ -46,6 +59,7 @@ void freePrefetchCommandsBatch(void) { } zfree(batch->clients); + zfree(batch->key_members); zfree(batch->keys); zfree(batch->keys_tables); zfree(batch->slots); @@ -65,6 +79,7 @@ void prefetchCommandsBatchInit(void) { batch = zcalloc(sizeof(PrefetchCommandsBatch)); batch->max_prefetch_size = max_prefetch_size; batch->clients = zcalloc(max_prefetch_size * sizeof(client *)); + batch->key_members = zcalloc(max_prefetch_size * sizeof(robj *)); batch->keys = zcalloc(max_prefetch_size * sizeof(void *)); batch->keys_tables = zcalloc(max_prefetch_size * sizeof(hashtable *)); batch->slots = zcalloc(max_prefetch_size * sizeof(int)); @@ -105,6 +120,7 @@ static KeyPrefetchInfo *getNextPrefetchInfo(void) { return NULL; } +/* Initialize per-key state and start the main-hashtable find for each key. */ static void initBatchInfo(hashtable **tables) { /* Initialize the prefetch info */ for (size_t i = 0; i < batch->key_count; i++) { @@ -115,29 +131,43 @@ static void initBatchInfo(hashtable **tables) { continue; } info->state = PREFETCH_ENTRY; + info->member = batch->key_members[i]; + info->inner_is_zset = 0; + info->nested_phase = NESTED_PREFETCH_INIT; hashtableIncrementalFindInit(&info->hashtab_state, tables[i], batch->keys[i]); } } +/* A key is eligible for nested prefetch when its command supplied a member and the + * value is backed by a hashtable we can look the member up in. */ +static inline int canNestedPrefetch(KeyPrefetchInfo *info, robj *val) { + return info->member != NULL && (val->encoding == OBJ_ENCODING_HASHTABLE || + (val->type == OBJ_ZSET && val->encoding == OBJ_ENCODING_BTREE)); +} + +/* Advance the main-hashtable find and pick the next state once the entry is found. */ static void prefetchEntry(KeyPrefetchInfo *info) { if (hashtableIncrementalFindStep(&info->hashtab_state)) { /* Not done yet */ moveToNextKey(); - } else if (server.io_threads_num >= server.min_io_threads_copy_avoid) { - /* Copy avoidance should be more efficient without value prefetch - * starting certain number of I/O threads */ - markKeyAsdone(info); } else { info->state = PREFETCH_VALUE; } } -/* Prefetch the entry's value. If the value is found.*/ +/* Prefetch the entry's value object, then hand hash and zset keys to the nested path. */ static void prefetchValue(KeyPrefetchInfo *info) { void *entry; if (hashtableIncrementalFindGetResult(&info->hashtab_state, &entry)) { robj *val = entry; - if (val->encoding == OBJ_ENCODING_RAW && val->type == OBJ_STRING) { + if (canNestedPrefetch(info, val)) { + valkey_prefetch(objectGetVal(val)); + info->state = PREFETCH_VALUE_NESTED; + info->nested_phase = NESTED_PREFETCH_INIT; + moveToNextKey(); + return; + } + if (server.io_threads_num < server.min_io_threads_copy_avoid && val->encoding == OBJ_ENCODING_RAW && val->type == OBJ_STRING) { valkey_prefetch(objectGetVal(val)); } } @@ -145,6 +175,82 @@ static void prefetchValue(KeyPrefetchInfo *info) { markKeyAsdone(info); } +/* Nested prefetch: walk the inner hashtable for hash/zset types using a phased + * approach (INIT -> STEP [-> VALUE]) to amortize cache misses across commands + * in the batch. Prefetches the single member supplied by the command. The VALUE + * phase runs only for non-embedded hash values. */ +static void prefetchValueNested(KeyPrefetchInfo *info) { + void *entry; + if (!hashtableIncrementalFindGetResult(&info->hashtab_state, &entry)) { + markKeyAsdone(info); + return; + } + robj *val = entry; + + switch (info->nested_phase) { + case NESTED_PREFETCH_INIT: { + /* The header is warm now, so the inner hashtable pointer can be read. */ + hashtable *inner_ht = NULL; + if (val->encoding == OBJ_ENCODING_HASHTABLE) { + inner_ht = objectGetVal(val); + } else if (val->type == OBJ_ZSET && val->encoding == OBJ_ENCODING_BTREE) { + zset *zs = objectGetVal(val); + inner_ht = zs->ht; + info->inner_is_zset = 1; + } + if (!inner_ht || hashtableSize(inner_ht) == 0) { + markKeyAsdone(info); + return; + } + /* The zset hashtable stores packed [score][element] items, so a plain sds + * lookup key must be marked for the callbacks to read it as an element. */ + sds member = objectGetVal(info->member); + if (info->inner_is_zset) zsetMarkLookupKey(member); + hashtableIncrementalFindInit(&info->inner_hashtab_state, inner_ht, member); + if (info->inner_is_zset) zsetUnmarkLookupKey(member); + info->nested_phase = NESTED_PREFETCH_STEP; + moveToNextKey(); + return; + } + + case NESTED_PREFETCH_STEP: { + /* A step may invoke the compare callback, so mark the lookup key here too. */ + sds step_member = objectGetVal(info->member); + if (info->inner_is_zset) zsetMarkLookupKey(step_member); + int more = hashtableIncrementalFindStep(&info->inner_hashtab_state); + if (info->inner_is_zset) zsetUnmarkLookupKey(step_member); + if (more) { + moveToNextKey(); + return; + } + /* Only non-embedded hash values have a separate value pointer worth + * prefetching; embedded values and zset/set skip the VALUE phase. */ + if (val->type == OBJ_HASH) { + void *inner_entry; + if (hashtableIncrementalFindGetResult(&info->inner_hashtab_state, &inner_entry) && inner_entry && + !entryHasEmbeddedValue(inner_entry)) { + info->nested_phase = NESTED_PREFETCH_VALUE; + moveToNextKey(); + return; + } + } + markKeyAsdone(info); + return; + } + + case NESTED_PREFETCH_VALUE: { + void *inner_entry; + if (hashtableIncrementalFindGetResult(&info->inner_hashtab_state, &inner_entry) && inner_entry) { + char *value = entryGetValue(inner_entry, NULL); + if (value) valkey_prefetch(value); + } + markKeyAsdone(info); + return; + } + default: serverPanic("Unknown nested prefetch phase %d", info->nested_phase); + } +} + /* Prefetch hashtable data for an array of keys. * * This function takes an array of tables and keys, attempting to bring @@ -162,6 +268,7 @@ static void hashtablePrefetch(hashtable **tables) { switch (info->state) { case PREFETCH_ENTRY: prefetchEntry(info); break; case PREFETCH_VALUE: prefetchValue(info); break; + case PREFETCH_VALUE_NESTED: prefetchValueNested(info); break; default: serverPanic("Unknown prefetch state %d", info->state); } } @@ -247,10 +354,14 @@ static void addCommandToBatch(struct serverCommand *cmd, robj **argv, int argc, getKeysResult result; initGetKeysResult(&result); int num_keys = getKeysFromCommand(cmd, argv, argc, &result); + int member_idx = cmd->member_arg_index; + robj *member = (member_idx > 0 && member_idx < argc) ? argv[member_idx] : NULL; for (int i = 0; i < num_keys && batch->key_count < batch->max_prefetch_size; i++) { batch->keys[batch->key_count] = argv[result.keys[i].pos]; batch->slots[batch->key_count] = slot >= 0 ? slot : 0; batch->keys_tables[batch->key_count] = kvstoreGetHashtable(db->keys, batch->slots[batch->key_count]); + batch->key_members[batch->key_count] = + (result.keys[i].flags & CMD_KEY_OW) && !(result.keys[i].flags & CMD_KEY_ACCESS) ? NULL : member; batch->key_count++; } getKeysFreeResult(&result); diff --git a/src/module.c b/src/module.c index 76c66e25b..4d3dc780e 100644 --- a/src/module.c +++ b/src/module.c @@ -58,6 +58,7 @@ #include "server.h" #include "ordered_index.h" #include "cluster.h" +#include "entry.h" #include "commandlog.h" #include "rdb.h" #include "monotonic.h" @@ -70,6 +71,8 @@ #include "io_threads.h" #include "scripting_engine.h" #include "cluster_migrateslots.h" +#include "bgiteration.h" +#include "forkless.h" #include #include #include @@ -607,6 +610,38 @@ char *VM_Strdup(const char *str) { return zstrdup(str); } +/* Report memory obtained outside the server allocator, such as an mmap()ed + * region, so it counts toward used_memory and maxmemory. Does not allocate. + * + * Report only resident memory, and only memory not already obtained from + * ValkeyModule_Alloc(). Each call must be matched by + * ValkeyModule_DecrExternalMemory() of the same size. + * + * May be called from a command callback or a thread-safe context. + * + * Returns VALKEYMODULE_OK, or VALKEYMODULE_ERR with errno set to ERANGE if the + * total would overflow, leaving the accounting unchanged. */ +int VM_IncrExternalMemory(size_t bytes) { + if (zmalloc_increase_used_memory_external(bytes) != 0) { + errno = ERANGE; + return VALKEYMODULE_ERR; + } + return VALKEYMODULE_OK; +} + +/* Stop accounting for memory reported with ValkeyModule_IncrExternalMemory(). + * Frees nothing. + * + * Returns VALKEYMODULE_OK, or VALKEYMODULE_ERR with errno set to ERANGE if + * `bytes` exceeds the reported total, leaving the accounting unchanged. */ +int VM_DecrExternalMemory(size_t bytes) { + if (zmalloc_decrease_used_memory_external(bytes) != 0) { + errno = ERANGE; + return VALKEYMODULE_ERR; + } + return VALKEYMODULE_OK; +} + /* -------------------------------------------------------------------------- * Pool allocator * -------------------------------------------------------------------------- */ @@ -2115,6 +2150,8 @@ int VM_SetCommandInfo(ValkeyModuleCommand *command, const ValkeyModuleCommandInf /* Update the legacy (first,last,step) spec and "movablekeys" flag used by the COMMAND command, * by trying to "glue" consecutive range key specs. */ populateCommandLegacyRangeSpec(cmd); + + detectWriteFirstkeyOnlyCommand(cmd); } if (info->args) { @@ -2438,7 +2475,7 @@ void VM_SetModuleAttribs(ValkeyModuleCtx *ctx, const char *name, int ver, int ap module->apiver = apiver; module->types = listCreate(); module->usedby = listCreate(); - module->using = listCreate(); + module->uses = listCreate(); module->filters = listCreate(); module->module_configs = listCreate(); listSetMatchMethod(module->module_configs, moduleListConfigMatch); @@ -2563,7 +2600,7 @@ void VM_Yield(ValkeyModuleCtx *ctx, int flags, const char *busy_reply) { if (flags & VALKEYMODULE_YIELD_FLAG_CLIENTS) server.busy_module_yield_flags |= BUSY_MODULE_YIELD_CLIENTS; /* Let the server process events */ - if (!pthread_equal(server.main_thread_id, pthread_self())) { + if (!onServerMainThread()) { /* If we are not in the main thread, we defer event loop processing to the main thread * after the main thread enters acquiring GIL state in order to protect the event * loop (ae.c) and avoid potential race conditions. */ @@ -4297,6 +4334,14 @@ static void moduleInitKeyTypeSpecific(ValkeyModuleKey *key) { * call ValkeyModule_CloseKey() and ValkeyModule_KeyType() on a NULL * value. * + * Valkey 9.2+: When opening a key with VALKEYMODULE_WRITE, NULL will be returned + * if the key is currently write-locked (i.e. if forkless operations are operating + * on the key). This change is non-breaking as: + * * Modules have to opt-in using VALKEYMODULE_OPTIONS_HANDLE_FORKLESS_SAVE + * * Module write commands are blocked (before execution), if a declared key is write-locked + * The risk is only for a module that performs VM_OpenKey() on a key which was NOT + * declared in the current command OR arbitrarily opens keys during a timer event. + * * Extra flags that can be pass to the API under the mode argument: * * VALKEYMODULE_OPEN_KEY_NOTOUCH - Avoid touching the LRU/LFU of the key when opened. * * VALKEYMODULE_OPEN_KEY_NONOTIFY - Don't trigger keyspace event on key misses. @@ -4315,6 +4360,7 @@ ValkeyModuleKey *VM_OpenKey(ValkeyModuleCtx *ctx, robj *keyname, int mode) { if (mode & VALKEYMODULE_WRITE) { value = lookupKeyWriteWithFlags(ctx->client->db, keyname, flags); + if (value && bgIteration_isEntryInuse(value)) return NULL; } else { value = lookupKeyReadWithFlags(ctx->client->db, keyname, flags); if (value == NULL) { @@ -6949,6 +6995,9 @@ static void moduleCallCommandHelper(ValkeyModuleCtx *ctx, client *c, robj **argv if (!(flags & VALKEYMODULE_CALL_ARGV_NO_AOF)) call_flags |= CMD_CALL_PROPAGATE_AOF; if (!(flags & VALKEYMODULE_CALL_ARGV_NO_REPLICAS)) call_flags |= CMD_CALL_PROPAGATE_REPL; } + /* Mirror processInputBuffer: set pending_command so that if the command + * blocks on keys, unblockClientOnKey will reprocess it on unblock. */ + c->flag.pending_command = 1; call(c, call_flags); /* Propagate database changes from the temporary client back to the context client @@ -6970,6 +7019,15 @@ static void moduleCallCommandHelper(ValkeyModuleCtx *ctx, client *c, robj **argv server.replication_allowed = prev_replication_allowed; if (c->flag.blocked) { + if (c->flag.deny_blocking) { + /* The module did not pass ALLOW_BLOCK — it does not expect the + * command to block. Unblock the client and return an error. */ + c->flag.pending_command = 0; + unblockClient(c, 0); + addReplyError(c, "-INUSE Key is being processed"); + goto cleanup; + } + /* Blocking commands are not allowed when calling commands in scripting engines. */ serverAssert(!is_running_script); serverAssert(flags & VALKEYMODULE_CALL_ARGV_ALLOW_BLOCK); @@ -7673,6 +7731,26 @@ int moduleVerifyAllAllowAtomicSlotMigrationOrReply(client *c) { return C_OK; } +/* Returns 0 if any loaded module did not declare + * VALKEYMODULE_OPTIONS_HANDLE_FORKLESS, in which case forkless operations + * should be blocked. Every module must opt in: a module that accesses a key for + * write during a forkless operation must acknowledge the possible behavior + * change, and a module that registers a data type must also confirm its RDB save + * callback is thread-safe. */ +int moduleAllModulesHandleForkless(void) { + listIter li; + listNode *ln; + + listRewind(modules, &li); + while ((ln = listNext(&li)) != NULL) { + struct ValkeyModule *module = listNodeValue(ln); + if (!(module->options & VALKEYMODULE_OPTIONS_HANDLE_FORKLESS)) { + return 0; + } + } + return 1; +} + /* Returns true if any previous IO API failed. * for `Load*` APIs the VALKEYMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with * ValkeyModule_SetModuleOptions first. */ @@ -8441,6 +8519,10 @@ ValkeyModuleBlockedClient *moduleBlockClient(ValkeyModuleCtx *ctx, c->bstate->timeout = timeout; blockClient(c, BLOCKED_MODULE); } + /* Module handles its own reply on unblock, so clear pending_command + * to prevent re-execution. Auth clients are the exception — they + * need re-execution after auth completes. */ + if (!auth_reply_callback) c->flag.pending_command = 0; /* Defer response until after being unblocked for a context originated from * keyspace notification events */ if (is_keyspace_notification) { @@ -9416,6 +9498,16 @@ int VM_SubscribeToKeyspaceEvents(ValkeyModuleCtx *ctx, int types, ValkeyModuleNo return VALKEYMODULE_OK; } +/* Whether any module post-execution-unit job is pending. Kept as a tiny + * accessor rather than exposing modulePostExecUnitJobs itself, so callers + * outside this file (postExecutionUnitOperations()) don't need to know it's + * backed by a list - if that representation ever changes, only this + * function needs to change with it. It's trivial enough that LTO can inline + * it at its (currently single) call site same as any other cross-TU call. */ +bool moduleHasPostExecUnitJobs(void) { + return listLength(modulePostExecUnitJobs) > 0; +} + void firePostExecutionUnitJobs(void) { /* Avoid propagation of commands. * In that way, postExecutionUnitOperations will prevent @@ -10382,7 +10474,9 @@ ValkeyModuleUser *VM_CreateModuleUser(const char *name) { } /* Frees a given user and disconnects all of the clients that have been - * authenticated with it. See VM_CreateModuleUser for detailed usage.*/ + * authenticated with it. See VM_CreateModuleUser for detailed usage. + * + * Returns VALKEYMODULE_OK. */ int VM_FreeModuleUser(ValkeyModuleUser *user) { if (user->free_user) ACLFreeUserAndKillClients(user->user); zfree(user); @@ -11437,7 +11531,7 @@ void *VM_GetSharedAPI(ValkeyModuleCtx *ctx, const char *apiname) { ValkeyModuleSharedAPI *sapi = dictGetVal(de); if (listSearchKey(sapi->module->usedby, ctx->module) == NULL) { listAddNodeTail(sapi->module->usedby, ctx->module); - listAddNodeTail(ctx->module->using, sapi->module); + listAddNodeTail(ctx->module->uses, sapi->module); } return sapi->func; } @@ -11474,7 +11568,7 @@ int moduleUnregisterUsedAPI(ValkeyModule *module) { listNode *ln; int count = 0; - listRewind(module->using, &li); + listRewind(module->uses, &li); while ((ln = listNext(&li))) { ValkeyModule *used = ln->value; listNode *ln = listSearchKey(used->usedby, module); @@ -12123,8 +12217,11 @@ static void moduleScanKeyHashtableCallback(void *privdata, void *entry) { * ValkeyModule_CloseKey(key); * ValkeyModule_ScanCursorDestroy(c); * - * The function will return 1 if there are more elements to scan and 0 otherwise, - * possibly setting errno if the call failed. + * The function will return 1 if there are more elements to scan and 0 otherwise. + * On a return value of 0, errno is set to distinguish the cases: + * - 0 - the scan completed successfully. + * - EINVAL - the key is NULL or not a hash, set or sorted set. + * - ENOENT - the cursor is already exhausted (a previous call returned 0). * It is also possible to restart an existing cursor using VM_ScanCursorRestart. * * NOTE: Certain operations are unsafe while iterating the object. For instance @@ -12201,6 +12298,171 @@ int VM_ScanKey(ValkeyModuleKey *key, ValkeyModuleScanCursor *cursor, ValkeyModul return ret; } +/* Callback for VM_ScanKeyRawBorrowed. See VM_ScanKeyRawBorrowed below for the + * (field, value) meaning per type and the pointer lifetime contract. */ +typedef void (*ValkeyModuleScanKeyRawBorrowedCB)(ValkeyModuleKey *key, + const char *field, + size_t field_len, + const char *value, + size_t value_len, + void *privdata); +typedef struct { + ValkeyModuleKey *key; + void *user_data; + ValkeyModuleScanKeyRawBorrowedCB fn; +} ScanKeyRawBorrowedCBData; + +/* Hashtable-encoded SET / HASH / ZSET(btree) callback: borrowed field/member + * (+ borrowed hash value, or materialized zset score). */ +static void moduleScanKeyRawBorrowedHashtableCallback(void *privdata, void *entry) { + ScanKeyRawBorrowedCBData *data = privdata; + robj *o = data->key->value; + if (objectGetType(o) == OBJ_SET) { + sds member = entry; + data->fn(data->key, member, sdslen(member), NULL, 0, data->user_data); + } else if (objectGetType(o) == OBJ_ZSET) { + const char *member; + size_t mlen; + orderedIndexItemGetElement((const OrderedIndexItem *)entry, &member, &mlen); + char scorebuf[MAX_D2STRING_CHARS]; /* materialized: callback-scoped */ + int slen = d2string(scorebuf, sizeof(scorebuf), orderedIndexItemGetScore((const OrderedIndexItem *)entry)); + data->fn(data->key, member, mlen, scorebuf, (size_t)slen, data->user_data); + } else if (objectGetType(o) == OBJ_HASH) { + sds field = entryGetField(entry); + size_t val_len; + char *val = entryGetValue(entry, &val_len); + data->fn(data->key, field, sdslen(field), val, val_len, data->user_data); + } else { + serverPanic("unexpected object type in ScanKeyRawBorrowed"); + } +} + +/* Like ValkeyModule_ScanKey, but each element is delivered to the callback as + * borrowed `(const char *, size_t)` byte ranges instead of allocating a + * ValkeyModuleString per element. This avoids a per-element allocation on the + * hot reply path. Works on the same key types as ValkeyModule_ScanKey: hash, + * set and sorted set. + * + * void scan_callback(ValkeyModuleKey *key, const char *field, size_t field_len, + * const char *value, size_t value_len, void *privdata); + * + * Per element the callback receives (field, field_len, value, value_len): + * - HASH: field = field name, value = field value. + * - SET: field = member, value = NULL, value_len = 0 (sets have no value). + * - ZSET: field = member, value = score as a decimal string. + * + * The score string uses the same form as `ZRANGE ... WITHSCORES`: e.g. the score + * 1.5 is delivered as "1.5" and 2.0 as "2". (ValkeyModule_ScanKey instead + * delivers a `%.17Lg`-style rendering for sorted-set scores.) + * + * POINTER LIFETIME: the field and value pointers are only guaranteed to be valid + * for the duration of the callback invocation. A typical use is to reply to the + * calling client from within the callback with ValkeyModule_ReplyWithStringBuffer. + * To keep a field or value beyond the callback, copy it (for example into a + * ValkeyModuleString with ValkeyModule_CreateString). + * + * The usage pattern, return value, errno semantics and iteration-safety notes + * are identical to ValkeyModule_ScanKey. */ +int VM_ScanKeyRawBorrowed(ValkeyModuleKey *key, ValkeyModuleScanCursor *cursor, ValkeyModuleScanKeyRawBorrowedCB fn, void *privdata) { + if (key == NULL || key->value == NULL) { + errno = EINVAL; + return 0; + } + hashtable *ht = NULL; + robj *o = key->value; + if (objectGetType(o) == OBJ_SET) { + if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) ht = objectGetVal(o); + } else if (objectGetType(o) == OBJ_HASH) { + if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) ht = objectGetVal(o); + } else if (objectGetType(o) == OBJ_ZSET) { + if (objectGetEncoding(o) == OBJ_ENCODING_BTREE) ht = ((zset *)objectGetVal(o))->ht; + } else { + errno = EINVAL; + return 0; + } + if (cursor->done) { + errno = ENOENT; + return 0; + } + int ret = 1; + if (ht) { + /* hashtable-encoded set/hash, or btree-encoded zset: incremental. */ + ScanKeyRawBorrowedCBData data = {key, privdata, fn}; + cursor->cursor = hashtableScan(ht, cursor->cursor, moduleScanKeyRawBorrowedHashtableCallback, &data); + if (cursor->cursor == 0) { + cursor->done = 1; + ret = 0; + } + } else if (objectGetType(o) == OBJ_SET) { + /* intset / listpack set: full scan. Listpack members are borrowed; + * intset integer members are materialized (callback-scoped). */ + setTypeIterator *si = setTypeInitIterator(o); + char *str; + size_t len; + int64_t llele; + char intbuf[LONG_STR_SIZE]; + while (setTypeNext(si, &str, &len, &llele) != -1) { + const char *m; + size_t mlen; + if (str != NULL) { + m = str; + mlen = len; + } else { + mlen = (size_t)ll2string(intbuf, sizeof(intbuf), llele); + m = intbuf; + } + fn(key, m, mlen, NULL, 0, privdata); + } + setTypeReleaseIterator(si); + cursor->cursor = 1; + cursor->done = 1; + ret = 0; + } else { + /* listpack-encoded zset or hash: (field/member, value/score) pairs. + * String entries are borrowed; integer-encoded entries are materialized + * (callback-scoped). Integers may be either field or value. */ + unsigned char *lp = objectGetVal(o); + unsigned char *p = lpSeek(lp, 0); + while (p) { + unsigned int flen; + long long fll; + char fbuf[LONG_STR_SIZE]; + unsigned char *fstr = lpGetValue(p, &flen, &fll); + const char *fp; + size_t fl; + if (fstr != NULL) { + fp = (char *)fstr; + fl = flen; + } else { + fl = (size_t)ll2string(fbuf, sizeof(fbuf), fll); + fp = fbuf; + } + p = lpNext(lp, p); + if (!p) break; + unsigned int vlen; + long long vll; + char vbuf[LONG_STR_SIZE]; + unsigned char *vstr = lpGetValue(p, &vlen, &vll); + const char *vp; + size_t vl; + if (vstr != NULL) { + vp = (char *)vstr; + vl = vlen; + } else { + vl = (size_t)ll2string(vbuf, sizeof(vbuf), vll); + vp = vbuf; + } + fn(key, fp, fl, vp, vl, privdata); + p = lpNext(lp, p); + } + cursor->cursor = 1; + cursor->done = 1; + ret = 0; + } + errno = 0; + return ret; +} + /* -------------------------------------------------------------------------- * ## Module fork API @@ -13227,7 +13489,7 @@ void moduleFreeModuleStructure(struct ValkeyModule *module) { listRelease(module->types); listRelease(module->filters); listRelease(module->usedby); - listRelease(module->using); + listRelease(module->uses); listRelease(module->module_configs); sdsfree(module->name); moduleLoadQueueEntryFree(module->loadmod); @@ -13476,9 +13738,9 @@ static int moduleInitPostOnLoadResolved(ModuleLoadFunc onload, ACLRecomputeCommandBitsFromCommandRulesAllUsers(); } if (is_static) { - serverLog(LL_NOTICE, "Static Module '%s' successfully loaded", ctx.module->name); + serverLog(LL_NOTICE, "Static Module '%s' successfully loaded (version %d)", ctx.module->name, ctx.module->ver); } else { - serverLog(LL_NOTICE, "Module '%s' loaded from %s", ctx.module->name, display_name); + serverLog(LL_NOTICE, "Module '%s' loaded from %s (version %d)", ctx.module->name, display_name, ctx.module->ver); } ctx.module->onload = 0; @@ -13515,6 +13777,12 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa ModuleLoadFunc onload; void *handle; + if (isForklessSaveInProgress()) { + serverLog(LL_WARNING, "Module %s failed to load: cannot load during forkless save.", path); + if (errmsg) *errmsg = "cannot load module during forkless save"; + return C_ERR; + } + if (server.async_loading) { serverLog(LL_WARNING, "Module %s failed to load: cannot load during async replication.", path); if (errmsg) *errmsg = "cannot load module during async replication"; @@ -13872,7 +14140,7 @@ sds genModulesInfoString(sds info) { struct ValkeyModule *module = listNodeValue(ln); sds usedby = genModulesInfoStringRenderModulesList(module->usedby); - sds using = genModulesInfoStringRenderModulesList(module->using); + sds using = genModulesInfoStringRenderModulesList(module->uses); sds options = genModulesInfoStringRenderModuleOptions(module); info = sdscatfmt(info, "module:name=%S,ver=%i,api=%i,filters=%i," @@ -14467,7 +14735,8 @@ int VM_RdbLoad(ValkeyModuleCtx *ctx, ValkeyModuleRdbStream *stream, int flags) { /* Kill existing RDB fork as it is saving outdated data. Also killing it * will prevent COW memory issue. */ - if (server.child_type == CHILD_TYPE_RDB) killRDBChild(); + if (isForkBgsaveInProgress()) killRDBChild(); + if (isForklessSaveInProgress()) forklessSaveCancel(); /* Kill existing slot migration fork as it is saving outdated data. Also killing it * will prevent COW memory issue. */ @@ -14987,13 +15256,19 @@ struct ValkeyModuleDefragCtx { /* Register a defrag callback for global data, i.e. anything that the module * may allocate that is not tied to a specific data type. + * + * The callback is invoked with a time limit: it should call VM_DefragShouldStop() periodically, and + * save its position with VM_DefragCursorSet() so a later invocation can resume (using VM_DefragCursorGet). + * + * If a non-zero cursor is set (VM_DefragCursorSet) the function will be invoked repeatedly until a zero + * cursor is returned. */ int VM_RegisterDefragFunc(ValkeyModuleCtx *ctx, ValkeyModuleDefragFunc cb) { ctx->module->defrag_cb = cb; return VALKEYMODULE_OK; } -/* When the data type defrag callback iterates complex structures, this +/* When a defrag callback iterates complex structures, this * function should be called periodically. A zero (false) return * indicates the callback may continue its work. A non-zero value (true) * indicates it should stop. @@ -15001,8 +15276,9 @@ int VM_RegisterDefragFunc(ValkeyModuleCtx *ctx, ValkeyModuleDefragFunc cb) { * When stopped, the callback may use VM_DefragCursorSet() to store its * position so it can later use VM_DefragCursorGet() to resume defragging. * - * When stopped and more work is left to be done, the callback should - * return 1. Otherwise, it should return 0. + * When stopped and more work is left to be done, the data type callback + * should return 1. Otherwise, it should return 0. The global callback has no + * return value and reports this through its cursor instead. * * NOTE: Modules should consider the frequency in which this function is called, * so it generally makes sense to do small batches of work in between calls. @@ -15013,18 +15289,18 @@ int VM_DefragShouldStop(ValkeyModuleDefragCtx *ctx) { /* Store an arbitrary cursor value for future re-use. * - * This should only be called if VM_DefragShouldStop() has returned a non-zero - * value and the defrag callback is about to exit without fully iterating its - * data type. + * For a data type callback, this should only be called if VM_DefragShouldStop() + * has returned a non-zero value and the defrag callback is about to exit without + * fully iterating its data type. * * This behavior is reserved to cases where late defrag is performed. Late * defrag is selected for keys that implement the `free_effort` callback and * return a `free_effort` value that is larger than the defrag * 'active-defrag-max-scan-fields' configuration directive. * - * Smaller keys, keys that do not implement `free_effort` or the global - * defrag callback are not called in late-defrag mode. In those cases, a - * call to this function will return VALKEYMODULE_ERR. + * Smaller keys and keys that do not implement `free_effort` are not called in + * late-defrag mode. In those cases, a call to this function will return + * VALKEYMODULE_ERR. * * The cursor may be used by the module to represent some progress into the * module's data type. Modules may also store additional cursor-related @@ -15032,6 +15308,15 @@ int VM_DefragShouldStop(ValkeyModuleDefragCtx *ctx) { * traversal of a new key begins. This is possible because the API makes * a guarantee that concurrent defragmentation of multiple keys will * not be performed. + * + * A global callback (registered with VM_RegisterDefragFunc) always has a cursor + * available, and the cursor is also how it reports completion: 0, the value a + * fresh pass starts from, means done, and non-zero means it will be invoked + * again. The server discards the cursor once the callback completes or the pass + * is interrupted, so a cursor saved before an interruption is never handed + * back. A flush or a database swap does not end defragmentation, so a + * module must still be able to restart when its cursor may be invalid, + * usually just returning a 0 cursor, indicating done. */ int VM_DefragCursorSet(ValkeyModuleDefragCtx *ctx, unsigned long cursor) { if (!ctx->cursor) return VALKEYMODULE_ERR; @@ -15042,7 +15327,7 @@ int VM_DefragCursorSet(ValkeyModuleDefragCtx *ctx, unsigned long cursor) { /* Fetch a cursor value that has been previously stored using VM_DefragCursorSet(). * - * If not called for a late defrag operation, VALKEYMODULE_ERR will be returned and + * If no cursor is available, VALKEYMODULE_ERR will be returned and * the cursor should be ignored. See VM_DefragCursorSet() for more details on * defrag cursors. */ @@ -15146,20 +15431,49 @@ int moduleDefragValue(robj *key, robj *value, int dbid) { return 1; } -/* Call registered module API defrag functions */ -void moduleDefragGlobals(void) { - if (listLength(modules) == 0) return; +/* Global defrag walks the modules one at a time. These two values are the whole resume state: the + * module currently being defragged (by position, since a module can be unloaded between invocations) + * and the cursor that module last stored. Both are reset at the start of every cycle. */ +static long defrag_module_position = 0; +static unsigned long defrag_module_cursor = 0; - listIter li; - listNode *ln; +/* Begin a fresh pass from the first module. Called internally when the stage starts (endtime==0); + * a cycle that was aborted mid-pass leaves stale values here, which this discards. */ +static void moduleDefragGlobalsStart(void) { + defrag_module_position = 0; + defrag_module_cursor = 0; +} + +/* Defrag module global data, forwarding 'endtime' so a callback can bound its own latency via + * VM_DefragShouldStop(). Each module is defragged to completion (its cursor back to 0) before we + * move to the next; walking off the end of the module list means every module is done. + * + * Returns true while work remains, false once the pass is complete. */ +bool moduleDefragGlobals(monotime endtime) { + if (endtime == 0) { + moduleDefragGlobalsStart(); + return true; + } + + /* Resolve the resume position to a node once; walking with listIndex per step would be quadratic + * in the number of loaded modules. The list can't change during a single call (defrag is + * single-threaded), so the node stays valid until we return. */ + listNode *ln = listIndex(modules, defrag_module_position); + while (ln != NULL) { + if (getMonotonicUs() >= endtime) return true; - listRewind(modules, &li); - while ((ln = listNext(&li)) != NULL) { struct ValkeyModule *module = listNodeValue(ln); - if (!module->defrag_cb) continue; - ValkeyModuleDefragCtx defrag_ctx = {0, NULL, NULL, -1}; - module->defrag_cb(&defrag_ctx); + if (module->defrag_cb) { + ValkeyModuleDefragCtx defrag_ctx = {endtime, &defrag_module_cursor, NULL, -1}; + module->defrag_cb(&defrag_ctx); + if (defrag_module_cursor != 0) continue; /* more work on this module */ + } + /* This module is done (or has no callback): advance and start the next one at cursor 0. */ + defrag_module_position++; + defrag_module_cursor = 0; + ln = ln->next; } + return false; } /* Returns the name of the key currently being processed. @@ -15226,6 +15540,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(TryRealloc); REGISTER_API(Free); REGISTER_API(Strdup); + REGISTER_API(IncrExternalMemory); + REGISTER_API(DecrExternalMemory); REGISTER_API(CreateCommand); REGISTER_API(GetCommand); REGISTER_API(CreateSubcommand); @@ -15535,6 +15851,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ScanCursorRestart); REGISTER_API(Scan); REGISTER_API(ScanKey); + REGISTER_API(ScanKeyRawBorrowed); REGISTER_API(CreateModuleUser); REGISTER_API(SetContextUser); REGISTER_API(SetModuleUserACL); diff --git a/src/module.h b/src/module.h index b411cf6a5..f1b1bc0bc 100644 --- a/src/module.h +++ b/src/module.h @@ -107,7 +107,7 @@ typedef struct ValkeyModule { int apiver; /* Module API version as requested during initialization.*/ list *types; /* Module data types. */ list *usedby; /* List of modules using APIs from this one. */ - list *using; /* List of modules we use some APIs of. */ + list *uses; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ list *module_configs; /* List of configurations the module has registered */ int configs_initialized; /* Have the module configurations been initialized? */ @@ -213,6 +213,7 @@ void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); unsigned long moduleNotifyKeyspaceSubscribersCnt(void); void firePostExecutionUnitJobs(void); +bool moduleHasPostExecUnitJobs(void); void moduleCallCommandFilters(client *c); void moduleFireCommandResultEvent(client *c, struct serverCommand *cmd, @@ -227,6 +228,7 @@ int TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(void); int moduleAllModulesHandleReplAsyncLoad(void); +int moduleAllModulesHandleForkless(void); int moduleVerifyAllAllowAtomicSlotMigrationOrReply(client *c); sds modulesCollectInfo(sds info, dict *sections_dict, int for_crash_report, int sections); void moduleFireServerEvent(uint64_t eid, int subid, void *data); @@ -242,7 +244,7 @@ size_t moduleGetMemUsage(robj *key, robj *val, size_t sample_size, int dbid); robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, int todb, robj *value); int moduleDefragValue(robj *key, robj *obj, int dbid); int moduleLateDefrag(robj *key, robj *value, unsigned long *cursor, monotime endtime, int dbid); -void moduleDefragGlobals(void); +bool moduleDefragGlobals(monotime endtime); void *moduleGetHandleByName(char *modulename); int moduleIsModuleCommand(void *module_handle, struct serverCommand *cmd); void freeClientModuleData(client *c); diff --git a/src/modules/lua/engine_lua.c b/src/modules/lua/engine_lua.c index bc1abe88a..82706eab4 100644 --- a/src/modules/lua/engine_lua.c +++ b/src/modules/lua/engine_lua.c @@ -502,7 +502,8 @@ LUA_MODULE_VISIBILITY int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, } ValkeyModule_SetModuleOptions(ctx, VALKEYMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD | - VALKEYMODULE_OPTIONS_HANDLE_ATOMIC_SLOT_MIGRATION); + VALKEYMODULE_OPTIONS_HANDLE_ATOMIC_SLOT_MIGRATION | + VALKEYMODULE_OPTIONS_HANDLE_FORKLESS); engine_ctx = createEngineContext(ctx); diff --git a/src/monotonic.h b/src/monotonic.h index b465f90b1..69285f45a 100644 --- a/src/monotonic.h +++ b/src/monotonic.h @@ -58,4 +58,8 @@ static inline uint64_t elapsedMs(monotime start_time) { return elapsedUs(start_time) / 1000; } +static inline uint64_t elapsedSec(monotime start_time) { + return elapsedUs(start_time) / 1000000; +} + #endif diff --git a/src/multi.c b/src/multi.c index 510be20f2..052b39751 100644 --- a/src/multi.c +++ b/src/multi.c @@ -188,8 +188,123 @@ void execCommandAbort(client *c, sds error) { /* Send EXEC to clients waiting data from MONITOR. We did send a MULTI * already, and didn't send any of the queued commands, now we'll just send - * EXEC so it is clear that the transaction is over. */ - replicationFeedMonitors(c, server.monitors, c->db->id, c->argv, c->argc); + * EXEC so it is clear that the transaction is over. If called from call(), + * it will feed monitors when it returns. */ + if (!c->flag.executing_command) { + replicationFeedMonitors(c, server.monitors, c->db->id, c->argv, c->argc); + } +} + +typedef enum { + EXEC_CONDITION_IFEQ, + EXEC_CONDITION_IFNE, + EXEC_CONDITION_NX, + EXEC_CONDITION_XX, +} execCondition; + +/* Parse the next condition in an EXEC command. */ +static int parseExecCondition(robj **argv, int argc, int *index, execCondition *condition) { + const char *token = objectGetVal(argv[*index]); + int args; + + if (!strcasecmp(token, "ifeq")) { + *condition = EXEC_CONDITION_IFEQ; + args = 2; + } else if (!strcasecmp(token, "ifne")) { + *condition = EXEC_CONDITION_IFNE; + args = 2; + } else if (!strcasecmp(token, "nx")) { + *condition = EXEC_CONDITION_NX; + args = 1; + } else if (!strcasecmp(token, "xx")) { + *condition = EXEC_CONDITION_XX; + args = 1; + } else { + return C_ERR; + } + + if (*index + args >= argc) return C_ERR; + *index += args + 1; + return C_OK; +} + +/* Return the condition keys in EXEC arguments for ACL and cluster routing. */ +int execGetKeys(struct serverCommand *cmd, robj **argv, int argc, getKeysResult *result) { + UNUSED(cmd); + int index = 1; + int numkeys = 0; + keyReference *keys; + + while (index < argc) { + int key_index = index + 1; + execCondition condition; + + if (parseExecCondition(argv, argc, &index, &condition) != C_OK) { + result->numkeys = 0; + return 0; + } + keys = getKeysPrepareResult(result, numkeys + 1); + keys[numkeys].pos = key_index; + keys[numkeys].flags = CMD_KEY_RO | CMD_KEY_ACCESS; + numkeys++; + } + result->numkeys = numkeys; + return numkeys; +} + +/* Check whether every condition supplied to EXEC matches the current database. */ +static int checkExecConditions(client *c) { + int index = 1; + + /* Validate the complete condition list before reading any keys. This keeps + * malformed commands from exposing the result of an earlier condition. */ + while (index < c->argc) { + execCondition condition; + + if (parseExecCondition(c->argv, c->argc, &index, &condition) != C_OK) { + execCommandAbort(c, "invalid check condition syntax"); + return -1; + } + } + + index = 1; + while (index < c->argc) { + int condition_index = index; + execCondition condition; + robj *key, *value = NULL, *o; + int matches; + + serverAssert(parseExecCondition(c->argv, c->argc, &index, &condition) == C_OK); + + key = c->argv[condition_index + 1]; + if (condition == EXEC_CONDITION_IFEQ || condition == EXEC_CONDITION_IFNE) { + value = c->argv[condition_index + 2]; + } + o = lookupKeyReadWithFlags(c->db, key, LOOKUP_NONOTIFY | LOOKUP_NOSTATS | LOOKUP_NOTOUCH); + + switch (condition) { + case EXEC_CONDITION_IFEQ: + if (o && objectGetType(o) != OBJ_STRING) { + execCommandAbort(c, objectGetVal(shared.wrongtypeerr)); + return -1; + } + matches = o && equalStringObjects(o, value); + break; + case EXEC_CONDITION_IFNE: + if (o && objectGetType(o) != OBJ_STRING) { + execCommandAbort(c, objectGetVal(shared.wrongtypeerr)); + return -1; + } + matches = !o || !equalStringObjects(o, value); + break; + case EXEC_CONDITION_NX: matches = !o; break; + case EXEC_CONDITION_XX: matches = o != NULL; break; + default: serverPanic("Unknown EXEC condition"); + } + + if (!matches) return 0; + } + return 1; } void execCommand(client *c) { @@ -225,6 +340,15 @@ void execCommand(client *c) { return; } + int conditions_match = checkExecConditions(c); + if (conditions_match == -1) return; + + if (!conditions_match) { + addReply(c, shared.nullarray[c->resp]); + discardTransaction(c); + return; + } + struct ClientFlags old_flags = c->flag; /* we do not want to allow blocking commands inside multi */ diff --git a/src/networking.c b/src/networking.c index 024486367..e104c6534 100644 --- a/src/networking.c +++ b/src/networking.c @@ -37,6 +37,10 @@ #include "fpconv_dtoa.h" #include "fmtargs.h" #include "io_threads.h" +#include "compression_stream.h" +#include "throttle.h" +#include "throttle_repl.h" +#include "stat_calc.h" #include "module.h" #include "connection.h" #include "zmalloc.h" @@ -233,6 +237,12 @@ void linkClient(client *c) { c->client_list_node = listLast(server.clients); uint64_t id = htonu64(c->id); raxInsert(server.clients_index, (unsigned char *)&id, sizeof(id), c, NULL); + + /* Increment active client counters. These counters are paired with decrements + * in unlinkClient() and track connected clients in the global active clients list. */ + if (connIsPriority(c->conn)) { + server.stat_num_active_priority_clients++; + } } /* Initialize client authentication state. */ @@ -384,6 +394,10 @@ client *createClient(connection *conn) { listSetFreeMethod(c->reply, freeClientReplyValue); listSetDupMethod(c->reply, dupClientReplyValue); c->repl_data = NULL; + c->throttler = NULL; + c->throttle_node = NULL; + c->throttle_start = 0; + c->cob_trend = NULL; c->bstate = NULL; c->pubsub_data = NULL; c->module_data = NULL; @@ -1803,6 +1817,15 @@ int clientHasPendingReplies(client *c) { /* Replicas use global shared replication buffer instead of * private output buffer. */ serverAssert(c->bufpos == 0 && listLength(c->reply) == 0); + + /* Unsent compressed data counts as pending. Skip while CLIENT_PENDING_IO: + * the IO thread owns the compression state; postWriteToReplica re-checks + * when the job completes. */ + if (c->repl_data->repl_compression && c->io_write_state != CLIENT_PENDING_IO && + c->repl_data->repl_compression->out_buf_pos < sdslen(c->repl_data->repl_compression->out_buf)) { + return 1; + } + if (c->repl_data->ref_repl_buf_node == NULL) return 0; /* If the last replication buffer block content is totally sent, @@ -1878,9 +1901,171 @@ void clientAcceptHandler(connection *conn) { moduleFireServerEvent(VALKEYMODULE_EVENT_CLIENT_CHANGE, VALKEYMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED, c); } +/* ==================================================================== + * Priority Subnets and Admission Control + * ==================================================================== */ + +/* parseSubnetList parses a string containing a list of subnets separated by spaces, tabs, or commas. + * On success, it allocates an array of anetSubnet, populates it, and sets *subnets and *count. + * Returns C_OK on success, and C_ERR on any parsing error. + * Caller is responsible for freeing *subnets using zfree() if it is non-NULL. */ +static int parseSubnetList(const char *raw_sources, anetSubnet **subnets, int *count) { + if (!subnets || !count) return C_ERR; + *subnets = NULL; + *count = 0; + + if (!raw_sources || raw_sources[0] == '\0') { + return C_OK; + } + + /* First pass: count non-empty tokens */ + char *sources_to_count = zstrdup(raw_sources); + char *token; + char *saveptr; + int sources_count = 0; + + token = strtok_r(sources_to_count, " \t,", &saveptr); + while (token != NULL) { + if (strlen(token) > 0) { + sources_count++; + } + token = strtok_r(NULL, " \t,", &saveptr); + } + zfree(sources_to_count); + + if (sources_count == 0) { + return C_OK; + } + + anetSubnet *new_subnets = zmalloc(sizeof(anetSubnet) * sources_count); + char *sources_to_parse = zstrdup(raw_sources); + + int source_index = 0; + int success = 1; + token = strtok_r(sources_to_parse, " \t,", &saveptr); + while (token != NULL) { + if (strlen(token) > 0) { + if (anetParseSubnet(NULL, token, &new_subnets[source_index++]) != ANET_OK) { + success = 0; + break; + } + } + token = strtok_r(NULL, " \t,", &saveptr); + } + zfree(sources_to_parse); + + if (!success) { + zfree(new_subnets); + return C_ERR; + } + + *subnets = new_subnets; + *count = sources_count; + return C_OK; +} + +/* Re-evaluate connection priority for all currently connected clients when + * priority-subnets is updated dynamically at runtime via CONFIG SET. + * + * 1. Immediate dynamic reclassification: Existing clients connecting before a + * subnet update that match the new configuration are immediately promoted + * to priority status without requiring a reconnect. Similarly, clients that + * no longer match are demoted to normal priority. + * 2. Strict counter reconciliation: Accurately recomputes + * server.stat_num_active_priority_clients to reflect the exact + * ground truth of active priority connections, preventing telemetry drift + * or underflow/overflow desync across dynamic config changes. + * 3. Safe transport handling: Fake clients (c->conn == NULL) and non-IP + * connections (such as UNIX domain sockets or unresolved peers) are safely + * classified as normal (non-priority) connections. */ +static void reclassifyClientsPriority(void) { + if (!server.clients) return; + + long long count = 0; + listIter li; + listNode *ln; + listRewind(server.clients, &li); + + while ((ln = listNext(&li)) != NULL) { + client *c = listNodeValue(ln); + if (!c->conn) continue; + + char ip[CONN_ADDR_STR_LEN]; + int port = 0; + if (connAddrPeerName(c->conn, ip, sizeof(ip), &port) != C_OK) { + connSetPriority(c->conn, false); + continue; + } + + bool is_prio = (server.priority_subnets_count > 0 && + anetMatchIpSubnet(ip, server.priority_subnets_array, server.priority_subnets_count)); + connSetPriority(c->conn, is_prio); + if (is_prio) count++; + } + + server.stat_num_active_priority_clients = count; +} + +/* Validate priority-subnets configuration string. + * Returns C_OK if valid, C_ERR otherwise and sets *err if provided. */ +int validatePrioritySubnets(const char *subnets_str, const char **err) { + anetSubnet *subnets = NULL; + int count = 0; + if (parseSubnetList(subnets_str, &subnets, &count) != C_OK) { + if (err) *err = "Invalid IP address or CIDR subnet in priority-subnets"; + return C_ERR; + } + if (subnets) zfree(subnets); + return C_OK; +} + +/* Update compiled priority-subnets from configuration string and reclassify clients. + * Returns C_OK on success, C_ERR on parsing failure. */ +int updatePrioritySubnets(const char *subnets_str) { + anetSubnet *new_subnets = NULL; + int new_count = 0; + if (parseSubnetList(subnets_str, &new_subnets, &new_count) != C_OK) { + return C_ERR; + } + zfree(server.priority_subnets_array); + server.priority_subnets_array = new_subnets; + server.priority_subnets_count = new_count; + reclassifyClientsPriority(); + return C_OK; +} + +/* Admission Control: + * 1. Total clients can never exceed maxclients. + * 2. Normal clients are capped at max(0, maxclients - maxclients-reserved). + * 3. Priority clients originating from priority-subnets can take up to maxclients. + * 4. maxclients-reserved connection slots are guaranteed for priority clients. + */ +static bool hasMaxClientsLimitReached(bool is_prioritized) { + long long total_clients = (long long)listLength(server.clients) + + (long long)getClusterConnectionsCount(); + if (total_clients >= (long long)server.maxclients) { + return true; + } + + if (is_prioritized) { + return false; + } + + if (server.maxclients_reserved > 0 && server.priority_subnets_count > 0) { + long long normal_limit = 0; + if (server.maxclients > server.maxclients_reserved) { + normal_limit = (long long)server.maxclients - (long long)server.maxclients_reserved; + } + long long prioritized_clients = server.stat_num_active_priority_clients; + long long normal_clients = (total_clients > prioritized_clients) ? (total_clients - prioritized_clients) : 0; + return normal_clients >= normal_limit; + } + + return false; +} + void acceptCommonHandler(connection *conn, struct ClientFlags flags, char *ip) { client *c; - UNUSED(ip); char addr[CONN_ADDR_STR_LEN] = {0}; char laddr[CONN_ADDR_STR_LEN] = {0}; @@ -1899,7 +2084,9 @@ void acceptCommonHandler(connection *conn, struct ClientFlags flags, char *ip) { * Admission control will happen before a client is created and connAccept() * called, because we don't want to even start transport-level negotiation * if rejected. */ - if (listLength(server.clients) + getClusterConnectionsCount() >= server.maxclients) { + bool is_prioritized = (server.priority_subnets_count > 0 && ip != NULL && + anetMatchIpSubnet(ip, server.priority_subnets_array, server.priority_subnets_count)); + if (hasMaxClientsLimitReached(is_prioritized)) { char *err; if (server.cluster_enabled) err = "-ERR max number of clients + cluster " @@ -1914,10 +2101,14 @@ void acceptCommonHandler(connection *conn, struct ClientFlags flags, char *ip) { /* Nothing to do, Just to avoid the warning... */ } server.stat_rejected_conn++; + if (is_prioritized) { + server.stat_rejected_priority_conn++; + } connClose(conn); return; } - + /* Set the priority of the connection */ + connSetPriority(conn, is_prioritized); /* Create connection and client */ if ((c = createClient(conn)) == NULL) { serverLog(LL_WARNING, "Error registering fd event for the new client connection: %s (addr=%s laddr=%s)", @@ -2021,6 +2212,15 @@ void unlinkClient(client *c) { raxRemove(server.clients_index, (unsigned char *)&id, sizeof(id), NULL); listDelNode(server.clients, c->client_list_node); c->client_list_node = NULL; + + /* Decrement active client counters. Fake clients (where c->conn is NULL) + * and unlinked clients (c->client_list_node is NULL) do not increment these + * counters on creation, so we only decrement here for linked, active connections. */ + if (connIsPriority(c->conn)) { + if (server.stat_num_active_priority_clients > 0) { + server.stat_num_active_priority_clients--; + } + } } removeClientFromPendingCommandsBatch(c); @@ -2064,6 +2264,8 @@ void unlinkClient(client *c) { c->conn = NULL; } + throttle_removeClient(c); + /* Remove from the list of pending writes if needed. */ if (c->flag.pending_write) { serverAssert(server.clients_pending_write->len > 0); @@ -2084,6 +2286,10 @@ void unlinkClient(client *c) { /* Clear the tracking status. */ if (c->flag.tracking) disableTracking(c); + + /* Client must not be in blocked or unblocked state at this point. + * Guaranteed by freeClient ordering: unblockClient -> freeClientBlockingState -> unlinkClient. */ + serverAssert(!c->flag.blocked && !c->flag.unblocked); } /* Clear the client state to resemble a newly connected client. */ @@ -2214,7 +2420,7 @@ int freeClient(client *c) { /* Deallocate structures used to block on blocking ops. */ /* If there is any in-flight command, we don't record their duration. */ c->duration = 0; - if (c->flag.blocked) unblockClient(c, 1); + if (c->flag.blocked) unblockClient(c, 0); freeClientBlockingState(c); freeClientPubSubData(c); @@ -2268,6 +2474,7 @@ int freeClient(client *c) { if (c->lib_name) decrRefCount(c->lib_name); if (c->lib_ver) decrRefCount(c->lib_ver); freeClientMultiState(c); + if (c->cob_trend) trendCalculator_free(c->cob_trend); sdsfree(c->peerid); sdsfree(c->sockname); zfree(c); @@ -2461,61 +2668,226 @@ client *lookupClientByID(uint64_t id) { return c; } +/* Bound compression work and staging memory for one write dispatch. */ +#define REPL_COMPRESSION_BATCH_SIZE (1024 * 1024) + +/* Advance the replica's replication-buffer cursor (ref_repl_buf_node / + * ref_block_pos) past consumed raw bytes, releasing the reference on each + * fully-sent block. Shared by the compressed and plaintext post-write paths. */ +static void advanceReplicaReplBufferCursor(client *c, size_t consumed) { + listNode *node = c->repl_data->ref_repl_buf_node; + listNode *next_node = NULL; + size_t remaining = consumed + c->repl_data->ref_block_pos; + replBufBlock *block = listNodeValue(node); + + while (remaining >= block->used) { + next_node = listNextNode(node); + if (!next_node) break; /* End of list */ + + remaining -= block->used; + block->refcount--; + + node = next_node; + block = listNodeValue(node); + block->refcount++; + } + + serverAssert(remaining <= block->used); + c->repl_data->ref_repl_buf_node = node; + c->repl_data->ref_block_pos = remaining; +} + static void postWriteToReplica(client *c) { + replicaCompressionState *compression = c->repl_data->repl_compression; + if (c->write_flags & WRITE_FLAGS_COMPRESSION_ERROR) { + serverAssert(compression != NULL); + serverLog(LL_WARNING, "Compression error on replica %s (algo=%s, batch_uncompressed_bytes=%zu), disconnecting", + replicationGetReplicaName(c), compressionAlgoName(compression->compressor.algo), + compression->batch_uncompressed_bytes); + freeClientAsync(c); + return; + } + if (c->nwritten <= 0) return; server.stat_net_repl_output_bytes += c->nwritten; - /* Locate the last node which has leftover data and - * decrement reference counts of all nodes in front of it. - * Set c->ref_repl_buf_node to point to the last node and - * c->ref_block_pos to the offset within that node */ - listNode *curr = c->repl_data->ref_repl_buf_node; - listNode *next = NULL; - size_t nwritten = c->nwritten + c->repl_data->ref_block_pos; - replBufBlock *o = listNodeValue(curr); + if (compression) { + /* An IO thread may send the batch but cannot update replication block + * refcounts while the main thread appends and trims them. Advance the + * cursor here only after the whole compressed batch has been sent. */ + if (compression->out_buf_pos == sdslen(compression->out_buf)) { + size_t batch_uncompressed_bytes = compression->batch_uncompressed_bytes; - while (nwritten >= o->used) { - next = listNextNode(curr); - if (!next) break; /* End of list */ + advanceReplicaReplBufferCursor(c, batch_uncompressed_bytes); - nwritten -= o->used; - o->refcount--; + compression->uncompressed_bytes += batch_uncompressed_bytes; + compression->compressed_bytes += sdslen(compression->out_buf); - curr = next; - o = listNodeValue(curr); - o->refcount++; + /* Start the next batch. One batch's compressed output is bounded + * by REPL_COMPRESSION_BATCH_SIZE plus the codec's small + * worst-case expansion margin, so the allocation is retained. */ + sdsclear(compression->out_buf); + compression->out_buf_pos = 0; + compression->batch_uncompressed_bytes = 0; + + incrementalTrimReplicationBacklog(REPL_BACKLOG_TRIM_BLOCKS_PER_CALL); + } + return; } - serverAssert(nwritten <= o->used); - c->repl_data->ref_repl_buf_node = curr; - c->repl_data->ref_block_pos = nwritten; + advanceReplicaReplBufferCursor(c, c->nwritten); incrementalTrimReplicationBacklog(REPL_BACKLOG_TRIM_BLOCKS_PER_CALL); } -static void writeToReplica(client *c) { +/* Resolve the replication-buffer range available to replica c: the + * last block to send and the end position within it (the start is the + * replica's own cursor, ref_repl_buf_node/ref_block_pos). The main thread + * reads the live buffer tail; an IO thread uses the snapshot taken when the + * write job was dispatched. Returns false only when the buffer has no blocks. + * Shared by the plaintext and compressed write paths. */ +static bool getReplicaWriteRange(client *c, listNode **last_node, size_t *last_pos) { + if (inMainThread()) { + *last_node = listLast(server.repl_buffer_blocks); + if (!*last_node) return false; + *last_pos = ((replBufBlock *)listNodeValue(*last_node))->used; + } else { + *last_node = c->io_last_reply_block; + serverAssert(*last_node != NULL); + *last_pos = c->io_last_bufpos; + } + return true; +} + +/* Append compressed input to the link's staging buffer. The first call emits + * the replication envelope; a sync flush makes the batch writable without + * ending the frame. */ +static int compressReplicaDataToOutputBuffer(replicaCompressionState *compression, + const uint8_t *input, + size_t input_len, + compressFlushMode flush_mode) { + if (!compression->compressor.stream_started) { + uint8_t envelope[VCS_ENVELOPE_SIZE]; + if (vcsBuildEnvelope(envelope, compression->compressor.algo, VCS_STREAM_REPL) == C_ERR) return C_ERR; + compression->out_buf = sdscatlen(compression->out_buf, envelope, sizeof(envelope)); + } + size_t bound = streamCompressorOutputBound(&compression->compressor, input_len); + serverAssert(bound > 0); + compression->out_buf = sdsMakeRoomFor(compression->out_buf, bound); + ssize_t compressed = + streamCompressorFeed(&compression->compressor, (uint8_t *)compression->out_buf + sdslen(compression->out_buf), + sdsavail(compression->out_buf), input, input_len, flush_mode); + if (compressed < 0) return C_ERR; + sdsIncrLen(compression->out_buf, (size_t)compressed); + return C_OK; +} + +/* Compressed write path for replicas on either the IO thread or the main thread. */ +static void writeToReplicaCompressed(client *c) { + replicaCompressionState *compression = c->repl_data->repl_compression; + + /* Finish sending the previous batch's leftover first; compressed bytes + * must reach the socket in order. */ + if (compression->out_buf_pos < sdslen(compression->out_buf)) { + size_t avail = sdslen(compression->out_buf) - compression->out_buf_pos; + c->nwritten = connWrite(c->conn, + compression->out_buf + compression->out_buf_pos, + avail); + if (c->nwritten <= 0) { + c->write_flags |= WRITE_FLAGS_WRITE_ERROR; + return; + } + compression->out_buf_pos += c->nwritten; + /* Skip trim here; postWriteToReplica trims only after the batch fully + * drains. Worst-case trim delay is one batch (REPL_COMPRESSION_BATCH_SIZE). */ + return; + } + + /* postWriteToReplica resets the batch once it fully drains, so a fresh + * batch always starts from an empty buffer. */ + serverAssert(sdslen(compression->out_buf) == 0 && compression->out_buf_pos == 0 && + compression->batch_uncompressed_bytes == 0); + listNode *last_node; - size_t bufpos; + size_t last_pos; + if (!getReplicaWriteRange(c, &last_node, &last_pos)) return; + listNode *first_node = c->repl_data->ref_repl_buf_node; + /* Compress new replication-backlog bytes, capped at + * REPL_COMPRESSION_BATCH_SIZE raw bytes per cycle to bound per-batch + * latency and keep out_buf size predictable. */ + size_t batch_uncompressed_bytes = 0; + for (listNode *cur = first_node; cur != NULL; cur = listNextNode(cur)) { + replBufBlock *block = listNodeValue(cur); + size_t start = (cur == first_node) ? c->repl_data->ref_block_pos : 0; + size_t end = (cur == last_node) ? last_pos : block->used; + + serverAssert(end >= start); + if (end == start) { + if (cur == last_node) break; + continue; + } + + size_t len = end - start; + /* Cap this write at the remaining batch budget; the cursor resumes mid-block next cycle. */ + size_t remaining = REPL_COMPRESSION_BATCH_SIZE - batch_uncompressed_bytes; + if (len > remaining) len = remaining; + if (compressReplicaDataToOutputBuffer(compression, (const uint8_t *)block->buf + start, len, + COMPRESS_FLUSH_CONTINUE) == C_ERR) { + c->write_flags |= WRITE_FLAGS_COMPRESSION_ERROR | WRITE_FLAGS_WRITE_ERROR; + return; + } + batch_uncompressed_bytes += len; + if (batch_uncompressed_bytes >= REPL_COMPRESSION_BATCH_SIZE) break; + if (cur == last_node) break; + } + + if (batch_uncompressed_bytes == 0) return; + + /* Drain codec-buffered bytes so the whole batch lands in out_buf. */ + if (compressReplicaDataToOutputBuffer(compression, NULL, 0, COMPRESS_FLUSH_SYNC) != C_OK) { + c->write_flags |= WRITE_FLAGS_COMPRESSION_ERROR | WRITE_FLAGS_WRITE_ERROR; + return; + } + + compression->batch_uncompressed_bytes = batch_uncompressed_bytes; + + /* Send out_buf. The backlog cursor advances only after a full send + * (postWriteToReplica), so a partial send keeps it pinned to the start of + * the batch. */ + size_t avail = sdslen(compression->out_buf); + serverAssert(avail > 0); + + c->nwritten = connWrite(c->conn, compression->out_buf, avail); + if (c->nwritten <= 0) { + c->write_flags |= WRITE_FLAGS_WRITE_ERROR; + return; + } + compression->out_buf_pos = c->nwritten; +} + +static void writeToReplica(client *c) { serverAssert(c->bufpos == 0 && listLength(c->reply) == 0); - /* Determine the last block and buffer position based on thread context */ - if (inMainThread()) { - last_node = listLast(server.repl_buffer_blocks); - if (!last_node) return; - bufpos = ((replBufBlock *)listNodeValue(last_node))->used; - } else { - last_node = c->io_last_reply_block; - serverAssert(last_node != NULL); - bufpos = c->io_last_bufpos; + + /* Compressed replicas use the framed write path; the decision lives here so + * callers do not branch on the per-replica compression state. */ + if (c->repl_data->repl_compression != NULL) { + writeToReplicaCompressed(c); + return; } + listNode *last_node; + size_t last_pos; + + if (!getReplicaWriteRange(c, &last_node, &last_pos)) return; + listNode *first_node = c->repl_data->ref_repl_buf_node; /* Handle the single block case */ if (first_node == last_node) { replBufBlock *b = listNodeValue(first_node); - c->nwritten = connWrite(c->conn, b->buf + c->repl_data->ref_block_pos, bufpos - c->repl_data->ref_block_pos); + c->nwritten = connWrite(c->conn, b->buf + c->repl_data->ref_block_pos, last_pos - c->repl_data->ref_block_pos); if (c->nwritten <= 0) { c->write_flags |= WRITE_FLAGS_WRITE_ERROR; } @@ -2532,7 +2904,7 @@ static void writeToReplica(client *c) { for (listNode *cur_node = first_node; cur_node != NULL && iovcnt < iovmax; cur_node = listNextNode(cur_node)) { replBufBlock *cur_block = listNodeValue(cur_node); size_t start = (cur_node == first_node) ? c->repl_data->ref_block_pos : 0; - size_t len = (cur_node == last_node) ? bufpos : cur_block->used; + size_t len = (cur_node == last_node) ? last_pos : cur_block->used; len -= start; /* For TLS, we should not call SSL_write() with num=0 */ @@ -3202,7 +3574,9 @@ int handleReadResult(client *c) { c->last_interaction = server.unixtime; c->net_input_bytes += c->nread; if (isReplicatedClient(c)) { - c->repl_data->read_reploff += c->nread; + /* A reader-active primary link advances read_reploff with decoded + * bytes (replDecodeToQueryBuf); c->nread counts wire bytes here. */ + if (!(c->flag.primary && server.repl_stream_reader)) c->repl_data->read_reploff += c->nread; if (getClientType(c) == CLIENT_TYPE_PRIMARY) { server.stat_net_repl_input_bytes += c->nread; } else { @@ -3420,6 +3794,7 @@ void resetClient(client *c) { c->flag.replication_done = 0; c->flag.buffered_reply = 0; c->flag.keyspace_notified = 0; + c->flag.throttle_checked = 0; c->net_output_bytes_curr_cmd = 0; /* Make sure the duration has been recorded to some command. */ @@ -3928,8 +4303,9 @@ void commandProcessed(client *c) { * The client will be reset in unblockClient(). * 2. Don't update replication offset or propagate commands to replicas, * since we have not applied the command. */ - if (c->flag.blocked) return; + if (c->flag.blocked || c->flag.throttled) return; + c->flag.pending_command = 0; reqresAppendResponse(c); clusterSlotStatsAddNetworkBytesInForUserClient(c); resetClient(c); @@ -4012,8 +4388,8 @@ int processPendingCommandAndInputBuffer(client *c) { * But in case of a module blocked client (see RM_Call 'K' flag) we do not reach this code path. * So whenever we change the code here we need to consider if we need this change on module * blocked client as well */ + if (c->flag.close_asap) return C_ERR; if (c->flag.pending_command) { - c->flag.pending_command = 0; if (processCommandAndResetClient(c) == C_ERR) { return C_ERR; } @@ -4305,6 +4681,7 @@ int processInputBuffer(client *c) { } /* We are finally ready to execute the command. */ + c->flag.pending_command = 1; if (processCommandAndResetClient(c) == C_ERR) { /* If the client is no longer valid, we avoid exiting this * loop and trimming the client buffer later. So we return @@ -4406,6 +4783,26 @@ static bool readToQueryBuf(client *c) { } #define REPL_MAX_READS_PER_IO_EVENT 25 + +/* Keep the wire scratch buffer off the stack of ordinary client reads. */ +__attribute__((noinline)) static bool readAndDecodePrimaryStream(client *primary, + size_t output_budget, + ssize_t *decoded_bytes, + bool *full_read) { + uint8_t wire_buf[PROTO_IOBUF_LEN]; + if (primary->flag.close_asap) { + primary->nread = 0; + *full_read = false; + } else { + primary->nread = connRead(primary->conn, wire_buf, sizeof(wire_buf)); + *full_read = primary->nread == (int)sizeof(wire_buf); + } + if (handleReadResult(primary) != C_OK) return false; + + *decoded_bytes = replDecodeToQueryBuf(primary, wire_buf, (size_t)primary->nread, output_budget); + return true; +} + void readQueryFromClient(connection *conn) { client *c = connGetPrivateData(conn); /* Check if we can send the client to be handled by the IO-thread */ @@ -4415,15 +4812,38 @@ void readQueryFromClient(connection *conn) { bool repeat = false; int iter = 0; + size_t decoded_bytes = 0; do { - bool full_read = readToQueryBuf(c); - if (handleReadResult(c) == C_OK) { + ssize_t decoded_this_iteration = 0; + bool full_read; + bool read_ok = true; + bool decode_repl_stream = c->flag.primary && server.repl_stream_reader; + bool resume_decode = decode_repl_stream && replStreamHasPendingDecode(); + size_t decode_budget = REPL_DECODE_EVENT_BUDGET - decoded_bytes; + if (resume_decode) { + decoded_this_iteration = replDecodeToQueryBuf(c, NULL, 0, decode_budget); + full_read = !replStreamHasPendingDecode(); + } else if (decode_repl_stream) { + read_ok = readAndDecodePrimaryStream(c, decode_budget, &decoded_this_iteration, &full_read); + } else { + full_read = readToQueryBuf(c); + read_ok = handleReadResult(c) == C_OK; + } + if (read_ok) { + if (decoded_this_iteration < 0) { + serverLog(LL_WARNING, "Disconnecting primary due to replication stream decompression failure"); + freeClientAsync(c); + return; + } + decoded_bytes += (size_t)decoded_this_iteration; if (processInputBuffer(c) == C_ERR) return; trimCommandQueue(c); + if (decode_repl_stream && replStreamHasPendingDecode()) full_read = false; } repeat = (c->flag.primary && !c->flag.close_asap && ++iter < REPL_MAX_READS_PER_IO_EVENT && + decoded_bytes < REPL_DECODE_EVENT_BUDGET && full_read); beforeNextClient(c); } while (repeat); @@ -4494,7 +4914,7 @@ int isClientConnIpV6(client *c) { * readable format, into the sds string 's'. */ sds catClientInfoString(sds s, client *client, int hide_user_data) { if (!server.crashed) waitForClientIO(client); - char flags[17], events[3], capa[9], conninfo[CONN_INFO_LEN], *p; + char flags[32], events[3], capa[9], conninfo[CONN_INFO_LEN], *p; p = flags; if (client->flag.replica) { @@ -4519,9 +4939,11 @@ sds catClientInfoString(sds s, client *client, int hide_user_data) { if (client->flag.readonly) *p++ = 'r'; if (client->flag.no_evict) *p++ = 'e'; if (client->flag.no_touch) *p++ = 'T'; + if (client->flag.throttled) *p++ = 'h'; if (client->flag.import_source) *p++ = 'I'; if (client->slot_migration_job && isImportSlotMigrationJob(client->slot_migration_job)) *p++ = 'i'; if (client->slot_migration_job && !isImportSlotMigrationJob(client->slot_migration_job)) *p++ = 'E'; + if (connIsPriority(client->conn)) *p++ = 'H'; if (p == flags) *p++ = 'N'; *p++ = '\0'; @@ -5045,9 +5467,11 @@ static int validateClientFlagFilter(sds flag_filter) { case 'r': case 'e': case 'T': + case 'h': case 'I': case 'i': case 'E': + case 'H': case 'N': /* Valid flag, do nothing. */ break; @@ -5199,6 +5623,9 @@ static int clientMatchesFlagFilter(client *c, sds flag_filter) { case 'T': /* client will not touch the LRU/LFU of the keys it accesses */ if (!c->flag.no_touch) return 0; break; + case 'h': /* client is throttled */ + if (!c->flag.throttled) return 0; + break; case 'I': /* Import source flag */ if (!c->flag.import_source) return 0; break; @@ -5208,6 +5635,9 @@ static int clientMatchesFlagFilter(client *c, sds flag_filter) { case 'E': /* Slot migration export flag */ if (!c->slot_migration_job || isImportSlotMigrationJob(c->slot_migration_job)) return 0; break; + case 'H': /* High priority connection */ + if (!connIsPriority(c->conn)) return 0; + break; case 'N': /* Check for no flags */ if (c->flag.replica || c->flag.primary || c->flag.pubsub || c->flag.multi || c->flag.blocked || c->flag.tracking || @@ -5215,8 +5645,9 @@ static int clientMatchesFlagFilter(client *c, sds flag_filter) { c->flag.dirty_cas || c->flag.close_after_reply || c->flag.unblocked || c->flag.close_asap || c->flag.unix_socket || c->flag.readonly || - c->flag.no_evict || c->flag.no_touch || - c->flag.import_source || c->slot_migration_job) { + c->flag.no_evict || c->flag.no_touch || c->flag.throttled || + c->flag.import_source || c->slot_migration_job || + connIsPriority(c->conn)) { return 0; } break; @@ -6161,6 +6592,8 @@ size_t getClientOutputBufferMemoryUsage(client *c) { repl_buf_size = last->repl_offset + last->size - cur->repl_offset; repl_node_num = last->id - cur->id + 1; } + /* A compressed batch keeps this cursor pinned until the staged output + * drains, so repl_buf_size already represents all unsent data. */ return repl_buf_size + (repl_node_size * repl_node_num); } @@ -6186,6 +6619,14 @@ size_t getClientMemoryUsage(client *c, size_t *output_buffer_mem_usage) { mem += c->querybuf ? sdsAllocSize(c->querybuf) : 0; mem += zmalloc_size(c); mem += c->buf_usable_size; + /* Compression staging capacity is retained for reuse, so account it as + * client memory rather than pending output. Skip while an IO thread may + * reallocate the buffer. */ + if (getClientType(c) == CLIENT_TYPE_REPLICA && c->repl_data->repl_compression && + c->io_write_state != CLIENT_PENDING_IO) { + replicaCompressionState *compression = c->repl_data->repl_compression; + mem += zmalloc_size(compression) + sdsAllocSize(compression->out_buf); + } /* For efficiency (less work keeping track of the argv memory), it doesn't include the used memory * i.e. unused sds space and internal fragmentation, just the string length. but this is enough to * spot problematic clients. */ @@ -6291,6 +6732,10 @@ int checkClientOutputBufferLimits(client *c) { } else { c->obuf_soft_limit_reached_time = 0; } + /* The steady-state throttle may exempt a replica from the soft limit to give throttling + * time to converge; the hard limit is never suppressed, so a replica that reaches it is + * always disconnected. */ + if (soft && !hard && throttleRepl_isClientExemptFromCobLimits(c)) return 0; return soft || hard; } diff --git a/src/object.c b/src/object.c index eeb6e06e3..7ff0a43db 100644 --- a/src/object.c +++ b/src/object.c @@ -39,6 +39,7 @@ #include "zmalloc.h" #include "sds.h" #include "module.h" +#include "bgiteration.h" #include #include @@ -80,6 +81,98 @@ void objectSetLRU(robj *o, unsigned int lru) { o->lru = lru; } +/* Get beginning of embedded data, which may contain expire, metadata, key, and/or value. + * Embedded data flags must be accurate when called. */ +static unsigned char *objectEmbeddedData(const robj *o) { + unsigned char *data = (void *)(o + 1); + if (o->hasembval) data -= sizeof(void *); + return data; +} + +/* ===================== Object Metadata Management ========================= */ + +/* Static variable to store metadata size. Set once at server initialization. */ +static size_t object_metadata_size = 0; + +/* Set the metadata size. + * Size should not be changed once set. */ +void objectSetMetadataSize(size_t size) { + /* Metadata size already set - only allow setting to the same value */ + if (object_metadata_size == size) return; + + /* When current size is 0 and the incoming size is not - setting for the first time */ + serverAssert(object_metadata_size == 0); + + /* Check that all databases are empty */ + if (server.db != NULL) { + for (int j = 0; j < server.dbnum; j++) { + if (server.db[j] != NULL) { + serverAssert(kvstoreSize(server.db[j]->keys) == 0); + } + } + } + + object_metadata_size = size; +} + +/* Calculate the size of metadata for an object. + * Returns the configured metadata size if the object has an embedded key, 0 otherwise. */ +size_t objectGetMetadataSize(const robj *o) { + if (o->hasembkey) return object_metadata_size; + return 0; +} + +/* Get a void pointer to the metadata for an object. + * Returns NULL if the object doesn't have metadata. + * The caller must cast this to the appropriate metadata structure type. + * + * Memory layout visualization for objects: + * + * ┌─────────────────────────────────────────────────────────────────┐ + * │ robj (struct serverObject) │ + * │ - type, encoding, lru, hasexpire, hasembkey, hasembval... │ + * ├─────────────────────────────────────────────────────────────────┤ + * │ expire field (optional, if hasexpire == 1) │ + * │ - long long (8 bytes) │ + * ├─────────────────────────────────────────────────────────────────┤ + * │ metadata (optional, if hasembkey == 1 && metadata_size > 0) │ + * │ - (object_metadata_size) │ ← objectGetMetadata returns pointer here + * ├─────────────────────────────────────────────────────────────────┤ + * │ embedded key (if hasembkey == 1) │ + * ├─────────────────────────────────────────────────────────────────┤ + * │ embedded value (if hasembval == 1) │ + * └─────────────────────────────────────────────────────────────────┘ + */ +void *objectGetMetadata(const robj *o) { + if (object_metadata_size == 0 || !o->hasembkey) return NULL; + + /* The memory after the struct where we embedded metadata. */ + unsigned char *data = objectEmbeddedData(o); + + /* If expire field exists, metadata is after it */ + if (o->hasexpire) { + data += sizeof(long long); + } + + return (void *)data; +} + +/* Copy the opaque metadata region from one object to another. Used when an + * object is reallocated (e.g. objectSetKeyAndExpire) so that metadata attached + * by subsystems such as background iteration (forkless save) survives the move. + * + * The copy happens only when both objects actually carry metadata (both have an + * embedded key and the configured metadata size is non-zero). If the source has + * no metadata there is nothing to preserve, and the destination keeps its + * zero-initialized metadata. */ +void objectCopyMetadata(robj *dst, const robj *src) { + if (object_metadata_size == 0) return; + void *src_md = objectGetMetadata(src); + void *dst_md = objectGetMetadata(dst); + if (src_md == NULL || dst_md == NULL) return; + memcpy(dst_md, src_md, object_metadata_size); +} + /* ===================== Creation and parsing of objects ==================== */ /* Creates an object, optionally with embedded key and expire fields. The key @@ -93,10 +186,12 @@ static robj *createUnembeddedObjectWithKeyAndExpire(int type, void *val, const_s size_t key_sds_len = has_embkey ? sdslen(key) : 0; char key_sds_type = has_embkey ? sdsReqType(key_sds_len) : 0; size_t key_sds_size = has_embkey ? sdsReqSize(key_sds_len, key_sds_type) : 0; + size_t metadata_size = has_embkey ? object_metadata_size : 0; size_t min_size = sizeof(robj); if (has_expire) { min_size += sizeof(long long); } + min_size += metadata_size; if (has_embkey) { /* Size of embedded key, incl. 1 byte for prefixed sds hdr size. */ min_size += 1 + key_sds_size; @@ -129,6 +224,12 @@ static robj *createUnembeddedObjectWithKeyAndExpire(int type, void *val, const_s data += sizeof(long long); } + /* Initialize metadata to zero */ + if (metadata_size > 0) { + memset(data, 0, metadata_size); + data += metadata_size; + } + /* Copy embedded key. */ if (o->hasembkey) { *data++ = sdsHdrSize(key_sds_type); @@ -171,12 +272,6 @@ robj *createRawStringObject(const char *ptr, size_t len) { return createObject(OBJ_STRING, sdsnewlen(ptr, len)); } -/* Get beginning of embedded data, which may contain expire, key, and/or value. Embedded data flags must be accurate when called. */ -static unsigned char *objectEmbeddedData(const robj *o) { - unsigned char *data = (void *)(o + 1); - if (o->hasembval) data -= sizeof(void *); - return data; -} /* Creates a new embedded string object and copies the content of key, val_ptr * and expire to the new object. LRU is set to 0. */ @@ -190,6 +285,7 @@ static robj *createEmbeddedStringObjectWithKeyAndExpire(const char *val_ptr, char key_sds_type = has_embkey ? sdsReqType(key_sds_len) : 0; size_t key_sds_size = has_embkey ? sdsReqSize(key_sds_len, key_sds_type) : 0; size_t val_sds_size = sdsReqSize(val_len, SDS_TYPE_8); + size_t metadata_size = has_embkey ? object_metadata_size : 0; if (val_sds_size < sizeof(void *)) { val_sds_size = sizeof(void *); /* Ensure it's possible to "unembed" value later */ } @@ -199,6 +295,7 @@ static robj *createEmbeddedStringObjectWithKeyAndExpire(const char *val_ptr, if (expire != EXPIRY_NONE) { min_size += sizeof(long long); } + min_size += metadata_size; if (has_embkey) { /* Size of embedded key, incl. 1 byte for prefixed sds hdr size. */ min_size += 1 + key_sds_size; @@ -232,6 +329,12 @@ static robj *createEmbeddedStringObjectWithKeyAndExpire(const char *val_ptr, data += sizeof(long long); } + /* Initialize metadata to zero */ + if (metadata_size > 0) { + memset(data, 0, metadata_size); + data += metadata_size; + } + /* Copy embedded key. */ if (o->hasembkey) { *data++ = sdsHdrSize(key_sds_type); @@ -265,6 +368,7 @@ static bool shouldEmbedStringObject(size_t val_len, const_sds key, long long exp if (key) { size_t key_len = sdslen(key); size += sdsReqSize(key_len, sdsReqType(key_len)) + 1; /* 1 byte for prefixed sds hdr size */ + size += object_metadata_size; } size += (expire != EXPIRY_NONE) * sizeof(long long); size += sdsReqSize(val_len, SDS_TYPE_8); @@ -284,7 +388,7 @@ robj *createStringObjectFromSds(const_sds s) { return createStringObject(s, sdslen(s)); } -static robj *createStringObjectWithKeyAndExpire(const char *ptr, size_t len, const_sds key, long long expire) { +robj *createStringObjectWithKeyAndExpire(const char *ptr, size_t len, const_sds key, long long expire) { if (shouldEmbedStringObject(len, key, expire)) { return createEmbeddedStringObjectWithKeyAndExpire(ptr, len, key, expire); } else { @@ -300,6 +404,8 @@ void *objectGetVal(const robj *o) { data += sizeof(long long); } if (o->hasembkey) { + /* Skip metadata */ + data += objectGetMetadataSize(o); /* Skip embedded key */ uint8_t hdr_size = *(uint8_t *)data; data += 1 + hdr_size; /* +1 for header size byte */ @@ -319,6 +425,9 @@ sds objectGetKey(const robj *o) { data += sizeof(long long); } if (o->hasembkey) { + /* Skip metadata */ + data += objectGetMetadataSize(o); + /* Skip header size byte */ uint8_t hdr_size = *(uint8_t *)data; data += 1 + hdr_size; return (sds)data; @@ -388,6 +497,8 @@ robj *objectSetKeyAndExpire(robj *o, const_sds key, long long expire) { if (objectGetType(o) == OBJ_STRING && objectGetEncoding(o) == OBJ_ENCODING_EMBSTR) { robj *new = createStringObjectWithKeyAndExpire(objectGetVal(o), sdslen(objectGetVal(o)), key, expire); objectSetLRU(new, objectGetLRU(o)); + objectCopyMetadata(new, o); + bgIteration_updateDbEntryPtr(o, new); decrRefCount(o); return new; } @@ -413,6 +524,8 @@ robj *objectSetKeyAndExpire(robj *o, const_sds key, long long expire) { robj *new = createUnembeddedObjectWithKeyAndExpire(objectGetType(o), ptr, key, expire); objectSetEncoding(new, objectGetEncoding(o)); objectSetLRU(new, objectGetLRU(o)); + objectCopyMetadata(new, o); + bgIteration_updateDbEntryPtr(o, new); decrRefCount(o); return new; } @@ -1461,7 +1574,7 @@ struct serverMemOverhead *getMemoryOverheadData(void) { mh->aof_buffer = mem; mem_total += mem; - mem = evalScriptsMemory(); + mem = scriptsMemoryOverhead(); mh->lua_caches = mem; mem_total += mem; mh->functions_caches = functionsMemoryOverhead(); @@ -1581,7 +1694,7 @@ sds getMemoryDoctorReport(void) { } /* Too many scripts are cached? */ - if (dictSize(evalScriptsDict()) > 1000) { + if (dictSize(evalCtxScriptsDict()) > 1000) { many_scripts = 1; num_reports++; } @@ -1716,7 +1829,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle_secs) { /* This is a helper function for the OBJECT command. We need to lookup keys * without any modification of LRU or other parameters. */ robj *objectCommandLookup(client *c, robj *key) { - return lookupKeyReadWithFlags(c->db, key, LOOKUP_NOTOUCH | LOOKUP_NONOTIFY); + return lookupKeyReadWithFlags(c->db, key, LOOKUP_NOTOUCH | LOOKUP_NONOTIFY | LOOKUP_NOHOTKEYS); } robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply) { diff --git a/src/ordered_index.c b/src/ordered_index.c index 8ee652a7c..c0bc269e6 100644 --- a/src/ordered_index.c +++ b/src/ordered_index.c @@ -182,9 +182,13 @@ unsigned long orderedIndexDeleteRangeByLex(OrderedIndex *oi, const_sds min, cons sds min_packed, max_packed; + bool min_ex_eff = min_ex; if (min == shared.minstring) { min_packed = sdsnewlen(NULL, SCORE_SIZE); memcpy(min_packed, &score_prefix, SCORE_SIZE); + /* The bare score prefix equals the packed form of the empty-string + * element; the sentinel bound is always inclusive. */ + min_ex_eff = false; } else { size_t min_len = sdslen(min); min_packed = sdsnewlen(NULL, SCORE_SIZE + min_len); @@ -213,7 +217,7 @@ unsigned long orderedIndexDeleteRangeByLex(OrderedIndex *oi, const_sds min, cons } rangeDeleteArgs args = {on_delete, privdata}; - unsigned long deleted = fbtreeDeleteRangeByValue(fbt, min_packed, max_packed, min_ex, max_ex_eff, rangeDeleteCallback, &args); + unsigned long deleted = fbtreeDeleteRangeByValue(fbt, min_packed, max_packed, min_ex_eff, max_ex_eff, rangeDeleteCallback, &args); sdsfree(min_packed); sdsfree(max_packed); return deleted; @@ -283,10 +287,14 @@ unsigned long orderedIndexCountLexRange(const OrderedIndex *oi, const_sds min, c * sorts before every real element; the maxstring sentinel is bounded by * the next score bucket's prefix, exclusive. */ sds min_packed, max_packed; + bool min_ex_eff = min_ex; bool max_ex_eff = max_ex; if (min == shared.minstring) { min_packed = sdsnewlen(NULL, SCORE_SIZE); memcpy(min_packed, &score_prefix, SCORE_SIZE); + /* The bare score prefix equals the packed form of the empty-string + * element; the sentinel bound is always inclusive. */ + min_ex_eff = false; } else { min_packed = packLexBound(score_prefix, min); } @@ -303,7 +311,7 @@ unsigned long orderedIndexCountLexRange(const OrderedIndex *oi, const_sds min, c max_packed = packLexBound(score_prefix, max); } - unsigned long count = fbtreeCountRangeByValue(fbt, min_packed, max_packed, min_ex, max_ex_eff); + unsigned long count = fbtreeCountRangeByValue(fbt, min_packed, max_packed, min_ex_eff, max_ex_eff); sdsfree(min_packed); sdsfree(max_packed); @@ -480,6 +488,16 @@ void orderedIndexSeekToLexRange(OrderedIndexIterator *iter, const_sds min, const unsigned long len = fbtreeLength(fbt); int reverse = (offset < 0); + /* A crossed sentinel bound (min is the positively infinite string or max + * is the negatively infinite string) admits no elements; the sentinels + * are identity values whose bytes must never be packed as an element. + * Park the iterator where the first step in the iteration direction + * yields nothing. */ + if (min == shared.maxstring || max == shared.minstring) { + fbtreeSeekToRank(fbt_iter, reverse ? 0 : len); + return; + } + if (!reverse) { /* Forward: seek to min bound */ if (min == shared.minstring) { diff --git a/src/rdb.c b/src/rdb.c index 951321db4..499e3bb23 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -49,6 +49,7 @@ #include "cluster_migrateslots.h" #include "compression.h" #include "compression_stream.h" +#include "forkless.h" #include #include @@ -62,8 +63,11 @@ #include #include -/* Size of the static buffer used for rdbcompression */ -#define LZF_STATIC_BUFFER_SIZE (8 * 1024) +/* Minimum size of the rdbcompression output buffer. Short-lived temporaries are + * over-allocated to a fixed size so they come from a single jemalloc bin; sizing + * them exactly spreads allocations across bins and inflates copy-on-write in + * fork children. */ +#define LZF_MIN_BUFFER_SIZE (8 * 1024) /* This macro is called when the internal RDB structure is corrupt */ #define rdbReportCorruptRDB(...) rdbReportError(1, __LINE__, __VA_ARGS__) @@ -159,6 +163,13 @@ void rdbReportError(int corruption_error, int linenum, char *reason, ...) { exit(1); } +/* Route a corrupt compressed frame through the parser's fatal path, so a bad + * stream logs and terminates instead of driving a full-sync retry loop. */ +void rdbReportCorruptCompressedStream(const char *source) { + serverLog(LL_WARNING, "Corrupt streaming-compressed RDB input. Unrecoverable error, aborting now."); + rdbReportCorruptRDB("Corrupt compressed RDB stream from %s", source); +} + typedef struct { rdbAuxFieldEncoder encoder; rdbAuxFieldDecoder decoder; @@ -444,20 +455,18 @@ ssize_t rdbSaveLzfBlob(rio *rdb, void *data, size_t compress_len, size_t origina ssize_t rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) { size_t comprlen, outlen; void *out; - static void *buffer = NULL; /* We require at least four bytes compression for this to be worth it */ if (len <= 4) return 0; outlen = len - 4; - if (outlen < LZF_STATIC_BUFFER_SIZE) { - if (!buffer) buffer = zmalloc(LZF_STATIC_BUFFER_SIZE); - out = buffer; - } else { - if ((out = zmalloc(outlen + 1)) == NULL) return 0; - } + /* Over-allocate to a fixed minimum so every allocation is served from the + * same jemalloc bin. Exact-sized allocations spread across many bins, and in + * a fork child that reuses pages still shared with the parent, driving + * copy-on-write up to roughly the size of the dataset. */ + out = zmalloc(outlen + 1 > LZF_MIN_BUFFER_SIZE ? outlen + 1 : LZF_MIN_BUFFER_SIZE); comprlen = lzf_compress(s, len, out, outlen); ssize_t nwritten = comprlen ? rdbSaveLzfBlob(rdb, out, comprlen, len) : 0; - if (out != buffer) zfree(out); + zfree(out); return nwritten; } @@ -763,18 +772,16 @@ int rdbGetObjectType(robj *o, int rdbver) { else serverPanic("Unknown sorted set encoding"); case OBJ_HASH: - if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) - return RDB_TYPE_HASH_LISTPACK; - else if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) - if (hashTypeHasVolatileFields(o)) - if (rdbver >= 80) - return RDB_TYPE_HASH_2; - else - return -1; /* can't be stored in old RDB */ - else - return RDB_TYPE_HASH; - else - serverPanic("Unknown hash encoding"); + if (hashTypeHasVolatileFields(o)) { + /* Field TTLs need a TTL-capable RDB type: HASH_2 triplets for + * RDB 80 (9.0) and newer targets, regardless of the in-memory + * encoding; older targets can't store them. */ + if (rdbver >= 80) return RDB_TYPE_HASH_2; + return -1; /* can't be stored in old RDB */ + } + if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) return RDB_TYPE_HASH_LISTPACK; + if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) return RDB_TYPE_HASH; + serverPanic("Unknown hash encoding"); case OBJ_STREAM: return RDB_TYPE_STREAM_LISTPACKS_3; case OBJ_MODULE: return RDB_TYPE_MODULE_2; default: serverPanic("Unknown object type"); @@ -1008,7 +1015,34 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid, unsigned char rdbt } } else if (objectGetType(o) == OBJ_HASH) { /* Save a hash value */ - if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { + if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK && rdbtype == RDB_TYPE_HASH_2) { + /* A listpack hash with field TTLs: write the field/value/expiry + * triplet format without converting the in-memory object. */ + unsigned char *zl = objectGetVal(o); + unsigned char field_intbuf[LP_INTBUF_SIZE], value_intbuf[LP_INTBUF_SIZE]; + + if ((n = rdbSaveLen(rdb, hashTypeLength(o))) == -1) return -1; + nwritten += n; + + unsigned char *p = lpFirst(zl); + while (p) { + int64_t flen, vlen; + unsigned char *field = lpGet(p, &flen, field_intbuf); + unsigned char *vptr = lpNext(zl, p); + serverAssert(vptr != NULL); + unsigned char *value = lpGet(vptr, &vlen, value_intbuf); + long long expiry = hashTypeListpackGetExpiry(zl, vptr); + + if ((n = rdbSaveRawString(rdb, field, flen)) == -1) return -1; + nwritten += n; + if ((n = rdbSaveRawString(rdb, value, vlen)) == -1) return -1; + nwritten += n; + if ((n = rdbSaveMillisecondTime(rdb, expiry)) == -1) return -1; + nwritten += n; + + p = lpNext(zl, vptr); + } + } else if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { size_t l = lpBytes((unsigned char *)objectGetVal(o)); if ((n = rdbSaveRawString(rdb, objectGetVal(o), l)) == -1) return -1; @@ -1430,36 +1464,15 @@ ssize_t rdbSaveDb(rio *rdb, int dbid, int rdbflags, int rdbver, long *key_counte if ((res = rdbSaveLen(rdb, dbid)) < 0) goto werr; written += res; - /* Write the RESIZE DB opcode. */ - unsigned long long expires_size = kvstoreSize(db->expires) + kvstoreImportingSize(db->expires); - if ((res = rdbSaveType(rdb, RDB_OPCODE_RESIZEDB)) < 0) goto werr; - written += res; - if ((res = rdbSaveLen(rdb, db_size)) < 0) goto werr; - written += res; - if ((res = rdbSaveLen(rdb, expires_size)) < 0) goto werr; + /* Write the RESIZE DB opcode and slot-info hints. */ + if ((res = rdbSaveDbSizeHints(rdb, db, 1)) < 0) goto werr; written += res; kvs_it = kvstoreIteratorInit(db->keys, HASHTABLE_ITER_SAFE | HASHTABLE_ITER_PREFETCH_VALUES | HASHTABLE_ITER_INCLUDE_IMPORTING); - int last_slot = -1; /* Iterate this DB writing every entry */ void *next; while (kvstoreIteratorNext(kvs_it, &next)) { robj *o = next; - int curr_slot = kvstoreIteratorGetCurrentHashtableIndex(kvs_it); - /* Save slot info. */ - if (server.cluster_enabled && curr_slot != last_slot) { - sds slot_info = sdscatprintf(sdsempty(), "%i,%lu,%lu,%lu", curr_slot, - kvstoreHashtableSize(db->keys, curr_slot), - kvstoreHashtableSize(db->expires, curr_slot), - kvstoreHashtableSize(db->keys_with_volatile_items, curr_slot)); - if ((res = rdbSaveAuxFieldStrStr(rdb, "slot-info", slot_info)) < 0) { - sdsfree(slot_info); - goto werr; - } - written += res; - last_slot = curr_slot; - sdsfree(slot_info); - } sds keystr = objectGetKey(o); robj key; long long expire; @@ -1504,22 +1517,10 @@ ssize_t rdbSaveDb(rio *rdb, int dbid, int rdbflags, int rdbver, long *key_counte * integer pointed by 'error' is set to the value of errno just after the I/O * error. */ int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) { - char magic[10]; - uint64_t cksum; long key_counter = 0; int j; - if (server.rdb_checksum && !(rdb->flags & RIO_FLAG_SKIP_RDB_CHECKSUM)) - rdb->update_cksum = rioGenericUpdateChecksum; - const char *magic_prefix = rdbUseValkeyMagic(rdbver) ? "VALKEY" : "REDIS0"; - serverAssert(rdbver >= 0 && rdbver <= RDB_VERSION); - snprintf(magic, sizeof(magic), "%s%03d", magic_prefix, rdbver); - if (rdbWriteRaw(rdb, magic, 9) == -1) goto werr; - if (rdbSaveInfoAuxFields(rdb, rdbflags, rsi) == -1) goto werr; - if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, VALKEYMODULE_AUX_BEFORE_RDB) == -1) goto werr; - - /* save functions */ - if (!(req & REPLICA_REQ_RDB_EXCLUDE_FUNCTIONS) && rdbSaveFunctions(rdb) == -1) goto werr; + if (rdbWriteHeader(rdb, req, rdbver, rdbflags, rsi) == C_ERR) goto werr; /* save all databases, skip this if we're in functions-only mode */ if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA)) { @@ -1531,16 +1532,7 @@ int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveI } } - if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, VALKEYMODULE_AUX_AFTER_RDB) == -1) goto werr; - - /* EOF opcode */ - if (rdbSaveType(rdb, RDB_OPCODE_EOF) == -1) goto werr; - - /* RDB checksum field. It will be zero if checksum computation is disabled, the - * loading code skips the check in this case. */ - cksum = rdb->cksum; - memrev64ifbe(&cksum); - if (rioWrite(rdb, &cksum, 8) == 0) goto werr; + if (rdbWriteFooter(rdb, req) == C_ERR) goto werr; return C_OK; werr: @@ -1548,6 +1540,9 @@ int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveI return C_ERR; } +static int rdbCompressionInit(rio *rdb, streamWriter *writer, compressionAlgo algo, bool codec_checksum); +static void rdbCompressionFree(rio *rdb, streamWriter *writer); + /* This helper function is only used for diskless replication. * This is just a wrapper to rdbSaveRio() that additionally adds a prefix * and a suffix to the generated RDB dump. The prefix is: @@ -1557,8 +1552,10 @@ int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveI * While the suffix is the 40 bytes hex string we announced in the prefix. * This way processes receiving the payload can understand when it ends * without doing any processing of the content. */ -int rdbSaveRioWithEOFMark(int req, int rdbver, rio *rdb, int *error, rdbSaveInfo *rsi) { +static int rdbSaveRioWithEOFMark(int req, int rdbver, rio *rdb, int *error, rdbSaveInfo *rsi, compressionAlgo compression_algo) { char eofmark[RDB_EOF_MARK_SIZE]; + streamWriter compression_writer; + bool compression_initialized = false; startSaving(RDBFLAGS_REPLICATION); getRandomHexChars(eofmark, RDB_EOF_MARK_SIZE); @@ -1566,7 +1563,31 @@ int rdbSaveRioWithEOFMark(int req, int rdbver, rio *rdb, int *error, rdbSaveInfo if (rioWrite(rdb, "$EOF:", 5) == 0) goto werr; if (rioWrite(rdb, eofmark, RDB_EOF_MARK_SIZE) == 0) goto werr; if (rioWrite(rdb, "\r\n", 2) == 0) goto werr; + + /* Compress only the RDB body; the $EOF prefix/suffix stay plaintext. The + * VCS frame owns checksum policy, so drop the outer RDB CRC64. */ + if (compression_algo != ALGO_NONE) { + if (rdbCompressionInit(rdb, &compression_writer, compression_algo, server.rdb_checksum) == C_ERR) { + if (error && *error == 0) *error = EIO; + goto werr; + } + compression_initialized = true; + rdb->flags |= RIO_FLAG_SKIP_RDB_CHECKSUM; + rdb->update_cksum = NULL; + rdb->cksum = 0; + } + if (rdbSaveRio(req, rdbver, rdb, error, RDBFLAGS_REPLICATION, rsi) == C_ERR) goto werr; + + if (compression_initialized) { + if (streamWriterFinish(&compression_writer) == C_ERR) { + if (error && *error == 0) *error = EIO; + goto werr; + } + rdbCompressionFree(rdb, &compression_writer); + compression_initialized = false; + } + if (rioWrite(rdb, eofmark, RDB_EOF_MARK_SIZE) == 0) goto werr; stopSaving(1); return C_OK; @@ -1574,6 +1595,7 @@ int rdbSaveRioWithEOFMark(int req, int rdbver, rio *rdb, int *error, rdbSaveInfo werr: /* Write error. */ /* Set 'error' only if not already set by rdbSaveRio() call. */ if (error && *error == 0) *error = errno; + if (compression_initialized) rdbCompressionFree(rdb, &compression_writer); stopSaving(0); return C_ERR; } @@ -1604,10 +1626,11 @@ static int rdbSaveInternal(int req, const char *filename, rdbSaveInfo *rsi, int char *err_op; /* For a detailed log */ compressionAlgo compression_algo = rdbCompressionAlgorithm(server.rdb_compression); bool use_streaming_compression = compression_algo == ALGO_LZ4; - /* Keep replication snapshots plain until full sync negotiates compression. - * Disk-based sync snapshots can also become AOF bases, which currently do - * not record whether the reused RDB has whole-stream compression. */ - if (rdbflags & RDBFLAGS_REPLICATION) use_streaming_compression = false; + /* Replication full sync uses the codec selected before the child was forked. */ + if (rdbflags & RDBFLAGS_REPLICATION) { + compression_algo = server.rdb_child_sync_algo; + use_streaming_compression = compression_algo != ALGO_NONE; + } streamWriter compression_writer; bool compression_initialized = false; @@ -1764,15 +1787,50 @@ int rdbSave(int req, char *filename, rdbSaveInfo *rsi, int rdbflags) { return C_OK; } +int isForkBgsaveInProgress(void) { + return server.child_type == CHILD_TYPE_RDB; +} + +int isSaveInProgress(void) { + return isForkBgsaveInProgress() || isForklessSaveInProgress(); +} + +/* Choose the background save method based on configuration. Returns forkless + * only when it is configured, the infrastructure is enabled, and every loaded + * module can handle a forkless save. Otherwise fall back to a fork-based save + * and log why, so the fallback is not silent. */ +int resolveBgsaveType(void) { + if (server.bgsave_default_method != RDB_BGSAVE_TYPE_FORKLESS) return RDB_BGSAVE_TYPE_FORK; + + /* bgsave-default-method can only be set to forkless when the infrastructure + * is enabled (enforced by config validation), so it must be enabled here. */ + serverAssert(server.forkless_infrastructure_enabled); + + if (!moduleAllModulesHandleForkless()) { + serverLog(LL_WARNING, "Falling back to fork-based save: forkless is configured but a loaded " + "module has not declared VALKEYMODULE_OPTIONS_HANDLE_FORKLESS"); + return RDB_BGSAVE_TYPE_FORK; + } + return RDB_BGSAVE_TYPE_FORKLESS; +} + +/* Start a background save, choosing fork or forkless based on bgsave_type. */ +int rdbStartBgsave(int bgsave_type) { + if (bgsave_type == RDB_BGSAVE_TYPE_FORKLESS) { + return forklessSaveToDisk(server.rdb_filename); + } else { + rdbSaveInfo rsi, *rsiptr; + rsiptr = rdbPopulateSaveInfo(&rsi); + return rdbSaveBackground(REPLICA_REQ_NONE, server.rdb_filename, rsiptr, RDBFLAGS_NONE); + } +} + int rdbSaveBackground(int req, char *filename, rdbSaveInfo *rsi, int rdbflags) { pid_t childpid; if (hasActiveChildProcess()) return C_ERR; server.stat_rdb_saves++; - server.dirty_before_bgsave = server.dirty; - server.lastbgsave_try = time(NULL); - if ((childpid = serverFork(CHILD_TYPE_RDB)) == 0) { int retval; @@ -1791,13 +1849,14 @@ int rdbSaveBackground(int req, char *filename, rdbSaveInfo *rsi, int rdbflags) { } else { /* Parent */ if (childpid == -1) { + server.rdb_child_sync_algo = ALGO_NONE; /* Roll back the caller's sync-algo assignment. */ server.lastbgsave_status = C_ERR; + server.lastbgsave_try = time(NULL); serverLog(LL_WARNING, "Can't save in background: fork: %s", strerror(errno)); return C_ERR; } serverLog(LL_NOTICE, "Background saving started by pid %ld", (long)childpid); - server.rdb_save_time_start = time(NULL); - server.rdb_child_type = RDB_CHILD_TYPE_DISK; + rdbRecordStartMetrics(RDB_BGSAVE_TYPE_FORK); return C_OK; } return C_OK; /* unreached */ @@ -1962,13 +2021,38 @@ static int _listZiplistEntryConvertAndValidate(unsigned char *p, unsigned int he return 1; } +/* State shared between lpValidateIntegrityAndDups and its per-entry + * callback _lpEntryValidation. */ +typedef struct lpValidationData { + int pairs; + int allow_metadata; + long count; + long entries_seen; + long long expected_volatile; /* -1: no aggregate header present. */ + long long seen_volatile; + hashtable *fields; /* Initialisation at the first callback. */ +} lpValidationData; + /* callback to check the listpack doesn't have duplicate records */ static int _lpEntryValidation(unsigned char *p, unsigned int head_count, void *userdata) { - struct { - int pairs; - long count; - hashtable *fields; - } *data = userdata; + lpValidationData *data = userdata; + + /* Metadata (tagged) entries are only legal in hash listpacks. When allowed, + * skip them (they're not real field/value records); otherwise reject the + * listpack, since their presence in a set/zset payload indicates corruption. + * A tagged entry in the leading position is the aggregate header carrying + * the volatile-field count; every other one is a per-field expiry, tallied + * so the caller can cross-check the header. */ + if (lpIsMetadata(p)) { + if (!data->allow_metadata) return 0; + if (data->entries_seen == 0) + data->expected_volatile = lpGetMetadataValue(p); + else + data->seen_volatile++; + data->entries_seen++; + return 1; + } + data->entries_seen++; if (data->fields == NULL) { data->fields = hashtableCreate(&setHashtableType); @@ -1997,20 +2081,27 @@ static int _lpEntryValidation(unsigned char *p, unsigned int head_count, void *u /* Validate the integrity of the listpack structure and check for duplicates. * when `pairs` is 0, all elements need to be unique (it's a set) - * when `pairs` is 1, odd elements need to be unique (it's a key-value map) */ -int lpValidateIntegrityAndDups(unsigned char *lp, size_t size, int pairs) { + * when `pairs` is 1, odd elements need to be unique (it's a key-value map) + * `allow_metadata` must only be set for hash listpacks, which may carry tagged + * metadata (field expiration) entries; for sets/zsets it stays 0 so that such + * entries are treated as corruption. */ +int lpValidateIntegrityAndDups(unsigned char *lp, size_t size, int pairs, int allow_metadata) { /* Keep track of the field names to locate duplicate ones */ - struct { - int pairs; - long count; - hashtable *fields; /* Initialisation at the first callback. */ - } data = {pairs, 0, NULL}; + lpValidationData data = {pairs, allow_metadata, 0, 0, -1, 0, NULL}; - int ret = lpValidateIntegrity(lp, size, _lpEntryValidation, &data); + int ret = lpValidateIntegrity(lp, size, _lpEntryValidation, &data, allow_metadata); /* make sure we have an even number of records. */ if (pairs && data.count & 1) ret = 0; + /* Cross-check the aggregate volatile-count header against the per-field + * expiry entries actually present: a mismatch (or per-field entries with + * no header at all) indicates corruption. */ + if (ret && allow_metadata) { + long long expected = (data.expected_volatile == -1) ? 0 : data.expected_volatile; + if (expected != data.seen_volatile) ret = 0; + } + if (data.fields) hashtableRelease(data.fields); return ret; } @@ -2225,8 +2316,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd o = createHashObject(); - /* Too many entries or hash object contains elements with expiry? Use a hash table right from the start. */ - if (len > server.hash_max_listpack_entries || rdbtype == RDB_TYPE_HASH_2) + /* Too many entries? Use a hash table right from the start. A HASH_2 + * hash (field TTLs) that is small enough is loaded as a listpack with + * tagged metadata entries: the triplet format already carries the + * expiry, so no dedicated RDB type is needed to preserve the listpack + * encoding across a save/load cycle. */ + if (len > server.hash_max_listpack_entries) hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); else { /* Guarantee that the server won't crash later when the listpack @@ -2238,6 +2333,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd /* Load every field and value into the ziplist */ + long long volatile_fields = 0; /* fields loaded into the listpack carrying a TTL */ while (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK && len > 0) { len--; /* Load raw strings */ @@ -2253,6 +2349,19 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd return NULL; } + /* Also load the entry expiry */ + long long itemexpiry = EXPIRY_NONE; + if (rdbtype == RDB_TYPE_HASH_2) { + itemexpiry = rdbLoadMillisecondTime(rdb, RDB_VERSION); + if (itemexpiry < EXPIRY_NONE || rioGetReadError(rdb)) { + sdsfree(field); + sdsfree(value); + decrRefCount(o); + if (dupSearchHashtable) hashtableRelease(dupSearchHashtable); + return NULL; + } + } + if (dupSearchHashtable) { sds field_dup = sdsdup(field); if (!hashtableAdd(dupSearchHashtable, field_dup)) { @@ -2266,12 +2375,39 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd } } - /* Convert to hash table if size threshold is exceeded */ + /* If this is a non-preamble RDB being loaded on the primary, and this + * field is already expired relative to 'now', skip it */ + if (iAmPrimary() && !(rdbflags & RDBFLAGS_AOF_PREAMBLE) && now != 0 && + itemexpiry != EXPIRY_NONE && itemexpiry < now) { + /* Emit HDEL to replicas. */ + if ((rdbflags & RDBFLAGS_FEED_REPL) && server.repl_backlog) { + robj keyobj, fieldobj; + initStaticStringObject(keyobj, key); + initStaticStringObject(fieldobj, field); + robj *argv[3]; + argv[0] = shared.hdel; + argv[1] = &keyobj; + argv[2] = &fieldobj; + replicationFeedReplicas(dbid, argv, 3); + } + sdsfree(field); + sdsfree(value); + continue; + } + + /* Convert to hash table if size threshold is exceeded. A field + * carrying a TTL also adds a tagged metadata entry, which + * lpSafeToAdd knows nothing about, so account for its worst case + * here. */ + size_t add_bytes = sdslen(field) + sdslen(value); + if (itemexpiry != EXPIRY_NONE) add_bytes += LP_METADATA_MAX_ENTRY_BYTES; if (objectGetEncoding(o) != OBJ_ENCODING_HASHTABLE && (sdslen(field) > server.hash_max_listpack_value || sdslen(value) > server.hash_max_listpack_value || - !lpSafeToAdd(objectGetVal(o), sdslen(field) + sdslen(value)))) { + !lpSafeToAdd(objectGetVal(o), add_bytes))) { + /* hashTypeConvert carries the TTLs of the pairs already in the + * listpack into the volatile set; no header is needed for that. */ hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); - entry *entry = entryCreate(field, value, EXPIRY_NONE); + entry *entry = entryCreate(field, value, itemexpiry); sdsfree(field); if (!hashtableAdd((hashtable *)objectGetVal(o), entry)) { rdbReportCorruptRDB("Duplicate hash fields detected"); @@ -2280,13 +2416,24 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd decrRefCount(o); return NULL; } + if (itemexpiry != EXPIRY_NONE) hashTypeTrackEntry(o, entry); break; } - /* Add pair to listpack */ + /* Add pair to listpack, with a trailing tagged metadata entry + * when the field carries a TTL. */ objectSetVal(o, lpAppend(objectGetVal(o), (unsigned char *)field, sdslen(field))); objectSetVal(o, lpAppend(objectGetVal(o), (unsigned char *)value, sdslen(value))); + if (itemexpiry != EXPIRY_NONE) { + unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; + uint64_t enclen; + lpEncodeIntegerGetType(itemexpiry, intenc, &enclen); + unsigned char *zl = objectGetVal(o); + unsigned char *eofptr = zl + lpGetTotalBytes(zl) - 1; + objectSetVal(o, lpInsertMetadata(zl, intenc, enclen, eofptr, LP_BEFORE, NULL)); + volatile_fields++; + } sdsfree(field); sdsfree(value); @@ -2299,6 +2446,19 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd dupSearchHashtable = NULL; } + /* Install the aggregate volatile-count header in one pass (per-pair + * updates would rewrite it on every insert). This must happen before + * any load-time reaping, which gates on the O(1) header peek in + * hashTypeHasVolatileFields(). */ + if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { + if (volatile_fields > 0) hashTypeUpdateVolatileCount(o, volatile_fields); + /* Normalize the allocation to the exact listpack size, like the + * blob-loading path does; the incremental build can leave a + * larger-than-needed chunk (visible via MEMORY USAGE with libc + * malloc). */ + objectSetVal(o, lpShrinkToFit(objectGetVal(o))); + } + if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) { if (!hashtableTryExpand(objectGetVal(o), len)) { rdbReportCorruptRDB("OOM in hashtableTryExpand %llu", (unsigned long long)len); @@ -2415,7 +2575,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd if (rdbtype == RDB_TYPE_LIST_QUICKLIST_2) { lp = data; server.stat_dump_payload_sanitizations++; - if (!lpValidateIntegrity(lp, encoded_len, NULL, NULL)) { + if (!lpValidateIntegrity(lp, encoded_len, NULL, NULL, 0)) { rdbReportCorruptRDB("Listpack integrity check failed."); decrRefCount(o); zfree(lp); @@ -2560,7 +2720,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd break; case RDB_TYPE_SET_LISTPACK: server.stat_dump_payload_sanitizations++; - if (!lpValidateIntegrityAndDups(encoded, encoded_len, 0)) { + if (!lpValidateIntegrityAndDups(encoded, encoded_len, 0, 0)) { rdbReportCorruptRDB("Set listpack integrity check failed."); zfree(encoded); objectSetVal(o, NULL); @@ -2618,7 +2778,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd } case RDB_TYPE_ZSET_LISTPACK: server.stat_dump_payload_sanitizations++; - if (!lpValidateIntegrityAndDups(encoded, encoded_len, 1)) { + if (!lpValidateIntegrityAndDups(encoded, encoded_len, 1, 0)) { rdbReportCorruptRDB("Zset listpack integrity check failed."); zfree(encoded); objectSetVal(o, NULL); @@ -2671,9 +2831,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd objectSetVal(o, lpShrinkToFit(objectGetVal(o))); break; } - case RDB_TYPE_HASH_LISTPACK: + case RDB_TYPE_HASH_LISTPACK: { + /* Tagged metadata (field TTLs) never appears in this type; + * it indicates corruption. */ server.stat_dump_payload_sanitizations++; - if (!lpValidateIntegrityAndDups(encoded, encoded_len, 1)) { + if (!lpValidateIntegrityAndDups(encoded, encoded_len, 1, 0)) { rdbReportCorruptRDB("Hash listpack integrity check failed."); zfree(encoded); objectSetVal(o, NULL); @@ -2682,6 +2844,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd } objectSetType(o, OBJ_HASH); objectSetEncoding(o, OBJ_ENCODING_LISTPACK); + + /* A hash that is already empty on load (e.g. an empty or corrupt + * dump) is skipped as an empty key, preserving historic behavior. */ if (hashTypeLength(o) == 0) { decrRefCount(o); goto emptykey; @@ -2689,6 +2854,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd if (hashTypeLength(o) > server.hash_max_listpack_entries) hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); break; + } default: /* totally unreachable */ rdbReportCorruptRDB("Unknown RDB encoding type %d", rdbtype); @@ -3161,14 +3327,19 @@ void stopLoading(int success) { void startSaving(int rdbflags) { /* Fire the persistence modules start event. */ int subevent; - if (rdbflags & RDBFLAGS_AOF_PREAMBLE && getpid() != server.pid) - subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_AOF_START; - else if (rdbflags & RDBFLAGS_AOF_PREAMBLE) - subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START; - else if (getpid() != server.pid) - subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_RDB_START; - else - subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START; + if (rdbflags & RDBFLAGS_AOF_PREAMBLE) { + if (getpid() != server.pid) { + subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_AOF_START; + } else { + subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START; + } + } else { + if (getpid() != server.pid || (rdbflags & RDBFLAGS_FORKLESS_SAVE)) { + subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_RDB_START; + } else { + subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START; + } + } moduleFireServerEvent(VALKEYMODULE_EVENT_PERSISTENCE, subevent, NULL); } @@ -3203,8 +3374,13 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { processEventsWhileBlocked(); processModuleLoadingProgressEvent(0); } - if (server.repl_state == REPL_STATE_TRANSFER && rioCheckType(r) == RIO_TYPE_CONN) { - server.stat_net_repl_input_bytes += len; + /* Dual-channel loads on the rdb channel before REPL_STATE_TRANSFER, so count those bytes too. */ + if ((server.repl_state == REPL_STATE_TRANSFER || server.repl_rdb_channel_state == REPL_DUAL_CHANNEL_RDB_LOAD) && + rioCheckType(r) == RIO_TYPE_CONN) { + /* The stream reader accounts encoded bytes in rdbStreamReadRaw(). */ + if (!r->stream_reader) { + server.stat_net_repl_input_bytes += len; + } } } @@ -3220,7 +3396,14 @@ bool rdbRioHasInternalStreamReaderError(rio *rdb) { } static ssize_t rdbStreamReadRaw(void *ctx, void *buf, size_t len) { - return rioReadRawPartial((rio *)ctx, buf, len); + rio *rdb = ctx; + ssize_t nread = rioReadRawPartial(rdb, buf, len); + if (nread > 0 && rioCheckType(rdb) == RIO_TYPE_CONN && + (server.repl_state == REPL_STATE_TRANSFER || + server.repl_rdb_channel_state == REPL_DUAL_CHANNEL_RDB_LOAD)) { + server.stat_net_repl_input_bytes += nread; + } + return nread; } rdbStreamReaderInitResult rdbInitStreamReader(rio *rdb, @@ -3231,6 +3414,8 @@ rdbStreamReaderInitResult rdbInitStreamReader(rio *rdb, .allow_passthrough = true, .skip_codec_checksum_validation = skip_codec_checksum_validation, .buffer_size = STREAM_READER_BUFFER_SIZE_DEFAULT, + /* A file is fully present, so a short frame there is corruption. */ + .eof_mid_frame_is_truncation = (rioCheckType(rdb) == RIO_TYPE_CONN), }; compressionAlgo detected_algo = ALGO_NONE; @@ -3834,7 +4019,7 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags) { rio rdb; streamReader stream_reader; bool stream_reader_initialized = false; - compressionAlgo streaming_algo = ALGO_NONE; + compressionAlgo compression_algo = ALGO_NONE; int retval = RDB_FAILED; struct stat sb; int rdb_fd; @@ -3861,7 +4046,7 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags) { * For VCS input the parser sees the header produced by the decoder. */ bool skip_codec_checksum_validation = !server.rdb_checksum || server.skip_checksum_validation; rdbStreamReaderInitResult init_rc = - rdbInitStreamReader(&rdb, &stream_reader, skip_codec_checksum_validation, &streaming_algo); + rdbInitStreamReader(&rdb, &stream_reader, skip_codec_checksum_validation, &compression_algo); if (init_rc == RDB_STREAM_READER_INIT_INCOMPATIBLE) { serverLog(LL_WARNING, "Invalid or unsupported RDB stream envelope in %s. " @@ -3876,15 +4061,23 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags) { goto done; } stream_reader_initialized = true; + if (rsi) { + rsi->loaded_compressed = compression_algo != ALGO_NONE; + } if (rdb.flags & RIO_FLAG_STREAMING_COMPRESSION) { serverLog(LL_NOTICE, "Loading compressed RDB (algo=%s) from %s", - compressionAlgoName(streaming_algo), filename); + compressionAlgoName(compression_algo), filename); } retval = rdbLoadRio(&rdb, rdbflags, rsi); if (retval == RDB_OK && streamReaderFinish(&stream_reader) == C_ERR) { - serverLog(LL_WARNING, "Compressed RDB stream in %s did not end cleanly", filename); + if (stream_reader.error_kind == STREAM_READER_ERROR_CORRUPT) { + /* Treat a corrupt frame end like mid-parse corruption via the fatal path. */ + rdbReportCorruptCompressedStream(filename); + } else { + serverLog(LL_WARNING, "Compressed RDB stream in %s did not end cleanly", filename); + } retval = RDB_FAILED; } @@ -3905,14 +4098,13 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags) { /* A background saving child (BGSAVE) terminated its work. Handle this. * This function covers the case of actual BGSAVEs. */ static void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal, time_t save_end) { - if (!bysignal && exitcode == 0) { - serverLog(LL_NOTICE, "Background saving terminated with success"); - server.dirty = server.dirty - server.dirty_before_bgsave; - server.lastsave = save_end; - server.lastbgsave_status = C_OK; - } else if (!bysignal && exitcode != 0) { - serverLog(LL_WARNING, "Background saving error"); - server.lastbgsave_status = C_ERR; + if (!bysignal) { + if (exitcode == 0) { + serverLog(LL_NOTICE, "Background saving terminated with success"); + } else { + serverLog(LL_WARNING, "Background saving error"); + } + rdbRecordEndMetrics(RDB_BGSAVE_TYPE_FORK, (exitcode == 0) ? C_OK : C_ERR, save_end); } else { mstime_t latency; @@ -3924,7 +4116,7 @@ static void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal, time_t sav latencyTraceIfNeeded(rdb, rdb_unlink_temp_file, latency); /* SIGUSR1 is whitelisted, so we have a way to kill a child without * triggering an error condition. */ - if (bysignal != SIGUSR1) server.lastbgsave_status = C_ERR; + if (bysignal != SIGUSR1) rdbRecordEndMetrics(RDB_BGSAVE_TYPE_FORK, C_ERR, save_end); } } @@ -3957,18 +4149,18 @@ static void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { /* When a background RDB saving/transfer terminates, call the right handler. */ void backgroundSaveDoneHandler(int exitcode, int bysignal) { - int type = server.rdb_child_type; + int type = server.rdb_write_target; time_t save_end = time(NULL); - switch (server.rdb_child_type) { - case RDB_CHILD_TYPE_DISK: backgroundSaveDoneHandlerDisk(exitcode, bysignal, save_end); break; - case RDB_CHILD_TYPE_SOCKET: backgroundSaveDoneHandlerSocket(exitcode, bysignal); break; + switch (server.rdb_write_target) { + case RDB_WRITE_TARGET_DISK: backgroundSaveDoneHandlerDisk(exitcode, bysignal, save_end); break; + case RDB_WRITE_TARGET_SOCKET: backgroundSaveDoneHandlerSocket(exitcode, bysignal); break; default: serverPanic("Unknown RDB child type."); break; } - server.rdb_child_type = RDB_CHILD_TYPE_NONE; - server.rdb_save_time_last = save_end - server.rdb_save_time_start; - server.rdb_save_time_start = -1; + rdbClearSaveState(save_end); + server.rdb_child_sync_algo = ALGO_NONE; + /* Possibly there are replicas waiting for a BGSAVE in order to be served * (the first stage of SYNC is a bulk transfer of dump.rdb) */ updateReplicasWaitingBgsave((!bysignal && exitcode == 0) ? C_OK : C_ERR, type); @@ -4032,6 +4224,7 @@ int rdbSaveToReplicasSockets(int req, int rdbver, rdbSaveInfo *rsi) { * Otherwise, use checksum for this RDB transfer. */ int skip_rdb_checksum = 1; + int common_capa = -1; /* Collect the connections of the replicas we want to transfer * the RDB to, which are in WAIT_BGSAVE_START state. */ int connsnum = 0; @@ -4051,6 +4244,8 @@ int rdbSaveToReplicasSockets(int req, int rdbver, rdbSaveInfo *rsi) { if (replica->repl_data->replica_req != req) continue; if (replicaRdbVersion(replica) != rdbver) continue; + common_capa &= replica->repl_data->replica_capa; + conns[connsnum++] = replica->conn; if (dual_channel) { connSendTimeout(replica->conn, server.repl_timeout * 1000); @@ -4072,6 +4267,11 @@ int rdbSaveToReplicasSockets(int req, int rdbver, rdbSaveInfo *rsi) { skip_rdb_checksum = 0; } + compressionAlgo sync_compression_algo = + connsnum > 0 ? replSelectFullSyncCompression(common_capa, true) : ALGO_NONE; + if (sync_compression_algo != ALGO_NONE) + serverLog(LL_NOTICE, "Diskless full sync with compression: %s", compressionAlgoName(sync_compression_algo)); + /* Create the child process. */ if ((childpid = serverFork(CHILD_TYPE_RDB)) == 0) { /* Child */ @@ -4095,16 +4295,17 @@ int rdbSaveToReplicasSockets(int req, int rdbver, rdbSaveInfo *rsi) { if (skip_rdb_checksum) rdb.flags |= RIO_FLAG_SKIP_RDB_CHECKSUM; - retval = rdbSaveRioWithEOFMark(req, rdbver, &rdb, NULL, rsi); + retval = rdbSaveRioWithEOFMark(req, rdbver, &rdb, NULL, rsi, sync_compression_algo); if (retval == C_OK && rioFlush(&rdb) == 0) retval = C_ERR; if (retval == C_OK) { sendChildCowInfo(CHILD_INFO_TYPE_RDB_COW_SIZE, "RDB"); - if (dual_channel) { - sendChildInfoGeneric(CHILD_INFO_TYPE_REPL_OUTPUT_BYTES, 0, rdb.processed_bytes, -1, "RDB"); - } } if (dual_channel) { + /* Bytes actually written across all replica sockets: correct whether + * or not compressed, and reported on both success and failure so a + * mid-transfer failure still counts what went out. */ + sendChildInfoGeneric(CHILD_INFO_TYPE_REPL_OUTPUT_BYTES, 0, rdb.io.connset.net_output_bytes, -1, "RDB"); rioFreeConnset(&rdb); } else { rioFreeFd(&rdb); @@ -4152,7 +4353,8 @@ int rdbSaveToReplicasSockets(int req, int rdbver, rdbSaveInfo *rsi) { skip_rdb_checksum ? " while skipping RDB checksum for this transfer" : ""); server.rdb_save_time_start = time(NULL); - server.rdb_child_type = RDB_CHILD_TYPE_SOCKET; + server.rdb_write_target = RDB_WRITE_TARGET_SOCKET; + server.cur_bgsave_type = RDB_BGSAVE_TYPE_FORK; if (dual_channel) { /* For dual channel sync, the main process no longer requires these RDB connections. */ zfree(conns); @@ -4171,7 +4373,7 @@ int rdbSaveToReplicasSockets(int req, int rdbver, rdbSaveInfo *rsi) { } void saveCommand(client *c) { - if (server.child_type == CHILD_TYPE_RDB) { + if (isSaveInProgress()) { addReplyError(c, "Background save already in progress"); return; } @@ -4187,25 +4389,38 @@ void saveCommand(client *c) { } } -/* BGSAVE [SCHEDULE] */ +/* BGSAVE [SCHEDULE] | BGSAVE CANCEL */ void bgsaveCommand(client *c) { int schedule = 0; - /* The SCHEDULE option changes the behavior of BGSAVE when an AOF rewrite - * is in progress. Instead of returning an error a BGSAVE gets scheduled. */ - if (c->argc > 1) { - if (c->argc == 2 && !strcasecmp(objectGetVal(c->argv[1]), "schedule")) { + /* BGSAVE can be invoked with the following options: + * - CANCEL: terminates an in-progress or scheduled BGSAVE + * - SCHEDULE: schedules a BGSAVE when an AOF rewrite is in progress. + * Instead of returning an error, the BGSAVE is scheduled to run + * when the AOF rewrite completes. */ + for (int i = 1; i < c->argc; i++) { + char *arg = objectGetVal(c->argv[i]); + if (!strcasecmp(arg, "schedule")) { schedule = 1; - } else if (c->argc == 2 && !strcasecmp(objectGetVal(c->argv[1]), "cancel")) { + } else if (!strcasecmp(arg, "cancel")) { + if (c->argc != 2) { + addReplyError(c, "Cancel cannot be combined with other options"); + return; + } /* Terminates an in progress BGSAVE */ - if (server.child_type == CHILD_TYPE_RDB) { - /* There is an ongoing bgsave */ - serverLog(LL_NOTICE, "Background saving will be aborted due to user request"); + if (isForkBgsaveInProgress()) { + /* There is an ongoing fork-based bgsave */ + serverLog(LL_NOTICE, "Background saving (fork) will be aborted due to user request"); killRDBChild(); addReplyStatus(c, "Background saving cancelled"); - } else if (server.rdb_bgsave_scheduled == 1) { + } else if (isForklessSaveInProgress()) { + /* There is an ongoing forkless save */ + serverLog(LL_NOTICE, "Background saving (forkless) will be aborted due to user request"); + forklessSaveCancel(); + addReplyStatus(c, "Background saving cancelled"); + } else if (server.rdb_bgsave_scheduled != RDB_BGSAVE_TYPE_NONE) { serverLog(LL_NOTICE, "Scheduled background saving will be cancelled due to user request"); - server.rdb_bgsave_scheduled = 0; + server.rdb_bgsave_scheduled = RDB_BGSAVE_TYPE_NONE; addReplyStatus(c, "Scheduled background saving cancelled"); } else { addReplyError(c, "Background saving is currently not in progress or scheduled"); @@ -4217,14 +4432,16 @@ void bgsaveCommand(client *c) { } } + int chosen_save_type = resolveBgsaveType(); + rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); - if (server.child_type == CHILD_TYPE_RDB) { + if (isSaveInProgress()) { addReplyError(c, "Background save already in progress"); } else if (hasActiveChildProcess() || server.in_exec) { if (schedule || server.in_exec) { - server.rdb_bgsave_scheduled = 1; + server.rdb_bgsave_scheduled = chosen_save_type; if (schedule) { serverLog(LL_NOTICE, "Background saving scheduled due to user request"); } else { @@ -4236,6 +4453,12 @@ void bgsaveCommand(client *c) { "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " "possible."); } + } else if (chosen_save_type == RDB_BGSAVE_TYPE_FORKLESS) { + if (forklessSaveToDisk(server.rdb_filename) == C_OK) { + addReplyStatus(c, "Background saving started"); + } else { + addReplyErrorObject(c, shared.err); + } } else if (rdbSaveBackground(REPLICA_REQ_NONE, server.rdb_filename, rsiptr, RDBFLAGS_NONE) == C_OK) { addReplyStatus(c, "Background saving started"); } else { @@ -4291,3 +4514,114 @@ rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi) { } return NULL; } + + +/* Write RESIZEDB and slot-info size hints for a single database. + * If include_importing is set, importing slot sizes are included (for fork-based save during migration). + * Returns bytes written on success, -1 on error. */ +ssize_t rdbSaveDbSizeHints(rio *rdb, serverDb *db, int include_importing) { + ssize_t res, written = 0; + + unsigned long long db_size = kvstoreSize(db->keys); + unsigned long long expires_size = kvstoreSize(db->expires); + if (include_importing) { + db_size += kvstoreImportingSize(db->keys); + expires_size += kvstoreImportingSize(db->expires); + } + + if ((res = rdbSaveType(rdb, RDB_OPCODE_RESIZEDB)) < 0) { + serverLog(LL_WARNING, "rdbSaveDbSizeHints: error writing OPCODE_RESIZEDB"); + return -1; + } + written += res; + if ((res = rdbSaveLen(rdb, db_size)) < 0) { + serverLog(LL_WARNING, "rdbSaveDbSizeHints: error writing db_size"); + return -1; + } + written += res; + if ((res = rdbSaveLen(rdb, expires_size)) < 0) { + serverLog(LL_WARNING, "rdbSaveDbSizeHints: error writing expires_size"); + return -1; + } + written += res; + + if (server.cluster_enabled) { + int slot = kvstoreGetFirstNonEmptyHashtableIndex(db->keys); + while (slot != -1) { + sds slot_info = sdscatprintf(sdsempty(), "%i,%lu,%lu,%lu", slot, + kvstoreHashtableSize(db->keys, slot), + kvstoreHashtableSize(db->expires, slot), + kvstoreHashtableSize(db->keys_with_volatile_items, slot)); + if ((res = rdbSaveAuxFieldStrStr(rdb, "slot-info", slot_info)) < 0) { + serverLog(LL_WARNING, "rdbSaveDbSizeHints: error writing slot-info for slot %d", slot); + sdsfree(slot_info); + return -1; + } + written += res; + sdsfree(slot_info); + slot = kvstoreGetNextNonEmptyHashtableIndex(db->keys, slot); + } + } + + return written; +} + +/* Write the RDB header: magic string, aux fields, module aux (before RDB), and functions. + * Returns C_OK on success, C_ERR on error. */ +int rdbWriteHeader(rio *rdb, int req, int rdbver, int rdbflags, rdbSaveInfo *rsi) { + char magic[10]; + if (server.rdb_checksum && !(rdb->flags & RIO_FLAG_SKIP_RDB_CHECKSUM)) { + rdb->update_cksum = rioGenericUpdateChecksum; + } + + const char *magic_prefix = rdbUseValkeyMagic(rdbver) ? "VALKEY" : "REDIS0"; + serverAssert(rdbver >= 0 && rdbver <= RDB_VERSION); + snprintf(magic, sizeof(magic), "%s%03d", magic_prefix, rdbver); + if (rdbWriteRaw(rdb, magic, 9) == -1) return C_ERR; + if (rdbSaveInfoAuxFields(rdb, rdbflags, rsi) == -1) return C_ERR; + if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, VALKEYMODULE_AUX_BEFORE_RDB) == -1) return C_ERR; + /* Save functions */ + if (!(req & REPLICA_REQ_RDB_EXCLUDE_FUNCTIONS) && rdbSaveFunctions(rdb) == -1) return C_ERR; + return C_OK; +} + +/* Write the RDB footer: module aux (after RDB), EOF opcode, and checksum. + * Returns C_OK on success, C_ERR on error. */ +int rdbWriteFooter(rio *rdb, int req) { + if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, VALKEYMODULE_AUX_AFTER_RDB) == -1) return C_ERR; + if (rdbSaveType(rdb, RDB_OPCODE_EOF) == -1) return C_ERR; + /* RDB checksum field. It will be zero if checksum computation is disabled, the + * loading code skips the check in this case. */ + uint64_t cksum = rdb->cksum; + memrev64ifbe(&cksum); + if (rioWrite(rdb, &cksum, 8) == 0) return C_ERR; + return C_OK; +} + +/* Common state updates when a background save starts. */ +void rdbRecordStartMetrics(int bgsave_type) { + server.dirty_before_bgsave = server.dirty; + server.lastbgsave_try = time(NULL); + server.rdb_save_time_start = time(NULL); + server.rdb_write_target = RDB_WRITE_TARGET_DISK; + server.cur_bgsave_type = bgsave_type; +} + +/* Reset save timing and target state. Called after any background save or + * transfer completes, regardless of whether it was a persistence event. */ +void rdbClearSaveState(time_t save_end) { + server.rdb_save_time_last = save_end - server.rdb_save_time_start; + server.rdb_save_time_start = -1; + server.rdb_write_target = RDB_WRITE_TARGET_NONE; + server.cur_bgsave_type = RDB_BGSAVE_TYPE_NONE; +} + +/* Record persistence metrics when a background save completes. */ +void rdbRecordEndMetrics(int bgsave_type, int status, time_t save_end) { + server.lastbgsave_status = status; + server.lastbgsave_type = bgsave_type; + if (status == C_OK) { + server.dirty = server.dirty - server.dirty_before_bgsave; + server.lastsave = save_end; + } +} diff --git a/src/rdb.h b/src/rdb.h index 7bceb4cc9..4f30a381f 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -172,13 +172,14 @@ enum RdbType { #define RDB_LOAD_SDS (1 << 2) /* flags on the purpose of rdb save or load */ -#define RDBFLAGS_NONE 0 /* No special RDB loading or saving. */ -#define RDBFLAGS_AOF_PREAMBLE (1 << 0) /* Load/save the RDB as AOF preamble. */ -#define RDBFLAGS_REPLICATION (1 << 1) /* Load/save for SYNC. */ -#define RDBFLAGS_ALLOW_DUP (1 << 2) /* Allow duplicated keys when loading.*/ -#define RDBFLAGS_FEED_REPL (1 << 3) /* Feed replication stream when loading.*/ -#define RDBFLAGS_KEEP_CACHE (1 << 4) /* Don't reclaim cache after rdb file is generated */ -#define RDBFLAGS_EMPTY_DATA (1 << 5) /* Flush the database after validating magic and rdb version*/ +#define RDBFLAGS_NONE 0 /* No special RDB loading or saving. */ +#define RDBFLAGS_AOF_PREAMBLE (1 << 0) /* Load/save the RDB as AOF preamble. */ +#define RDBFLAGS_REPLICATION (1 << 1) /* Load/save for SYNC. */ +#define RDBFLAGS_ALLOW_DUP (1 << 2) /* Allow duplicated keys when loading.*/ +#define RDBFLAGS_FEED_REPL (1 << 3) /* Feed replication stream when loading.*/ +#define RDBFLAGS_KEEP_CACHE (1 << 4) /* Don't reclaim cache after rdb file is generated */ +#define RDBFLAGS_EMPTY_DATA (1 << 5) /* Flush the database after validating magic and rdb version*/ +#define RDBFLAGS_FORKLESS_SAVE (1 << 6) /* Save is performed by forkless save (background thread). */ /* When rdbLoadObject() returns NULL, the err flag is * set to hold the type of error that occurred */ @@ -201,6 +202,8 @@ int rdbGetObjectType(robj *o, int rdbver); int rdbLoadObjectType(rio *rdb); int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags); int rdbSaveBackground(int req, char *filename, rdbSaveInfo *rsi, int rdbflags); +int rdbStartBgsave(int bgsave_type); +int resolveBgsaveType(void); int rdbSaveToReplicasSockets(int req, int rdbver, rdbSaveInfo *rsi); void rdbRemoveTempFile(pid_t childpid, int from_signal); int rdbSaveToFile(const char *filename); @@ -224,6 +227,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi); int rdbLoadRioWithLoadingCtxScopedRdb(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadingCtx *rdb_loading_ctx); bool rdbRioHasCorruptCompressedInput(rio *rdb); bool rdbRioHasInternalStreamReaderError(rio *rdb); +void rdbReportCorruptCompressedStream(const char *source); typedef enum { RDB_STREAM_READER_INIT_ERROR = -1, @@ -245,5 +249,11 @@ int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveI ssize_t rdbSaveFunctions(rio *rdb); rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi); void replicationEmptyDbCallback(hashtable *ht); +ssize_t rdbSaveDbSizeHints(rio *rdb, serverDb *db, int include_importing); +int rdbWriteHeader(rio *rdb, int req, int rdbver, int rdbflags, rdbSaveInfo *rsi); +int rdbWriteFooter(rio *rdb, int req); +void rdbRecordStartMetrics(int bgsave_type); +void rdbRecordEndMetrics(int bgsave_type, int status, time_t save_end); +void rdbClearSaveState(time_t save_end); #endif diff --git a/src/rdma.c b/src/rdma.c index fe3886e75..795853534 100644 --- a/src/rdma.c +++ b/src/rdma.c @@ -191,7 +191,7 @@ static int rdmaPostRecv(RdmaContext *ctx, struct rdma_cm_id *cm_id, ValkeyRdmaCm ret = ibv_post_recv(cm_id->qp, &recv_wr, &bad_wr); if (ret && (ret != EAGAIN)) { - serverLog(LL_WARNING, "RDMA: post recv failed: %d", ret); + serverLog(LL_WARNING, "RDMA: post recv failed: %s (%d)", strerror(ret), ret); return C_ERR; } @@ -286,7 +286,7 @@ static int rdmaSetupIoBuf(RdmaContext *ctx, struct rdma_cm_id *cm_id) { ctx->cmd_buf = rdmaMemoryAlloc(length); ctx->cmd_mr = ibv_reg_mr(ctx->pd, ctx->cmd_buf, length, access); if (!ctx->cmd_mr) { - serverLog(LL_WARNING, "RDMA: reg mr for CMD failed"); + serverLog(LL_WARNING, "RDMA: reg mr for CMD failed: %s", strerror(errno)); goto destroy_iobuf; } @@ -294,7 +294,6 @@ static int rdmaSetupIoBuf(RdmaContext *ctx, struct rdma_cm_id *cm_id) { cmd = ctx->cmd_buf + i; if (rdmaPostRecv(ctx, cm_id, cmd) == C_ERR) { - serverLog(LL_WARNING, "RDMA: post recv failed"); goto destroy_iobuf; } } @@ -311,7 +310,7 @@ static int rdmaSetupIoBuf(RdmaContext *ctx, struct rdma_cm_id *cm_id) { ctx->rx.length = length; ctx->rx.mr = ibv_reg_mr(ctx->pd, ctx->rx.addr, length, access); if (!ctx->rx.mr) { - serverLog(LL_WARNING, "RDMA: reg mr for recv buffer failed"); + serverLog(LL_WARNING, "RDMA: reg mr for recv buffer failed: %s", strerror(errno)); goto destroy_iobuf; } @@ -331,14 +330,15 @@ static int rdmaCreateResource(RdmaContext *ctx, struct rdma_cm_id *cm_id) { struct ibv_pd *pd = NULL; int comp_vector = rdma_config->completion_vector; - if (ibv_query_device(cm_id->verbs, &device_attr)) { - serverLog(LL_WARNING, "RDMA: ibv ibv query device failed"); + ret = ibv_query_device(cm_id->verbs, &device_attr); + if (ret) { + serverLog(LL_WARNING, "RDMA: ibv query device failed: %s (%d)", strerror(ret), ret); return C_ERR; } pd = ibv_alloc_pd(cm_id->verbs); if (!pd) { - serverLog(LL_WARNING, "RDMA: ibv alloc pd failed"); + serverLog(LL_WARNING, "RDMA: ibv alloc pd failed: %s", strerror(errno)); return C_ERR; } @@ -346,7 +346,7 @@ static int rdmaCreateResource(RdmaContext *ctx, struct rdma_cm_id *cm_id) { comp_channel = ibv_create_comp_channel(cm_id->verbs); if (!comp_channel) { - serverLog(LL_WARNING, "RDMA: ibv create comp channel failed"); + serverLog(LL_WARNING, "RDMA: ibv create comp channel failed: %s", strerror(errno)); return C_ERR; } @@ -360,7 +360,7 @@ static int rdmaCreateResource(RdmaContext *ctx, struct rdma_cm_id *cm_id) { cq = ibv_create_cq(cm_id->verbs, VALKEY_RDMA_MAX_WQE * 2, NULL, comp_channel, comp_vector % cm_id->verbs->num_comp_vectors); if (!cq) { - serverLog(LL_WARNING, "RDMA: ibv create cq failed"); + serverLog(LL_WARNING, "RDMA: ibv create cq failed: %s", strerror(errno)); return C_ERR; } @@ -377,7 +377,7 @@ static int rdmaCreateResource(RdmaContext *ctx, struct rdma_cm_id *cm_id) { init_attr.recv_cq = cq; ret = rdma_create_qp(cm_id, pd, &init_attr); if (ret) { - serverLog(LL_WARNING, "RDMA: create qp failed"); + serverLog(LL_WARNING, "RDMA: create qp failed: %s", strerror(errno)); return C_ERR; } @@ -423,7 +423,7 @@ static int rdmaAdjustSendbuf(RdmaContext *ctx, unsigned int length) { ctx->tx_length = length; ctx->tx.mr = ibv_reg_mr(ctx->pd, ctx->tx.addr, length, access); if (!ctx->tx.mr) { - serverRdmaError(server.neterr, "RDMA: reg send mr failed"); + serverRdmaError(server.neterr, "RDMA: reg send mr failed: %s", strerror(errno)); serverLog(LL_WARNING, "RDMA: FATAL error, recv corrupted cmd"); zlibc_free(ctx->tx.addr); ctx->tx.addr = NULL; @@ -602,7 +602,7 @@ static int connRdmaHandleCq(rdma_connection *rdma_conn) { if (ibv_get_cq_event(ctx->comp_channel, &ev_cq, &ev_ctx) < 0) { if (errno != EAGAIN) { - serverLog(LL_WARNING, "RDMA: get CQ event error"); + serverLog(LL_WARNING, "RDMA: get CQ event error: %s", strerror(errno)); return C_ERR; } @@ -610,15 +610,16 @@ static int connRdmaHandleCq(rdma_connection *rdma_conn) { } ibv_ack_cq_events(ctx->cq, 1); - if (ibv_req_notify_cq(ev_cq, 0)) { - serverLog(LL_WARNING, "RDMA: notify CQ error"); + ret = ibv_req_notify_cq(ev_cq, 0); + if (ret) { + serverLog(LL_WARNING, "RDMA: notify CQ error: %s (%d)", strerror(ret), ret); return C_ERR; } pollcq: ret = ibv_poll_cq(ctx->cq, 1, &wc); if (ret < 0) { - serverLog(LL_WARNING, "RDMA: poll recv CQ error"); + serverLog(LL_WARNING, "RDMA: poll recv CQ error: %s (%d)", strerror(-ret), ret); return C_ERR; } else if (ret == 0) { return C_OK; @@ -814,7 +815,7 @@ static int rdmaHandleConnect(aeEventLoop *el, char *err, struct rdma_cm_event *e ret = rdma_accept(cm_id, &conn_param); if (ret) { - serverRdmaError(err, "RDMA: accept failed"); + serverRdmaError(err, "RDMA: accept failed: %s", strerror(errno)); goto free_rdma; } @@ -901,7 +902,7 @@ rdmaAccept(aeEventLoop *el, connListener *listener, char *err, int fd, char *ip, } if (rdma_ack_cm_event(ev)) { - serverLog(LL_WARNING, "ack cm event failed\n"); + serverLog(LL_WARNING, "RDMA: ack CM event failed: %s", strerror(errno)); return ANET_ERR; } @@ -1077,7 +1078,7 @@ static void rdmaCMeventHandler(struct aeEventLoop *el, int fd, void *clientData, } if (rdma_ack_cm_event(ev)) { - serverLog(LL_NOTICE, "RDMA: ack cm event failed\n"); + serverLog(LL_NOTICE, "RDMA: ack CM event failed: %s", strerror(errno)); } /* connection error or closed by remote peer */ @@ -1578,19 +1579,19 @@ static int rdmaServer(char *err, int port, char *bindaddr, int af, rdma_listener } if (rdma_create_id(listen_channel, &listen_cmid, NULL, RDMA_PS_TCP)) { - serverRdmaError(err, "RDMA: create listen cm id error"); + serverRdmaError(err, "RDMA: create listen cm id error: %s", strerror(errno)); goto error; } rdma_set_option(listen_cmid, RDMA_OPTION_ID, RDMA_OPTION_ID_AFONLY, &afonly, sizeof(afonly)); if (rdma_bind_addr(listen_cmid, (struct sockaddr *)&sock_addr)) { - serverRdmaError(err, "RDMA: bind addr error"); + serverRdmaError(err, "RDMA: bind addr error: %s", strerror(errno)); goto error; } if (rdma_listen(listen_cmid, 0)) { - serverRdmaError(err, "RDMA: listen addr error"); + serverRdmaError(err, "RDMA: listen addr error: %s", strerror(errno)); goto error; } @@ -1876,6 +1877,7 @@ static ConnectionType CT_RDMA = { /* Miscellaneous */ .connIntegrityChecked = NULL, + .is_closing = NULL, }; ConnectionType *connectionTypeRdma(void) { @@ -1925,7 +1927,7 @@ int ValkeyModule_OnLoad(void *ctx, ValkeyModuleString **argv, int argc) { return VALKEYMODULE_ERR; } - ValkeyModule_SetModuleOptions(ctx, VALKEYMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD | VALKEYMODULE_OPTIONS_HANDLE_ATOMIC_SLOT_MIGRATION); + ValkeyModule_SetModuleOptions(ctx, VALKEYMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD | VALKEYMODULE_OPTIONS_HANDLE_ATOMIC_SLOT_MIGRATION | VALKEYMODULE_OPTIONS_HANDLE_FORKLESS); if (connTypeRegister(&CT_RDMA) != C_OK) return VALKEYMODULE_ERR; diff --git a/src/replication.c b/src/replication.c index 11bbb5289..32f42b241 100644 --- a/src/replication.c +++ b/src/replication.c @@ -34,6 +34,7 @@ */ #include "server.h" +#include "sds.h" #include "cluster.h" #include "cluster_slot_stats.h" #include "bio.h" @@ -41,8 +42,11 @@ #include "connection.h" #include "module.h" #include "cluster_migrateslots.h" +#include "io_threads.h" +#include "compression_stream.h" #include +#include #include #include #include @@ -80,6 +84,178 @@ ConnectionType *connTypeOfReplication(void) { return connectionTypeTcp(); } +static compressionAlgo replCompressionAlgorithm(void) { + switch ((repl_compression_mode)server.repl_compression) { + case REPL_COMPRESSION_NO: return ALGO_NONE; + case REPL_COMPRESSION_YES: + case REPL_COMPRESSION_LZ4: return ALGO_LZ4; + default: serverPanic("Unknown repl compression mode: %d", server.repl_compression); + } +} + +/* Whether a replica accepts the codec used for a streaming-compressed payload. */ +static bool replicaAcceptsCompressionAlgorithm(int replica_capa, compressionAlgo compression_algo) { + switch (compression_algo) { + case ALGO_NONE: return true; + case ALGO_LZ4: return replica_capa & REPLICA_CAPA_LZ4; + default: return false; + } +} + +static compressionAlgo replicaNegotiatedCompressionAlgorithm(client *replica) { + compressionAlgo configured_algo = replCompressionAlgorithm(); + return replicaAcceptsCompressionAlgorithm(replica->repl_data->replica_capa, configured_algo) ? configured_algo : ALGO_NONE; +} + +/* True when the replica's live transport no longer matches what the current + * config would negotiate for it. */ +static bool replicaCompressionNeedsRenegotiation(client *replica) { + compressionAlgo active = replica->repl_data->repl_compression ? replica->repl_data->repl_compression->compressor.algo : ALGO_NONE; + return replicaNegotiatedCompressionAlgorithm(replica) != active; +} + +/* Runtime repl-compression changes converge by reconnect, since a live link + * cannot switch between plaintext and compressed mid-stream; the reconnects + * attempt partial resync before falling back to a full sync. Both reconcile + * steps run from replicationCron: that is always after the CONFIG SET that + * requested them has finished, and they pause during a failover so a transport + * change never disconnects the failover target. */ + +/* Disconnect at most one online replica whose transport mismatches the + * current setting. Stateless and idempotent, called every cron tick: at most + * one link converges per tick, so a config change never disconnects the + * whole fleet at once. Still-syncing replicas keep their frozen decision and + * become eligible for reconciliation after they are online. */ +static void reconcileReplicaCompression(void) { + listIter li; + listNode *ln; + + listRewind(server.replicas, &li); + while ((ln = listNext(&li))) { + client *replica = ln->value; + if (replica->repl_data->repl_state != REPLICA_STATE_ONLINE) continue; + if (replica->repl_data->repl_start_cmd_stream_on_ack || replica->flag.close_asap) continue; + if (!replicaCompressionNeedsRenegotiation(replica)) continue; + + serverLog(LL_NOTICE, "Disconnecting replica %s to renegotiate replication compression (now %s)", + replicationGetReplicaName(replica), server.repl_compression != REPL_COMPRESSION_NO ? "enabled" : "disabled"); + freeClientAsync(replica); + return; + } +} + +/* Drop an established upstream link when its advertised capability no longer + * matches the current configuration. A change made during handshake or full + * sync waits until the link is connected, preserving the in-progress sync. */ +static void reconcileUpstreamCompression(void) { + if (!server.primary_host || server.repl_state != REPL_STATE_CONNECTED) return; + if (server.repl_compression_advertised == REPL_COMPRESSION_CAPA_UNKNOWN) return; + serverAssert(server.primary != NULL); + if (server.primary->flag.close_asap) return; + + int compression_enabled = replCompressionAlgorithm() != ALGO_NONE; + if (compression_enabled == server.repl_compression_advertised) return; + + /* A retired reader means the live stream was classified as plaintext. + * Disabling compression then needs no reconnect; a future handshake will + * advertise the new setting. Keep probing or compressed links eligible for + * reconnect because their transport may still need to change. */ + if (!compression_enabled && !server.repl_stream_reader) return; + + serverLog(LL_NOTICE, "Disconnecting from primary to renegotiate replication compression (now %s)", + compression_enabled ? "enabled" : "disabled"); + server.repl_compression_advertised = REPL_COMPRESSION_CAPA_UNKNOWN; + freeClientAsync(server.primary); +} + +/* Enable framed transport compression for a replica at PSYNC completion when + * both sides opted in: repl-compression is enabled here and the replica + * advertised the capability. The write path emits the VCS envelope with the + * first compressed batch. No-ops when the link stays plaintext or is already + * compressed (dual-channel reaches both the +CONTINUE and put-online paths). + * Returns C_ERR when initialization failed; the caller drops the link. */ +static int replicaEnableCompressionIfNegotiated(client *replica) { + compressionAlgo algo = replicaNegotiatedCompressionAlgorithm(replica); + if (algo == ALGO_NONE) return C_OK; + if (replica->repl_data->repl_compression) return C_OK; + + serverAssert(replica->io_write_state == CLIENT_IDLE); + + replicaCompressionState *compression = zcalloc(sizeof(*compression)); + if (streamCompressorInit(&compression->compressor, algo, 0, STREAM_CHECKSUM_BLOCK) != C_OK) { + zfree(compression); + serverLog(LL_WARNING, "Failed to initialize compression for replica %s", replicationGetReplicaName(replica)); + return C_ERR; + } + compression->out_buf = sdsempty(); + + replica->repl_data->repl_compression = compression; + + serverLog(LL_NOTICE, "Replication compression enabled for replica %s (algo=%s)", replicationGetReplicaName(replica), + compressionAlgoName(algo)); + return C_OK; +} + +static void replFreeStreamReader(void) { + if (server.repl_stream_reader) { + streamPushReaderFree(server.repl_stream_reader); + zfree(server.repl_stream_reader); + server.repl_stream_reader = NULL; + } +} + +/* (Re)create the replica-side push reader for a fresh primary stream. A primary + * can compress only when this replica advertised LZ4 in the current handshake, + * so keep the ordinary read path when it did not. When LZ4 was advertised, the + * primary may still choose plaintext; classify that from the leading bytes. */ +static void replResetStreamReader(void) { + replFreeStreamReader(); + serverAssert(server.repl_compression_advertised != REPL_COMPRESSION_CAPA_UNKNOWN); + if (!server.repl_compression_advertised) return; + server.repl_stream_reader = zmalloc(sizeof(*server.repl_stream_reader)); + streamPushReaderInit(server.repl_stream_reader, VCS_STREAM_REPL); +} + +bool replStreamHasPendingDecode(void) { + return server.repl_stream_reader && streamPushReaderHasPendingDecode(server.repl_stream_reader); +} + +/* Decode wire bytes into primary->querybuf, advancing read_reploff by the + * decoded byte count. handleReadResult accounts the encoded bytes in network + * statistics but skips read_reploff while this reader is active. Returns the + * decoded byte count (0 means a partial envelope or compressed block was + * buffered), or -1 when the stream is corrupt and the caller should disconnect + * the link. Once the probe classifies the stream as plaintext, the reader + * retires itself: later reads take the regular read path (callers gate on + * server.repl_stream_reader) and may use IO threads. */ +ssize_t replDecodeToQueryBuf(client *primary, const void *wire_buf, size_t wire_len, size_t output_budget) { + streamPushReader *reader = server.repl_stream_reader; + serverAssert(reader != NULL); + if (wire_len == 0 && !streamPushReaderHasPendingDecode(reader)) return 0; + serverAssert(output_budget > 0); + + /* Decoded output grows the query buffer, so it must be private. The + * reader path never assigns the thread-shared query buffer, so the + * buffer here is always private or NULL. */ + if (primary->querybuf == NULL) primary->querybuf = sdsempty(); + + size_t before = sdslen(primary->querybuf); + streamPushReaderResult result = streamPushReaderFeed(reader, wire_buf, wire_len, &primary->querybuf, output_budget); + if (result == STREAM_PUSH_READER_ERR || result == STREAM_PUSH_READER_FRAME_DONE) { + if (result == STREAM_PUSH_READER_FRAME_DONE) + serverLog(LL_WARNING, "Primary closed compressed replication frame unexpectedly"); + return -1; + } + size_t produced = sdslen(primary->querybuf) - before; + if (primary->querybuf_peak < sdslen(primary->querybuf)) primary->querybuf_peak = sdslen(primary->querybuf); + + primary->repl_data->read_reploff += (long long)produced; + + /* Plaintext stream confirmed: the reader is pure overhead from here on. */ + if (reader->state == STREAM_PUSH_READER_PASSTHROUGH) replFreeStreamReader(); + return (ssize_t)produced; +} + /* Return the pointer to a string representing the replica ip:listening_port * pair. Mostly useful for logging, since we want to log a replica using its * IP address and its listening port which is more clear for the user, for @@ -971,6 +1147,17 @@ int primaryTryPartialResynchronization(client *c, long long psync_offset) { freeClientAsync(c); return C_OK; } + + /* Initialize compression after +CONTINUE (plaintext) and before + * addReplyReplicationBacklog so backlog data goes through the compressed + * path. The command stream starts here; a dual-channel replica reaches + * put-online later in REPLICA_STATE_BG_RDB_LOAD, which tells put-online + * this decision is live and must not be re-made mid-stream. */ + if (replicaEnableCompressionIfNegotiated(c) != C_OK) { + freeClientAsync(c); + return C_OK; + } + psync_len = addReplyReplicationBacklog(c, psync_offset); serverLog( LL_NOTICE, @@ -995,6 +1182,14 @@ int primaryTryPartialResynchronization(client *c, long long psync_offset) { return C_ERR; } +compressionAlgo replSelectFullSyncCompression(int replica_capa, bool socket_target) { + /* Diskless full sync follows repl-compression. A disk-based sync follows + * rdbcompression because it also creates the persisted snapshot. */ + compressionAlgo configured_algo = + socket_target ? replCompressionAlgorithm() : (server.rdb_compression == RDB_COMPRESSION_LZ4 ? ALGO_LZ4 : ALGO_NONE); + return replicaAcceptsCompressionAlgorithm(replica_capa, configured_algo) ? configured_algo : ALGO_NONE; +} + /* Start a BGSAVE for replication goals, which is, selecting the disk or * socket target depending on the configuration, and making sure that * the script cache is flushed before to start. @@ -1019,6 +1214,7 @@ int primaryTryPartialResynchronization(client *c, long long psync_offset) { int startBgsaveForReplication(int mincapa, int req, int rdbver) { int retval; int socket_target = 0; + compressionAlgo sync_compression_algo = ALGO_NONE; listIter li; listNode *ln; @@ -1048,6 +1244,12 @@ int startBgsaveForReplication(int mincapa, int req, int rdbver) { if (socket_target) retval = rdbSaveToReplicasSockets(req, rdbver, rsiptr); else { + /* mincapa is the trigger's own mask on the eager path but the group AND on the cron path, so a mixed group downgrades to plain. */ + sync_compression_algo = replSelectFullSyncCompression(mincapa, false); + if (sync_compression_algo != ALGO_NONE) + serverLog(LL_NOTICE, "Disk-based full sync with compression: %s", compressionAlgoName(sync_compression_algo)); + /* The forked child reads this global to pick the sync codec. */ + server.rdb_child_sync_algo = sync_compression_algo; /* Keep the page cache since it'll get used soon */ retval = rdbSaveBackground(req, server.rdb_filename, rsiptr, RDBFLAGS_REPLICATION | RDBFLAGS_KEEP_CACHE); } @@ -1096,6 +1298,10 @@ int startBgsaveForReplication(int mincapa, int req, int rdbver) { /* Check replica has the exact requirements */ if (replica->repl_data->replica_req != req) continue; if (replicaRdbVersion(replica) != rdbver) continue; + /* A non-capable waiter must not receive a compressed sync; it + * stays parked and the next cron round, whose AND includes it, + * is plain. A capable waiter may still join a plain round. */ + if (!replicaAcceptsCompressionAlgorithm(replica->repl_data->replica_capa, sync_compression_algo)) continue; replicationSetupReplicaForFullResync(replica, getPsyncInitialOffset()); } } @@ -1169,6 +1375,11 @@ void syncCommand(client *c) { } serverLog(LL_NOTICE, "Replica %s asks for synchronization", replicationGetReplicaName(c)); + /* Upgrade incoming replica connection to high priority so that replication + * command streaming and ACK heartbeats are not delayed by normal client commands. */ + if (connSetPriority(c->conn, true) == C_ERR) { + serverLog(LL_WARNING, "Failed to upgrade priority for replica connection %d", c->conn->fd); + } /* Try a partial resynchronization if this is a PSYNC command. * If it fails, we continue with usual full resynchronization, however @@ -1244,7 +1455,7 @@ void syncCommand(client *c) { } /* CASE 1: BGSAVE is in progress, with disk target. */ - if (server.child_type == CHILD_TYPE_RDB && server.rdb_child_type == RDB_CHILD_TYPE_DISK) { + if (server.rdb_write_target == RDB_WRITE_TARGET_DISK) { /* Ok a background save is in progress. Let's check if it is a good * one for replication, i.e. if there is another replica that is * registering differences since the server forked to save. */ @@ -1262,10 +1473,15 @@ void syncCommand(client *c) { break; } /* To attach this replica, we check that it has at least all the - * capabilities of the replica that triggered the current BGSAVE - * and its exact requirements. */ - if (ln && ((c->repl_data->replica_capa & replica->repl_data->replica_capa) == replica->repl_data->replica_capa) && - c->repl_data->replica_req == replica->repl_data->replica_req) { + * capabilities of the replica that triggered the current BGSAVE and its + * exact requirements. Compression is asymmetric: a plain running save is + * joinable by anyone, but a compressed one only by a capable replica. + * The LZ4 capability is masked out of the capability superset check so a + * capable newcomer can still join a plain running save. */ + int trigger_capa = ln ? (replica->repl_data->replica_capa & ~REPLICA_CAPA_LZ4) : 0; + if (ln && ((c->repl_data->replica_capa & trigger_capa) == trigger_capa) && + c->repl_data->replica_req == replica->repl_data->replica_req && + replicaAcceptsCompressionAlgorithm(c->repl_data->replica_capa, server.rdb_child_sync_algo)) { /* Perfect, the server is already registering differences for * another replica. Set the right state, and copy the buffer. * We don't copy buffer if clients don't want. */ @@ -1279,7 +1495,7 @@ void syncCommand(client *c) { } /* CASE 2: BGSAVE is in progress, with socket target. */ - } else if (server.child_type == CHILD_TYPE_RDB && server.rdb_child_type == RDB_CHILD_TYPE_SOCKET) { + } else if (server.rdb_write_target == RDB_WRITE_TARGET_SOCKET) { /* There is an RDB child process but it is writing directly to * children sockets. We need to wait for the next BGSAVE * in order to synchronize. */ @@ -1332,6 +1548,13 @@ void initClientReplicationData(client *c) { void freeClientReplicationData(client *c) { if (!c->repl_data) return; + serverAssert(!clientHasPendingIO(c)); + if (c->repl_data->repl_compression) { + replicaCompressionState *compression = c->repl_data->repl_compression; + streamCompressorFree(&compression->compressor); + sdsfree(compression->out_buf); + zfree(compression); + } freeReplicaReferencedReplBuffer(c); /* Primary/replica cleanup Case 1: * we lost the connection with a replica. */ @@ -1344,7 +1567,7 @@ void freeClientReplicationData(client *c) { * should not remove directly since that means RDB is important for users * to keep data safe and we may delay configured 'save' for full sync. */ if (server.saveparamslen == 0 && c->repl_data->repl_state == REPLICA_STATE_WAIT_BGSAVE_END && - server.child_type == CHILD_TYPE_RDB && server.rdb_child_type == RDB_CHILD_TYPE_DISK && + server.child_type == CHILD_TYPE_RDB && server.rdb_write_target == RDB_WRITE_TARGET_DISK && anyOtherReplicaWaitRdb(c) == 0) { serverLog(LL_NOTICE, "Background saving, persistence disabled, last replica dropped, killing fork child."); killRDBChild(); @@ -1389,13 +1612,14 @@ void freeClientReplicationData(client *c) { * the primary can accurately lists replicas and their listening ports in the * INFO output. * - * - capa + * - capa * What is the capabilities of this instance. * eof: supports EOF-style RDB transfer for diskless replication. * psync2: supports PSYNC v2, so understands +CONTINUE . * dual-channel: supports full sync using rdb channel. * skip-rdb-checksum: supports skipping RDB checksum calculations during diskless sync using * a connection that has integrity checks (such as TLS). + * lz4: accepts LZ4 streaming-compressed replication payloads. * * - ack [fack ] * Replica informs the primary the amount of replication stream that it @@ -1477,6 +1701,9 @@ void replconfCommand(client *c) { } } else if (!strcasecmp(objectGetVal(c->argv[j + 1]), REPLICA_CAPA_SKIP_RDB_CHECKSUM_STR)) c->repl_data->replica_capa |= REPLICA_CAPA_SKIP_RDB_CHECKSUM; + /* "lz4": the replica accepts LZ4 streaming-compressed replication payloads. */ + else if (!strcasecmp(objectGetVal(c->argv[j + 1]), REPLICA_CAPA_LZ4_STR)) + c->repl_data->replica_capa |= REPLICA_CAPA_LZ4; } else if (!strcasecmp(objectGetVal(c->argv[j]), "ack")) { /* REPLCONF ACK is used by replica to inform the primary the amount * of replication stream that it processed so far. It is an @@ -1503,7 +1730,7 @@ void replconfCommand(client *c) { checkChildrenDone(); if (c->repl_data->repl_start_cmd_stream_on_ack && c->repl_data->repl_state == REPLICA_STATE_ONLINE) replicaStartCommandStream(c); if (c->repl_data->repl_state == REPLICA_STATE_BG_RDB_LOAD) { - replicaPutOnline(c); + if (!replicaPutOnline(c)) freeClientAsync(c); } /* Note: this command does not reply anything! */ return; @@ -1622,6 +1849,12 @@ int replicaPutOnline(client *replica) { replicationGetReplicaName(replica)); return 0; } + /* A dual-channel command stream started at +CONTINUE while the RDB was + * loading, so keep that transport until cron reconciles it after the + * replica is online. */ + bool command_stream_already_started = replica->repl_data->repl_state == REPLICA_STATE_BG_RDB_LOAD; + if (!command_stream_already_started && replicaEnableCompressionIfNegotiated(replica) != C_OK) return 0; + replica->repl_data->repl_state = REPLICA_STATE_ONLINE; replica->repl_data->repl_ack_time = server.unixtime; /* Prevent false timeout. */ @@ -2056,7 +2289,7 @@ void updateReplicasWaitingBgsave(int bgsaveerr, int type) { * already an RDB -> Replicas socket transfer, used in the case of * diskless replication, our work is trivial, we can just put * the replica online. */ - if (type == RDB_CHILD_TYPE_SOCKET) { + if (type == RDB_WRITE_TARGET_SOCKET) { serverLog(LL_NOTICE, "Streamed RDB transfer with replica %s succeeded (socket). Waiting for REPLCONF ACK from " "replica to enable streaming", @@ -2424,6 +2657,9 @@ void replicaAfterLoadPrimaryRDB(connection *conn, rdbSaveInfo *rsi, int disk_bas replicationCreatePrimaryClient(server.repl_transfer_s, rsi->repl_stream_db); server.repl_state = REPL_STATE_CONNECTED; server.repl_down_since = 0; + /* The ACK allows the primary to start the command stream, so install + * the reader first even though this path does not currently yield. */ + replResetStreamReader(); /* Send the initial ACK immediately to put this replica in online state. */ replicationSendAck(); /* Finalize full sync duration here for single channel replication. @@ -2461,11 +2697,17 @@ void replicaAfterLoadPrimaryRDB(connection *conn, rdbSaveInfo *rsi, int disk_bas * directly, avoiding a redundant bgrewriteaof. Otherwise (diskless * sync or rdb-preamble disabled), fall back to bgrewriteaof. */ if (server.aof_enabled) { - if (disk_based_sync && server.aof_use_rdb_preamble) { + bool aof_rdb_base_candidate = disk_based_sync && server.aof_use_rdb_preamble; + if (aof_rdb_base_candidate && !rsi->loaded_compressed) { if (restartAOFWithSyncRdb() == C_ERR) { restartAOFAfterSYNC(); } } else { + if (aof_rdb_base_candidate) { + serverLog(LL_NOTICE, + "Sync RDB file %s is streaming-compressed, falling back to BGREWRITEAOF instead of reusing it as an AOF base", + server.rdb_filename); + } restartAOFAfterSYNC(); } } @@ -2480,6 +2722,9 @@ void replicaAfterLoadPrimaryRDB(connection *conn, rdbSaveInfo *rsi, int disk_bas int replicaLoadPrimaryRDBFromSocket(connection *conn, char *buf, char *eofmark, int *usemark, rdbSaveInfo *rsi) { rio rdb; + streamReader stream_reader; + bool stream_reader_initialized = false; + compressionAlgo compression_algo = ALGO_NONE; serverDb **dbarray; functionsLibCtx *functions_lib_ctx; serverDb **diskless_load_tempDb = NULL; @@ -2522,26 +2767,84 @@ int replicaLoadPrimaryRDBFromSocket(connection *conn, char *buf, char *eofmark, startLoading(server.repl_transfer_size, RDBFLAGS_REPLICATION, asyncLoading); if (replicationSupportSkipRDBChecksum(conn, 1, *usemark)) rdb.flags |= RIO_FLAG_SKIP_RDB_CHECKSUM; int loadingFailed = 0; + int retval = RDB_FAILED; + /* Always attach a stream reader that probes the envelope: decode a compressed RDB, or pass through plaintext. */ + bool skip_codec_checksum = (rdb.flags & RIO_FLAG_SKIP_RDB_CHECKSUM) != 0; + rdbStreamReaderInitResult init_rc = + rdbInitStreamReader(&rdb, &stream_reader, skip_codec_checksum, &compression_algo); + if (init_rc == RDB_STREAM_READER_INIT_INCOMPATIBLE) { + serverLog(LL_WARNING, + "Unsupported RDB stream envelope from primary. This replica " + "cannot decode it; a Valkey version with streaming RDB " + "compression support may be required on the replica."); + /* Nothing loaded yet: take the data-preserving incompatibility path below. */ + retval = RDB_INCOMPATIBLE; + loadingFailed = 1; + } else if (init_rc == RDB_STREAM_READER_INIT_ERROR) { + serverLog(LL_WARNING, "Failed to initialize RDB stream reader from primary"); + loadingFailed = 1; + } else { + stream_reader_initialized = true; + /* rdbInitStreamReader sets RIO_FLAG_SKIP_RDB_CHECKSUM on a codec match. */ + if (compression_algo != ALGO_NONE) + serverLog(LL_NOTICE, "Loading compressed RDB (algo=%s) from primary", compressionAlgoName(compression_algo)); + } rdbLoadingCtx loadingCtx = {.dbarray = dbarray, .functions_lib_ctx = functions_lib_ctx}; /* If we aren't using the swapdb method, then we want to empty the data before loading the rdb */ int flags = RDBFLAGS_REPLICATION; if (server.repl_diskless_load != REPL_DISKLESS_LOAD_SWAPDB) flags |= RDBFLAGS_EMPTY_DATA; - int retval = rdbLoadRioWithLoadingCtxScopedRdb(&rdb, flags, rsi, &loadingCtx); + if (!loadingFailed) retval = rdbLoadRioWithLoadingCtxScopedRdb(&rdb, flags, rsi, &loadingCtx); if (retval != RDB_OK) { /* RDB loading failed. */ serverLog(LL_WARNING, "Failed trying to load the PRIMARY synchronization DB " "from socket, check server logs."); loadingFailed = 1; - } else if (*usemark) { - /* Verify the end mark is correct. */ - if (!rioRead(&rdb, buf, RDB_EOF_MARK_SIZE) || memcmp(buf, eofmark, RDB_EOF_MARK_SIZE) != 0) { - serverLog(LL_WARNING, "Replication stream EOF marker is broken"); - loadingFailed = 1; + } else { + if (compression_algo != ALGO_NONE) { + /* Close the compressed frame; streamReaderFinish stops at the frame boundary. */ + int finish_rc = streamReaderFinish(&stream_reader); + if (finish_rc == C_ERR) { + if (stream_reader.error_kind == STREAM_READER_ERROR_TRUNCATED) { + serverLog(LL_WARNING, "Compressed RDB stream from primary was truncated; will resync"); + } else if (stream_reader.error_kind == STREAM_READER_ERROR_CORRUPT) { + /* Same fatal path as parse-time corruption, so a corrupt stream cannot retry-loop. */ + rdbReportCorruptCompressedStream("primary socket"); + } else { + serverLog(LL_WARNING, "Compressed RDB stream from primary did not end cleanly"); + } + loadingFailed = 1; + } + } + if (!loadingFailed) { + /* Detach so trailing-framing reads hit the raw socket. */ + if (stream_reader_initialized) { + rdbFreeStreamReader(&rdb, &stream_reader); + stream_reader_initialized = false; + } + if (*usemark) { + /* Verify the end mark is correct (plaintext, follows any frame). */ + if (!rioRead(&rdb, buf, RDB_EOF_MARK_SIZE) || memcmp(buf, eofmark, RDB_EOF_MARK_SIZE) != 0) { + serverLog(LL_WARNING, "Replication stream EOF marker is broken"); + loadingFailed = 1; + } + } else if (compression_algo != ALGO_NONE) { + /* Size-framed compressed: consumed wire bytes must equal the announced + * transfer size; an early close otherwise looks like success. */ + if (rdb.io.conn.read_so_far != rdb.io.conn.read_limit) { + serverLog(LL_WARNING, + "Compressed RDB stream from primary ended before the announced " + "transfer size; got %llu of %llu bytes", + (unsigned long long)rdb.io.conn.read_so_far, + (unsigned long long)rdb.io.conn.read_limit); + loadingFailed = 1; + } + } } } if (loadingFailed) { stopLoading(0); + if (stream_reader_initialized) rdbFreeStreamReader(&rdb, &stream_reader); rioFreeConn(&rdb, NULL); if (server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB) { @@ -2597,6 +2900,7 @@ int replicaLoadPrimaryRDBFromSocket(connection *conn, char *buf, char *eofmark, /* Cleanup and restore the socket to the original state to continue * with the normal replication. */ + if (stream_reader_initialized) rdbFreeStreamReader(&rdb, &stream_reader); rioFreeConn(&rdb, NULL); connNonBlock(conn); connRecvTimeout(conn, 0); @@ -3089,17 +3393,17 @@ int sendCurrentOffsetToReplica(client *replica) { return C_OK; } -sds replicationSendAuth(connection *conn) { +sds replicationSendAuth(connection *conn, const char *user, size_t user_len, const char *pass, size_t pass_len) { char *args[] = {"AUTH", NULL, NULL}; size_t lens[] = {4, 0, 0}; int argc = 1; - if (server.primary_user) { - args[argc] = server.primary_user; - lens[argc] = strlen(server.primary_user); + if (user) { + args[argc] = (char *)user; + lens[argc] = user_len; argc++; } - args[argc] = server.primary_auth; - lens[argc] = sdslen(server.primary_auth); + args[argc] = (char *)pass; + lens[argc] = pass_len; argc++; return sendCommandArgv(conn, argc, args, lens); } @@ -3120,7 +3424,9 @@ static int dualChannelReplHandleHandshake(connection *conn, sds *err) { dualChannelServerLog(LL_DEBUG, "Received first reply from primary using rdb connection."); /* AUTH with the primary if required. */ if (server.primary_auth) { - *err = replicationSendAuth(conn); + const char *user = server.primary_user; + size_t user_len = user ? strlen(user) : 0; + *err = replicationSendAuth(conn, user, user_len, server.primary_auth, sdslen(server.primary_auth)); if (*err) { dualChannelServerLog(LL_WARNING, "Sending command to primary in dual channel replication handshake: %s", *err); return C_ERR; @@ -3128,9 +3434,14 @@ static int dualChannelReplHandleHandshake(connection *conn, sds *err) { } /* Send replica listening port to primary for clarification */ sds portstr = getReplicaPortString(); - /* Also inform the primary of our (replica) version */ - *err = sendCommand(conn, "REPLCONF", "capa", "eof", "rdb-only", "1", "rdb-channel", "1", "listening-port", portstr, - "version", VALKEY_VERSION, NULL); + /* Also inform the primary of our version and advertise LZ4 when enabled. */ + if (replCompressionAlgorithm() != ALGO_NONE) { + *err = sendCommand(conn, "REPLCONF", "capa", "eof", "rdb-only", "1", "rdb-channel", "1", "listening-port", + portstr, "version", VALKEY_VERSION, "capa", REPLICA_CAPA_LZ4_STR, NULL); + } else { + *err = sendCommand(conn, "REPLCONF", "capa", "eof", "rdb-only", "1", "rdb-channel", "1", "listening-port", + portstr, "version", VALKEY_VERSION, NULL); + } sdsfree(portstr); if (*err) { dualChannelServerLog(LL_WARNING, "Sending command to primary in dual channel replication handshake: %s", *err); @@ -3335,10 +3646,10 @@ void replDataBufInit(void) { /* Replication: Replica side. * Track the local repl-data buffer streaming progress and serve clients from time to time */ -void replStreamProgressCallback(size_t offset, int readlen, time_t *last_progress_callback) { +void replStreamProgressCallback(size_t offset, size_t length, time_t *last_progress_callback) { time_t now = mstime(); if (server.loading_process_events_interval_bytes && - ((offset + readlen) / server.loading_process_events_interval_bytes > + ((offset + length) / server.loading_process_events_interval_bytes > offset / server.loading_process_events_interval_bytes) && (now - *last_progress_callback > server.loading_process_events_interval_ms)) { replicationSendNewlineToPrimary(); @@ -3435,21 +3746,54 @@ void bufferReplData(connection *conn) { int streamReplDataBufToDb(client *c) { serverAssert(c->flag.primary); blockingOperationStarts(); - size_t used, offset = 0; + size_t used, processed_bytes = 0; listNode *cur = NULL; time_t last_progress_callback = mstime(); while (server.pending_repl_data.blocks && (cur = listFirst(server.pending_repl_data.blocks))) { - /* Read and process repl data block */ + /* Buffered blocks hold wire bytes: with the push reader active they + * decode straight into the query buffer (which also advances + * read_reploff by the decoded count); once the reader has retired to + * passthrough, blocks append as-is. */ replDataBufBlock *o = listNodeValue(cur); used = o->used; - c->querybuf = sdscatlen(c->querybuf, o->buf, used); - c->repl_data->read_reploff += used; - processInputBuffer(c); + if (server.repl_stream_reader) { + const void *input = o->buf; + size_t input_len = used; + do { + ssize_t decoded = replDecodeToQueryBuf(c, input, input_len, REPL_DECODE_EVENT_BUDGET); + if (decoded < 0) { + serverLog(LL_WARNING, "Dual-channel replication stream decompression failure"); + blockingOperationEnds(); + return C_ERR; + } + processInputBuffer(c); + input = NULL; + input_len = 0; + + /* Compressed wire bytes can expand far beyond their input + * size. Account progress in decoded bytes so highly + * compressible streams still yield at the configured rate. */ + replStreamProgressCallback(processed_bytes, (size_t)decoded, &last_progress_callback); + processed_bytes += (size_t)decoded; + if (!server.pending_repl_data.blocks) { + blockingOperationEnds(); + return C_ERR; + } + } while (replStreamHasPendingDecode()); + } else { + c->querybuf = sdscatlen(c->querybuf, o->buf, used); + c->repl_data->read_reploff += used; + processInputBuffer(c); + replStreamProgressCallback(processed_bytes, used, &last_progress_callback); + processed_bytes += used; + if (!server.pending_repl_data.blocks) { + blockingOperationEnds(); + return C_ERR; + } + } server.pending_repl_data.mem -= (used + sizeof(replDataBufBlock) + sizeof(listNode)); server.pending_repl_data.len -= used; - offset += used; listDelNode(server.pending_repl_data.blocks, cur); - replStreamProgressCallback(offset, used, &last_progress_callback); } blockingOperationEnds(); if (!server.pending_repl_data.blocks) { @@ -3476,7 +3820,7 @@ void dualChannelSyncSuccess(void) { /* Verify sync is still in progress */ if (server.repl_rdb_channel_state != REPL_DUAL_CHANNEL_STATE_NONE) { replicationAbortDualChannelSyncTransfer(); - replicationUnsetPrimary(); + freeClientAsync(server.primary); } return; } @@ -3808,6 +4152,7 @@ int dualChannelReplMainConnRecvPsyncReply(connection *conn, sds *err) { serverCommunicateSystemd("STATUS=PRIMARY <-> REPLICA sync: Partial Resynchronization accepted. Ready to " "accept connections in read-write mode.\n"); } + replResetStreamReader(); dualChannelSyncHandlePsync(); return C_OK; } @@ -3902,7 +4247,9 @@ int syncWithPrimaryHandleSendHandshakeState(connection *conn) { sds err; /* AUTH with the primary if required. */ if (server.primary_auth) { - err = replicationSendAuth(conn); + const char *user = server.primary_user; + size_t user_len = user ? strlen(user) : 0; + err = replicationSendAuth(conn, user, user_len, server.primary_auth, sdslen(server.primary_auth)); if (err) goto err; } @@ -3937,8 +4284,8 @@ int syncWithPrimaryHandleSendHandshakeState(connection *conn) { // we can ignore primary's conditions when sending capa (is_primary_stream_verified=1) int send_skip_rdb_checksum_capa = replicationSupportSkipRDBChecksum(conn, useDisklessLoad(), 1); - char *argv[9] = {"REPLCONF", "capa", "eof", "capa", "psync2", NULL, NULL, NULL, NULL}; - size_t lens[9] = {8, 4, 3, 4, 6, 0, 0, 0, 0}; + char *argv[11] = {"REPLCONF", "capa", "eof", "capa", "psync2", NULL, NULL, NULL, NULL, NULL, NULL}; + size_t lens[11] = {8, 4, 3, 4, 6, 0, 0, 0, 0, 0, 0}; int argc = 5; if (send_skip_rdb_checksum_capa) { argv[argc] = "capa"; @@ -3956,8 +4303,19 @@ int syncWithPrimaryHandleSendHandshakeState(connection *conn) { lens[argc] = strlen("dual-channel"); argc++; } + /* Advertise LZ4 only when this replica enables replication compression. */ + int advertise_lz4 = replCompressionAlgorithm() != ALGO_NONE; + if (advertise_lz4) { + argv[argc] = "capa"; + lens[argc] = strlen("capa"); + argc++; + argv[argc] = REPLICA_CAPA_LZ4_STR; + lens[argc] = strlen(REPLICA_CAPA_LZ4_STR); + argc++; + } err = sendCommandArgv(conn, argc, argv, lens); if (err) goto err; + server.repl_compression_advertised = advertise_lz4; /* Inform the primary of our (replica) version. */ err = sendCommand(conn, "REPLCONF", "version", VALKEY_VERSION, NULL); @@ -4333,6 +4691,7 @@ void syncWithPrimary(connection *conn) { serverCommunicateSystemd("STATUS=PRIMARY <-> REPLICA sync: Partial Resynchronization accepted. Ready to " "accept connections in read-write mode.\n"); } + replResetStreamReader(); return; } @@ -4379,6 +4738,9 @@ void syncWithPrimary(connection *conn) { if (psync_result == PSYNC_FULLRESYNC_DUAL_CHANNEL) { /* Create RDB connection */ server.repl_rdb_transfer_s = connCreate(connTypeOfReplication()); + /* Tag connection as high-priority before connecting so non-blocking connect and + * subsequent RDB transfer events are registered with QoS priority. */ + connSetPriority(server.repl_rdb_transfer_s, true); if (connConnect(server.repl_rdb_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, dualChannelFullSyncWithPrimary) == C_ERR) { dualChannelServerLog(LL_WARNING, "Unable to connect to Primary: %s", @@ -4428,7 +4790,11 @@ void syncWithPrimary(connection *conn) { } int connectWithPrimary(void) { + server.repl_compression_advertised = REPL_COMPRESSION_CAPA_UNKNOWN; server.repl_transfer_s = connCreate(connTypeOfReplication()); + /* Tag main replication connection as high-priority before connecting so handshake, + * heartbeat pings, and PSYNC streaming are processed with QoS priority. */ + connSetPriority(server.repl_transfer_s, true); if (connConnect(server.repl_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, syncWithPrimary) == C_ERR) { serverLog(LL_WARNING, "Unable to connect to PRIMARY: %s", connGetLastError(server.repl_transfer_s)); @@ -4618,6 +4984,7 @@ void replicationUnsetPrimary(void) { * the replicas will be able to partially resync with us, so it will be * a very fast reconnection. */ disconnectReplicas(); + replFreeStreamReader(); server.repl_state = REPL_STATE_NONE; /* We need to make sure the new primary will start the replication stream @@ -4680,6 +5047,11 @@ void replicationHandlePrimaryDisconnection(void) { /* Any other repl_state means the state machine already moved on * (e.g. REPL_STATE_CONNECT, CONNECTING, NONE) — leave it untouched. */ + /* Tear down the replication stream reader so a later (possibly + * uncompressed) primary stream isn't fed into stale frame state. + * Idempotent when no reader exists. */ + replFreeStreamReader(); + /* We lost connection with our primary, don't disconnect replicas yet, * maybe we'll be able to PSYNC with our primary later. We'll disconnect * the replicas only if we'll have to do a full resync with our primary. */ @@ -5149,7 +5521,10 @@ void waitCommand(client *c) { } /* Otherwise, block the client and put it into our list of clients - * waiting for ack from replicas. */ + * waiting for ack from replicas. WAIT handles its own reply in + * processClientsWaitingReplicas, so clear pending_command to avoid + * being mistaken for a command that needs re-execution. */ + c->flag.pending_command = 0; blockClientForReplicaAck(c, timeout, offset, numreplicas, 0); /* Make sure that the server will send an ACK request to all the replicas @@ -5191,7 +5566,10 @@ void waitaofCommand(client *c) { } /* Otherwise, block the client and put it into our list of clients - * waiting for ack from replicas. */ + * waiting for ack from replicas. WAITAOF handles its own reply in + * processClientsWaitingReplicas, so clear pending_command to avoid + * being mistaken for a command that needs re-execution. */ + c->flag.pending_command = 0; blockClientForReplicaAck(c, timeout, offset, numreplicas, numlocal); /* Make sure that the server will send an ACK request to all the replicas @@ -5359,8 +5737,16 @@ void handleBioThreadFinishedRDBDownload(void) { void replicationCron(void) { static long long replication_cron_loops = 0; - /* Check failover status first, to see if we need to start - * handling the failover. */ + /* Converge replication transports to the current repl-compression + * setting (see the reconcile functions for the model). Paused during + * failover so a transport change never disconnects the failover target. */ + if (server.failover_state == NO_FAILOVER) { + reconcileUpstreamCompression(); + reconcileReplicaCompression(); + } + + /* Check failover status to see if we need to start handling the + * failover. */ updateFailoverStatus(); /* Non blocking connection timeout? */ @@ -5440,7 +5826,7 @@ void replicationCron(void) { int is_presync = (replica->repl_data->repl_state == REPLICA_STATE_WAIT_BGSAVE_START || - (replica->repl_data->repl_state == REPLICA_STATE_WAIT_BGSAVE_END && server.rdb_child_type != RDB_CHILD_TYPE_SOCKET)); + (replica->repl_data->repl_state == REPLICA_STATE_WAIT_BGSAVE_END && server.rdb_write_target != RDB_WRITE_TARGET_SOCKET)); if (is_presync) { connWrite(replica->conn, "\n", 1); @@ -5469,7 +5855,7 @@ void replicationCron(void) { * by the fork child so if a disk-based replica is stuck it doesn't prevent the fork child * from terminating. */ if (replica->repl_data->repl_state == REPLICA_STATE_WAIT_BGSAVE_END && - server.rdb_child_type == RDB_CHILD_TYPE_SOCKET) { + server.rdb_write_target == RDB_WRITE_TARGET_SOCKET) { if (replica->repl_data->repl_last_partial_write != 0 && (server.unixtime - replica->repl_data->repl_last_partial_write) > server.repl_timeout) { serverLog(LL_WARNING, "Disconnecting timedout replica (full sync): %s", @@ -5548,7 +5934,7 @@ int shouldStartChildReplication(int *mincapa_out, int *req_out, int *rdbver_out) * In case of diskless replication, we make sure to wait the specified * number of seconds (according to configuration) so that other replicas * have the time to arrive before we start streaming. */ - if (!hasActiveChildProcess()) { + if (!hasActiveSaveOrChild()) { time_t idle, max_idle = 0; int replicas_waiting = 0; int mincapa; diff --git a/src/rio.c b/src/rio.c index 68e56cb24..b585e9c1f 100644 --- a/src/rio.c +++ b/src/rio.c @@ -225,51 +225,88 @@ static size_t rioConnWrite(rio *r, const void *buf, size_t len) { return 0; /* Error, this target does not yet support writing. */ } -/* Returns 1 or 0 for success/failure. */ -static size_t rioConnRead(rio *r, void *buf, size_t len) { +/* Fill the connection read buffer until at least min_available bytes are available. + * Returns 1 on success, 0 on EOF, -1 on error. */ +static int rioConnEnsureBuffered(rio *r, size_t min_available) { size_t avail = sdslen(r->io.conn.buf) - r->io.conn.pos; /* If the buffer is too small for the entire request: realloc. */ - if (sdslen(r->io.conn.buf) + sdsavail(r->io.conn.buf) < len) - r->io.conn.buf = sdsMakeRoomFor(r->io.conn.buf, len - sdslen(r->io.conn.buf)); + if (sdslen(r->io.conn.buf) + sdsavail(r->io.conn.buf) < min_available) + r->io.conn.buf = sdsMakeRoomFor(r->io.conn.buf, min_available - sdslen(r->io.conn.buf)); /* If the remaining unused buffer is not large enough: memmove so that we * can read the rest. */ - if (len > avail && sdsavail(r->io.conn.buf) < len - avail) { + if (min_available > avail && sdsavail(r->io.conn.buf) < min_available - avail) { sdsrange(r->io.conn.buf, r->io.conn.pos, -1); r->io.conn.pos = 0; } - /* Make sure the caller didn't request to read past the limit. - * If they didn't we'll buffer till the limit, if they did, we'll - * return an error. */ - if (r->io.conn.read_limit != 0 && r->io.conn.read_limit < r->io.conn.read_so_far + len) { - errno = EOVERFLOW; - return 0; - } - - /* If we don't already have all the data in the sds, read more */ - while (len > sdslen(r->io.conn.buf) - r->io.conn.pos) { - size_t buffered = sdslen(r->io.conn.buf) - r->io.conn.pos; - size_t needs = len - buffered; + while (avail < min_available) { + size_t needs = min_available - avail; /* Read either what's missing, or PROTO_IOBUF_LEN, the bigger of * the two. */ size_t toread = needs < PROTO_IOBUF_LEN ? PROTO_IOBUF_LEN : needs; if (toread > sdsavail(r->io.conn.buf)) toread = sdsavail(r->io.conn.buf); - if (r->io.conn.read_limit != 0 && r->io.conn.read_so_far + buffered + toread > r->io.conn.read_limit) { - toread = r->io.conn.read_limit - r->io.conn.read_so_far - buffered; + if (r->io.conn.read_limit != 0) { + size_t remaining = + r->io.conn.read_so_far >= r->io.conn.read_limit ? 0 : r->io.conn.read_limit - r->io.conn.read_so_far; + if (avail >= remaining) { + toread = 0; + } else if (toread > remaining - avail) { + toread = remaining - avail; + } } + if (toread == 0) return 0; + int retval = connRead(r->io.conn.conn, (char *)r->io.conn.buf + sdslen(r->io.conn.buf), toread); if (retval == 0) { return 0; } else if (retval < 0) { if (connLastErrorRetryable(r->io.conn.conn)) continue; if (errno == EWOULDBLOCK) errno = ETIMEDOUT; - return 0; + return -1; } sdsIncrLen(r->io.conn.buf, retval); + avail = sdslen(r->io.conn.buf) - r->io.conn.pos; } + return 1; +} + +/* Partial-read variant: returns bytes read, 0 on EOF, -1 on error. */ +static ssize_t rioConnReadSome(rio *r, void *buf, size_t len) { + size_t avail = sdslen(r->io.conn.buf) - r->io.conn.pos; + + if (r->io.conn.read_limit != 0 && r->io.conn.read_so_far >= r->io.conn.read_limit) { + return 0; + } + if (avail == 0) { + int fill_rc = rioConnEnsureBuffered(r, 1); + if (fill_rc <= 0) return fill_rc; + avail = sdslen(r->io.conn.buf) - r->io.conn.pos; + } + if (r->io.conn.read_limit != 0) { + size_t remaining = r->io.conn.read_limit - r->io.conn.read_so_far; + if (len > remaining) len = remaining; + } + + size_t got = avail < len ? avail : len; + memcpy(buf, (char *)r->io.conn.buf + r->io.conn.pos, got); + r->io.conn.read_so_far += got; + r->io.conn.pos += got; + return (ssize_t)got; +} + +/* Returns 1 or 0 for success/failure. */ +static size_t rioConnRead(rio *r, void *buf, size_t len) { + if (r->io.conn.read_limit != 0 && + (r->io.conn.read_so_far > r->io.conn.read_limit || + len > r->io.conn.read_limit - r->io.conn.read_so_far)) { + errno = EOVERFLOW; + return 0; + } + if (rioConnEnsureBuffered(r, len) != 1) return 0; + memcpy(buf, (char *)r->io.conn.buf + r->io.conn.pos, len); r->io.conn.read_so_far += len; r->io.conn.pos += len; @@ -294,7 +331,7 @@ static const rio rioConnIO = { .write = rioConnWrite, .tell = rioConnTell, .flush = rioConnFlush, - .read_some = NULL, + .read_some = rioConnReadSome, .update_cksum = NULL, .cksum = 0, .flags = 0, @@ -636,6 +673,9 @@ static size_t rioConnsetWrite(rio *r, const void *buf, size_t len) { if (retval == -1 && errno == EWOULDBLOCK) errno = ETIMEDOUT; break; } + /* Count bytes actually written to this connection, including a + * partial write before an error. */ + r->io.connset.net_output_bytes += retval; nwritten += retval; } @@ -701,6 +741,7 @@ void rioInitWithConnset(rio *r, connection **conns, int numconns) { r->io.connset.numconns = numconns; r->io.connset.pos = 0; r->io.connset.buf = sdsempty(); + r->io.connset.net_output_bytes = 0; } /* release the rio stream. */ diff --git a/src/rio.h b/src/rio.h index 1512a27a4..626c4a947 100644 --- a/src/rio.h +++ b/src/rio.h @@ -72,6 +72,12 @@ struct _rio { * computation. */ void (*update_cksum)(struct _rio *, const void *buf, size_t len); + /* Optional callback invoked between write chunks. If it returns + * non-zero, the write is aborted (rioWrite returns 0 to caller). + * Allows long-running operations that issue many writes (e.g. + * serializing a large collection) to be interrupted. */ + int (*check_abort_between_writes)(struct _rio *); + /* The current checksum and flags (see RIO_FLAG_*) */ uint64_t cksum, flags; @@ -132,6 +138,7 @@ struct _rio { int numconns; off_t pos; sds buf; + size_t net_output_bytes; /* Total bytes written across all connections. */ } connset; } io; }; @@ -168,6 +175,7 @@ static inline size_t rioWriteRaw(rio *r, const void *buf, size_t len) { static inline size_t rioWrite(rio *r, const void *buf, size_t len) { if (r->flags & RIO_FLAG_WRITE_ERROR || r->flags & RIO_FLAG_CLOSE_ASAP) return 0; while (len) { + if (r->check_abort_between_writes && r->check_abort_between_writes(r)) return 0; size_t bytes_to_write = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; if (r->update_cksum) r->update_cksum(r, buf, bytes_to_write); diff --git a/src/scripting_engine.c b/src/scripting_engine.c index bc21f2fb6..f3753ad30 100644 --- a/src/scripting_engine.c +++ b/src/scripting_engine.c @@ -73,11 +73,6 @@ dictType engineDictType = { .entryDestructor = zfree, }; -static int isCalledFromAsyncThread(void) { - pthread_t curr_thread = pthread_self(); - return !pthread_equal(server.main_thread_id, curr_thread); -} - /* Initializes the scripting engine manager. * The engine manager is responsible for managing the several scripting engines * that are loaded in the server and implemented by Valkey Modules. @@ -313,7 +308,7 @@ void scriptingEngineCallFreeFunction(scriptingEngine *engine, subsystemType type, compiledFunction *compiled_func) { serverAssert(type == VMSE_EVAL || type == VMSE_FUNCTION); - int is_async = isCalledFromAsyncThread(); + int is_async = !onServerMainThread(); /* We need to acquire the module GIL when running from an async thread while * flushing the script functions. */ diff --git a/src/server.c b/src/server.c index 3f6825033..993ada521 100644 --- a/src/server.c +++ b/src/server.c @@ -32,6 +32,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ #include "server.h" +#include "hotkeys.h" #include "ordered_index.h" #include "connection.h" #include "monotonic.h" @@ -48,13 +49,18 @@ #include "threads_mngr.h" #include "fmtargs.h" #include "io_threads.h" +#include "compression.h" #include "tls.h" #include "sds.h" #include "module.h" #include "scripting_engine.h" +#include "throttle.h" +#include "throttle_repl.h" #include "util.h" +#include "forkless.h" #include "eval.h" +#include "bgiteration.h" #include "trace/trace_commands.h" @@ -904,6 +910,12 @@ int hasActiveChildProcess(void) { return server.child_pid != -1; } +/* Returns true if a background save (fork or forkless) or child process is + * active. */ +int hasActiveSaveOrChild(void) { + return hasActiveChildProcess() || isSaveInProgress(); +} + void resetChildState(void) { server.child_type = CHILD_TYPE_NONE; server.child_pid = -1; @@ -1214,6 +1226,27 @@ void getExpensiveClientsInfo(size_t *in_usage, size_t *out_usage) { *out_usage = o; } +/* Detect and free zombie connections whose read handler was removed (e.g. + * BLOCKED_INUSE). Without a read handler the event loop won't notice the + * remote side closing, so these fds would leak until the fd limit is hit. */ +static bool clientsCronTcpIsClosing(client *c) { + if (!c->conn) return false; + + /* If the fd is still watched by the event loop, it detects the close and frees the client itself. */ + if (connHasReadHandler(c->conn) || connHasWriteHandler(c->conn)) return false; + + if (!connIsClosing(c->conn)) return false; + + if (server.verbosity <= LL_VERBOSE) { + sds client_info = catClientInfoString(sdsempty(), c, server.hide_user_data_from_log); + serverLog(LL_VERBOSE, "Client closed connection while blocked %s", client_info); + sdsfree(client_info); + } + + freeClientAsync(c); + return true; +} + /* This function is called by clientsTimeProc() and is used in order to perform * operations on clients that are important to perform constantly. For instance * we use this function in order to disconnect clients after a timeout, including @@ -1265,9 +1298,11 @@ static void clientsCron(int clients_this_cycle) { * The protocol is that they return non-zero if the client was * terminated. */ if (clientsCronHandleTimeout(c, now)) continue; + if (clientsCronTcpIsClosing(c)) continue; if (clientsCronResizeQueryBuffer(c)) continue; if (clientsCronResizeOutputBuffer(c, now)) continue; if (clientsCronTrackExpensiveClients(c, curr_peak_mem_usage_slot)) continue; + if (clientsCronTcpIsClosing(c)) continue; /* Iterating all the clients in getMemoryOverheadData() is too slow and * in turn would make the INFO command too slow. So we perform this @@ -1393,6 +1428,10 @@ void databasesCron(void) { } } } + + /* Close any elapsed hot-key detection window, so a completed window is + * frozen on schedule even when there is no traffic. */ + hotkeysCron(); } static inline void updateCachedTimeWithUs(int update_daylight_info, const ustime_t ustime) { @@ -1653,8 +1692,10 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa databasesCron(); /* Start a scheduled AOF rewrite if this was requested by the user while - * a BGSAVE was in progress. */ - if (!hasActiveChildProcess() && server.aof_rewrite_scheduled && !aofRewriteLimited()) { + * a BGSAVE was in progress. We don't start the rewrite if there is an + * active child process (to avoid multiple concurrent fork children) or if + * a forkless save is in progress (to avoid potential copy-on-write). */ + if (!hasActiveSaveOrChild() && server.aof_rewrite_scheduled && !aofRewriteLimited()) { rewriteAppendOnlyFileBackground(); } @@ -1662,7 +1703,7 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa if (hasActiveChildProcess() || scriptingEngineDebuggerPendingChildren()) { run_with_period(1000) receiveChildInfo(); checkChildrenDone(); - } else { + } else if (!isSaveInProgress()) { /* If there is not a background saving/rewrite in progress check if * we have to save/rewrite now. */ for (j = 0; j < server.saveparamslen; j++) { @@ -1676,15 +1717,14 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa (server.unixtime - server.lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY || server.lastbgsave_status == C_OK)) { serverLog(LL_NOTICE, "%d changes in %d seconds. Saving...", sp->changes, (int)sp->seconds); - rdbSaveInfo rsi, *rsiptr; - rsiptr = rdbPopulateSaveInfo(&rsi); - rdbSaveBackground(REPLICA_REQ_NONE, server.rdb_filename, rsiptr, RDBFLAGS_NONE); + rdbStartBgsave(resolveBgsaveType()); break; } } - /* Trigger an AOF rewrite if needed. */ - if (server.aof_state == AOF_ON && !hasActiveChildProcess() && server.aof_rewrite_perc && + /* Trigger an AOF rewrite if needed. Avoid starting while another child process + * is active. Also avoid when forkless save is in progress to prevent potential copy-on-write. */ + if (server.aof_state == AOF_ON && !hasActiveSaveOrChild() && server.aof_rewrite_perc && server.aof_current_size > server.aof_rewrite_min_size) { long long base = server.aof_rewrite_base_size ? server.aof_rewrite_base_size : 1; long long growth = (server.aof_current_size * 100 / base) - 100; @@ -1729,6 +1769,8 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa run_with_period(1000) replicationCron(); } + run_with_period(100) throttleRepl_adjustThrottling(); + /* Run the Cluster cron. */ if (server.cluster_enabled) { run_with_period(CLUSTER_CRON_PERIOD_MS) clusterCron(); @@ -1755,12 +1797,9 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa * Note: this code must be after the replicationCron() call above so * make sure when refactoring this file to keep this order. This is useful * because we want to give priority to RDB savings for replication. */ - if (!hasActiveChildProcess() && server.rdb_bgsave_scheduled && + if (!hasActiveSaveOrChild() && server.rdb_bgsave_scheduled && (server.unixtime - server.lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY || server.lastbgsave_status == C_OK)) { - rdbSaveInfo rsi, *rsiptr; - rsiptr = rdbPopulateSaveInfo(&rsi); - if (rdbSaveBackground(REPLICA_REQ_NONE, server.rdb_filename, rsiptr, RDBFLAGS_NONE) == C_OK) - server.rdb_bgsave_scheduled = 0; + if (rdbStartBgsave(server.rdb_bgsave_scheduled) == C_OK) server.rdb_bgsave_scheduled = RDB_BGSAVE_TYPE_NONE; } /* TLS auto-reload if enabled (only when TLS is built-in). */ @@ -1863,6 +1902,21 @@ static void sendGetackToReplicas(void) { extern int ProcessingEventsWhileBlocked; +/* Process one buffered decompression slice before the event loop sleeps. + * Returning true lets processEventsWhileBlocked count the slice as progress. */ +static bool processPendingReplStreamDecode(void) { + client *primary = server.primary; + if (!primary || primary->flag.close_asap || !replStreamHasPendingDecode()) return false; + /* streamReplDataBufToDb owns the reader while replaying dual-channel + * buffers. Resuming it here could read newer socket bytes before the + * remaining buffered blocks. */ + if (server.pending_repl_data.blocks) return false; + if (primary->io_write_state != CLIENT_IDLE || primary->io_read_state != CLIENT_IDLE) return false; + + readQueryFromClient(primary->conn); + return true; +} + /* This function gets called every time the server is entering the * main loop of the event driven library, that is, before to sleep * for ready file descriptors. @@ -1895,6 +1949,9 @@ void beforeSleep(struct aeEventLoop *eventLoop) { uint64_t processed = 0; processed += processIOThreadsResponses(); processed += connTypeProcessPendingData(); + /* Keep an online compressed primary draining when a long-running + * command yields to the event loop. */ + processed += processPendingReplStreamDecode(); if (server.aof_state == AOF_ON || server.aof_state == AOF_WAIT_REWRITE) flushAppendOnlyFile(0); processed += handleClientsWithPendingWrites(); int last_processed = 0; @@ -1918,6 +1975,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) { /* If any connection type(typical TLS) still has pending unread data don't sleep at all. */ int dont_sleep = connTypeHasPendingData(); + if (processPendingReplStreamDecode()) { + server.el_iteration_active = true; + if (replStreamHasPendingDecode()) dont_sleep = 1; + } /* Call the Cluster before sleep function. Note that this function * may change the state of Cluster (from ok to fail or vice versa), @@ -1925,6 +1986,8 @@ void beforeSleep(struct aeEventLoop *eventLoop) { * later in this function, must be done before blockedBeforeSleep. */ if (server.cluster_enabled) clusterBeforeSleep(); + /* Release keys from bgIteration before processing unblocked clients. */ + bgIteration_beforeSleep(); /* Handle blocked clients. * must be done before flushAppendOnlyFile, in case of appendfsync=always, * since the unblocked clients may write data. */ @@ -2070,7 +2133,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) { /* Before we are going to sleep, let the threads access the dataset by * releasing the GIL. The server main thread will not touch anything at this * time. */ - if (moduleCount()) moduleReleaseGIL(); + if (moduleCount()) { + atomic_store_explicit(&server.module_gil_acquired, 0, memory_order_relaxed); + moduleReleaseGIL(); + } /********************* WARNING ******************** * Do NOT add anything below moduleReleaseGIL !!! * ***************************** ********************/ @@ -2092,6 +2158,7 @@ void afterSleep(struct aeEventLoop *eventLoop, int numevents) { atomic_store_explicit(&server.module_gil_acquiring, 1, memory_order_relaxed); moduleAcquireGIL(); atomic_store_explicit(&server.module_gil_acquiring, 0, memory_order_relaxed); + atomic_store_explicit(&server.module_gil_acquired, 1, memory_order_relaxed); moduleFireServerEvent(VALKEYMODULE_EVENT_EVENTLOOP, VALKEYMODULE_SUBEVENT_EVENTLOOP_AFTER_SLEEP, NULL); latencyEndMonitor(latency); latencyAddSampleIfNeeded("module-acquire-GIL", latency); @@ -2119,6 +2186,21 @@ void afterSleep(struct aeEventLoop *eventLoop, int numevents) { IOThreadsAfterSleep(numevents); } +/* Callback invoked by the event loop after draining priority events. + * Records priority eventloop duration and updates peak commands executed per priority cycle. */ +static void qosStatsCallback(struct aeEventLoop *el, uint64_t duration_us) { + UNUSED(el); + durationAddSample(EL_DURATION_TYPE_PRIORITY_EL, duration_us); + unsigned long long priority_cmds = server.duration_stats[EL_DURATION_TYPE_PRIORITY_CMD].cnt; + if (priority_cmds > (unsigned long long)server.priority_el_cmd_cnt_prev) { + long long el_cmd_cnt = priority_cmds - server.priority_el_cmd_cnt_prev; + if (el_cmd_cnt > server.priority_el_cmd_cnt_max) { + server.priority_el_cmd_cnt_max = el_cmd_cnt; + } + server.priority_el_cmd_cnt_prev = priority_cmds; + } +} + /* =========================== Server initialization ======================== */ static inline robj *createSharedString(const char *string) { @@ -2257,6 +2339,8 @@ void createSharedObjects(void) { shared.srem = createSharedString("SREM"); shared.xgroup = createSharedString("XGROUP"); shared.xclaim = createSharedString("XCLAIM"); + shared.xdel = createSharedString("XDEL"); + shared.xack = createSharedString("XACK"); shared.script = createSharedString("SCRIPT"); shared.replconf = createSharedString("REPLCONF"); shared.pexpireat = createSharedString("PEXPIREAT"); @@ -2354,6 +2438,7 @@ void initServerConfig(void) { for (j = 0; j < CONFIG_DEFAULT_BINDADDR_COUNT; j++) server.bindaddr[j] = zstrdup(default_bindaddr[j]); memset(server.listeners, 0x00, sizeof(server.listeners)); server.active_expire_enabled = 1; + server.forkless_infrastructure_enabled = 0; server.lazy_expire_disabled = 0; server.skip_checksum_validation = 0; server.loading = 0; @@ -2384,6 +2469,8 @@ void initServerConfig(void) { server.shutdown_flags = 0; server.shutdown_mstime = 0; server.cluster_module_flags = CLUSTER_MODULE_FLAG_NONE; + atomic_store_explicit(&server.cluster_config_save_status, C_OK, memory_order_relaxed); + atomic_store_explicit(&server.cluster_config_last_save_time, time(NULL), memory_order_relaxed); server.migrate_cached_sockets = dictCreate(&migrateCacheDictType); server.next_client_id = 1; /* Client IDs, start from 1 .*/ server.page_size = sysconf(_SC_PAGESIZE); @@ -2400,9 +2487,11 @@ void initServerConfig(void) { server.latency_tracking_info_percentiles[2] = 99.9; /* p999 */ server.tls_server_cert_expire_time = 0; + server.tls_server_alt_cert_expire_time = 0; server.tls_client_cert_expire_time = 0; server.tls_ca_cert_expire_time = 0; server.tls_server_cert_serial = NULL; + server.tls_server_alt_cert_serial = NULL; server.tls_client_cert_serial = NULL; server.tls_ca_cert_serial = NULL; @@ -2423,6 +2512,7 @@ void initServerConfig(void) { server.repl_transfer_tmpfile = NULL; server.repl_transfer_fd = -1; server.repl_transfer_s = NULL; + server.repl_compression_advertised = REPL_COMPRESSION_CAPA_UNKNOWN; server.repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT; server.repl_down_since = 0; /* Never connected, repl is down since EVER. */ server.primary_repl_offset = 0; @@ -2857,6 +2947,7 @@ void resetServerStats(void) { server.stat_fork_rate = 0; server.stat_total_forks = 0; server.stat_rejected_conn = 0; + server.stat_rejected_priority_conn = 0; server.stat_sync_full = 0; server.stat_sync_partial_ok = 0; server.stat_sync_partial_err = 0; @@ -2890,9 +2981,15 @@ void resetServerStats(void) { server.stat_dump_payload_sanitizations = 0; server.aof_delayed_fsync = 0; server.stat_reply_buffer_shrinks = 0; + server.stat_cluster_threaded_reads_processed = 0; + server.stat_cluster_threaded_writes_processed = 0; + server.stat_cluster_threaded_accepts_processed = 0; + server.stat_cluster_io_main_thread_fallbacks = 0; server.stat_reply_buffer_expands = 0; memset(server.duration_stats, 0, sizeof(durationStats) * EL_DURATION_TYPE_NUM); server.el_cmd_cnt_max = 0; + server.priority_el_cmd_cnt_max = 0; + server.priority_el_cmd_cnt_prev = 0; server.stat_active_time = 0; server.el_iteration_active = false; server.stat_total_prefetch_batches = 0; @@ -3012,6 +3109,8 @@ void initServer(void) { server.cluster_drop_packet_filter = -1; server.debug_cluster_disable_random_ping = 0; server.debug_cluster_disable_reconnection = 0; + server.debug_cluster_failover_delay = -1; + server.debug_cluster_failover_epoch = -1; server.reply_buffer_peak_reset_time = REPLY_BUFFER_DEFAULT_PEAK_RESET_TIME; server.reply_buffer_resizing_enabled = 1; server.client_mem_usage_buckets = NULL; @@ -3044,9 +3143,24 @@ void initServer(void) { serverLog(LL_WARNING, "Failed creating the event loop. Error message: '%s'", strerror(errno)); exit(1); } - + /* Setup QoS event loop if multiplexer backend supports secondary polling. + * If secondary polling is unsupported (e.g. evport, select), gracefully fallback to standard event processing without QoS. */ + if (aeActuateQoSEventLoopIfSupported(server.el, server.priority_preemptive_poll_interval_us, qosStatsCallback) == AE_ERR) { + serverLog(LL_NOTICE, "QoS event prioritization not supported on %s multiplexer, falling back to standard event processing", + aeGetApiName()); + } server.dbnum = server.cluster_enabled ? server.config_databases_cluster : server.config_databases; server.db = zcalloc(sizeof(serverDb *) * server.dbnum); + + /* Set object metadata size before creating any database key objects */ + if (server.forkless_infrastructure_enabled) { + /* NOTE: At this time, there is only one reason for dbEntry metadata: bgIteration. However, + * if/when new metadata options are added, we will need to compute the size of a variable + * size metadata, and provide appropriate accessors to access the specific portion of the + * metadata (each of which may/may not exist, based on immutable startup parameters). */ + objectSetMetadataSize(BGITERATION_ENTRY_METADATA_SIZE); + } + createDatabaseIfNeeded(0); /* The default database should always exist */ evictionPoolAlloc(); /* Initialize the LRU keys pool. */ @@ -3061,18 +3175,19 @@ void initServer(void) { server.watching_clients = 0; server.cronloops = 0; server.in_exec = 0; + server.in_call = 0; server.busy_module_yield_flags = BUSY_MODULE_YIELD_NONE; server.busy_module_yield_reply = NULL; server.client_pause_in_transaction = 0; server.child_pid = -1; server.child_type = CHILD_TYPE_NONE; - server.rdb_child_type = RDB_CHILD_TYPE_NONE; + server.rdb_write_target = RDB_WRITE_TARGET_NONE; server.rdb_pipe_conns = NULL; server.rdb_pipe_numconns = 0; server.rdb_pipe_numconns_writing = 0; server.rdb_pipe_buff = NULL; server.rdb_pipe_bufflen = 0; - server.rdb_bgsave_scheduled = 0; + server.rdb_bgsave_scheduled = RDB_BGSAVE_TYPE_NONE; server.child_info_pipe[0] = -1; server.child_info_pipe[1] = -1; server.child_info_nread = 0; @@ -3101,12 +3216,18 @@ void initServer(void) { server.stat_module_progress = 0; for (int j = 0; j < CLIENT_TYPE_COUNT; j++) server.stat_clients_type_memory[j] = 0; server.stat_cluster_links_memory = 0; + server.stat_cluster_threaded_reads_processed = 0; + server.stat_cluster_threaded_writes_processed = 0; + server.stat_cluster_threaded_accepts_processed = 0; + server.stat_cluster_io_main_thread_fallbacks = 0; server.cron_malloc_stats.zmalloc_used = 0; server.cron_malloc_stats.process_rss = 0; server.cron_malloc_stats.allocator_allocated = 0; server.cron_malloc_stats.allocator_active = 0; server.cron_malloc_stats.allocator_resident = 0; server.lastbgsave_status = C_OK; + server.lastbgsave_type = RDB_BGSAVE_TYPE_NONE; + server.cur_bgsave_type = RDB_BGSAVE_TYPE_NONE; server.aof_last_write_status = C_OK; server.aof_last_write_errno = 0; server.repl_good_replicas_count = 0; @@ -3157,7 +3278,9 @@ void initServer(void) { commandlogInit(); latencyMonitorInit(); + throttle_init(); initSharedQueryBuf(); + bgIteration_init(); /* Initialize ACL default password if it exists */ ACLUpdateDefaultUserPassword(server.requirepass); @@ -3173,6 +3296,14 @@ void initServer(void) { applyWatchdogPeriod(); if (server.maxmemory_clients != 0) initServerClientMemUsageBuckets(); + + /* Initialization hotkey */ + hotkeysInit(); + + /* Initialize priority subnets if configured */ + if (updatePrioritySubnets(server.priority_subnets) != C_OK) { + serverPanic("Failed parsing priority-subnets on startup, check the server logs."); + } } void initListeners(void) { @@ -3384,6 +3515,27 @@ void commandAddSubcommand(struct serverCommand *parent, struct serverCommand *su serverAssert(hashtableAdd(parent->subcommands_ht, subcommand)); } +/* Automatically set CMD_WRITE_FIRSTKEY_ONLY for write commands where the first + * key is written, and other keys are read only. */ +void detectWriteFirstkeyOnlyCommand(struct serverCommand *c) { + c->flags &= ~CMD_WRITE_FIRSTKEY_ONLY; // Override if set elsewhere + if (!(c->flags & CMD_WRITE)) return; + if (c->key_specs_num < 2) return; + if (!(c->key_specs[0].flags & (CMD_KEY_OW | CMD_KEY_RW))) return; + if (c->key_specs[0].find_keys_type != KSPEC_FK_RANGE) return; + if (c->key_specs[0].fk.range.lastkey != 0) return; + + bool write_first_key_only = true; + for (int i = 1; i < c->key_specs_num; i++) { + if (!(c->key_specs[i].flags & CMD_KEY_RO) || (c->key_specs[i].flags & (CMD_KEY_RW | CMD_KEY_OW | CMD_KEY_RM))) { + write_first_key_only = false; + break; + } + } + + if (write_first_key_only) c->flags |= CMD_WRITE_FIRSTKEY_ONLY; +} + /* Recursively populate the command structure. * * On success, the function return C_OK. Otherwise, C_ERR is returned and we won't @@ -3407,6 +3559,8 @@ int populateCommandStructure(struct serverCommand *c) { /* Handle the legacy range spec and the "movablekeys" flag (must be done after populating all key specs). */ populateCommandLegacyRangeSpec(c); + detectWriteFirstkeyOnlyCommand(c); + /* Assign the ID used for ACL. */ c->id = ACLGetCommandID(c->fullname); @@ -3696,6 +3850,58 @@ static void propagateNow(int dbid, robj **argv, int argc, int target, int slot) if (propagate_to_slot_migration) clusterFeedSlotExportJobs(dbid, argv, argc, slot); } +/* BgIteration requires that replication is sent after each command, however the + * alsoPropagate mechanism queues replication until the end of the transaction + * (when propagatePendingCommands is invoked). Also, the propagation mechanism + * strips out multi/exec, adding them back during propagatePendingCommands (if + * necessary). This function ensures that replication, including multi/exec are + * sequenced with the commands for bgIteration. + * + * Called from alsoPropagate with regular params. + * Called from propagatePendingCommands with dbid = -1 (to close multi/exec). */ +static void propagateToBgIteration(int dbid, int argc, robj **argv, int target) { + /* STATIC indicates that we have sent the MULTI, and need to match it with + * an EXEC during propagatePendingCommands. */ + static bool sentMultiToBgIterator = false; + /* STATIC indicates that last DBID that was sent, so that we can use the + * same DBID when sending a generated EXEC. */ + static int lastDbidSentToBgIterator; + + if (dbid >= 0) { + // Called from alsoPropagate() to replicate a command + if (target & PROPAGATE_REPL && bgIteration_iterationActive()) { + if (!sentMultiToBgIterator && (scriptIsRunning() || server.in_exec)) { + /* For a script or multi/exec, we should be sending the MULTI at + * the beginning of the execution unit. There shouldn't be any + * commands in the propagation queue yet. */ + serverAssert(server.also_propagate.numops == 0); + /* If this is the first propagated command of a script or multi, + * make it a transaction. It may turn out that there is only 1 + * command in the MULTI block, but we can't know that now. + * Unlike regular replication, we can't defer all of the + * replication until we know for sure. We must call bgIteration + * after each command. */ + static struct serverCommand *cmd_multi = NULL; // STATIC + if (cmd_multi == NULL) cmd_multi = lookupCommandOrOriginal(&shared.multi, 1); + bgIteration_handleCommandReplication(dbid, cmd_multi, 1, &shared.multi); + sentMultiToBgIterator = true; + } + struct serverCommand *cmd = lookupCommandOrOriginal(argv, argc); + bgIteration_handleCommandReplication(dbid, cmd, argc, argv); + lastDbidSentToBgIterator = dbid; + } + } else { + // Called from propagatePendingCommands() to finalize a transaction + if (sentMultiToBgIterator) { + // If a MULTI was sent to bgIterator via alsoPropagate(), then send the matching EXEC. + static struct serverCommand *cmd_exec = NULL; // STATIC + if (cmd_exec == NULL) cmd_exec = lookupCommandOrOriginal(&shared.exec, 1); + bgIteration_handleCommandReplication(lastDbidSentToBgIterator, cmd_exec, 1, &shared.exec); + sentMultiToBgIterator = false; + } + } +} + /* Used inside commands to schedule the propagation of additional commands * after the current command is propagated to AOF / Replication. * @@ -3708,6 +3914,8 @@ static void propagateNow(int dbid, robj **argv, int argc, int target, int slot) * stack allocated). The function automatically increments ref count of * passed objects, so the caller does not need to. */ void alsoPropagate(int dbid, robj **argv, int argc, int target, int slot) { + propagateToBgIteration(dbid, argc, argv, target); + robj **argvcopy; int j; @@ -3774,6 +3982,12 @@ void updateCommandLatencyHistogram(struct hdr_histogram **latency_histogram, int * multiple separated commands. Note that alsoPropagate() is not affected * by CLIENT_PREVENT_PROP flag. */ static void propagatePendingCommands(void) { + /* This is done before the check on server.also_propagate.numops. Numops + * might be zero if there is no replica but we might be running bgIteration + * for something other than replication. If we sent the multi (to + * bgIteration), we need to send the matching exec. */ + propagateToBgIteration(-1, 0, NULL, 0); + if (server.also_propagate.numops == 0) return; int j; @@ -3812,6 +4026,21 @@ static void propagatePendingCommands(void) { serverOpArrayFree(&server.also_propagate); } +/* Whether any of the module-jobs / propagation / module-yield post-execution- + * unit work is pending. Shared by postExecutionUnitOperations() and + * afterCommand() so the "is there anything to do?" condition for these three + * sub-systems has a single home instead of being duplicated at both call + * sites - static inline costs nothing here since both callers are in this + * same translation unit. moduleHasPostExecUnitJobs() is a tiny cross-TU + * accessor rather than reaching into module.c's list directly, so this file + * doesn't need to know how the module subsystem tracks its pending jobs. + * + * Must stay an OR of every condition below - never drop one as an + * optimization, since that would silently skip real pending work. */ +static inline int hasPostExecutionUnitPendingWork(void) { + return moduleHasPostExecUnitJobs() || server.also_propagate.numops || server.busy_module_yield_flags; +} + /* Performs operations that should be performed after an execution unit ends. * Execution unit is a code that should be done atomically. * Execution units can be nested and do not necessarily start with a server command. @@ -3829,14 +4058,27 @@ static void propagatePendingCommands(void) { void postExecutionUnitOperations(void) { if (server.execution_nesting) return; - firePostExecutionUnitJobs(); + /* Combined pending-work gate: in the overwhelming majority of calls + * (e.g. after a plain read-only command like GET) none of the three + * sub-systems below have anything queued. Fold all of their "is there + * anything to do?" checks into a single branch here so the common case + * pays for one memory read + one branch instead of three separate + * (partly cross-translation-unit, non-inlinable) function calls. + * + * Deliberately NOT hinted unlikely() here: unlike the afterCommand() + * gate below, this function is also called right after queuing a + * propagation (expire.c, evict.c, db.c) where the condition is + * typically true, so a fixed hint would be wrong for those call sites. */ + if (hasPostExecutionUnitPendingWork()) { + firePostExecutionUnitJobs(); - /* If we are at the top-most call() and not inside an active module - * context (e.g. within a module timer) we can propagate what we accumulated. */ - propagatePendingCommands(); + /* If we are at the top-most call() and not inside an active module + * context (e.g. within a module timer) we can propagate what we accumulated. */ + propagatePendingCommands(); - /* Module subsystem post-execution-unit logic */ - modulePostExecutionUnitOperations(); + /* Module subsystem post-execution-unit logic */ + modulePostExecutionUnitOperations(); + } } /* Increment the command failure counters (either rejected_calls or failed_calls). @@ -3903,6 +4145,10 @@ int incrCommandStatsOnError(struct serverCommand *cmd, int flags) { * */ void call(client *c, int flags) { + if (bgIteration_blockClientIfRequired(c)) return; + + server.in_call++; + long long dirty; struct ClientFlags client_old_flags = c->flag; @@ -4058,7 +4304,13 @@ void call(client *c, int flags) { } else { latencyTraceIfNeeded(server, command, duration); } - if (server.execution_nesting == 0) durationAddSample(EL_DURATION_TYPE_CMD, duration); + if (server.execution_nesting == 0) { + durationAddSample(EL_DURATION_TYPE_CMD, duration); + /* Attribute command execution latency for high-priority client connections. */ + if (connIsPriority(c->conn)) { + durationAddSample(EL_DURATION_TYPE_PRIORITY_CMD, duration); + } + } } /* Log the command into the commandlog if needed. @@ -4080,7 +4332,7 @@ void call(client *c, int flags) { if (update_command_stats && !c->flag.blocked) { real_cmd->calls++; real_cmd->microseconds += c->duration; - if (server.latency_tracking_enabled && !c->flag.blocked) + if (server.latency_tracking_enabled) updateCommandLatencyHistogram(&(real_cmd->latency_histogram), c->duration * 1000); clusterSlotStatsAddCpuDuration(c, c->duration); } @@ -4169,6 +4421,7 @@ void call(client *c, int flags) { } server.executing_client = prev_client; + server.in_call--; } /* Used when a command that is ready for execution needs to be rejected, due to @@ -4221,18 +4474,37 @@ void rejectCommandFormat(client *c, int notify_modules, const char *fmt, ...) { /* This is called after a command in call, we can do some maintenance job in it. */ void afterCommand(client *c) { UNUSED(c); - /* Should be done before trackingHandlePendingKeyInvalidations so that we - * reply to client before invalidating cache (makes more sense) */ - postExecutionUnitOperations(); - /* Flush pending tracking invalidations. */ - trackingHandlePendingKeyInvalidations(); + /* Combined pending-work gate for command-completion tail work. + * See postExecutionUnitOperations() for why each callee below still + * needs its own defensive check - this gate exists purely so the + * overwhelmingly common "nothing pending" case (e.g. after a plain + * GET, or after each sub-command of a MULTI/EXEC or script) short- + * circuits before paying for any of the calls below. + * + * Unlike the shared hasPostExecutionUnitPendingWork() conditions, the + * unlikely() hint here is safe: this is the single call site reached + * from a plain top-level command, where "nothing pending" genuinely + * dominates. clusterSlotStatsAddNetworkBytesOutForUserClient() is + * intentionally excluded from the gate: it is not deferred/queued + * work, it must run for every command whenever slot-stats accounting + * is enabled. */ + if (server.execution_nesting == 0 && + unlikely(hasPostExecutionUnitPendingWork() || trackingHasPendingKeyInvalidations() || + listLength(server.pending_push_messages))) { + /* Should be done before trackingHandlePendingKeyInvalidations so that we + * reply to client before invalidating cache (makes more sense) */ + postExecutionUnitOperations(); + + /* Flush pending tracking invalidations. */ + trackingHandlePendingKeyInvalidations(); + + /* Flush other pending push messages. Not interleaved with + * transaction response since we're already outside nesting here. */ + listJoin(c->reply, server.pending_push_messages); + } clusterSlotStatsAddNetworkBytesOutForUserClient(c); - - /* Flush other pending push messages. only when we are not in nested call. - * So the messages are not interleaved with transaction response. */ - if (!server.execution_nesting) listJoin(c->reply, server.pending_push_messages); } /* Check if c->cmd exists, fills `err` with details in case it doesn't. @@ -4351,6 +4623,8 @@ void unprepareCommand(client *c) { * other operations can be performed by the caller. Otherwise * if C_ERR is returned the client was destroyed (i.e. after QUIT). */ int processCommand(client *c) { + serverAssert(!c->flag.blocked && !c->flag.unblocked); + if (!scriptIsTimedout()) { /* Both EXEC and scripts call call() directly so there should be * no way in_exec or scriptIsRunning() is 1. @@ -4736,6 +5010,8 @@ int processCommand(client *c) { return C_OK; } + if (throttle_throttleClientIfNeeded(c)) return C_OK; + /* Exec the command */ if (c->flag.multi && c->cmd->proc != execCommand && c->cmd->proc != discardCommand && c->cmd->proc != quitCommand && @@ -4949,7 +5225,7 @@ int finishShutdown(void) { /* Kill the saving child if there is a background saving in progress. We want to avoid race conditions, for instance our saving child may overwrite the synchronous saving did by SHUTDOWN. */ - if (server.child_type == CHILD_TYPE_RDB) { + if (isForkBgsaveInProgress()) { serverLog(LL_WARNING, "There is a child saving an .rdb. Killing it!"); killRDBChild(); /* Note that, in killRDBChild normally has backgroundSaveDoneHandler @@ -4960,6 +5236,10 @@ int finishShutdown(void) { * but OS will close this fd when process exits. */ rdbRemoveTempFile(server.child_pid, 0); } + if (isForklessSaveInProgress()) { + serverLog(LL_WARNING, "There is a thread saving an .rdb. Cancelling it!"); + forklessSaveCancel(); + } /* Kill module child if there is one. */ if (server.child_type == CHILD_TYPE_MODULE) { @@ -6268,6 +6548,11 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { tls_server_seconds_remaining = server.tls_server_cert_expire_time - (long long)server.unixtime; if (tls_server_seconds_remaining < 0) tls_server_seconds_remaining = 0; } + long long tls_server_alt_seconds_remaining = 0; + if (server.tls_server_alt_cert_expire_time > 0) { + tls_server_alt_seconds_remaining = server.tls_server_alt_cert_expire_time - (long long)server.unixtime; + if (tls_server_alt_seconds_remaining < 0) tls_server_alt_seconds_remaining = 0; + } long long tls_client_seconds_remaining = 0; if (server.tls_client_cert_expire_time > 0) { tls_client_seconds_remaining = server.tls_client_cert_expire_time - (long long)server.unixtime; @@ -6283,6 +6568,8 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "# TLS\r\n" FMTARGS( "tls_server_cert_serial:%s\r\n", server.tls_server_cert_serial ? server.tls_server_cert_serial : "none", "tls_server_cert_expires_in_seconds:%lld\r\n", tls_server_seconds_remaining, + "tls_server_alt_cert_serial:%s\r\n", server.tls_server_alt_cert_serial ? server.tls_server_alt_cert_serial : "none", + "tls_server_alt_cert_expires_in_seconds:%lld\r\n", tls_server_alt_seconds_remaining, "tls_client_cert_serial:%s\r\n", server.tls_client_cert_serial ? server.tls_client_cert_serial : "none", "tls_client_cert_expires_in_seconds:%lld\r\n", tls_client_seconds_remaining, "tls_ca_cert_serial:%s\r\n", server.tls_ca_cert_serial ? server.tls_ca_cert_serial : "none", @@ -6315,6 +6602,7 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { info, "# Clients\r\n" FMTARGS( "connected_clients:%lu\r\n", listLength(server.clients) - listLength(server.replicas), + "connected_priority_clients:%lld\r\n", server.stat_num_active_priority_clients, "cluster_connections:%lu\r\n", getClusterConnectionsCount(), "maxclients:%u\r\n", server.maxclients, "client_recent_max_input_buffer:%zu\r\n", maxin, @@ -6389,7 +6677,7 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "used_memory_vm_eval:%lld\r\n", memory_lua, "used_memory_lua_human:%s\r\n", used_memory_lua_hmem, /* deprecated */ "used_memory_scripts_eval:%lld\r\n", (long long)mh->lua_caches, - "number_of_cached_scripts:%zu\r\n", dictSize(evalScriptsDict()), + "number_of_cached_scripts:%zu\r\n", dictSize(evalCtxScriptsDict()), "number_of_functions:%lu\r\n", functionsNum(), "number_of_libraries:%lu\r\n", functionsLibNum(), "used_memory_vm_functions:%lld\r\n", memory_functions, @@ -6434,14 +6722,30 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { /* Persistence */ if (all_sections || (dictFind(section_dict, "persistence") != NULL)) { if (sections++) info = sdscat(info, "\r\n"); - double fork_perc = 0; + double save_perc = 0; if (server.stat_module_progress) { - fork_perc = server.stat_module_progress * 100; + save_perc = server.stat_module_progress * 100; } else if (server.stat_current_save_keys_total) { - fork_perc = ((double)server.stat_current_save_keys_processed / server.stat_current_save_keys_total) * 100; + save_perc = ((double)server.stat_current_save_keys_processed / server.stat_current_save_keys_total) * 100; } int aof_bio_fsync_status = atomic_load_explicit(&server.aof_bio_fsync_status, memory_order_relaxed); + /* Determine current bgsave type */ + const char *current_bgsave_type; + switch (server.cur_bgsave_type) { + case RDB_BGSAVE_TYPE_FORK: current_bgsave_type = "fork"; break; + case RDB_BGSAVE_TYPE_FORKLESS: current_bgsave_type = "forkless"; break; + default: current_bgsave_type = "none"; break; + } + + /* Determine last bgsave type */ + const char *last_bgsave_type; + switch (server.lastbgsave_type) { + case RDB_BGSAVE_TYPE_FORK: last_bgsave_type = "fork"; break; + case RDB_BGSAVE_TYPE_FORKLESS: last_bgsave_type = "forkless"; break; + default: last_bgsave_type = "none"; break; + } + info = sdscatprintf( info, "# Persistence\r\n" FMTARGS( @@ -6450,15 +6754,17 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "current_cow_peak:%zu\r\n", server.stat_current_cow_peak, "current_cow_size:%zu\r\n", server.stat_current_cow_bytes, "current_cow_size_age:%lu\r\n", (server.stat_current_cow_updated ? (unsigned long)elapsedMs(server.stat_current_cow_updated) / 1000 : 0), - "current_fork_perc:%.2f\r\n", fork_perc, + "current_fork_perc:%.2f\r\n", save_perc, "current_save_keys_processed:%zu\r\n", server.stat_current_save_keys_processed, "current_save_keys_total:%zu\r\n", server.stat_current_save_keys_total, "rdb_changes_since_last_save:%lld\r\n", server.dirty, - "rdb_bgsave_in_progress:%d\r\n", server.child_type == CHILD_TYPE_RDB, + "rdb_bgsave_in_progress:%d\r\n", isSaveInProgress(), + "rdb_current_bgsave_type:%s\r\n", current_bgsave_type, + "rdb_last_bgsave_type:%s\r\n", last_bgsave_type, "rdb_last_save_time:%jd\r\n", (intmax_t)server.lastsave, "rdb_last_bgsave_status:%s\r\n", (server.lastbgsave_status == C_OK) ? "ok" : "err", "rdb_last_bgsave_time_sec:%jd\r\n", (intmax_t)server.rdb_save_time_last, - "rdb_current_bgsave_time_sec:%jd\r\n", (intmax_t)((server.child_type != CHILD_TYPE_RDB) ? -1 : time(NULL) - server.rdb_save_time_start), + "rdb_current_bgsave_time_sec:%jd\r\n", (intmax_t)(isSaveInProgress() ? time(NULL) - server.rdb_save_time_start : -1), "rdb_saves:%lld\r\n", server.stat_rdb_saves, "rdb_last_cow_size:%zu\r\n", server.stat_rdb_cow_bytes, "rdb_last_load_keys_expired:%lld\r\n", server.rdb_last_load_keys_expired, @@ -6523,6 +6829,9 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "loading_loaded_perc:%.2f\r\n", perc, "loading_eta_seconds:%jd\r\n", (intmax_t)eta)); } + + /* Forkless / bgiteration metrics */ + info = forkless_catInfo(info); } /* Stats */ @@ -6550,6 +6859,7 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "instantaneous_input_repl_kbps:%.2f\r\n", (float)getInstantaneousMetric(STATS_METRIC_NET_INPUT_REPLICATION) / 1024, "instantaneous_output_repl_kbps:%.2f\r\n", (float)getInstantaneousMetric(STATS_METRIC_NET_OUTPUT_REPLICATION) / 1024, "rejected_connections:%lld\r\n", server.stat_rejected_conn, + "rejected_priority_connections:%lld\r\n", server.stat_rejected_priority_conn, "sync_full:%lld\r\n", server.stat_sync_full, "sync_partial_ok:%lld\r\n", server.stat_sync_partial_ok, "sync_partial_err:%lld\r\n", server.stat_sync_partial_err, @@ -6602,7 +6912,10 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "eventloop_duration_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_EL].sum, "eventloop_duration_cmd_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_CMD].sum, "instantaneous_eventloop_cycles_per_sec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_EL_CYCLE), - "instantaneous_eventloop_duration_usec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_EL_DURATION))); + "instantaneous_eventloop_duration_usec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_EL_DURATION), + "eventloop_priority_cycles:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].cnt, + "eventloop_priority_duration_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].sum, + "eventloop_priority_duration_cmd_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_PRIORITY_CMD].sum)); info = genValkeyInfoStringACLStats(info); } @@ -6705,12 +7018,22 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { info = sdscatprintf(info, "slave%d:ip=%s,port=%d,state=%s," - "offset=%lld,lag=%ld,type=%s\r\n", + "offset=%lld,lag=%ld,type=%s", replica_id, replica_ip, replica->repl_data->replica_listening_port, state, replica->repl_data->repl_ack_off, lag, replica->flag.repl_rdb_channel ? "rdb-channel" : replica->repl_data->repl_state == REPLICA_STATE_BG_RDB_LOAD ? "main-channel" : "replica"); + if (replica->repl_data->repl_compression) { + info = sdscatprintf(info, + ",repl_compression=%s" + ",repl_compressed_bytes=%lld" + ",repl_uncompressed_bytes=%lld", + compressionAlgoName(replica->repl_data->repl_compression->compressor.algo), + replica->repl_data->repl_compression->compressed_bytes, + replica->repl_data->repl_compression->uncompressed_bytes); + } + info = sdscat(info, "\r\n"); replica_id++; } } @@ -6853,6 +7176,21 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { } } + /* Hotkeys */ + if (all_sections || (dictFind(section_dict, "hotkeys") != NULL)) { + if (sections++) info = sdscat(info, "\r\n"); + info = sdscatprintf(info, "# Hotkeys\r\n"); + info = genHotkeysInfoString(info); + } + + /* Throttling */ + if (all_sections || (dictFind(section_dict, "throttling") != NULL)) { + if (sections++) info = sdscat(info, "\r\n"); + info = sdscat(info, "# Throttling\r\n"); + info = throttle_sdscatInfoMetrics(info); + info = throttleRepl_sdscatInfoMetrics(info); + } + /* Get info from modules. * Returned when the user asked for "everything", "modules", or a specific module section. * We're not aware of the module section names here, and we rather avoid the search when we can. @@ -6866,16 +7204,23 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { } if (dictFind(section_dict, "debug") != NULL) { + size_t module_external_memory = zmalloc_used_external_memory(); if (sections++) info = sdscat(info, "\r\n"); info = sdscatprintf( info, "# Debug\r\n" FMTARGS( + "used_memory_module_external:%zu\r\n", module_external_memory, "eventloop_duration_aof_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_AOF].sum, "eventloop_duration_cron_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_CRON].sum, "eventloop_duration_max:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_EL].max, "eventloop_cmd_per_cycle_max:%lld\r\n", server.el_cmd_cnt_max, + "eventloop_priority_duration_max:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].max, + "eventloop_priority_cmd_per_cycle_max:%lld\r\n", server.priority_el_cmd_cnt_max, "io_threaded_reads_pending:%lld\r\n", server.stat_io_reads_pending, "io_threaded_writes_pending:%lld\r\n", server.stat_io_writes_pending)); + + info = forkless_catDebugInfo(info); + info = throttleRepl_sdscatInfoDebugMetrics(info); } return info; @@ -7173,7 +7518,7 @@ void closeChildUnusedResourceAfterFork(void) { /* purpose is one of CHILD_TYPE_ types */ int serverFork(int purpose) { if (isMutuallyExclusiveChildType(purpose)) { - if (hasActiveChildProcess()) { + if (hasActiveSaveOrChild()) { errno = EALREADY; return -1; } @@ -7904,10 +8249,12 @@ __attribute__((weak)) int main(int argc, char **argv) { * MSET specific command extended options - XX/NX * HGET specific command extended options - PERSIST * HSET specific command extended options - NX/XX/FXX/FNX + * INCREX specific command extended options - BYINT/BYFLOAT * Common command extended options - EX/EXAT/PX/PXAT/KEEPTTL * * Function takes pointers to client, flags, unit, expire_idx, pointer to pointer of expire obj, - * pointer to pointer of compare obj if needed to be determined and command_type which can be COMMAND_*. + * pointer to pointer of compare obj, pointer to pointer of incrby obj, and command_type + * which can be COMMAND_*. * * If there are any syntax violations C_ERR is returned else C_OK is returned. * @@ -7917,7 +8264,7 @@ __attribute__((weak)) int main(int argc, char **argv) { * start_idx provides a way to start scanning from a specific index. * max_args provides a way to limit the scan to a specific range of arguments. */ -int parseExtendedCommandArgumentsOrReply(client *c, int command_type, int start_idx, int max_args, int *flags, int *unit, int *expire_idx, robj **expire, robj **compare_val) { +int parseExtendedCommandArgumentsOrReply(client *c, int command_type, int start_idx, int max_args, int *flags, int *unit, int *expire_idx, robj **expire, robj **compare_val, robj **incrby_val) { int j = start_idx; if (expire_idx) *expire_idx = -1; for (; j < max_args; j++) { @@ -7927,14 +8274,14 @@ int parseExtendedCommandArgumentsOrReply(client *c, int command_type, int start_ /* clang-format off */ if ((opt[0] == 'n' || opt[0] == 'N') && (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && - !(*flags & ARGS_SET_XX || *flags & ARGS_SET_IFEQ) && - (command_type == COMMAND_SET || command_type == COMMAND_HSET || command_type == COMMAND_MSET)) + !(*flags & (ARGS_SET_CONDITIONAL & ~ARGS_SET_NX)) && /* Repeated NX allowed */ + (command_type == COMMAND_SET || command_type == COMMAND_HSET || command_type == COMMAND_MSET || command_type == COMMAND_INCREX)) { *flags |= ARGS_SET_NX; } else if ((opt[0] == 'x' || opt[0] == 'X') && (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' && - !(*flags & ARGS_SET_NX || *flags & ARGS_SET_IFEQ) && - (command_type == COMMAND_SET || command_type == COMMAND_HSET || command_type == COMMAND_MSET)) + !(*flags & (ARGS_SET_CONDITIONAL & ~ARGS_SET_XX)) && /* Repeated XX allowed */ + (command_type == COMMAND_SET || command_type == COMMAND_HSET || command_type == COMMAND_MSET || command_type == COMMAND_INCREX)) { *flags |= ARGS_SET_XX; } else if ((opt[0] == 'f' || opt[0] == 'F') && @@ -7954,11 +8301,21 @@ int parseExtendedCommandArgumentsOrReply(client *c, int command_type, int start_ (opt[2] == 'e' || opt[2] == 'E') && (opt[3] == 'q' || opt[3] == 'Q') && opt[4] == '\0' && next && - !(*flags & ARGS_SET_NX || *flags & ARGS_SET_XX || *flags & ARGS_SET_IFEQ) && (command_type == COMMAND_SET)) + !(*flags & ARGS_SET_CONDITIONAL) && (command_type == COMMAND_SET)) { *flags |= ARGS_SET_IFEQ; *compare_val = next; j++; + } else if ((opt[0] == 'i' || opt[0] == 'I') && + (opt[1] == 'f' || opt[1] == 'F') && + (opt[2] == 'n' || opt[2] == 'N') && + (opt[3] == 'e' || opt[3] == 'E') && opt[4] == '\0' && + next && + !(*flags & ARGS_SET_CONDITIONAL) && (command_type == COMMAND_SET)) + { + *flags |= ARGS_SET_IFNE; + *compare_val = next; + j++; } else if ((opt[0] == 'g' || opt[0] == 'G') && (opt[1] == 'e' || opt[1] == 'E') && (opt[2] == 't' || opt[2] == 'T') && opt[3] == '\0' && @@ -8023,6 +8380,30 @@ int parseExtendedCommandArgumentsOrReply(client *c, int command_type, int start_ *expire = next; if (expire_idx) *expire_idx = j; j++; + } else if ((opt[0] == 'b' || opt[0] == 'B') && + (opt[1] == 'y' || opt[1] == 'Y') && + (opt[2] == 'i' || opt[2] == 'I') && + (opt[3] == 'n' || opt[3] == 'N') && + (opt[4] == 't' || opt[4] == 'T') && opt[5] == '\0' && + command_type == COMMAND_INCREX && + !(*flags & ARGS_BYINT) && !(*flags & ARGS_BYFLOAT) && next) + { + *flags |= ARGS_BYINT; + if (incrby_val) *incrby_val = next; + j++; + } else if ((opt[0] == 'b' || opt[0] == 'B') && + (opt[1] == 'y' || opt[1] == 'Y') && + (opt[2] == 'f' || opt[2] == 'F') && + (opt[3] == 'l' || opt[3] == 'L') && + (opt[4] == 'o' || opt[4] == 'O') && + (opt[5] == 'a' || opt[5] == 'A') && + (opt[6] == 't' || opt[6] == 'T') && opt[7] == '\0' && + command_type == COMMAND_INCREX && + !(*flags & ARGS_BYINT) && !(*flags & ARGS_BYFLOAT) && next) + { + *flags |= ARGS_BYFLOAT; + if (incrby_val) *incrby_val = next; + j++; } else { addReplyErrorObject(c, shared.syntaxerr); return C_ERR; diff --git a/src/server.h b/src/server.h index d34fed685..23c3de2d9 100644 --- a/src/server.h +++ b/src/server.h @@ -32,6 +32,7 @@ #include "fmacros.h" #include "config.h" +#include "compression.h" #include "solarisfixes.h" #include "rio.h" #include "commands.h" @@ -103,7 +104,19 @@ static_assert(sizeof(off_t) >= 8, "off_t must be 64-bit; ensure _FILE_OFFSET_BIT #define dismissMemory zmadvise_dontneed #define VALKEYMODULE_CORE 1 -typedef struct serverObject robj; + +/* serverObject (aka robj) is currently overloaded for 2 purposes. This is a legacy artifact. + * 1. It's carries a reference counted STRING (a keyless value) during parsing and command execution. + * 2. It's also used to carry a key/value pair which is inserted into the DB. In this form, the + * value is not limited to being a string. + * + * The typedef "dbEntry" is used to explicitly connote the latter form. It indicates a key/value + * pair which is suitable to exist in the DB. It might be active in the DB, or may be unlinked from + * the DB (but still contains a key/value). The value may be any of the Valkey data types/encodings. + */ +typedef struct serverObject robj; // A keyless string OR a key/value pair +typedef struct serverObject dbEntry; // Explicitly a key/value pair + #include "valkeymodule.h" /* Modules API defines. */ /* Following includes allow test functions to be called from main() */ @@ -347,6 +360,7 @@ typedef enum blocking_type { BLOCKED_ZSET, /* BZPOP et al. */ BLOCKED_POSTPONE, /* Blocked by processCommand, re-try processing later. */ BLOCKED_SHUTDOWN, /* SHUTDOWN. */ + BLOCKED_INUSE, /* Key in use by background thread. */ BLOCKED_NUM, /* Number of blocked states. */ BLOCKED_END /* End of enumeration */ } blocking_type; @@ -452,9 +466,11 @@ typedef enum { #define REPLICA_CAPA_PSYNC2 (1 << 1) /* Supports PSYNC2 protocol. */ #define REPLICA_CAPA_DUAL_CHANNEL (1 << 2) /* Supports dual channel replication sync */ #define REPLICA_CAPA_SKIP_RDB_CHECKSUM (1 << 3) /* Supports skipping RDB checksum for sync requests. */ +#define REPLICA_CAPA_LZ4 (1 << 4) /* Accepts LZ4 streaming-compressed replication payloads. */ /* Replica capability strings */ #define REPLICA_CAPA_SKIP_RDB_CHECKSUM_STR "skip-rdb-checksum" /* Supports skipping RDB checksum for sync requests. */ +#define REPLICA_CAPA_LZ4_STR "lz4" /* Accepts LZ4 streaming-compressed replication payloads. */ /* Replica requirements */ #define REPLICA_REQ_NONE 0 @@ -610,6 +626,14 @@ typedef enum { RDB_COMPRESSION_LZ4 /* Pin whole-stream LZ4 compression. */ } rdb_compression_mode; +typedef enum { + REPL_COMPRESSION_NO = 0, /* Disable replication compression. */ + REPL_COMPRESSION_YES, /* Use the default compression algorithm (currently LZ4). */ + REPL_COMPRESSION_LZ4 /* Pin whole-stream LZ4 compression. */ +} repl_compression_mode; + +#define REPL_COMPRESSION_CAPA_UNKNOWN -1 + /* Structure representing a non-owning view of a buffer. * A stringRef struct does not manage the underlying memory, so its destruction * will not free the buffer. */ @@ -648,13 +672,29 @@ typedef enum { /* Cluster persist config mode. */ typedef enum { CLUSTER_CONFIGFILE_SAVE_BEHAVIOR_SYNC = 0, /* Perform a synchronous save, exit the process if it fails. */ - CLUSTER_CONFIGFILE_SAVE_BEHAVIOR_BEST_EFFORT, /* Attempt to save on a "best-effort" basis, process will not exit if it fails. */ + CLUSTER_CONFIGFILE_SAVE_BEHAVIOR_BEST_EFFORT, /* Save asynchronously via BIO thread on a "best-effort" basis, process will not exit if it fails. */ } cluster_persist_config_mode; -/* RDB active child save type. */ -#define RDB_CHILD_TYPE_NONE 0 -#define RDB_CHILD_TYPE_DISK 1 /* RDB is written to disk. */ -#define RDB_CHILD_TYPE_SOCKET 2 /* RDB is written to replica socket. */ +/* RDB write target type. */ +typedef enum { + RDB_WRITE_TARGET_NONE = 0, + RDB_WRITE_TARGET_DISK = 1, /* RDB is written to disk. */ + RDB_WRITE_TARGET_SOCKET = 2 /* RDB is written to replica socket. */ +} rdbWriteTarget; + +/* RDB bgsave type. */ +typedef enum { + RDB_BGSAVE_TYPE_NONE = 0, + RDB_BGSAVE_TYPE_FORK = 1, /* Fork-based bgsave. */ + RDB_BGSAVE_TYPE_FORKLESS = 2 /* Forkless bgsave. */ +} rdbBgsaveType; + +/* Replica failover policy for server.cluster_replica_no_failover. */ +typedef enum { + CLUSTER_REPLICA_NO_FAILOVER_NO = 0, /* Allow automatic failover (default). */ + CLUSTER_REPLICA_NO_FAILOVER_YES, /* Never start a failover; sets CLUSTER_NODE_NOFAILOVER. */ + CLUSTER_REPLICA_NO_FAILOVER_IF_EMPTY, /* Refuse automatic failover only while the replica is empty. */ +} cluster_replica_no_failover_policy; /* Keyspace changes notification classes. Every class is associated with a * character for configuration purposes. */ @@ -723,20 +763,26 @@ typedef enum { /* Generic set command string object set flags */ #define ARGS_NO_FLAGS 0 -#define ARGS_SET_NX (1 << 0) /* Set if key not exists. */ -#define ARGS_SET_XX (1 << 1) /* Set if key exists. */ -#define ARGS_EX (1 << 2) /* Set if time in seconds is given */ -#define ARGS_PX (1 << 3) /* Set if time in ms in given */ -#define ARGS_KEEPTTL (1 << 4) /* Set and keep the ttl */ -#define ARGS_SET_GET (1 << 5) /* Set if want to get key before set */ -#define ARGS_EXAT (1 << 6) /* Set if timestamp in second is given */ -#define ARGS_PXAT (1 << 7) /* Set if timestamp in ms is given */ -#define ARGS_PERSIST (1 << 8) /* Set if we need to remove the ttl */ -#define ARGS_SET_IFEQ (1 << 9) /* Set if we need compare and set */ -#define ARGS_ARGV3 (1 << 10) /* Set if the value is at argv[3]; otherwise it's \ - * at argv[2]. */ -#define ARGS_SET_FNX (1 << 11) /* Set if key item not exists. */ -#define ARGS_SET_FXX (1 << 12) /* Set if key item exists. */ +#define ARGS_SET_NX (1 << 0) /* Set if key not exists. */ +#define ARGS_SET_XX (1 << 1) /* Set if key exists. */ +#define ARGS_EX (1 << 2) /* Set if time in seconds is given */ +#define ARGS_PX (1 << 3) /* Set if time in ms in given */ +#define ARGS_KEEPTTL (1 << 4) /* Set and keep the ttl */ +#define ARGS_SET_GET (1 << 5) /* Set if want to get key before set */ +#define ARGS_EXAT (1 << 6) /* Set if timestamp in second is given */ +#define ARGS_PXAT (1 << 7) /* Set if timestamp in ms is given */ +#define ARGS_PERSIST (1 << 8) /* Set if we need to remove the ttl */ +#define ARGS_SET_IFEQ (1 << 9) /* Set if we need compare and set */ +#define ARGS_ARGV3 (1 << 10) /* Set if the value is at argv[3]; otherwise it's \ + * at argv[2]. */ +#define ARGS_SET_FNX (1 << 11) /* Set if key item not exists. */ +#define ARGS_SET_FXX (1 << 12) /* Set if key item exists. */ +#define ARGS_SET_IFNE (1 << 13) /* Set only if values are not equal */ +#define ARGS_BYINT (1 << 14) /* Set if the value needs to be incremented by an integer. */ +#define ARGS_BYFLOAT (1 << 15) /* Set if the value needs to be incremented by a float. */ + +#define ARGS_SET_CONDITIONAL \ + (ARGS_SET_NX | ARGS_SET_XX | ARGS_SET_IFEQ | ARGS_SET_IFNE) /* An Object, that is a type able to hold a string / list / set */ @@ -782,6 +828,7 @@ typedef struct ValkeyModuleType moduleType; #define OBJ_ENCODING_QUICKLIST 9 /* Encoded as linked list of listpacks */ #define OBJ_ENCODING_STREAM 10 /* Encoded as a radix tree of listpacks */ #define OBJ_ENCODING_LISTPACK 11 /* Encoded as a listpack */ +#define OBJ_ENCODING_LISTPACK2 12 /* Encoded as a listpack with metadata tag */ #define OBJ_REFCOUNT_BITS 29 #define OBJ_SHARED_REFCOUNT ((1 << OBJ_REFCOUNT_BITS) - 1) /* Global object never destroyed. */ @@ -799,25 +846,27 @@ typedef struct ValkeyModuleType moduleType; * The optional variable-sized embedded data has 2 possible layouts. If value is embedded (hasembval == 1) * the `val_ptr` pointer is not used - instead the val data is embedded: * - * +------+----------+-----+------------+----------+--------+-----------------+---------+------------+ - * | type | encoding | lru | has* flags | refcount | expire | key_header_size | key sds | value data | - * +------+----------+-----+------------+----------+--------+-----------------+---------+------------+ - * ^ ^ ^ ^ - * | | | | - * | | | +--- present because hasembval == 1 - * | | | - * | +-----------------+--- present if hasembkey == 1 + * +------+----------+-----+------------+----------+--------+----------+-----------------+---------+------------+ + * | type | encoding | lru | has* flags | refcount | expire | metadata | key_header_size | key sds | value data | + * +------+----------+-----+------------+----------+--------+----------+-----------------+---------+------------+ + * ^ ^ ^ ^ ^ + * | | | | | + * | | | | +--- present because hasembval == 1 + * | | | | + * | +----------+-----------------+--- present if hasembkey == 1 + * | * | * +--- present if hasexpire == 1 * * Otherwise value is not embedded and we use the `val_ptr` pointer: * - * +------+----------+-----+------------+----------+---------+--------+-----------------+---------+ - * | type | encoding | lru | has* flags | refcount | val_ptr | expire | key_header_size | key sds | - * +------+----------+-----+------------+----------+---------+--------+-----------------+---------+ - * ^ ^ ^ ^ - * | | | | - * | | +-----------------+--- present if hasembkey == 1 + * +------+----------+-----+------------+----------+---------+--------+----------+-----------------+---------+ + * | type | encoding | lru | has* flags | refcount | val_ptr | expire | metadata | key_header_size | key sds | + * +------+----------+-----+------------+----------+---------+--------+----------+-----------------+---------+ + * ^ ^ ^ ^ ^ + * | | | | | + * | | +----------+-----------------+--- present if hasembkey == 1 + * | | * | | * | +--- present if hasexpire == 1 * | @@ -1033,6 +1082,9 @@ typedef struct readyList { no AUTH is needed, and every \ connection is immediately \ authenticated. */ +#define USER_FLAG_ROLE (1 << 3) /* This user entry represents a role, \ + not a regular user. Stored in the \ + Roles rax instead of Users. */ #define SELECTOR_FLAG_ROOT (1 << 0) /* This is the root user permission \ * selector. */ @@ -1046,10 +1098,15 @@ typedef struct readyList { typedef struct user { sds name; /* The username as an SDS string. */ uint32_t flags; /* See USER_FLAG_* */ - list *passwords; /* A list of SDS valid passwords for this user. */ + list *passwords; /* A list of SDS valid passwords for this user (NULL for roles). */ list *selectors; /* A list of selectors this user validates commands against. This list will always contain at least one selector for backwards compatibility. */ + list *roles; /* For users: the roles held by the user, kept in the + order they were assigned. Elements are `user *` + pointers owned by the Roles rax (NULL for roles). */ + dict *members; /* For roles: the users holding this role, keyed by their + `user *` pointer (NULL for users). */ robj *acl_string; /* cached string represent of ACLs */ } user; @@ -1206,6 +1263,9 @@ typedef struct ClientFlags { uint64_t keyspace_notified : 1; /* Indicates that a keyspace notification was triggered during the execution of the current command. */ uint64_t argv_borrowed : 1; /* The argv array and its elements are borrowed from the caller (VM_CallArgv) and must not be freed. */ + uint64_t throttled : 1; /* Currently queued in a throttler */ + uint64_t throttle_checked : 1; /* Already passed throttle check for this command */ + uint64_t throttle_multi : 1; /* Matches multiple throttlers */ } ClientFlags; /* Ensure ClientFlags never silently grows beyond two uint64_t words. * If this fires, move a flag to a separate field or widen the limit. */ @@ -1226,6 +1286,20 @@ typedef struct ClientPubSubData { context of client side caching. */ } ClientPubSubData; +/* Max decoded bytes processed before yielding to the event loop. This is + * shared by steady-state and dual-channel replication paths. */ +#define REPL_DECODE_EVENT_BUDGET (1024 * 1024) + +/* Primary-side compression state for one replica link. */ +typedef struct replicaCompressionState { + streamCompressor compressor; /* The frame stays open for the lifetime of the link. */ + sds out_buf; /* Compressed bytes waiting for the socket. */ + size_t out_buf_pos; /* Next byte to send from out_buf. */ + size_t batch_uncompressed_bytes; /* Backlog bytes represented by out_buf. */ + long long compressed_bytes; /* Completed batches, for INFO replication. */ + long long uncompressed_bytes; /* Completed batches, for INFO replication. */ +} replicaCompressionState; + typedef struct ClientReplicationData { int repl_state; /* Replication state if this is a replica. */ int repl_start_cmd_stream_on_ack; /* Install replica write handler on first ACK. */ @@ -1257,6 +1331,8 @@ typedef struct ClientReplicationData { size_t ref_block_pos; /* Access position of referenced buffer block, i.e. the next offset to send. */ sds replica_nodeid; /* Node id in cluster mode. */ + + replicaCompressionState *repl_compression; /* Primary-side compression state for this link, or NULL for plaintext. */ } ClientReplicationData; typedef struct ClientModuleData { @@ -1415,6 +1491,11 @@ typedef struct client { list *deferred_reply; /* List of reply objects to be sent to the client, typically after the client has been unblocked. */ unsigned long long deferred_reply_bytes; /* Total bytes of objects in the blocked client pending list.*/ + /* Throttling */ + struct throttler *throttler; /* Current throttler this client is queued in, or NULL */ + listNode *throttle_node; /* Node in throttler's client_queue */ + monotime throttle_start; /* When this client was queued for throttling */ + struct trendCalculator *cob_trend; /* Per-replica COB size trend (NULL if not replica) */ #ifdef LOG_REQ_RES clientReqResInfo reqres; #endif @@ -1493,7 +1574,7 @@ struct sharedObjectsStruct { *execaborterr, *noautherr, *noreplicaserr, *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink, *rpop, *lpop, *lpush, *zadd, *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax, *emptyscan, *multi, *exec, *left, *right, *hset, *hsetex, *hdel, *hpexpireat, *hpersist, *srem, - *xgroup, *xclaim, *script, *replconf, *eval, *cluster, *syncslots, *persist, *set, *pexpireat, *pexpire, *time, *pxat, *absttl, + *xgroup, *xclaim, *xdel, *xack, *script, *replconf, *eval, *cluster, *syncslots, *persist, *set, *pexpireat, *pexpire, *time, *pxat, *absttl, *retrycount, *force, *justid, *entriesread, *lastid, *ping, *setid, *keepttl, *load, *createconsumer, *getack, *special_asterisk, *special_equals, *default_username, *redacted, *ssubscribebulk, *sunsubscribebulk, *fields, *finish, *state, *success, *failed, *name, *message, @@ -1633,9 +1714,10 @@ typedef struct rdbSaveInfo { int repl_id_is_set; /* True if repl_id field is set. */ char repl_id[CONFIG_RUN_ID_SIZE + 1]; /* Replication ID. */ long long repl_offset; /* Replication offset. */ + bool loaded_compressed; /* True if rdbLoad() read a streaming-compressed file. */ } rdbSaveInfo; -#define RDB_SAVE_INFO_INIT {-1, 0, "0000000000000000000000000000000000000000", -1} +#define RDB_SAVE_INFO_INIT {-1, 0, "0000000000000000000000000000000000000000", -1, false} struct malloc_stats { size_t zmalloc_used; @@ -1670,6 +1752,9 @@ typedef struct serverTLSContextConfig { char *client_cert_file; /* Certificate to use as a client; if none, use cert_file */ char *client_key_file; /* Private key filename for client_cert_file */ char *client_key_file_pass; /* Optional password for client_key_file */ + char *alt_cert_file; /* Secondary server side cert file name */ + char *alt_key_file; /* Private key filename for alt_cert_file */ + char *alt_key_file_pass; /* Optional password for alt_key_file */ int client_auth_user; /* Field to be used for automatic TLS authentication based on client TLS certificate */ char *dh_params_file; char *ca_cert_file; @@ -1772,31 +1857,32 @@ struct valkeyServer { hashtable *commands; /* Command table */ hashtable *orig_commands; /* Command table before command renaming. */ sds command_response_cache[RESP_CACHE_INDEX_MAX]; /* Cached COMMAND response: [0]=RESP2, [1]=RESP3 */ - aeEventLoop *el; - _Atomic(AeIoState) io_poll_state; /* Indicates the state of the IO polling. */ - int io_ae_fired_events; /* Number of poll events received by the IO thread. */ - rax *errors; /* Errors table */ - volatile sig_atomic_t shutdown_asap; /* Shutdown ordered by signal handler. */ - mstime_t shutdown_mstime; /* Timestamp to limit graceful shutdown. */ - int last_sig_received; /* Indicates the last SIGNAL received, if any (e.g., SIGINT or SIGTERM). */ - int shutdown_flags; /* Flags passed to prepareForShutdown(). */ - int activerehashing; /* Incremental rehash in serverCron() */ - int active_defrag_cpu_percent; /* Current desired CPU percentage for active defrag */ - char *pidfile; /* PID file path */ - int arch_bits; /* 32 or 64 depending on sizeof(long) */ - int cronloops; /* Number of times the cron function run */ - char runid[CONFIG_RUN_ID_SIZE + 1]; /* ID always different at every exec. */ - int sentinel_mode; /* True if this instance is a Sentinel. */ - size_t initial_memory_usage; /* Bytes used after initialization. */ - int always_show_logo; /* Show logo even for non-stdout logging. */ - int in_exec; /* Are we inside EXEC? */ - int busy_module_yield_flags; /* Are we inside a busy module? (triggered by RM_Yield). see BUSY_MODULE_YIELD_ flags. */ - const char *busy_module_yield_reply; /* When non-null, we are inside RM_Yield. */ - char *ignore_warnings; /* Config: warnings that should be ignored. */ - int client_pause_in_transaction; /* Was a client pause executed during this Exec? */ - int server_del_keys_in_slot; /* The server is deleting the keys in the dirty slot. */ - int thp_enabled; /* If true, THP is enabled. */ - size_t page_size; /* The page size of OS. */ + aeEventLoop *el; /* Main event loop */ + _Atomic(AeIoState) io_poll_state; /* Indicates the state of the IO polling. */ + int io_ae_fired_events; /* Number of poll events received by the IO thread. */ + rax *errors; /* Errors table */ + volatile sig_atomic_t shutdown_asap; /* Shutdown ordered by signal handler. */ + mstime_t shutdown_mstime; /* Timestamp to limit graceful shutdown. */ + int last_sig_received; /* Indicates the last SIGNAL received, if any (e.g., SIGINT or SIGTERM). */ + int shutdown_flags; /* Flags passed to prepareForShutdown(). */ + int activerehashing; /* Incremental rehash in serverCron() */ + int active_defrag_cpu_percent; /* Current desired CPU percentage for active defrag */ + char *pidfile; /* PID file path */ + int arch_bits; /* 32 or 64 depending on sizeof(long) */ + int cronloops; /* Number of times the cron function run */ + char runid[CONFIG_RUN_ID_SIZE + 1]; /* ID always different at every exec. */ + int sentinel_mode; /* True if this instance is a Sentinel. */ + size_t initial_memory_usage; /* Bytes used after initialization. */ + int always_show_logo; /* Show logo even for non-stdout logging. */ + int in_exec; /* Are we inside EXEC? */ + int in_call; /* Nesting level within the call() function. */ + int busy_module_yield_flags; /* Are we inside a busy module? (triggered by RM_Yield). see BUSY_MODULE_YIELD_ flags. */ + const char *busy_module_yield_reply; /* When non-null, we are inside RM_Yield. */ + char *ignore_warnings; /* Config: warnings that should be ignored. */ + int client_pause_in_transaction; /* Was a client pause executed during this Exec? */ + int server_del_keys_in_slot; /* The server is deleting the keys in the dirty slot. */ + int thp_enabled; /* If true, THP is enabled. */ + size_t page_size; /* The page size of OS. */ /* Modules */ dict *moduleapi; /* Exported core APIs dictionary for modules. */ dict *sharedapi; /* Like moduleapi but containing the APIs that @@ -1808,6 +1894,7 @@ struct valkeyServer { pid_t child_pid; /* PID of current child */ int child_type; /* Type of current child */ _Atomic(int) module_gil_acquiring; /* Indicates whether the GIL is being acquiring by the main thread. */ + _Atomic(int) module_gil_acquired; /* Indicates if the main thread has the GIL acquired. */ /* Networking */ int port; /* TCP listening port */ int tls_port; /* TLS listening port */ @@ -1909,6 +1996,8 @@ struct valkeyServer { double stat_fork_rate; /* Fork rate in GB/sec. */ long long stat_total_forks; /* Total count of fork. */ long long stat_rejected_conn; /* Clients rejected because of maxclients */ + long long stat_rejected_priority_conn; /* Prioritized clients rejected because of maxclients */ + long long stat_num_active_priority_clients; /* Number of active prioritized clients */ long long stat_sync_full; /* Number of full resyncs with replicas. */ long long stat_sync_partial_ok; /* Number of accepted PSYNC requests. */ long long stat_sync_partial_err; /* Number of unaccepted PSYNC requests. */ @@ -1924,8 +2013,8 @@ struct valkeyServer { size_t stat_current_cow_peak; /* Peak size of copy on write bytes. */ size_t stat_current_cow_bytes; /* Copy on write bytes while child is active. */ monotime stat_current_cow_updated; /* Last update time of stat_current_cow_bytes */ - size_t stat_current_save_keys_processed; /* Processed keys while child is active. */ - size_t stat_current_save_keys_total; /* Number of keys when child started. */ + _Atomic(size_t) stat_current_save_keys_processed; /* Processed keys while save is active. */ + _Atomic(size_t) stat_current_save_keys_total; /* Number of keys when save started. */ size_t stat_rdb_cow_bytes; /* Copy on write bytes during RDB saving. */ size_t stat_aof_cow_bytes; /* Copy on write bytes during AOF rewrite. */ size_t stat_module_cow_bytes; /* Copy on write bytes during module fork. */ @@ -1933,6 +2022,10 @@ struct valkeyServer { double stat_module_progress; /* Module save progress. */ size_t stat_clients_type_memory[CLIENT_TYPE_COUNT]; /* Mem usage by type */ size_t stat_cluster_links_memory; /* Mem usage by cluster links */ + long long stat_cluster_threaded_reads_processed; /* Cluster reads completed by I/O threads */ + long long stat_cluster_threaded_writes_processed; /* Cluster writes completed by I/O threads */ + long long stat_cluster_threaded_accepts_processed; /* Cluster accepts completed by I/O threads */ + long long stat_cluster_io_main_thread_fallbacks; /* Cluster I/O ops handled on the main thread because dispatch failed */ long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to primary, etc.) error replies */ long long stat_total_error_replies; /* Total number of issued error replies ( command + rejected errors ) */ @@ -1968,6 +2061,9 @@ struct valkeyServer { * Note that commands in transactions are also counted. */ long long el_cmd_cnt_start; long long el_cmd_cnt_max; + /* Record the previous baseline and peak number of priority commands executed in one priority cycle. */ + long long priority_el_cmd_cnt_prev; + long long priority_el_cmd_cnt_max; /* The sum of active-expire, active-defrag and all other tasks done by cron and beforeSleep, but excluding read, write and AOF, which are counted by other sets of metrics. */ monotime el_cron_duration; @@ -2011,9 +2107,10 @@ struct valkeyServer { double *latency_tracking_info_percentiles; /* Extended latency tracking info output percentile list configuration. */ int latency_tracking_info_percentiles_len; unsigned int max_new_tls_conns_per_cycle; /* The maximum number of tls connections that will be accepted during each - invocation of the event loop. */ + invocation of the event loop. */ unsigned int max_new_conns_per_cycle; /* The maximum number of tcp connections that will be accepted during each - invocation of the event loop. */ + invocation of the event loop. */ + int priority_preemptive_poll_interval_us; /* Priority event loop preemptive poll interval in microseconds */ /* AOF persistence */ int aof_enabled; /* AOF configuration */ int aof_state; /* AOF_(ON|OFF|WAIT_REWRITE) */ @@ -2063,15 +2160,20 @@ struct valkeyServer { int saveparamslen; /* Number of saving points */ char *rdb_filename; /* Name of RDB file */ int rdb_compression; /* RDB compression mode */ + int repl_compression; /* Replication compression mode */ int rdb_checksum; /* Use RDB checksum? */ int rdb_del_sync_files; /* Remove RDB files used only for SYNC if the instance does not use persistence. */ + int forkless_infrastructure_enabled; /* Enable forkless options support. */ time_t lastsave; /* Unix time of last successful save */ time_t lastbgsave_try; /* Unix time of last attempted bgsave */ time_t rdb_save_time_last; /* Time used by last RDB save run. */ time_t rdb_save_time_start; /* Current RDB save start time. */ - int rdb_bgsave_scheduled; /* BGSAVE when possible if true. */ - int rdb_child_type; /* Type of save by active child. */ + rdbBgsaveType rdb_bgsave_scheduled; /* BGSAVE when possible if non-zero. */ + rdbWriteTarget rdb_write_target; /* Type of save by active child. */ + rdbBgsaveType cur_bgsave_type; /* Current bgsave type. */ + rdbBgsaveType lastbgsave_type; /* Last completed bgsave type. */ + compressionAlgo rdb_child_sync_algo; /* Streaming compression used by the active replication disk child. */ int lastbgsave_status; /* C_OK or C_ERR */ int stop_writes_on_bgsave_err; /* Don't allow writes if can't BGSAVE */ int rdb_pipe_read; /* RDB pipe used to transfer the rdb data */ @@ -2085,6 +2187,7 @@ struct valkeyServer { int rdb_key_save_delay; /* Delay in microseconds between keys while * writing aof or rdb. (for testings). negative * value means fractions of microseconds (on average). */ + int bgsave_default_method; /* Default bgsave method: RDB_BGSAVE_TYPE_FORK or RDB_BGSAVE_TYPE_FORKLESS */ int key_load_delay; /* Delay in microseconds between keys while * loading aof or rdb. (for testings). negative * value means fractions of microseconds (on average). */ @@ -2205,6 +2308,10 @@ struct valkeyServer { int repl_ignore_disk_write_error; /* Configures whether replicas panic when unable to * persist writes to AOF. */ + int repl_compression_advertised; /* Whether this replica advertised LZ4 in the current upstream + * handshake, or REPL_COMPRESSION_CAPA_UNKNOWN before REPLCONF capa. */ + struct streamPushReader *repl_stream_reader; /* Decoder for the upstream command stream, or NULL for plaintext. */ + /* The following two fields is where we store primary PSYNC replid/offset * while the PSYNC is in progress. At the end we'll copy the fields into * the server->primary client structure. */ @@ -2220,8 +2327,13 @@ struct valkeyServer { int get_ack_from_replicas; /* If true we send REPLCONF GETACK. */ /* Limits */ unsigned int maxclients; /* Max number of simultaneous clients */ + unsigned int maxclients_reserved; /* Client connection slots reserved for priority subnets */ + char *priority_subnets; /* Raw priority-subnets string config */ + anetSubnet *priority_subnets_array; /* Compiled priority subnets array */ + int priority_subnets_count; /* Count of compiled priority subnets */ unsigned long long maxmemory; /* Max number of memory bytes to use */ ssize_t maxmemory_clients; /* Memory limit for total client buffers */ + ssize_t maxmemory_scripts; /* Memory limit for cached EVAL scripts */ int maxmemory_policy; /* Policy for key eviction */ int maxmemory_samples; /* Precision of random sampling */ int maxmemory_eviction_tenacity; /* Aggressiveness of eviction processing */ @@ -2284,14 +2396,15 @@ struct valkeyServer { int cluster_message_gossip_perc; /* A configuration for setting the percentage of peer nodes to be gossiped in ping/pong messages. */ char *cluster_configfile; /* Cluster auto-generated config file name. */ int cluster_configfile_save_behavior; /* Cluster config file save behavior. */ + _Atomic(int) cluster_config_save_status; /* Status of cluster config save. */ + _Atomic(time_t) cluster_config_last_save_time; /* Unix time of last successful cluster config save. */ struct clusterState *cluster; /* State of the cluster */ int cluster_migration_barrier; /* Cluster replicas migration barrier. */ int cluster_allow_replica_migration; /* Automatic replica migrations to orphaned primaries and from empty primaries */ int cluster_replica_validity_factor; /* Replica max data age for failover. */ int cluster_require_full_coverage; /* If true, put the cluster down if there is at least an uncovered slot.*/ - int cluster_replica_no_failover; /* Prevent replica from starting a failover - if the primary is in failure state. */ + int cluster_replica_no_failover; /* Replica failover policy (NO/YES/IF_EMPTY). */ char *cluster_announce_ip; /* IP address to announce on cluster bus. */ char *cluster_announce_client_ipv4; /* IPv4 for clients, to announce on cluster bus. */ char *cluster_announce_client_ipv6; /* IPv6 for clients, to announce on cluster bus. */ @@ -2339,6 +2452,11 @@ struct valkeyServer { /* Debug config to expose intermediary slot migration states. */ uint32_t debug_slot_migration_prevent_pause : 1; uint32_t debug_slot_migration_prevent_failover : 1; + /* Debug config to override the failover delay (in ms). */ + int debug_cluster_failover_delay; + /* Debug config to force the next failover election to run in a specific + * epoch (testing only). -1 means don't override; consumed once. */ + long long debug_cluster_failover_epoch; sds cached_cluster_slot_info[CACHE_CONN_TYPE_MAX]; /* Index in array is a bitwise or of CACHE_CONN_TYPE_* */ /* Scripting */ mstime_t busy_reply_threshold; /* Script / module timeout in milliseconds */ @@ -2373,9 +2491,11 @@ struct valkeyServer { int tls_auth_clients; serverTLSContextConfig tls_ctx_config; long long tls_server_cert_expire_time; + long long tls_server_alt_cert_expire_time; long long tls_client_cert_expire_time; long long tls_ca_cert_expire_time; sds tls_server_cert_serial; + sds tls_server_alt_cert_serial; sds tls_client_cert_serial; sds tls_ca_cert_serial; serverUnixContextConfig unix_ctx_config; @@ -2405,6 +2525,11 @@ struct valkeyServer { char *locale_collate; char *debug_context; /* A free-form string that has no impact on server except being included in a crash report. */ int debug_force_tls_write_error; + /* Hot key detection parameters */ + int hotkeys_sampling_percentage; /* Percentage (1-100) of key accesses sampled for hot-key detection. */ + int hotkeys_top_k; /* Number of top keys to track (Space-Saving K); 0 disables detection. */ + int hotkeys_window_seconds; /* Length of the QPS accounting window in seconds. */ + struct spaceSavingManager *hotkeys_manager; }; #define MAX_KEYS_BUFFER 256 @@ -2675,6 +2800,9 @@ typedef int *commandDbIdArgs(robj **argv, int argc, int *count); * * CMD_ALL_DBS: The command works with all databases. * + * CMD_WRITE_FIRSTKEY_ONLY: The command must be CMD_WRITE. It only modifies the first key. + * Other keys are read-only. Example: SUNIONSTORE + * * The following additional flags are only used in order to put commands * in a specific ACL category. Commands can have multiple ACL categories. * See valkey.conf for the exact meaning of each. @@ -2722,6 +2850,9 @@ struct serverCommand { * Used for Cluster redirect (may be NULL) */ serverGetKeysProc *getkeys_proc; int num_args; /* Length of args array. */ + /* Nested prefetch: argv index of the field/member used for the inner hashtable + * lookup. 0 means disabled. Used by the prefetch system to find the lookup key. */ + int member_arg_index; /* Array of subcommands (may be NULL) */ struct serverCommand *subcommands; /* Array of arguments (may be NULL) */ @@ -2801,6 +2932,13 @@ typedef struct { unsigned char *lpi; /* listpack iterator */ } setTypeIterator; +/* Enum for the available hashTypeIterator's */ +typedef enum { + HASH_ITER_ALL = 0, /* Iterate all fields */ + HASH_ITER_VOLATILE, /* Iterate only fields which carry a ttl */ + HASH_ITER_PERSISTENT, /* Iterate only fields which do not carry a ttl */ +} hashIteratorType; + /* Structure to hold hash iteration abstraction. Note that iteration over * hashes involves both fields and values. Because it is possible that * not both are required, store pointers in the iterator to avoid @@ -2808,7 +2946,7 @@ typedef struct { typedef struct { robj *subject; int encoding; - bool volatile_items_iter; + hashIteratorType iterator_type; unsigned char *fptr, *vptr; hashtableIterator iter; @@ -2845,6 +2983,11 @@ typedef struct clusterScanCtx { *----------------------------------------------------------------------------*/ extern struct valkeyServer server; + +static inline bool onServerMainThread(void) { + return pthread_equal(server.main_thread_id, pthread_self()) != 0; +} + extern struct sharedObjectsStruct shared; extern dictType objectKeyPointerValueDictType; extern hashtableType objectHashtableType; @@ -2871,6 +3014,7 @@ extern list *modules; /* Command metadata */ void populateCommandLegacyRangeSpec(struct serverCommand *c); +void detectWriteFirstkeyOnlyCommand(struct serverCommand *c); /* Utils */ mstime_t commandTimeSnapshot(void); @@ -2927,6 +3071,9 @@ void dictVanillaFree(void *val); /* Write flags for various write errors and states */ #define WRITE_FLAGS_WRITE_ERROR (1 << 0) #define WRITE_FLAGS_IS_REPLICA (1 << 1) +/* Unlike a retryable socket write error, a compression error is fatal. The IO + * thread reports it here for the main thread to disconnect the replica. */ +#define WRITE_FLAGS_COMPRESSION_ERROR (1 << 2) client *createClient(connection *conn); int freeClient(client *c); @@ -2951,6 +3098,8 @@ void setDeferredAttributeLen(client *c, void *node, long length); void setDeferredPushLen(client *c, void *node, long length); int processInputBuffer(client *c); void acceptCommonHandler(connection *conn, struct ClientFlags flags, char *ip); +int validatePrioritySubnets(const char *subnets_str, const char **err); +int updatePrioritySubnets(const char *subnets_str); void readQueryFromClient(connection *conn); int prepareClientToWrite(client *c); writePreparedClient *prepareClientForFutureWrites(client *c); @@ -3072,7 +3221,7 @@ void releaseReplyReferences(client *c); void resetLastWrittenBuf(client *c); int clientConnPostponeMask(client *c); -int parseExtendedCommandArgumentsOrReply(client *c, int command_type, int start_idx, int max_args, int *flags, int *unit, int *expire_idx, robj **expire, robj **compare_val); +int parseExtendedCommandArgumentsOrReply(client *c, int command_type, int start_idx, int max_args, int *flags, int *unit, int *expire_idx, robj **expire, robj **compare_val, robj **incrby_val); /* logreqres.c - logging of requests and responses */ void reqresReset(client *c, int free_buf); @@ -3097,6 +3246,7 @@ void trackingRememberKeys(client *tracking, client *executing); void trackingInvalidateKey(client *c, robj *keyobj, int bcast); void trackingScheduleKeyInvalidation(uint64_t client_id, robj *keyobj); void trackingHandlePendingKeyInvalidations(void); +bool trackingHasPendingKeyInvalidations(void); void trackingInvalidateKeysOnFlush(int async); void freeTrackingRadixTree(rax *rt); void freeTrackingRadixTreeAsync(rax *rt); @@ -3147,6 +3297,7 @@ void touchAllWatchedKeysInDb(serverDb *emptied, serverDb *replaced_with); void discardTransaction(client *c); void flagTransaction(client *c); void execCommandAbort(client *c, sds error); +int execGetKeys(struct serverCommand *cmd, robj **argv, int argc, getKeysResult *result); /* Object implementation */ void decrRefCount(robj *o); @@ -3226,6 +3377,11 @@ void objectSetEncoding(robj *o, int encoding); unsigned int objectGetRefcount(const robj *o); unsigned int objectGetLRU(const robj *o); void objectSetLRU(robj *o, unsigned int lru); +/* Object metadata management */ +void objectSetMetadataSize(size_t size); +size_t objectGetMetadataSize(const robj *o); +void *objectGetMetadata(const robj *o); +void objectCopyMetadata(robj *dst, const robj *src); /* Synchronous I/O with timeout */ ssize_t syncWrite(int fd, char *ptr, ssize_t size, long long timeout); @@ -3280,11 +3436,15 @@ const char *getFailoverStateString(void); sds getReplicaPortString(void); int sendCurrentOffsetToReplica(client *replica); int replicaRdbVersion(client *replica); +/* Full-sync compression policy: select the codec and gate replica eligibility on capability. */ +compressionAlgo replSelectFullSyncCompression(int replica_capa, bool socket_target); void addRdbReplicaToPsyncWait(client *replica); void initClientReplicationData(client *c); void freeClientReplicationData(client *c); +ssize_t replDecodeToQueryBuf(client *primary, const void *wire_buf, size_t wire_len, size_t output_budget); +bool replStreamHasPendingDecode(void); void replicaReceiveRDBFromPrimaryToDisk(connection *conn, int is_dual_channel); -sds replicationSendAuth(connection *conn); +sds replicationSendAuth(connection *conn, const char *user, size_t user_len, const char *pass, size_t pass_len); sds receiveSynchronousResponse(connection *conn); ConnectionType *connTypeOfReplication(void); robj *generateSelectCommand(int dictid); @@ -3341,11 +3501,15 @@ void receiveChildInfo(void); /* Fork helpers */ int serverFork(int purpose); int hasActiveChildProcess(void); +int isSaveInProgress(void); +int hasActiveSaveOrChild(void); +int isForkBgsaveInProgress(void); void resetChildState(void); int isMutuallyExclusiveChildType(int type); /* acl.c -- Authentication related prototypes. */ extern rax *Users; +extern rax *Roles; extern user *DefaultUser; void ACLInit(void); int ACLModuleHasCommandRules(const struct ValkeyModule *module, sds *rule_out); @@ -3394,7 +3558,7 @@ uint64_t ACLGetCommandCategoryFlagByName(const char *name); int ACLAddCommandCategory(const char *name, uint64_t flag); void ACLCleanupCategoriesOnFailure(size_t num_acl_categories_added); int ACLAppendUserForLoading(sds *argv, int argc, int *argc_err); -const char *ACLSetUserStringError(void); +const char *ACLSetStringError(void); robj *ACLDescribeUser(user *u); void ACLLoadUsersAtStartup(void); void addReplyCommandCategories(client *c, struct serverCommand *cmd); @@ -3405,6 +3569,8 @@ sds getAclErrorMessage(int acl_res, user *user, struct serverCommand *cmd, sds e void ACLUpdateDefaultUserPassword(sds password); sds genValkeyInfoStringACLStats(sds info); void ACLRecomputeCommandBitsFromCommandRulesAllUsers(void); +user *ACLGetRoleByName(const char *name, size_t namelen); +int ACLAppendRoleForLoading(sds *argv, int argc, int *argc_err); /* Sorted sets data type */ @@ -3546,6 +3712,8 @@ void resetServerStats(void); void monitorActiveDefrag(void); void defragWhileBlocked(void); const char *evictPolicyToString(void); +size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid); +robj *createStringObjectWithKeyAndExpire(const char *ptr, size_t len, const_sds key, long long expire); struct serverMemOverhead *getMemoryOverheadData(void); void freeMemoryOverheadData(struct serverMemOverhead *mh); void checkChildrenDone(void); @@ -3603,8 +3771,12 @@ robj *setTypeDup(robj *o); #define HASH_SET_COPY 0 -void hashTypeFreeVolatileSet(robj *o); /* needed only for freeHashObject */ -void hashTypeTrackEntry(robj *o, entry *entry); /* needed only for rdbLoadObject */ +long long hashTypeVolatileCount(robj *o); /* total volatile fields, incl. expired-unreaped */ +long long hashTypeListpackGetExpiry(unsigned char *zl, unsigned char *vptr); /* expiry of the pair whose value entry is vptr, or EXPIRY_NONE */ +bool hashTypeListpackFieldIsValid(long long expiry); /* listpack mirror of validateEntry: is a field with this expiry visible now */ +void hashTypeFreeVolatileSet(robj *o); /* needed only for freeHashObject */ +void hashTypeTrackEntry(robj *o, entry *entry); /* needed only for rdbLoadObject */ +void hashTypeUpdateVolatileCount(robj *o, long delta); /* exported only for rdbLoadObject's HASH_2-to-listpack path */ size_t hashTypeScanDefrag(robj *ob, size_t cursor, void *(*defragAlloc)(void *)); size_t hashTypeDeleteExpiredFields(robj *o, mstime_t now, unsigned long max_fields, robj **out_fields); @@ -3615,6 +3787,7 @@ bool hashTypeDelete(robj *o, sds key); unsigned long hashTypeLength(const robj *o); void hashTypeInitIterator(robj *subject, hashTypeIterator *hi); void hashTypeInitVolatileIterator(robj *subject, hashTypeIterator *hi); +void hashTypeInitPersistentIterator(robj *subject, hashTypeIterator *hi); void hashTypeResetIterator(hashTypeIterator *hi); int hashTypeNext(hashTypeIterator *hi); void hashTypeCurrentFromListpack(hashTypeIterator *hi, @@ -3624,6 +3797,7 @@ void hashTypeCurrentFromListpack(hashTypeIterator *hi, long long *vll); char *hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what, size_t *len); sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what); +long long hashTypeCurrentExpiry(robj *o, hashTypeIterator *hi); robj *hashTypeLookupWriteOrCreate(client *c, robj *key); robj *hashTypeGetValueObject(robj *o, sds field); int hashTypeSet(robj *o, sds field, sds value, mstime_t expiry, int flags, bool *expired_overwritten); @@ -3774,13 +3948,14 @@ robj *objectCommandLookup(client *c, robj *key); robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply); int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle_secs); #define LOOKUP_NONE 0 -#define LOOKUP_NOTOUCH (1 << 0) /* Don't update LRU. */ -#define LOOKUP_NONOTIFY (1 << 1) /* Don't trigger keyspace event on key misses. */ -#define LOOKUP_NOSTATS (1 << 2) /* Don't update keyspace hits/misses counters. */ -#define LOOKUP_WRITE (1 << 3) /* Delete expired keys even in replicas. */ -#define LOOKUP_NOEXPIRE (1 << 4) /* Avoid deleting lazy expired keys. */ +#define LOOKUP_NOTOUCH (1 << 0) /* Don't update LRU. */ +#define LOOKUP_NONOTIFY (1 << 1) /* Don't trigger keyspace event on key misses. */ +#define LOOKUP_NOSTATS (1 << 2) /* Don't update keyspace hits/misses counters. */ +#define LOOKUP_WRITE (1 << 3) /* Delete expired keys even in replicas. */ +#define LOOKUP_NOEXPIRE (1 << 4) /* Avoid deleting lazy expired keys. */ +#define LOOKUP_NOHOTKEYS (1 << 5) /* Don't feed hot-key detection (introspection). */ #define LOOKUP_NOEFFECTS \ - (LOOKUP_NONOTIFY | LOOKUP_NOSTATS | LOOKUP_NOTOUCH | LOOKUP_NOEXPIRE) /* Avoid any effects from fetching the key */ + (LOOKUP_NONOTIFY | LOOKUP_NOSTATS | LOOKUP_NOTOUCH | LOOKUP_NOEXPIRE | LOOKUP_NOHOTKEYS) /* Avoid any effects from fetching the key */ void dbAdd(serverDb *db, robj *key, robj **valref); int dbAddRDBLoad(serverDb *db, sds key, robj **valref); @@ -3805,6 +3980,8 @@ typedef int(emptyDataHashtableFilter)(int didx); long long emptyData(int dbnum, int flags, void(callback)(hashtable *)); long long emptyDbStructure(serverDb **dbarray, int dbnum, int async, void(callback)(hashtable *)); void resetDbExpiryState(serverDb *db); +int parseFlushCommandFlags(client *c, int *flags); +int parseFlushCommandFlagsOrReply(client *c, int *flags); void flushAllDataAndResetRDB(int flags); long long dbTotalServerKeyCount(void); serverDb *initTempDb(int id); @@ -3888,8 +4065,10 @@ void freeEvalScriptsAsync(dict *scripts, list *scripts_lru_list, list *engine_ca void freeFunctionsAsync(functionsLibCtx *lib_ctx, list *engine_callbacks); void sha1hex(char *digest, char *script, size_t len); unsigned long evalMemory(void); -dict *evalScriptsDict(void); -unsigned long evalScriptsMemory(void); +dict *evalCtxScriptsDict(void); +unsigned long scriptsMemoryOverhead(void); +unsigned long evalScriptsMemoryOverhead(void); +void startScriptsEvictionTimeProc(void); uint64_t evalGetCommandFlags(client *c, uint64_t orig_flags); uint64_t fcallGetCommandFlags(client *c, uint64_t orig_flags); int isInsideYieldingLongCommand(void); @@ -3917,6 +4096,9 @@ void signalKeyAsReady(serverDb *db, robj *key, int type); void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, int unblock_on_nokey); void blockClientShutdown(client *c); void blockPostponeClient(client *c); +void blockClientInUseOnKeys(client *c, int num_keys, robj *keys[]); +void unblockClientsInUseOnKey(robj *key); +void unblockClientsInUseOnAllKeys(void); void blockClientForReplicaAck(client *c, mstime_t timeout, long long offset, int numreplicas, int numlocal); void replicationRequestAckFromReplicas(void); void signalDeletedKeyAsReady(serverDb *db, robj *key, int type); @@ -3947,11 +4129,13 @@ void startEvictionTimeProc(void); uint8_t *getConfigurableHashSeed(void); uint64_t dictSdsHash(const void *key); uint64_t dictSdsCaseHash(const void *key); +uint64_t dictObjHash(const void *key); uint64_t dictCStrHash(const void *key); uint64_t dictCStrCaseHash(const void *key); uint64_t dictEncObjHash(const void *key); int dictSdsKeyCompare(const void *key1, const void *key2); int dictSdsKeyCaseCompare(const void *key1, const void *key2); +int dictObjKeyCompare(const void *key1, const void *key2); int dictCStrKeyCompare(const void *key1, const void *key2); int dictCStrKeyCaseCompare(const void *key1, const void *key2); int dictEncObjKeyCompare(const void *key1, const void *key2); @@ -4003,6 +4187,7 @@ void decrCommand(client *c); void incrbyCommand(client *c); void decrbyCommand(client *c); void incrbyfloatCommand(client *c); +void increxCommand(client *c); void selectCommand(client *c); void swapdbCommand(client *c); void randomkeyCommand(client *c); @@ -4257,6 +4442,8 @@ void xclaimCommand(client *c); void xautoclaimCommand(client *c); void xinfoCommand(client *c); void xdelCommand(client *c); +void xackdelCommand(client *c); +void xdelexCommand(client *c); void xtrimCommand(client *c); void lolwutCommand(client *c); void aclCommand(client *c); @@ -4264,6 +4451,9 @@ void lcsCommand(client *c); void quitCommand(client *c); void resetCommand(client *c); void failoverCommand(client *c); +void hotkeysGetCommand(client *c); +void hotkeysResetCommand(client *c); +void hotkeysHelpCommand(client *c); /* Helper functions for getting database id args from argv, argc */ int *selectDbIdArgs(robj **argv, int argc, int *count); diff --git a/src/socket.c b/src/socket.c index c9f9cae04..a1b7f3a0c 100644 --- a/src/socket.c +++ b/src/socket.c @@ -30,6 +30,10 @@ #include "server.h" #include "connhelpers.h" #include "io_threads.h" +#include +#ifdef __APPLE__ +#include +#endif /* The connections module provides a lean abstraction of network connections * to avoid direct socket and async event management across the server code base. @@ -110,18 +114,23 @@ static int connSocketConnect(connection *conn, ConnectionCallbackFunc connect_handler) { int fd = anetTcpNonBlockBestEffortBindConnect(NULL, addr, port, src_addr, multipath); if (fd == -1) { - conn->state = CONN_STATE_ERROR; - conn->last_errno = errno; - return C_ERR; + goto error; } conn->fd = fd; conn->state = CONN_STATE_CONNECTING; conn->conn_handler = connect_handler; - aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE, conn->type->ae_handler, conn); + int priority_flag = connGetAEPriorityFlag(conn); + if (aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | priority_flag, conn->type->ae_handler, conn) == AE_ERR) + goto error; return C_OK; + +error: + conn->state = CONN_STATE_ERROR; + conn->last_errno = errno; + return C_ERR; } /* ------ Pure socket connections ------- */ @@ -156,8 +165,11 @@ static void connSocketClose(connection *conn) { } static int connSocketWrite(connection *conn, const void *data, size_t data_len) { - /* Assert the main thread is not writing to a connection that is currently offloaded. */ - debugServerAssert(!(conn->flags & CONN_FLAG_ALLOW_ACCEPT_OFFLOAD) || !inMainThread() || + /* Assert the main thread is not writing to a connection that is currently offloaded. + * Only applies to client-owned connections; cluster-link-owned connections use + * separate dispatch functions and do not carry client io_write_state. */ + debugServerAssert(connGetOwnerKind(conn) != CONN_OWNER_CLIENT || + !(conn->flags & CONN_FLAG_ALLOW_ACCEPT_OFFLOAD) || !inMainThread() || ((client *)connGetPrivateData(conn))->io_write_state != CLIENT_PENDING_IO); int ret = write(conn->fd, data, data_len); @@ -188,8 +200,11 @@ static int connSocketWritev(connection *conn, const struct iovec *iov, int iovcn } static int connSocketRead(connection *conn, void *buf, size_t buf_len) { - /* Assert the main thread is not reading from a connection that is currently offloaded. */ - debugServerAssert(!(conn->flags & CONN_FLAG_ALLOW_ACCEPT_OFFLOAD) || !inMainThread() || + /* Assert the main thread is not reading from a connection that is currently offloaded. + * Only applies to client-owned connections; cluster-link-owned connections use + * separate dispatch functions and do not carry client io_read_state. */ + debugServerAssert(connGetOwnerKind(conn) != CONN_OWNER_CLIENT || + !(conn->flags & CONN_FLAG_ALLOW_ACCEPT_OFFLOAD) || !inMainThread() || ((client *)connGetPrivateData(conn))->io_read_state != CLIENT_PENDING_IO); @@ -235,9 +250,10 @@ static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc fu conn->flags |= CONN_FLAG_WRITE_BARRIER; else conn->flags &= ~CONN_FLAG_WRITE_BARRIER; + int priority_flag = connGetAEPriorityFlag(conn); if (!conn->write_handler) aeDeleteFileEvent(server.el, conn->fd, AE_WRITABLE); - else if (aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE, conn->type->ae_handler, conn) == AE_ERR) + else if (aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | priority_flag, conn->type->ae_handler, conn) == AE_ERR) return C_ERR; return C_OK; } @@ -249,9 +265,10 @@ static int connSocketSetReadHandler(connection *conn, ConnectionCallbackFunc fun if (func == conn->read_handler) return C_OK; conn->read_handler = func; + int priority_flag = connGetAEPriorityFlag(conn); if (!conn->read_handler) aeDeleteFileEvent(server.el, conn->fd, AE_READABLE); - else if (aeCreateFileEvent(server.el, conn->fd, AE_READABLE, conn->type->ae_handler, conn) == AE_ERR) + else if (aeCreateFileEvent(server.el, conn->fd, AE_READABLE | priority_flag, conn->type->ae_handler, conn) == AE_ERR) return C_ERR; return C_OK; } @@ -274,7 +291,7 @@ static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientD conn->state = CONN_STATE_CONNECTED; } - if (!conn->write_handler) aeDeleteFileEvent(server.el, conn->fd, AE_WRITABLE); + if (!conn->write_handler) aeDeleteFileEvent(el, conn->fd, AE_WRITABLE); if (!callHandler(conn, conn->conn_handler)) return; conn->conn_handler = NULL; @@ -418,6 +435,28 @@ static int connSocketGetType(void) { return CONN_TYPE_SOCKET; } +int connTcpSocketIsClosing(connection *conn) { +#if defined(__linux__) + struct tcp_info info; + socklen_t infolen = sizeof(info); + if (getsockopt(conn->fd, IPPROTO_TCP, TCP_INFO, &info, &infolen) != 0 || + infolen < offsetof(struct tcp_info, tcpi_state) + sizeof(info.tcpi_state)) + return false; /* Cannot retrieve TCP info, or the state field was not returned. */ + return (info.tcpi_state == TCP_CLOSE_WAIT || info.tcpi_state == TCP_CLOSE); +#elif defined(__APPLE__) + struct tcp_connection_info info; + socklen_t infolen = sizeof(info); + if (getsockopt(conn->fd, IPPROTO_TCP, TCP_CONNECTION_INFO, &info, &infolen) != 0 || + infolen < offsetof(struct tcp_connection_info, tcpi_state) + sizeof(info.tcpi_state)) + return false; /* Cannot retrieve TCP info, or the state field was not returned. */ + return (info.tcpi_state == TCPS_CLOSE_WAIT || info.tcpi_state == TCPS_CLOSED); +#else + /* Unsupported platform: zombie connection detection is not available. */ + UNUSED(conn); + return false; +#endif +} + static ConnectionType CT_Socket = { /* connection type */ .get_type = connSocketGetType, @@ -465,6 +504,7 @@ static ConnectionType CT_Socket = { /* Miscellaneous */ .connIntegrityChecked = NULL, + .is_closing = connTcpSocketIsClosing, }; int connBlock(connection *conn) { diff --git a/src/space_saving.c b/src/space_saving.c new file mode 100644 index 000000000..a032e7d8a --- /dev/null +++ b/src/space_saving.c @@ -0,0 +1,341 @@ +/* + * Copyright Valkey Contributors. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * Space-Saving top-K over fixed time windows. See space_saving.h for the + * algorithm, the ownership model, and usage. + * + * Storage is a flat, unordered array of `capacity` slots. Membership lookup and + * smallest-count selection are done in a single linear scan; K is expected to + * be small (tens), so the scan is a handful of cache lines and beats the + * bookkeeping of a heap/linked structure at this size. + */ + +#include "zmalloc.h" /* zmalloc / zcalloc / zrealloc / zfree */ +#include "space_saving.h" + +/* =========================================================================== + * spaceSavingWindow — a single summary + * ==========================================================================*/ + +typedef struct { + sds key; /* Owned copy of the key name, or NULL if the slot is free */ + int dbid; /* Database the key was accessed in */ + uint32_t hash; /* Cached hash of (key, dbid) for fast reject before compare */ + uint64_t count; /* Estimated count (upper bound on the true count) */ + uint64_t error; /* Maximum overestimate vs. the true count */ +} spaceSavingSlot; + +/* A single Space-Saving summary. Internal to this file; callers use the + * spaceSavingManager (frozen-window) API in space_saving.h. */ +typedef struct spaceSavingWindow spaceSavingWindow; + +struct spaceSavingWindow { + spaceSavingSlot *slots; /* capacity-sized array */ + int capacity; /* K */ + int size; /* number of occupied slots (0..capacity) */ + uint64_t total; /* total observations recorded in this window (N) */ + uint64_t start_us; /* when this window started accepting observations */ + uint64_t end_us; /* when it was frozen (0 while still live) */ + int sampling_percentage; /* Sampling % these counts were gathered under (0 if unset) */ +}; + +/* FNV-1a over the key name, folded with the db id so identical names in + * different databases fast-reject without a full compare. */ +static uint32_t spaceSavingHashItem(const sds key, int dbid) { + uint32_t h = 2166136261u; + size_t klen = sdslen(key); + for (size_t i = 0; i < klen; i++) { + h ^= (unsigned char)key[i]; + h *= 16777619u; + } + return h ^ (uint32_t)dbid; +} + +static spaceSavingWindow *spaceSavingWindowCreate(int k) { + if (k <= 0) return NULL; + spaceSavingWindow *w = zcalloc(sizeof(*w)); + w->slots = zcalloc((size_t)k * sizeof(spaceSavingSlot)); + w->capacity = k; + w->size = 0; + w->total = 0; + w->start_us = 0; + w->end_us = 0; + w->sampling_percentage = 0; + return w; +} + +static void spaceSavingWindowReset(spaceSavingWindow *w) { + if (!w) return; + for (int i = 0; i < w->size; i++) { + sdsfree(w->slots[i].key); + w->slots[i].key = NULL; + } + w->size = 0; + w->total = 0; + w->start_us = 0; + w->end_us = 0; + w->sampling_percentage = 0; +} + +static void spaceSavingWindowRelease(spaceSavingWindow *w) { + if (!w) return; + spaceSavingWindowReset(w); + zfree(w->slots); + zfree(w); +} + +static void recordSpaceSavingWindowSample(spaceSavingWindow *w, sds key, int dbid) { + if (!w || !key) return; + w->total++; + uint32_t h = spaceSavingHashItem(key, dbid); + + /* Single pass: look for an existing slot (fast-rejecting on the cached hash + * before the full compare) while tracking the smallest-count slot for the + * eviction path. */ + int min_idx = 0; + uint64_t min_count = UINT64_MAX; + for (int i = 0; i < w->size; i++) { + spaceSavingSlot *e = &w->slots[i]; + if (e->hash == h && e->dbid == dbid && sdscmp(e->key, key) == 0) { + e->count += 1; + return; + } + if (e->count < min_count) { + min_count = e->count; + min_idx = i; + } + } + + /* Room available: insert with count = 1, error = 0. */ + if (w->size < w->capacity) { + spaceSavingSlot *e = &w->slots[w->size++]; + e->key = sdsdup(key); + e->dbid = dbid; + e->hash = h; + e->count = 1; + e->error = 0; + return; + } + + /* Full: evict the smallest-count slot. The new item inherits count = the + * evicted count + 1; error records the maximum possible overestimate. */ + spaceSavingSlot *e = &w->slots[min_idx]; + sdsfree(e->key); + e->key = sdsdup(key); + e->dbid = dbid; + e->hash = h; + e->count = min_count + 1; + e->error = min_count; +} + +static void spaceSavingWindowRemoveIf(spaceSavingWindow *w, int (*pred)(sds key, int dbid, void *arg), void *arg) { + if (!w || !pred) return; + int out = 0; + for (int i = 0; i < w->size; i++) { + if (pred(w->slots[i].key, w->slots[i].dbid, arg)) { + sdsfree(w->slots[i].key); + continue; /* drop: do not advance the write cursor */ + } + if (out != i) w->slots[out] = w->slots[i]; + out++; + } + w->size = out; +} + +/* Resize a window to `new_k` capacity, keeping the highest-count entries. + * Grow: preserve all entries; shrink: keep the top `new_k` by count. */ +static void spaceSavingWindowResize(spaceSavingWindow *w, int new_k) { + if (!w || new_k <= 0 || new_k == w->capacity) return; + if (new_k < w->size) { + /* Selection of the top new_k by count (K is small, O(K^2) is fine). */ + for (int i = 0; i < new_k; i++) { + int max_idx = i; + for (int j = i + 1; j < w->size; j++) + if (w->slots[j].count > w->slots[max_idx].count) max_idx = j; + if (max_idx != i) { + spaceSavingSlot t = w->slots[i]; + w->slots[i] = w->slots[max_idx]; + w->slots[max_idx] = t; + } + } + for (int i = new_k; i < w->size; i++) sdsfree(w->slots[i].key); + w->size = new_k; + } + w->slots = zrealloc(w->slots, (size_t)new_k * sizeof(spaceSavingSlot)); + for (int i = (w->capacity < new_k ? w->capacity : new_k); i < new_k; i++) { + w->slots[i].key = NULL; + w->slots[i].dbid = 0; + w->slots[i].hash = 0; + w->slots[i].count = 0; + w->slots[i].error = 0; + } + w->capacity = new_k; +} + +/* =========================================================================== + * spaceSavingManager — frozen-window top-K over fixed time windows. + * See space_saving.h for the model and usage. + * ==========================================================================*/ + +struct spaceSavingManager { + spaceSavingWindow *live; /* Current (open) window */ + spaceSavingWindow *frozen; /* Last completed window (read path) */ + uint64_t live_window_length_us; /* Configured length of a window, in microseconds */ +}; + +spaceSavingManager *spaceSavingManagerCreate(int k, uint64_t window_us, uint64_t now_us) { + spaceSavingManager *m = zcalloc(sizeof(*m)); + m->live = spaceSavingWindowCreate(k); + m->frozen = spaceSavingWindowCreate(k); + m->live_window_length_us = window_us; + if (!m->live || !m->frozen) { + spaceSavingManagerRelease(m); + return NULL; + } + m->live->start_us = now_us; + return m; +} + +void spaceSavingManagerRelease(spaceSavingManager *m) { + if (!m) return; + spaceSavingWindowRelease(m->live); + spaceSavingWindowRelease(m->frozen); + zfree(m); +} + +void spaceSavingManagerReset(spaceSavingManager *m, uint64_t now_us) { + if (!m) return; + /* The sampling percentage is configuration, not measurement: it describes + * how the NEXT observations will be gathered, so it outlives the data being + * dropped. Preserving it here is what makes this safe to call from the + * rotate path — clearing it would leave the live window at 0 and every + * subsequent estimate would come back as zero until a config change. */ + int live_pct = m->live->sampling_percentage; + spaceSavingWindowReset(m->live); + spaceSavingWindowReset(m->frozen); + m->live->sampling_percentage = live_pct; + m->live->start_us = now_us; +} + +/* Freeze the live window: the previous snapshot is discarded, the live window + * becomes the new frozen snapshot, and a fresh empty live window starts. + * A pointer swap + reset, so it is O(K) with no reallocation and transfers key + * ownership without copying. + * + * `now_us` is stamped as the outgoing window's real end and the incoming + * window's real start. Rotation is driven by a timer, so a window is closed at + * or after its nominal boundary, never before: recording the actual interval + * lets the reader divide by the traffic's real duration rather than the + * configured length, which would otherwise over-report by the rotation lag. */ +static void spaceSavingManagerFreeze(spaceSavingManager *m, uint64_t now_us) { + m->live->end_us = now_us; + spaceSavingWindow *tmp = m->frozen; + m->frozen = m->live; + m->live = tmp; + spaceSavingWindowReset(m->live); + m->live->start_us = now_us; + /* The frozen window (old live) carries the sampling percentage it ran under + * — it travels with the window on the swap. Carry it forward into the new + * live window so subsequent samples keep the current setting until the + * caller records a new one. */ + m->live->sampling_percentage = m->frozen->sampling_percentage; +} + +/* Close the live window once its configured length has fully elapsed. Rotation + * is timer-driven, so a window is closed at or after its nominal boundary and + * the snapshot carries the real interval it accumulated over. + * + * If the timer ran so late that the live window covers more than twice the + * configured length, its counts span too coarse an interval to publish as "the + * last window", so they are dropped rather than reported. That does discard + * whatever traffic arrived during the stall, which is the accepted cost of + * bounding how stale a report can be: a frozen window always spans + * [length, 2 * length), so HOTKEYS GET can never quietly return a long-run + * average under a one-window label. + * + * Boundaries are measured from when the live window really started, not from a + * nominal grid. A window is therefore never SHORTER than the configured length + * (it is length + however late this call ran), and a late rotation cannot + * shorten the following window — at the cost of the boundaries drifting against + * the wall clock, so a long run sees slightly fewer windows than + * elapsed / length. For a sampled estimator that reports its own measured span, + * never-shorter-than-configured is the more useful guarantee: it keeps N per + * window from collapsing after a hiccup. */ +void spaceSavingManagerRotate(spaceSavingManager *m, uint64_t now_us) { + if (!m || m->live_window_length_us == 0) return; + uint64_t len = m->live_window_length_us; + uint64_t start_us = m->live->start_us; + if (now_us < start_us + len) return; /* current window still open */ + if (now_us >= start_us + 2 * len) + spaceSavingManagerReset(m, now_us); /* too coarse to report: drop it */ + else + spaceSavingManagerFreeze(m, now_us); +} + +void recordSpaceSavingManagerSample(spaceSavingManager *m, sds key, int dbid) { + if (!m) return; + recordSpaceSavingWindowSample(m->live, key, dbid); +} + +int spaceSavingManagerCount(spaceSavingManager *m) { + return m ? m->frozen->size : 0; +} + +void spaceSavingManagerAt(spaceSavingManager *m, int i, sds *key, int *dbid, uint64_t *count, uint64_t *error) { + if (!m || i < 0 || i >= m->frozen->size) return; + spaceSavingSlot *e = &m->frozen->slots[i]; + if (key) *key = e->key; + if (dbid) *dbid = e->dbid; + if (count) *count = e->count; + if (error) *error = e->error; +} + +void spaceSavingManagerRemoveIf(spaceSavingManager *m, int (*pred)(sds key, int dbid, void *arg), void *arg) { + if (!m) return; + spaceSavingWindowRemoveIf(m->live, pred, arg); + spaceSavingWindowRemoveIf(m->frozen, pred, arg); +} + +uint64_t spaceSavingManagerFrozenTotal(spaceSavingManager *m) { + return m ? m->frozen->total : 0; +} + +void spaceSavingManagerSetLiveSamplingPercentage(spaceSavingManager *m, int sampling_percentage) { + if (m) m->live->sampling_percentage = sampling_percentage; +} + +int spaceSavingManagerFrozenSamplingPercentage(spaceSavingManager *m) { + return m ? m->frozen->sampling_percentage : 0; +} + +/* Real time the last completed window spent accumulating, in microseconds. This + * is the correct denominator for a rate: it is the window's actual span, which + * is its configured length plus however late the rotation ran. 0 when there is + * no completed window yet. */ +uint64_t spaceSavingManagerFrozenDurationUs(spaceSavingManager *m) { + if (!m || m->frozen->end_us <= m->frozen->start_us) return 0; + return m->frozen->end_us - m->frozen->start_us; +} + +/* Reconfigure the manager: reset only the live window (its counts were gathered + * under the previous config and are no longer comparable) and start a fresh + * window at `now_us` with the new capacity and window length, while KEEPING the + * last completed (frozen) window and its sampling config intact so an in-flight + * query still sees it. Use this instead of releasing/recreating on a config + * change. */ +void spaceSavingManagerReconfigure(spaceSavingManager *m, int new_k, uint64_t new_window_us, uint64_t now_us) { + if (!m) return; + int live_pct = m->live->sampling_percentage; /* configuration outlives the data */ + spaceSavingWindowReset(m->live); + m->live->sampling_percentage = live_pct; + if (new_k > 0 && new_k != m->live->capacity) { + spaceSavingWindowResize(m->live, new_k); + spaceSavingWindowResize(m->frozen, new_k); + } + m->live_window_length_us = new_window_us; + m->live->start_us = now_us; +} diff --git a/src/space_saving.h b/src/space_saving.h new file mode 100644 index 000000000..908d47642 --- /dev/null +++ b/src/space_saving.h @@ -0,0 +1,106 @@ +#ifndef SPACE_SAVING_H +#define SPACE_SAVING_H + +#include +#include + +#include "sds.h" + +/* + * space_saving — Space-Saving top-K frequency tracking over fixed time windows. + * + * Space-Saving (Metwally, Agrawal & El Abbadi, 2005) approximates the K most + * frequent items in a stream using O(K) memory. It keeps K (item, count, error) + * slots and, for each observation: + * 1. if the item is already tracked, increment its count; + * 2. else if a slot is free, insert it with count = 1, error = 0; + * 3. else evict the smallest-count slot and reuse it: the new item takes + * count = min_count + 1, error = min_count. + * Per-window guarantees: a tracked item's true count is in [count - error, + * count], and any item whose true frequency exceeds N/K (N = observations in + * the window) is guaranteed tracked. + * + * The tracked item is a (key name, database id) pair — the hot-key identity. + * Keeping it concrete keeps the hot path free of indirect calls: comparison, + * hashing and copying are all inlined here. The stored key is an owned `sds` + * copy, duplicated ONLY after the item is confirmed absent and a slot is + * committed to it, so the "already tracked" path performs no allocation. + * Anything derived from the key (for example the cluster hash slot) is computed + * on demand by the caller rather than stored per entry. + * + * The manager keeps two windows: a `live` one accumulating the current interval + * and a `frozen` snapshot of the last completed interval. Readers observe only + * the frozen window, never a partial one. Each window also records the real + * interval it accumulated over and the sampling percentage its counts were + * gathered under, so a reader can turn a frozen window into a rate correctly + * even after the configuration has since changed. + * + * The caller supplies a monotonic microsecond clock on each call, so this + * module has no global-clock dependency, and drives window boundaries by + * calling spaceSavingManagerRotate() on a timer — recording a sample never + * reads the clock. + * + * Not thread-safe: guard externally if shared across threads. + */ + +typedef struct spaceSavingManager spaceSavingManager; + +/* Create a manager tracking up to `k` items per window, with a window length of + * `window_us` microseconds. `now_us` seeds the first window start. */ +spaceSavingManager *spaceSavingManagerCreate(int k, uint64_t window_us, uint64_t now_us); +/* Free the manager and every key it owns. NULL-safe. */ +void spaceSavingManagerRelease(spaceSavingManager *m); +/* Clear both windows (including their recorded timing) and restart measuring at + * `now_us`. The configured sampling percentage is preserved, since it describes + * how the next observations will be gathered rather than the data dropped. */ +void spaceSavingManagerReset(spaceSavingManager *m, uint64_t now_us); +/* Close the live window if its configured length has fully elapsed (no-op if it + * is still open). A window that ran past TWICE the configured length is dropped + * instead of frozen: its counts span too coarse an interval to publish as "the + * last window". So a frozen window always spans [length, 2 * length) — never + * shorter than configured, and never a long-run average mislabelled as one + * window. Boundaries are measured from the live window's real start, so a late + * call cannot shorten the following window. */ +void spaceSavingManagerRotate(spaceSavingManager *m, uint64_t now_us); +/* Record one observation of (`key`, `dbid`) into the current (live) window. + * `key` is borrowed — it is copied only if a slot is committed to it. Does NOT + * rotate: the caller must drive boundaries via spaceSavingManagerRotate() on a + * timer, keeping this hot path free of any clock read. */ +void recordSpaceSavingManagerSample(spaceSavingManager *m, sds key, int dbid); +/* Number of items in the last completed (frozen) window. */ +int spaceSavingManagerCount(spaceSavingManager *m); +/* Read the i-th item of the frozen window (0 <= i < count). Out-params may be + * NULL; `*key` remains owned by the module and is valid until the next mutating + * call. Slots are unordered. */ +void spaceSavingManagerAt(spaceSavingManager *m, int i, sds *key, int *dbid, uint64_t *count, uint64_t *error); +/* Remove every item for which `pred(key, dbid, arg)` is non-zero, from BOTH the + * live and frozen windows. */ +void spaceSavingManagerRemoveIf(spaceSavingManager *m, int (*pred)(sds key, int dbid, void *arg), void *arg); +/* Total observations recorded in the last completed (frozen) window (N). */ +uint64_t spaceSavingManagerFrozenTotal(spaceSavingManager *m); + +/* Record the sampling percentage that the current (live) window's counts are + * being gathered under. It travels with the window when it is frozen, so a + * reader can scale the frozen counts by the percentage that produced them even + * after a later change. */ +void spaceSavingManagerSetLiveSamplingPercentage(spaceSavingManager *m, int sampling_percentage); +/* Sampling percentage that was in effect for the last completed (frozen) + * window; 0 when the window never had one (freshly created or reset). */ +int spaceSavingManagerFrozenSamplingPercentage(spaceSavingManager *m); +/* Real time the last completed (frozen) window spent accumulating, in + * microseconds, or 0 if there is no completed window yet — which also covers a + * window that was dropped for being too coarse (see spaceSavingManagerRotate), + * so 0 does not distinguish "just started" from "just dropped one". Rotation is + * driven by the caller's timer, so a window is closed at or after its nominal + * boundary and this is its configured length plus the rotation lag. Rates must + * be derived from THIS, not from the configured window length, or they + * over-report by that lag. */ +uint64_t spaceSavingManagerFrozenDurationUs(spaceSavingManager *m); + +/* Reset only the live window and restart it at `now_us` with the given capacity + * and window length, keeping the last completed (frozen) window and its + * sampling config. Use on a config change instead of releasing/recreating the + * manager. Shrinking the capacity keeps the highest-count entries. */ +void spaceSavingManagerReconfigure(spaceSavingManager *m, int new_k, uint64_t new_window_us, uint64_t now_us); + +#endif /* SPACE_SAVING_H */ diff --git a/src/stat_calc.c b/src/stat_calc.c new file mode 100644 index 000000000..79d0367d3 --- /dev/null +++ b/src/stat_calc.c @@ -0,0 +1,148 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include "stat_calc.h" +#include "monotonic.h" +#include "zmalloc.h" +#include + +static const long ONE_SECOND_IN_MICROS = 1000000; + +/* ------------- TPS Calculator ------------- */ +struct tpsCalculator { + double window_secs; + double window_us; + double trans_per_window; + monotime last_update; + long update_freq_us; + long uncounted_trans; + bool is_new; +}; + +tpsCalculator *tpsCalculator_create(int window_secs) { + tpsCalculator *calc = zmalloc(sizeof(tpsCalculator)); + calc->window_secs = (double)window_secs; + calc->window_us = (double)window_secs * 1000000.0; + calc->trans_per_window = 0.0; + calc->last_update = getMonotonicUs(); + /* Update at most 20 times per window for smooth results. */ + calc->update_freq_us = window_secs * (ONE_SECOND_IN_MICROS / 20); + calc->uncounted_trans = 0; + calc->is_new = true; + return calc; +} + +void tpsCalculator_free(tpsCalculator *calc) { + zfree(calc); +} + +void tpsCalculator_record(tpsCalculator *calc, unsigned long transactions) { + monotime now = getMonotonicUs(); + long elapsed_us = now - calc->last_update; + + calc->uncounted_trans += transactions; + if (elapsed_us < calc->update_freq_us) return; /* accumulate until update frequency is hit */ + + double total = (double)calc->uncounted_trans; + calc->uncounted_trans = 0; + calc->last_update = now; + + if (elapsed_us >= calc->window_us || calc->is_new) { + calc->trans_per_window = total * calc->window_us / elapsed_us; + calc->is_new = false; + } else { + /* Decay existing by fraction of window elapsed, add new. */ + calc->trans_per_window = + (calc->trans_per_window * (calc->window_us - elapsed_us) / calc->window_us) + total; + } +} + +double tpsCalculator_averageTps(tpsCalculator *calc) { + /* Flush any pending samples so the value reflects "now". */ + tpsCalculator_record(calc, 0); + return calc->trans_per_window / calc->window_secs; +} + +/* ------------- Trend Calculator ------------- */ + +#define DATA_POINTS 10 +struct trendCalculator { + int window_sec; + monotime last_update; + long update_freq_us; + bool is_new; + long metrics[DATA_POINTS]; + long uncounted_total; + int uncounted_samples; + double trend; + double trend_short; +}; + +trendCalculator *trendCalculator_create(int window_secs) { + trendCalculator *calc = zcalloc(sizeof(trendCalculator)); + calc->window_sec = window_secs; + calc->last_update = getMonotonicUs(); + calc->update_freq_us = window_secs * ONE_SECOND_IN_MICROS / DATA_POINTS; + calc->is_new = true; + return calc; +} + +void trendCalculator_free(trendCalculator *calc) { + zfree(calc); +} + +void trendCalculator_recordMetric(trendCalculator *calc, long metric_value) { + monotime now = getMonotonicUs(); + long elapsed_us = now - calc->last_update; + + calc->uncounted_total += metric_value; + calc->uncounted_samples++; + + if (elapsed_us < calc->update_freq_us) return; + + long new_value = calc->uncounted_total / calc->uncounted_samples; + calc->uncounted_total = 0; + calc->uncounted_samples = 0; + calc->last_update = now; + + if (calc->is_new) { + for (int i = 0; i < DATA_POINTS; i++) calc->metrics[i] = new_value; + calc->is_new = false; + } + + long older_total = 0; + for (int i = 0; i < DATA_POINTS / 2; i++) { + calc->metrics[i] = calc->metrics[i + 1]; + older_total += calc->metrics[i]; + } + long newer_total = 0; + for (int i = DATA_POINTS / 2; i < DATA_POINTS - 1; i++) { + calc->metrics[i] = calc->metrics[i + 1]; + newer_total += calc->metrics[i]; + } + calc->metrics[DATA_POINTS - 1] = new_value; + newer_total += new_value; + + /* Formula is the average of the newer data points, less the average of the older data + * points. The time is from the center of each half, + * resulting in half the window size (secs). So the formula is: + * (AveNewer - AveOlder) / (WindowSec/2) + * Where: + * AveNewer = newerTotal / (DATA_POINTS/2) + * AveOlder = olderTotal / (DATA_POINTS/2) */ + double older_avg = (double)older_total / (DATA_POINTS / 2); + double newer_avg = (double)newer_total / (DATA_POINTS / 2); + double time_between_centers = (double)calc->window_sec / 2.0; + calc->trend = (newer_avg - older_avg) / time_between_centers; + + /* Short-term: rate of change between last 2 datapoints. */ + long delta_short = calc->metrics[DATA_POINTS - 1] - calc->metrics[DATA_POINTS - 2]; + double time_between_slots = (double)calc->window_sec / DATA_POINTS; + calc->trend_short = delta_short / time_between_slots; +} + +double trendCalculator_changePerSecShortTerm(trendCalculator *calc) { + return calc->trend_short; +} diff --git a/src/stat_calc.h b/src/stat_calc.h new file mode 100644 index 000000000..c02ae13bd --- /dev/null +++ b/src/stat_calc.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Calculators for statistical values. + */ + +#ifndef STAT_CALC_H +#define STAT_CALC_H + +/* =========================== TPS Calculator =============================== */ + +/* A TPS calculator computes a rolling average TPS over a specified time window. + * This smooths jitter in the measurement, with the average slightly lagging + * instantaneous changes. This provides a stable measurement that is resilient + * to short-lived traffic spikes. + */ + +typedef struct tpsCalculator tpsCalculator; + +tpsCalculator *tpsCalculator_create(int window_secs); + +void tpsCalculator_free(tpsCalculator *calc); + +/* Add a datapoint of new transactions to the calculator. This should be called at minimum 10 times + * over the window for smooth results. */ +void tpsCalculator_record(tpsCalculator *calc, unsigned long transactions); + +/* Retrieve the average TPS over the calculator's window */ +double tpsCalculator_averageTps(tpsCalculator *calc); + + +/* ========================== Trend Calculator ============================== */ + +/* A trend calculator computes the rate of change of a metric over a specified + * time window, reported as average increase/decrease per second. + * Examples: + * - Data 1,2,1,2,1,2 — trend ≈ 0 (oscillating, no net change) + * - Data 0,0,0,10,10,10 — trend ≈ 3/sec (step increase) + * Conceptually similar to the slope of a linear regression, but uses a + * lightweight approximation suitable for high-frequency sampling. + */ +typedef struct trendCalculator trendCalculator; + +trendCalculator *trendCalculator_create(int window_secs); + +void trendCalculator_free(trendCalculator *calc); + +/* Add a datapoint to the calculator. Should be called at minimum 10 times + * over the window for smooth results. If the metric is highly volatile, + * calling more often reduces the impact of individual outliers. */ +void trendCalculator_recordMetric(trendCalculator *calc, long metric_value); + +/* Get the rate of change using only the final 10% of the window. + * More responsive to sudden changes but noisier than the full-window trend. */ +double trendCalculator_changePerSecShortTerm(trendCalculator *calc); + +#endif diff --git a/src/t_hash.c b/src/t_hash.c index 6c3049cec..d8a1a58ae 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -32,9 +32,12 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include "expire.h" #include "hashtable.h" +#include "listpack.h" #include "rax.h" #include "sds.h" +#include "util.h" #include "vset.h" #include "server.h" #include "zmalloc.h" @@ -67,9 +70,66 @@ static vset *hashTypeGetVolatileSet(robj *o) { return vsetIsValid(set) ? set : NULL; } +/* Maintain the aggregate volatile-count header of a listpack-encoded hash. + * + * The header is a single tagged entry leading the listpack whose integer payload + * is the number of fields carrying an expiry. It exists only while that + * count is > 0: created on the 0->1 transition, updated in place, and deleted + * on the 1->0 transition, so hashes without field TTLs pay nothing. All semantics + * live here; the listpack layer only provides the positional primitive. + * + * Must be called after the mutation it accounts for; it may reallocate the + * listpack, so callers must not reuse element pointers taken before it. */ +void hashTypeUpdateVolatileCount(robj *o, long delta) { + if (delta == 0) return; + serverAssert(objectGetEncoding(o) == OBJ_ENCODING_LISTPACK); + unsigned char *zl = objectGetVal(o); + unsigned char *head = lpStart(zl); + int has_head = lpIsMetadata(head); + long long count = (has_head ? lpGetMetadataValue(head) : 0) + delta; + serverAssert(count >= 0); + if (count == 0) { + if (has_head) zl = lpRemoveMetadata(zl, head); + } else { + unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; + uint64_t enclen; + lpEncodeIntegerGetType(count, intenc, &enclen); + /* head == lpStart(zl): replace the existing header in place, or insert + * a new one before the first physical entry / EOF. */ + zl = lpInsertMetadata(zl, intenc, enclen, head, has_head ? LP_REPLACE : LP_BEFORE, NULL); + } + objectSetVal(o, zl); +} + +/* Return the number of fields carrying an expiry, INCLUDING expired fields + * that have not been reaped yet. O(1) for listpack (aggregate header peek); + * O(buckets) for hashtable (vset walk). Returns 0 when none. */ +long long hashTypeVolatileCount(robj *o) { + serverAssert(objectGetType(o) == OBJ_HASH); + + if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { + unsigned char *p = lpStart(objectGetVal(o)); + unsigned char *header = lpIsMetadata(p) ? p : NULL; + return header ? lpGetMetadataValue(header) : 0; + } else if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) { + vset *set = hashTypeGetVolatileSet(o); + return set ? (long long)vsetSize(set) : 0; + } + serverPanic("Unknown hash encoding"); +} + bool hashTypeHasVolatileFields(robj *o) { if (o == NULL) return false; serverAssert(objectGetType(o) == OBJ_HASH); + + if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { + /* O(1): the aggregate header exists iff at least one field carries + * an expiry. The one exception is the RDB_TYPE_HASH_2 loader, which + * appends field expiries inside its loop and installs the header only + * after it, so do not reach this from inside that loop. */ + return lpIsMetadata(lpStart(objectGetVal(o))); + } + if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) { vset *set = hashTypeGetVolatileSet(o); if (set && !vsetIsEmpty(set)) @@ -78,9 +138,27 @@ bool hashTypeHasVolatileFields(robj *o) { return false; } +/* Transient "ignore TTL" state for the listpack encoding. The hashtable + * encoding hangs this state on the object itself (by swapping the hashtable + * type, see below); a listpack has nowhere to put it, so we use a file-scope + * flag consulted by hashTypeListpackFieldIsValid(). This is safe because + * command execution is single threaded and every ignore-bracket is a tight + * set(true)/.../set(false) pair that does not span commands. */ +static bool listpack_ttl_ignored = false; + /* make any access to the hash object elements ignore the specific elements expiration. * This is mainly in order to be able to access hash elements which are already expired. */ static inline void hashTypeIgnoreTTL(robj *o, bool ignore) { + if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { + listpack_ttl_ignored = ignore; + return; + } + /* Clearing is done regardless of encoding so that a bracket whose object + * was converted listpack->hashtable in between cannot leak the flag. + * Setting, however, must NOT touch the flag for hashtable objects: + * hashTypeFreeVolatileSet() uses ignore=true as steady-state (not + * bracketed) configuration for hashes without volatile fields. */ + if (!ignore) listpack_ttl_ignored = false; if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) { /* prevent placing access function if not needed */ if (!ignore && hashTypeGetVolatileSet(o) == NULL) { @@ -158,6 +236,22 @@ bool hashHashtableTypeValidate(hashtable *ht, void *entryptr) { return false; } +/* Listpack mirror of hashHashtableTypeValidate: whether a field whose stored + * expiry is 'expiry' is visible in the current execution context. The + * hashtable encoding applies this filter inside hashtableFind/Scan/Next via + * the validateEntry callback; listpack read paths must apply it explicitly + * so both encodings answer identically (notably under POLICY_IGNORE_EXPIRE: + * loading, replication stream, slot migration, import mode). */ +bool hashTypeListpackFieldIsValid(long long expiry) { + if (expiry == EXPIRY_NONE) return true; + /* Inside an ignore-TTL bracket (e.g. HSETEX force-deleting an already + * expired field) every field is visible, mirroring the hashtable + * encoding's type swap to the non-validating hashHashtableType. */ + if (listpack_ttl_ignored) return true; + if (getExpirationPolicyWithFlags(0) == POLICY_IGNORE_EXPIRE) return true; + return !timestampIsExpired(expiry); +} + /*----------------------------------------------------------------------------- * Hash type API *----------------------------------------------------------------------------*/ @@ -194,29 +288,62 @@ void hashTypeTryConversion(robj *o, robj **argv, int start, int end) { } /* Get the value from a listpack encoded hash, identified by field. - * Returns -1 when the field cannot be found. */ -int hashTypeGetFromListpack(robj *o, sds field, unsigned char **vstr, unsigned int *vlen, long long *vll) { - unsigned char *zl, *fptr = NULL, *vptr = NULL; + * Returns -1 when the field cannot be found (or is not visible in the + * current execution context, see hashTypeListpackFieldIsValid). + * If 'expiry' is not NULL it is set to the field's expiration time, or + * EXPIRY_NONE when the field has none, saving callers a second scan. */ +int hashTypeGetFromListpack(robj *o, sds field, unsigned char **vstr, unsigned int *vlen, long long *vll, mstime_t *expiry) { + unsigned char *zl, *fptr; serverAssert(objectGetEncoding(o) == OBJ_ENCODING_LISTPACK); zl = objectGetVal(o); fptr = lpFirst(zl); - if (fptr != NULL) { - fptr = lpFind(zl, fptr, (unsigned char *)field, sdslen(field), 1); - if (fptr != NULL) { - /* Grab pointer to the value (fptr points to the field) */ - vptr = lpNext(zl, fptr); - serverAssert(vptr != NULL); - } - } + if (fptr == NULL) return -1; + fptr = lpFind(zl, fptr, (unsigned char *)field, sdslen(field), 1); + if (fptr == NULL) return -1; + + /* Grab pointer to the value (fptr points to the field) */ + unsigned char *vptr = lpNext(zl, fptr); + serverAssert(vptr != NULL); + long long entry_expiry = hashTypeListpackGetExpiry(zl, vptr); + if (!hashTypeListpackFieldIsValid(entry_expiry)) return -1; + + *vstr = lpGetValue(vptr, vlen, vll); + if (expiry) *expiry = entry_expiry; + return 0; +} - if (vptr != NULL) { - *vstr = lpGetValue(vptr, vlen, vll); - return 0; - } +/* Expiry of the listpack field whose value entry is 'vptr': the integer + * payload of the pair's trailing metadata entry, or EXPIRY_NONE when the + * pair carries none. Purely a read of what is stored; callers decide how + * to treat expired-but-unreaped fields (and against which clock). */ +long long hashTypeListpackGetExpiry(unsigned char *zl, unsigned char *vptr) { + unsigned char *metadata_ptr = lpGetMetadata(zl, vptr); + return metadata_ptr ? lpGetMetadataValue(metadata_ptr) : EXPIRY_NONE; +} - return -1; +/* Returns the expiration time associated with the specified field. + * If the field is found C_OK is returned, otherwise C_ERR. + * The matching item expiration time is assigned to `expiry` memory location, if specified. + * In case the item has no assigned expiration time, -1 is returned. */ +int hashTypeGetExpiry(robj *o, sds field, mstime_t *expiry) { + if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { + unsigned char *vstr; + unsigned int vlen; + long long vll; + if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll, expiry) < 0) return C_ERR; + return C_OK; + } else if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) { + void *found_element = NULL; + if (hashtableFind(objectGetVal(o), field, &found_element)) { + if (expiry) *expiry = entryGetExpiry(found_element); + return C_OK; + } + } else { + serverPanic("Unknown hash encoding"); + } + return C_ERR; } /* Higher level function of hashTypeGet*() that returns the hash value @@ -234,8 +361,7 @@ int hashTypeGetFromListpack(robj *o, sds field, unsigned char **vstr, unsigned i int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vlen, long long *vll, mstime_t *expiry) { if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { *vstr = NULL; - if (hashTypeGetFromListpack(o, field, vstr, vlen, vll) == 0) { - if (expiry) *expiry = EXPIRY_NONE; + if (hashTypeGetFromListpack(o, field, vstr, vlen, vll, expiry) == 0) { return C_OK; } } else if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) { @@ -256,27 +382,6 @@ int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vle return C_ERR; } -/* Returns the expiration time associated with the specified field. - * If the field is found C_OK is returned, otherwise C_ERR. - * The matching item expiration time is assigned to `expiry` memory location, if specified. - * In case the item has no assigned expiration time, -1 is returned. */ -int hashTypeGetExpiry(robj *o, sds field, mstime_t *expiry) { - if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { - if (hashTypeExists(o, field)) { - if (expiry) *expiry = EXPIRY_NONE; - return C_OK; - } - } else if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) { - void *found_element = NULL; - if (hashtableFind(objectGetVal(o), field, &found_element)) { - if (expiry) *expiry = entryGetExpiry(found_element); - return C_OK; - } - } else { - serverPanic("Unknown hash encoding"); - } - return C_ERR; -} /* Like hashTypeGetValue() but returns an Object, which is useful for * interaction with the hash type outside t_hash.c. @@ -363,7 +468,6 @@ int hashTypeUpdateAsStringRef(robj *o, sds field, const char *buf, size_t len) { * * HASH_SET_COPY corresponds to no flags passed, and means the default * semantics of copying the values if needed. - * */ int hashTypeSet(robj *o, sds field, sds value, mstime_t expiry, int flags, bool *expired_overwritten) { int update = 0; @@ -372,34 +476,80 @@ int hashTypeSet(robj *o, sds field, sds value, mstime_t expiry, int flags, bool * This is needed for HINCRBY* case since in other commands this is handled early by * hashTypeTryConversion, so this check will be a NOP. */ if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { - if (expiry != EXPIRY_NONE || sdslen(field) > server.hash_max_listpack_value || sdslen(value) > server.hash_max_listpack_value) + if (sdslen(field) > server.hash_max_listpack_value || sdslen(value) > server.hash_max_listpack_value) hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); } if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { unsigned char *zl, *fptr, *vptr; + bool has_expiry = false; + int volatile_delta = 0; zl = objectGetVal(o); fptr = lpFirst(zl); if (fptr != NULL) { fptr = lpFind(zl, fptr, (unsigned char *)field, sdslen(field), 1); - if (fptr != NULL) { - /* Grab pointer to the value (fptr points to the field) */ - vptr = lpNext(zl, fptr); - serverAssert(vptr != NULL); - update = 1; + } - /* Replace value */ - zl = lpReplace(zl, &vptr, (unsigned char *)value, sdslen(value)); + /* If the field exists we update its metadata expiry */ + if (fptr != NULL) { + /* Grab pointer to the value (fptr points to the field) */ + vptr = lpNext(zl, fptr); + serverAssert(vptr != NULL); + /* Get pointer to the metadata field */ + unsigned char *metadata_ptr = lpGetMetadata(zl, vptr); + has_expiry = (metadata_ptr != NULL); + /* if we have a metadata expiry attached */ + if (has_expiry) { + long long entry_expiry = lpGetMetadataValue(metadata_ptr); + is_expired = checkAlreadyExpired(entry_expiry); + if (!is_expired && flags & HASH_SET_KEEP_EXPIRY) { + /* If HASH_SET_KEEP_EXPIRY is true keep the original expiry */ + expiry = entry_expiry; + } } - } - if (!update) { + /* Replace value. lpReplace keeps vptr valid, so the trailing + * metadata stays reachable without re-finding the field. */ + zl = lpReplace(zl, &vptr, (unsigned char *)value, sdslen(value)); + metadata_ptr = lpGetMetadata(zl, vptr); /* relocated by the replace */ + serverAssert(has_expiry == (metadata_ptr != NULL)); + + /* Metadata transition table (had expiry -> has expiry): + * had & has: refresh in place (LP_REPLACE re-encodes widths) + * !had & has: attach after the value + * had & !has: drop + * !had & !has: nothing to do */ + if (expiry != EXPIRY_NONE) { + unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; + uint64_t enclen; + lpEncodeIntegerGetType(expiry, intenc, &enclen); + if (has_expiry) { + zl = lpInsertMetadata(zl, intenc, enclen, metadata_ptr, LP_REPLACE, NULL); + } else { + zl = lpInsertMetadata(zl, intenc, enclen, vptr, LP_AFTER, NULL); + } + } else if (has_expiry) { + zl = lpRemoveMetadata(zl, metadata_ptr); + } + volatile_delta = (expiry != EXPIRY_NONE ? 1 : 0) - (has_expiry ? 1 : 0); + update = is_expired ? 0 : 1; + } else { /* Push new field/value pair onto the tail of the listpack */ zl = lpAppend(zl, (unsigned char *)field, sdslen(field)); zl = lpAppend(zl, (unsigned char *)value, sdslen(value)); + if (expiry != EXPIRY_NONE) { + unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; + uint64_t enclen; + lpEncodeIntegerGetType(expiry, intenc, &enclen); + unsigned char *eofptr = zl + lpGetTotalBytes(zl) - 1; + zl = lpInsertMetadata(zl, intenc, enclen, eofptr, LP_BEFORE, NULL); + volatile_delta = 1; + } } + objectSetVal(o, zl); + hashTypeUpdateVolatileCount(o, volatile_delta); /* Check if the listpack needs to be converted to a hash table */ if (hashTypeLength(o) > server.hash_max_listpack_entries) hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); @@ -472,82 +622,65 @@ static expiryModificationResult hashTypeSetExpire(robj *o, sds field, mstime_t e /* If no object we will return -2 */ if (o == NULL) return EXPIRATION_MODIFICATION_NOT_EXIST; - bool time_is_expired = checkAlreadyExpired(expiry); + /* 1. Locate the field and read its current expiry (per encoding). A + * missing or lazily-expired field is reported as NOT_EXIST: both the + * listpack lookup (via hashTypeListpackFieldIsValid) and the hashtable + * lookup (via the validateEntry callback inside hashtableFindRef) + * apply the same visibility rules. */ + mstime_t current_expire = EXPIRY_NONE; + void **entry_ref = NULL; /* hashtable only */ if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { unsigned char *vstr; unsigned int vlen; long long vll; - /* We do not want to convert to listpack for no good reason. - * So we first check if the item exists.*/ - if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) < 0) { + if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll, ¤t_expire) < 0) return EXPIRATION_MODIFICATION_NOT_EXIST; - } - /* When listpack representation is used, we consider it as infinite TTL, - * so expire command with gt always fail the GT as well as existence(XX). - * Else, if the ttl is set in the past, just delete the entry (we know it exists) - * Else, we already know we are going to set an expiration so we expend to hashtable encoding. */ - if (flag & EXPIRE_XX || flag & EXPIRE_GT) { - return EXPIRATION_MODIFICATION_FAILED_CONDITION; - } else if (time_is_expired) { - serverAssert(hashTypeDelete(o, field)); - return EXPIRATION_MODIFICATION_EXPIRE_ASAP; - } else { - hashTypeConvert(o, OBJ_ENCODING_HASHTABLE); - } + } else { + serverAssert(objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE); + entry_ref = hashtableFindRef(objectGetVal(o), field); + if (entry_ref == NULL) return EXPIRATION_MODIFICATION_NOT_EXIST; + current_expire = entryGetExpiry(*entry_ref); } - /* we must be hashtable encoded */ - serverAssert(objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE); - - hashtable *ht = objectGetVal(o); - void **entry_ref = NULL; - if ((entry_ref = hashtableFindRef(ht, field))) { - entry *current_entry = *entry_ref; - mstime_t current_expire = entryGetExpiry(current_entry); - if (flag) { - /* NX option is set, check no current expiry */ - if (flag & EXPIRE_NX) { - if (current_expire != EXPIRY_NONE) { - return EXPIRATION_MODIFICATION_FAILED_CONDITION; - } - } - - /* XX option is set, check current expiry */ - if (flag & EXPIRE_XX) { - if (current_expire == EXPIRY_NONE) { - return EXPIRATION_MODIFICATION_FAILED_CONDITION; - } - } + /* 2. Shared condition checks. EXPIRY_NONE is treated as +inf: GT can + * never beat it, LT always does. */ + if ((flag & EXPIRE_NX) && current_expire != EXPIRY_NONE) return EXPIRATION_MODIFICATION_FAILED_CONDITION; + if ((flag & EXPIRE_XX) && current_expire == EXPIRY_NONE) return EXPIRATION_MODIFICATION_FAILED_CONDITION; + if ((flag & EXPIRE_GT) && (current_expire == EXPIRY_NONE || expiry <= current_expire)) + return EXPIRATION_MODIFICATION_FAILED_CONDITION; + if ((flag & EXPIRE_LT) && current_expire != EXPIRY_NONE && expiry >= current_expire) + return EXPIRATION_MODIFICATION_FAILED_CONDITION; - /* GT option is set, check current expiry */ - if (flag & EXPIRE_GT) { - /* When current_expire is -1, we consider it as infinite TTL, - * so expire command with gt always fail the GT. */ - if (expiry <= current_expire || current_expire == EXPIRY_NONE) { - return EXPIRATION_MODIFICATION_FAILED_CONDITION; - } - } + /* If the ttl is set in the past, just delete the entry (we know it exists) */ + if (checkAlreadyExpired(expiry)) { + serverAssert(hashTypeDelete(o, field)); + return EXPIRATION_MODIFICATION_EXPIRE_ASAP; + } - /* LT option is set, check current expiry */ - if (flag & EXPIRE_LT) { - /* When current_expire -1, we consider it as infinite TTL, - * so if there is an expiry on the key and it's not less than current, we fail the LT. */ - if (current_expire != EXPIRY_NONE && expiry >= current_expire) { - return EXPIRATION_MODIFICATION_FAILED_CONDITION; - } - } - } - /* In case we are set to expire the entry after we went through all the validations, - * we can just delete the entry. */ - if (time_is_expired) { - serverAssert(hashTypeDelete(o, field)); - return EXPIRATION_MODIFICATION_EXPIRE_ASAP; + /* 3. Apply (per encoding). Nothing mutated the object since step 1 on + * this path, so the handles located there are still valid. */ + if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { + unsigned char *zl = objectGetVal(o); + unsigned char *fptr = lpFind(zl, lpFirst(zl), (unsigned char *)field, sdslen(field), 1); + serverAssert(fptr != NULL); + unsigned char *value_ptr = lpNext(zl, fptr); + unsigned char *metadata_ptr = lpGetMetadata(zl, value_ptr); + unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; + uint64_t enclen; + lpEncodeIntegerGetType(expiry, intenc, &enclen); + if (metadata_ptr) { /* refresh in place (LP_REPLACE re-encodes widths) */ + zl = lpInsertMetadata(zl, intenc, enclen, metadata_ptr, LP_REPLACE, NULL); + } else { /* attach after the value */ + zl = lpInsertMetadata(zl, intenc, enclen, value_ptr, LP_AFTER, NULL); } + objectSetVal(o, zl); + if (!metadata_ptr) hashTypeUpdateVolatileCount(o, 1); + } else { + entry *current_entry = *entry_ref; *entry_ref = entrySetExpiry(current_entry, expiry); hashTypeTrackUpdateEntry(o, current_entry, *entry_ref, current_expire, expiry); - return EXPIRATION_MODIFICATION_SUCCESSFUL; } - return EXPIRATION_MODIFICATION_NOT_EXIST; // we did not find anything to do. return -2 + return EXPIRATION_MODIFICATION_SUCCESSFUL; } @@ -556,11 +689,33 @@ static expiryModificationResult hashTypePersist(robj *o, sds field) { if (o == NULL || objectGetType(o) != OBJ_HASH) return EXPIRATION_MODIFICATION_NOT_EXIST; if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { - if (hashTypeExists(o, field)) - /* When listpack representation is used, All items are without expiry */ - return EXPIRATION_MODIFICATION_FAILED; - else - return EXPIRATION_MODIFICATION_NOT_EXIST; // Did not find any element return -2 + /* Unlike hashTypeSetExpire there are no flag conditions to evaluate, + * so locate/inspect/clear happens in a single pass over the listpack + * (routing through hashTypeGetFromListpack would add a second walk). */ + unsigned char *zl = objectGetVal(o); + unsigned char *fptr = lpFirst(zl); + if (fptr != NULL) fptr = lpFind(zl, fptr, (unsigned char *)field, sdslen(field), 1); + if (fptr == NULL) return EXPIRATION_MODIFICATION_NOT_EXIST; + + unsigned char *vptr = lpNext(zl, fptr); + serverAssert(vptr != NULL); + /* Check if the field has an expiration set, if not we fail */ + unsigned char *metadata_ptr = lpGetMetadata(zl, vptr); + if (metadata_ptr == NULL) return EXPIRATION_MODIFICATION_FAILED; + + /* A lazily-expired field is logically gone: report it as missing + * instead of resurrecting it, and leave the reaping (and its HDEL + * propagation) to the active-expiry cycle. The shared visibility + * predicate keeps the boundary and the expiration policy in sync + * with the hashtable encoding. */ + long long entry_expiry = lpGetMetadataValue(metadata_ptr); + if (!hashTypeListpackFieldIsValid(entry_expiry)) return EXPIRATION_MODIFICATION_NOT_EXIST; + + /* Remove the metadata entry; its presence implies the value exists. */ + zl = lpRemoveMetadata(zl, metadata_ptr); + objectSetVal(o, zl); + hashTypeUpdateVolatileCount(o, -1); + return EXPIRATION_MODIFICATION_SUCCESSFUL; } hashtable *ht = objectGetVal(o); @@ -591,9 +746,19 @@ bool hashTypeDelete(robj *o, sds field) { if (fptr != NULL) { fptr = lpFind(zl, fptr, (unsigned char *)field, sdslen(field), 1); if (fptr != NULL) { - /* Delete both field and value. */ + unsigned char *value_ptr = lpNext(zl, fptr); + serverAssert(value_ptr != NULL); + + long long entry_expiry = hashTypeListpackGetExpiry(zl, value_ptr); + bool was_volatile = lpGetMetadata(zl, value_ptr) != NULL; + if (!hashTypeListpackFieldIsValid(entry_expiry)) return false; + + /* Delete field and value; metadata entries trailing the pair + * are deleted along with it. */ zl = lpDeleteRangeWithEntry(zl, &fptr, 2); + objectSetVal(o, zl); + if (was_volatile) hashTypeUpdateVolatileCount(o, -1); deleted = true; } } @@ -627,7 +792,7 @@ unsigned long hashTypeLength(const robj *o) { void hashTypeInitIterator(robj *subject, hashTypeIterator *hi) { hi->subject = subject; hi->encoding = subject->encoding; - hi->volatile_items_iter = false; + hi->iterator_type = HASH_ITER_ALL; if (hi->encoding == OBJ_ENCODING_LISTPACK) { hi->fptr = NULL; @@ -642,10 +807,11 @@ void hashTypeInitIterator(robj *subject, hashTypeIterator *hi) { void hashTypeInitVolatileIterator(robj *subject, hashTypeIterator *hi) { hi->subject = subject; hi->encoding = subject->encoding; - hi->volatile_items_iter = true; + hi->iterator_type = HASH_ITER_VOLATILE; if (hi->encoding == OBJ_ENCODING_LISTPACK) { - return; + hi->fptr = NULL; + hi->vptr = NULL; } else if (hi->encoding == OBJ_ENCODING_HASHTABLE) { vsetInitIterator(hashTypeGetVolatileSet(subject), &hi->viter); } else { @@ -653,12 +819,28 @@ void hashTypeInitVolatileIterator(robj *subject, hashTypeIterator *hi) { } } +void hashTypeInitPersistentIterator(robj *subject, hashTypeIterator *hi) { + hi->subject = subject; + hi->encoding = subject->encoding; + hi->iterator_type = HASH_ITER_PERSISTENT; + + if (hi->encoding == OBJ_ENCODING_LISTPACK) { + hi->fptr = NULL; + hi->vptr = NULL; + } else if (hi->encoding == OBJ_ENCODING_HASHTABLE) { + hashtableInitIterator(&hi->iter, objectGetVal(subject), 0); + } else { + serverPanic("Unknown hash encoding"); + } +} + void hashTypeResetIterator(hashTypeIterator *hi) { if (hi->encoding == OBJ_ENCODING_HASHTABLE) { - if (!hi->volatile_items_iter) + if (hi->iterator_type == HASH_ITER_ALL || hi->iterator_type == HASH_ITER_PERSISTENT) { hashtableCleanupIterator(&hi->iter); - else + } else { vsetResetIterator(&hi->viter); + } } } @@ -666,39 +848,70 @@ void hashTypeResetIterator(hashTypeIterator *hi) { * could be found and C_ERR when the iterator reaches the end. */ int hashTypeNext(hashTypeIterator *hi) { if (hi->encoding == OBJ_ENCODING_LISTPACK) { - unsigned char *zl; - unsigned char *fptr, *vptr; - - /* listpack encoding does not have volatile items, so return as iteration end */ - if (hi->volatile_items_iter) return C_ERR; - - zl = objectGetVal(hi->subject); - fptr = hi->fptr; - vptr = hi->vptr; + while (1) { + unsigned char *zl; + unsigned char *fptr, *vptr; + + zl = objectGetVal(hi->subject); + fptr = hi->fptr; + vptr = hi->vptr; + + if (fptr == NULL) { + /* Initialize cursor */ + serverAssert(vptr == NULL); + fptr = lpFirst(zl); + } else { + /* Advance cursor (lpNext transparently skips metadata) */ + serverAssert(vptr != NULL); + fptr = lpNext(zl, vptr); + } + if (fptr == NULL) return C_ERR; - if (fptr == NULL) { - /* Initialize cursor */ - serverAssert(vptr == NULL); - fptr = lpFirst(zl); - } else { - /* Advance cursor */ + /* Grab pointer to the value (fptr points to the field) */ + vptr = lpNext(zl, fptr); serverAssert(vptr != NULL); - fptr = lpNext(zl, vptr); - } - if (fptr == NULL) return C_ERR; - /* Grab pointer to the value (fptr points to the field) */ - vptr = lpNext(zl, fptr); - serverAssert(vptr != NULL); + unsigned char *metadata_ptr = lpGetMetadata(zl, vptr); + /* Advance the cursor now, before any skip decision, so the next + * iteration resumes from this pair (the loop re-reads hi->vptr + * at the top to advance). */ + hi->fptr = fptr; + hi->vptr = vptr; + + if (hi->iterator_type == HASH_ITER_VOLATILE) { + /* VOLATILE skips pairs with no metadata */ + if (metadata_ptr == NULL) continue; + } else if (hi->iterator_type == HASH_ITER_PERSISTENT) { + /* PERSISTENT skips pairs with metadata */ + if (metadata_ptr != NULL) continue; + } - /* fptr, vptr now point to the first or next pair */ - hi->fptr = fptr; - hi->vptr = vptr; + /* Skip fields not visible in the current context (matches the + * hashtable iterator, which filters expired entries via + * validateEntry semantics). */ + if (metadata_ptr != NULL) { + int64_t expiry = lpGetMetadataValue(metadata_ptr); + if (!hashTypeListpackFieldIsValid(expiry)) continue; + } + break; + } } else if (hi->encoding == OBJ_ENCODING_HASHTABLE) { - if (!hi->volatile_items_iter) { - if (!hashtableNext(&hi->iter, &hi->next)) return C_ERR; + if (hi->iterator_type == HASH_ITER_ALL || hi->iterator_type == HASH_ITER_PERSISTENT) { + /* on a persistent iterator skip entries with expiry */ + if (hi->iterator_type == HASH_ITER_PERSISTENT) { + do { + if (!hashtableNext(&hi->iter, &hi->next)) return C_ERR; + } while (entryHasExpiry(hi->next)); + } else { + if (!hashtableNext(&hi->iter, &hi->next)) return C_ERR; + } } else { - if (!vsetNext(&hi->viter, &hi->next)) return C_ERR; + do { + /* vsetNext can return ghost entries if not reaped. Use the + * validateEntry callback (not raw entryIsExpired) so the skip + * honors the expiration policy, like hashtableNext does. */ + if (!vsetNext(&hi->viter, &hi->next)) return C_ERR; + } while (!hashHashtableTypeValidate(NULL, hi->next)); } } else { serverPanic("Unknown hash encoding"); @@ -766,6 +979,15 @@ robj *hashTypeLookupWriteOrCreate(client *c, robj *key) { return o; } +long long hashTypeCurrentExpiry(robj *o, hashTypeIterator *hi) { + if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { + return hashTypeListpackGetExpiry(objectGetVal(o), hi->vptr); + } else if (objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE) { + return entryGetExpiry(hi->next); + } + + serverPanic("Unknown encoding type"); +} void hashTypeConvertListpack(robj *o, int enc) { serverAssert(objectGetEncoding(o) == OBJ_ENCODING_LISTPACK); @@ -775,6 +997,11 @@ void hashTypeConvertListpack(robj *o, int enc) { } else if (enc == OBJ_ENCODING_HASHTABLE) { hashTypeIterator hi; + /* Whether any entry we actually carried over has an expiry. Derived + * from the entries themselves rather than from the aggregate header, + * which a half-built listpack does not have yet: the RDB loader + * installs it only after its listpack loop. */ + bool has_volatile = false; hashtable *ht = hashtableCreate(&hashHashtableType); @@ -785,7 +1012,10 @@ void hashTypeConvertListpack(robj *o, int enc) { while (hashTypeNext(&hi) != C_ERR) { sds field = hashTypeCurrentObjectNewSds(&hi, OBJ_HASH_FIELD); sds value = hashTypeCurrentObjectNewSds(&hi, OBJ_HASH_VALUE); - entry *entry = entryCreate(field, value, EXPIRY_NONE); + /* Get expiry for this field from the metadata value */ + long long expiry = hashTypeCurrentExpiry(o, &hi); + if (expiry != EXPIRY_NONE) has_volatile = true; + entry *entry = entryCreate(field, value, expiry); sdsfree(field); if (!hashtableAdd(ht, entry)) { entryFree(entry); @@ -801,6 +1031,22 @@ void hashTypeConvertListpack(robj *o, int enc) { zfree(objectGetVal(o)); objectSetEncoding(o, OBJ_ENCODING_HASHTABLE); objectSetVal(o, ht); + + /* Register entries carrying an expiry in the volatile set. This must + * happen after the object points at the new hashtable (the set lives + * in the hashtable metadata). Without it the fields would neither be + * actively reaped nor lazily hidden after conversion, and the + * db-level volatile-keys tracking would go stale, leaving a dangling + * object pointer for the active-expire cron. */ + if (has_volatile) { + hashtableIterator iter; + hashtableInitIterator(&iter, ht, 0); + void *next; + while (hashtableNext(&iter, &next)) { + if (entryGetExpiry(next) != EXPIRY_NONE) hashTypeTrackEntry(o, next); + } + hashtableCleanupIterator(&iter); + } } else { serverPanic("Unknown hash encoding"); } @@ -874,6 +1120,10 @@ void hashReplyFromListpackEntry(client *c, listpackEntry *e) { addReplyBulkLongLong(c, e->lval); } +/* Forward declaration; hashTypeCurrentToEntry is defined with the hash + * iterator helpers further below. */ +static inline void hashTypeCurrentToEntry(hashTypeIterator *hi, int withvalues, listpackEntry *f, listpackEntry *v); + /* Return random element from a non empty hash. * 'field' and 'val' will be set to hold the element. * The memory in them is not to be freed or modified by the caller. @@ -881,38 +1131,65 @@ void hashReplyFromListpackEntry(client *c, listpackEntry *e) { * Return C_ERR in case no random element was found (when all existing elements are expired). * Return C_OK otherwise. */ static int hashTypeRandomElement(robj *hashobj, unsigned long hashsize, listpackEntry *field, listpackEntry *val) { - int rc = C_OK; + if (hashsize == 0) return C_ERR; + + bool has_volatile = hashTypeHasVolatileFields(hashobj); + if (hashobj->encoding == OBJ_ENCODING_HASHTABLE) { + /* Fast path: O(1)-expected fair-random probe, rejecting expired + * ("ghost") entries. Also serves the non-volatile case, where the + * first probe is always live. */ void *e = NULL; int maxtries = 100; hashTypeIgnoreTTL(hashobj, true); - while (!e) { + while (maxtries--) { hashtableFairRandomEntry(objectGetVal(hashobj), &e); - if (entryIsExpired(e) && --maxtries) { - e = NULL; - continue; - } else if (maxtries == 0) { - /* in case we will not be able to locate an entry which is not expired, we will just not return any - * result. An alternative would have been that we end up returning an expired entry. */ - rc = C_ERR; - break; - } + if (!entryIsExpired(e)) break; /* found a live entry */ + e = NULL; + } + hashTypeIgnoreTTL(hashobj, false); + if (e != NULL) { sds sds_field = entryGetField(e); field->sval = (unsigned char *)sds_field; field->slen = sdslen(sds_field); - if (val) { - val->sval = (unsigned char *)entryGetValue(e, (size_t *)&val->slen); - } + if (val) val->sval = (unsigned char *)entryGetValue(e, (size_t *)&val->slen); + return C_OK; } - hashTypeIgnoreTTL(hashobj, false); + /* Probe defeated by dense ghosts: fall through to the reservoir. */ } else if (hashobj->encoding == OBJ_ENCODING_LISTPACK) { - lpRandomPair(objectGetVal(hashobj), hashsize, field, val); + if (!has_volatile) { + /* No volatile fields: every pair is live, fetch directly. */ + lpRandomPair(objectGetVal(hashobj), hashsize, field, val); + return C_OK; + } + /* Volatile listpack: fall through to the reservoir. */ } else { serverPanic("Unknown hash encoding"); } - return rc; -} + /* Only a hash with volatile fields can reach here; every non-volatile + * hash is served by the fast paths above. */ + serverAssert(has_volatile); + + /* Reservoir (k=1): we failed to locate a random non-expired element, so + * pick one uniformly in a single read-only pass over the live fields. */ + hashTypeIterator hi; + unsigned long seen = 0; + int found = 0; + listpackEntry cf, cv; + hashTypeInitIterator(hashobj, &hi); + while (hashTypeNext(&hi) != C_ERR) { + if (rand() % ++seen == 0) { + hashTypeCurrentToEntry(&hi, val != NULL, &cf, val ? &cv : NULL); + found = 1; + } + } + hashTypeResetIterator(&hi); + if (!found) return C_ERR; /* all fields expired */ + *field = cf; + if (val) *val = cv; + return C_OK; +} /*----------------------------------------------------------------------------- * Hash type commands @@ -1096,6 +1373,43 @@ static void addHashFieldToReply(client *c, robj *o, sds field) { } } +#define HMGET_FIND_BATCH_SIZE 16 +static_assert(HMGET_FIND_BATCH_SIZE <= HASHTABLE_FIND_BATCH_MAX_SIZE, + "HMGET batch size exceeds hashtable batch lookup limit"); + +static void addHashEntryToReply(client *c, const entry *hash_entry) { + if (hash_entry == NULL) { + addReplyNull(c); + return; + } + + size_t len = 0; + char *value = entryGetValue(hash_entry, &len); + serverAssert(value != NULL); + addReplyBulkCBuffer(c, value, len); +} + +static void hmgetReplyWithHashtable(client *c, hashtable *ht, robj **fields, size_t count) { + const void *keys[HMGET_FIND_BATCH_SIZE]; + void *found_entries[HMGET_FIND_BATCH_SIZE]; + while (count) { + size_t batch = count > HMGET_FIND_BATCH_SIZE ? HMGET_FIND_BATCH_SIZE : count; + + for (size_t i = 0; i < batch; i++) { + keys[i] = objectGetVal(fields[i]); + } + + uint32_t result = hashtableFindBatch(ht, (int)batch, keys, found_entries); + + for (size_t i = 0; i < batch; i++) { + addHashEntryToReply(c, (result >> i) & 1 ? found_entries[i] : NULL); + } + + fields += batch; + count -= batch; + } +} + void hgetCommand(client *c) { robj *o; @@ -1105,7 +1419,7 @@ void hgetCommand(client *c) { void hmgetCommand(client *c) { robj *o; - int i; + size_t count = c->argc - 2; /* Don't abort when the key cannot be found. Non-existing keys are empty * hashes, where HMGET should respond with a series of null bulks. */ @@ -1113,12 +1427,23 @@ void hmgetCommand(client *c) { if (checkType(c, o, OBJ_HASH)) return; - addReplyArrayLen(c, c->argc - 2); - for (i = 2; i < c->argc; i++) { - addHashFieldToReply(c, o, objectGetVal(c->argv[i])); + addReplyArrayLen(c, count); + + if (o == NULL) { + for (size_t i = 0; i < count; i++) { + addReplyNull(c); + } + return; } - if (o && hashTypeLength(o) == 0) { - dbDelete(c->db, c->argv[1]); + + /* Prefer hashtable batch lookup to improve performance. */ + if (o->encoding == OBJ_ENCODING_HASHTABLE && count > 1) { + hmgetReplyWithHashtable(c, objectGetVal(o), c->argv + 2, count); + return; + } + + for (size_t i = 0; i < count; i++) { + addHashFieldToReply(c, o, objectGetVal(c->argv[i + 2])); } } @@ -1389,7 +1714,7 @@ void hsetexCommand(client *c) { for (; fields_index < c->argc - 1; fields_index++) { if (!strcasecmp(objectGetVal(c->argv[fields_index]), "fields")) { /* checking optional flags */ - if (parseExtendedCommandArgumentsOrReply(c, COMMAND_HSET, 2, fields_index++, &flags, &unit, NULL, &expire, &comparison) != C_OK) return; + if (parseExtendedCommandArgumentsOrReply(c, COMMAND_HSET, 2, fields_index++, &flags, &unit, NULL, &expire, &comparison, NULL) != C_OK) return; if (getLongLongFromObjectOrReply(c, c->argv[fields_index++], &num_fields, NULL) != C_OK) return; break; } @@ -1647,7 +1972,7 @@ void hgetexCommand(client *c) { for (; fields_index < c->argc - 1; fields_index++) { if (!strcasecmp(objectGetVal(c->argv[fields_index]), "fields")) { /* checking optional flags */ - if (parseExtendedCommandArgumentsOrReply(c, COMMAND_HGET, 2, fields_index++, &flags, &unit, NULL, &expire, &comparison) != C_OK) return; + if (parseExtendedCommandArgumentsOrReply(c, COMMAND_HGET, 2, fields_index++, &flags, &unit, NULL, &expire, &comparison, NULL) != C_OK) return; if (getLongLongFromObjectOrReply(c, c->argv[fields_index++], &num_fields, NULL) != C_OK) return; break; } @@ -2159,6 +2484,41 @@ void hpexpiretimeCommand(client *c) { * the number of randoms per time. */ #define HRANDFIELD_RANDOM_SAMPLE_LIMIT 1000 +/* Store the field (and optionally value) at the iterator cursor into + * listpackEntry structs, for either encoding. The entries alias the hash + * object's memory and stay valid as long as it isn't mutated. */ +static inline void hashTypeCurrentToEntry(hashTypeIterator *hi, int withvalues, listpackEntry *f, listpackEntry *v) { + if (hi->encoding == OBJ_ENCODING_LISTPACK) { + f->sval = lpGetValue(hi->fptr, &f->slen, &f->lval); + if (withvalues) v->sval = lpGetValue(hi->vptr, &v->slen, &v->lval); + } else { + size_t len; + f->sval = (unsigned char *)hashTypeCurrentFromHashTable(hi, OBJ_HASH_FIELD, &len); + f->slen = len; + f->lval = 0; + if (withvalues) { + v->sval = (unsigned char *)hashTypeCurrentFromHashTable(hi, OBJ_HASH_VALUE, &len); + v->slen = len; + v->lval = 0; + } + } +} + +/* Collect every live (non-expired) field into the caller-provided arrays, + * which must have room for hashTypeLength(o) entries. Returns the number of + * live fields collected. */ +static unsigned long hashTypeCollectLive(robj *o, int withvalues, listpackEntry *fields, listpackEntry *values) { + hashTypeIterator hi; + unsigned long n = 0; + hashTypeInitIterator(o, &hi); + while (hashTypeNext(&hi) != C_ERR) { + hashTypeCurrentToEntry(&hi, withvalues, &fields[n], withvalues ? &values[n] : NULL); + n++; + } + hashTypeResetIterator(&hi); + return n; +} + void hrandfieldWithCountCommand(client *c, long l, int withvalues) { unsigned long count, size; int uniq = 1; @@ -2166,6 +2526,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { if ((hash = lookupKeyReadOrReply(c, c->argv[1], shared.emptyarray)) == NULL || checkType(c, hash, OBJ_HASH)) return; size = hashTypeLength(hash); + bool has_volatile = hashTypeHasVolatileFields(hash); if (l >= 0) { count = (unsigned long)l; @@ -2186,6 +2547,83 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { void *replylen = addReplyDeferredLen(c); unsigned long reply_size = 0; + /* Hashes with volatile fields take one generic, ghost-aware path over + * hashTypeIterator (both encodings): expired-unreaped fields are skipped + * and the expiration policy is honored. Hashes without field TTLs (the + * common case) fall through to the original CASE 1-4 samplers, which are + * only valid when every field is live. */ + if (has_volatile) { + if (count == 1) { + /* Single random field (HRANDFIELD key 1 and HRANDFIELD key -1): + * delegate to hashTypeRandomElement so all single-pick forms + * (including the no-count `HRANDFIELD key`) share identical + * behavior and its O(1)-expected fast path. Read-only. */ + listpackEntry field, value; + if (hashTypeRandomElement(hash, size, &field, withvalues ? &value : NULL) == C_OK) { + hrandfieldReplyWithListpack(wpc, 1, &field, withvalues ? &value : NULL); + reply_size = 1; + } + goto set_deferred_response; + } + if (!uniq) { + /* With replacement: collect the live fields once, then draw. */ + listpackEntry *fields = zmalloc(sizeof(listpackEntry) * size); + listpackEntry *values = withvalues ? zmalloc(sizeof(listpackEntry) * size) : NULL; + unsigned long live = hashTypeCollectLive(hash, withvalues, fields, values); + while (live > 0 && count--) { + unsigned long idx = rand() % live; + hrandfieldReplyWithListpack(wpc, 1, &fields[idx], values ? &values[idx] : NULL); + if (c->flag.close_asap) break; + reply_size++; + } + zfree(fields); + if (values) zfree(values); + } else if (count >= size) { + /* CASE 2 (volatile): the request is at least the physical size, + * hence at least every live field -- return them all in a single + * pass, no sampling or buffering. hashTypeNext skips expired + * fields, so only live fields are emitted. */ + hashTypeIterator hi; + hashTypeInitIterator(hash, &hi); + while (hashTypeNext(&hi) != C_ERR) { + if (withvalues && c->resp > 2) addWritePreparedReplyArrayLen(wpc, 2); + addHashIteratorCursorToReply(wpc, &hi, OBJ_HASH_FIELD); + if (withvalues) addHashIteratorCursorToReply(wpc, &hi, OBJ_HASH_VALUE); + reply_size++; + if (c->flag.close_asap) break; + } + hashTypeResetIterator(&hi); + } else { + /* Distinct sample (count < size): reservoir sampling (Algorithm R) + * in one pass. */ + listpackEntry *rf = zmalloc(sizeof(listpackEntry) * count); + listpackEntry *rv = withvalues ? zmalloc(sizeof(listpackEntry) * count) : NULL; + unsigned long seen = 0, filled = 0; + hashTypeIterator hi; + hashTypeInitIterator(hash, &hi); + while (hashTypeNext(&hi) != C_ERR) { + seen++; + if (filled < count) { + hashTypeCurrentToEntry(&hi, withvalues, &rf[filled], withvalues ? &rv[filled] : NULL); + filled++; + } else { + unsigned long j = rand() % seen; + if (j < count) hashTypeCurrentToEntry(&hi, withvalues, &rf[j], withvalues ? &rv[j] : NULL); + } + } + hashTypeResetIterator(&hi); + reply_size = filled; + hrandfieldReplyWithListpack(wpc, filled, rf, rv); + zfree(rf); + if (rv) zfree(rv); + } + goto set_deferred_response; + } + + /* Past this point every field is live: any hash with volatile fields was + * handled (and returned) by the generic path above. */ + serverAssert(!has_volatile); + /* CASE 1: The count was negative, so the extraction method is just: * "return N random elements" sampling the whole set every time. * This case is trivial and can be served without auxiliary data @@ -2430,8 +2868,47 @@ static int hashTypeExpireEntry(void *entry, void *c) { /* Extract expired entries from a hash object's volatile set. * Returns number of expired entries, populates `out_entries`. */ size_t hashTypeDeleteExpiredFields(robj *o, mstime_t now, unsigned long max_fields, robj **out_entries) { - serverAssert(objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE); + if (objectGetEncoding(o) == OBJ_ENCODING_LISTPACK) { + unsigned char *zl = objectGetVal(o); + unsigned char *p = lpFirst(zl); + size_t expired_count = 0; + unsigned char field_intbuf[LP_INTBUF_SIZE]; + + while (p && expired_count < max_fields) { + unsigned char *fptr = p; + int64_t flen; + unsigned char *field = lpGet(fptr, &flen, field_intbuf); + unsigned char *vptr = lpNext(zl, fptr); + if (!vptr) break; + + mstime_t expiry = hashTypeListpackGetExpiry(zl, vptr); + if (expiry != EXPIRY_NONE && expiry <= now) { + if (out_entries) { + out_entries[expired_count] = createStringObject((char *)field, flen); + } + /* Delete the field/value pair (trailing metadata goes with + * it). fptr is updated to the entry following the deleted + * range (NULL at EOF), so we resume the scan from there + * instead of restarting, keeping the reap linear. */ + zl = lpDeleteRangeWithEntry(zl, &fptr, 2); + objectSetVal(o, zl); + server.stat_expiredfields++; + expired_count++; + p = fptr; + continue; + } + + p = lpNext(zl, vptr); + } + /* Bulk-update the aggregate header once: doing it per deletion would + * reallocate the listpack under the scan cursor. */ + hashTypeUpdateVolatileCount(o, -(long)expired_count); + + return expired_count; + } + + serverAssert(objectGetEncoding(o) == OBJ_ENCODING_HASHTABLE); vset *vset = hashTypeGetVolatileSet(o); if (!vset) { return 0; diff --git a/src/t_set.c b/src/t_set.c index 26913eaad..c9fb0f3e7 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -700,33 +700,88 @@ void smoveCommand(client *c) { addReply(c, shared.cone); } +#define SMISMEMBER_FIND_BATCH_SIZE 16 +static_assert(SMISMEMBER_FIND_BATCH_SIZE <= HASHTABLE_FIND_BATCH_MAX_SIZE, + "SMISMEMBER batch size exceeds hashtable batch lookup limit"); + +static void sismemberReply(client *c, robj *set, robj *member) { + addReply(c, setTypeIsMember(set, objectGetVal(member)) ? shared.cone : shared.czero); +} + +static void smismemberReplyWithHashtable(client *c, hashtable *ht, robj **members, size_t count) { + const void *keys[SMISMEMBER_FIND_BATCH_SIZE]; + void *found_entries[SMISMEMBER_FIND_BATCH_SIZE]; + while (count) { + size_t batch = count > SMISMEMBER_FIND_BATCH_SIZE ? SMISMEMBER_FIND_BATCH_SIZE : count; + + for (size_t i = 0; i < batch; i++) { + keys[i] = objectGetVal(members[i]); + } + + uint32_t result = hashtableFindBatch(ht, (int)batch, keys, found_entries); + + for (size_t i = 0; i < batch; i++) { + addReply(c, (result >> i) & 1 ? shared.cone : shared.czero); + } + + members += batch; + count -= batch; + } +} + void sismemberCommand(client *c) { robj *set; + int xx = 0; - if ((set = lookupKeyReadOrReply(c, c->argv[1], shared.czero)) == NULL || checkType(c, set, OBJ_SET)) return; + if (c->argc == 4 && !strcasecmp(objectGetVal(c->argv[3]), "XX")) { + xx = 1; + } else if (c->argc > 3) { + addReplyErrorObject(c, shared.syntaxerr); + return; + } - if (setTypeIsMember(set, objectGetVal(c->argv[2]))) - addReply(c, shared.cone); - else - addReply(c, shared.czero); + set = lookupKeyRead(c->db, c->argv[1]); + if (set == NULL) { + if (xx) + /* If key doesn't exist and XX is specified, return -1 */ + addReplyLongLong(c, -1); + else + /* If key doesn't exist and XX is not specified, return 0 */ + addReply(c, shared.czero); + return; + } + + if (checkType(c, set, OBJ_SET)) return; + + sismemberReply(c, set, c->argv[2]); } void smismemberCommand(client *c) { robj *set; - int j; /* Don't abort when the key cannot be found. Non-existing keys are empty * sets, where SMISMEMBER should respond with a series of zeros. */ set = lookupKeyRead(c->db, c->argv[1]); - if (set && checkType(c, set, OBJ_SET)) return; + if (set == NULL) { + addReplyArrayLen(c, c->argc - 2); + for (int j = 2; j < c->argc; j++) { + addReply(c, shared.czero); + } + return; + } + if (checkType(c, set, OBJ_SET)) return; - addReplyArrayLen(c, c->argc - 2); + size_t count = c->argc - 2; + addReplyArrayLen(c, count); - for (j = 2; j < c->argc; j++) { - if (set && setTypeIsMember(set, objectGetVal(c->argv[j]))) - addReply(c, shared.cone); - else - addReply(c, shared.czero); + /* Prefer hashtable batch lookup to improve performance. */ + if (set->encoding == OBJ_ENCODING_HASHTABLE && count > 1) { + smismemberReplyWithHashtable(c, objectGetVal(set), c->argv + 2, count); + return; + } + + for (size_t i = 0; i < count; i++) { + sismemberReply(c, set, c->argv[i + 2]); } } diff --git a/src/t_stream.c b/src/t_stream.c index af92e6a03..2b66c4a09 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1625,6 +1625,47 @@ void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds decrRefCount(argv[4]); } +/* Propagate the deletion of stream entries as + * + * XDEL ... + * + * XDELEX & XACKDEL propagate their effects manually this way to ensure + * compatibility with any pre-9.2 replicas. */ +static void streamPropagateDelIDs(client *c, robj *key, streamID *ids, int count) { + if (count == 0) return; + + robj **argv = zmalloc(sizeof(robj *) * (2 + count)); + argv[0] = shared.xdel; + argv[1] = key; + for (int j = 0; j < count; j++) argv[2 + j] = createObjectFromStreamID(&ids[j]); + + alsoPropagate(c->db->id, argv, 2 + count, PROPAGATE_AOF | PROPAGATE_REPL, c->slot); + + for (int j = 0; j < count; j++) decrRefCount(argv[2 + j]); + zfree(argv); +} + +/* Propagate acknowledgement of 'count' ids for 'groupname' as + * + * XACK ... + * + * XDELEX & XACKDEL propagate their effects manually this way to ensure + * compatibility with any pre-9.2 replicas. */ +static void streamPropagateAckIDs(client *c, robj *key, robj *groupname, streamID *ids, int count) { + if (count == 0) return; + + robj **argv = zmalloc(sizeof(robj *) * (3 + count)); + argv[0] = shared.xack; + argv[1] = key; + argv[2] = groupname; + for (int j = 0; j < count; j++) argv[3 + j] = createObjectFromStreamID(&ids[j]); + + alsoPropagate(c->db->id, argv, 3 + count, PROPAGATE_AOF | PROPAGATE_REPL, c->slot); + + for (int j = 0; j < count; j++) decrRefCount(argv[3 + j]); + zfree(argv); +} + /* Send the stream items in the specified range to the client 'c'. The range * the client will receive is between start and end inclusive, if 'count' is * non zero, no more than 'count' elements are sent. @@ -2389,6 +2430,7 @@ void xreadCommand(client *c) { if (o == NULL) continue; stream *s = objectGetVal(o); streamID *gt = ids + i; /* ID must be greater than this. */ + int modified_stream = 0; int serve_synchronously = 0; int serve_history = 0; /* True for XREADGROUP with ID != ">". */ streamConsumer *consumer = NULL; /* Unused if XREAD */ @@ -2419,6 +2461,7 @@ void xreadCommand(client *c) { consumer = streamCreateConsumer(groups[i], objectGetVal(consumername), c->argv[streams_arg + i], c->db->id, SCC_DEFAULT); if (noack) streamPropagateConsumerCreation(c, spi.keyname, spi.groupname, consumer->name); + modified_stream = 1; } consumer->seen_time = commandTimeSnapshot(); } else if (s->length) { @@ -2448,9 +2491,14 @@ void xreadCommand(client *c) { int flags = 0; if (noack) flags |= STREAM_RWR_NOACK; if (serve_history) flags |= STREAM_RWR_HISTORY; - streamReplyWithRange(c, s, &start, NULL, count, 0, groups ? groups[i] : NULL, consumer, flags, &spi); - if (groups) server.dirty++; + size_t delivered = streamReplyWithRange(c, s, &start, NULL, count, 0, groups ? groups[i] : NULL, consumer, flags, &spi); + if (groups && delivered > 0) { + server.dirty++; + modified_stream = 1; + } } + + if (modified_stream) signalModifiedKey(c, c->db, c->argv[streams_arg + i]); } /* We replied synchronously! Set the top array len and return to caller. */ @@ -2520,6 +2568,21 @@ void streamFreeNACK(streamNACK *na) { zfree(na); } +/* Delete a pending entry from the group PEL and from the PEL of the consumer + * owning it, freeing the NACK. Returns 1 if entry was pending and was deleted, + * 0 otherwise leaving both group & individual consumer PEL untouched. */ +static int streamDeletePELEntry(rax *pel, streamID *id) { + unsigned char buf[sizeof(streamID)]; + streamEncodeID(buf, id); + void *result; + if (!raxFind(pel, buf, sizeof(buf), &result)) return 0; + streamNACK *nack = result; + raxRemove(pel, buf, sizeof(buf), NULL); + raxRemove(nack->consumer->pel, buf, sizeof(buf), NULL); + streamFreeNACK(nack); + return 1; +} + /* Free a consumer and associated data structures. Note that this function * will not reassign the pending messages associated with this consumer * nor will delete them from the stream, so when this function is called @@ -2735,11 +2798,11 @@ void xgroupCommand(client *c) { o = createStreamObject(); dbAdd(c->db, c->argv[2], &o); s = objectGetVal(o); - signalModifiedKey(c, c->db, c->argv[2]); } streamCG *cg = streamCreateCG(s, grpname, sdslen(grpname), &id, entries_read); if (cg) { + signalModifiedKey(c, c->db, c->argv[2]); server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM, "xgroup-create", c->argv[2], c->db->id); addReply(c, shared.ok); @@ -2755,6 +2818,7 @@ void xgroupCommand(client *c) { } cg->last_id = id; cg->entries_read = entries_read; + signalModifiedKey(c, c->db, c->argv[2]); server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM, "xgroup-setid", c->argv[2], c->db->id); addReply(c, shared.ok); @@ -2762,6 +2826,7 @@ void xgroupCommand(client *c) { if (cg) { raxRemove(s->cgroups, (unsigned char *)grpname, sdslen(grpname), NULL); streamFreeCG(cg); + signalModifiedKey(c, c->db, c->argv[2]); server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM, "xgroup-destroy", c->argv[2], c->db->id); addReply(c, shared.cone); @@ -2772,6 +2837,7 @@ void xgroupCommand(client *c) { } } else if (!strcasecmp(opt, "CREATECONSUMER") && c->argc == 5) { streamConsumer *created = streamCreateConsumer(cg, objectGetVal(c->argv[4]), c->argv[2], c->db->id, SCC_DEFAULT); + if (created) signalModifiedKey(c, c->db, c->argv[2]); addReplyLongLong(c, created ? 1 : 0); } else if (!strcasecmp(opt, "DELCONSUMER") && c->argc == 5) { long long pending = 0; @@ -2781,6 +2847,7 @@ void xgroupCommand(client *c) { * that were yet associated with such a consumer. */ pending = raxSize(consumer->pel); streamDelConsumer(cg, consumer); + signalModifiedKey(c, c->db, c->argv[2]); server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM, "xgroup-delconsumer", c->argv[2], c->db->id); } @@ -2857,6 +2924,7 @@ void xsetidCommand(client *c) { s->last_id = id; if (entries_added != -1) s->entries_added = entries_added; if (!streamIDEqZero(&max_xdel_id)) s->max_deleted_entry_id = max_xdel_id; + signalModifiedKey(c, c->db, c->argv[1]); server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM, "xsetid", c->argv[1], c->db->id); addReply(c, shared.ok); @@ -2904,16 +2972,12 @@ void xackCommand(client *c) { /* Lookup the ID in the group PEL: it will have a reference to the * NACK structure that will have a reference to the consumer, so that * we are able to remove the entry from both PELs. */ - void *result; - if (raxFind(group->pel, buf, sizeof(buf), &result)) { - streamNACK *nack = result; - raxRemove(group->pel, buf, sizeof(buf), NULL); - raxRemove(nack->consumer->pel, buf, sizeof(buf), NULL); - streamFreeNACK(nack); + if (streamDeletePELEntry(group->pel, &ids[j - 3])) { acknowledged++; server.dirty++; } } + if (acknowledged) signalModifiedKey(c, c->db, c->argv[1]); addReplyLongLong(c, acknowledged); cleanup: if (ids != static_ids) zfree(ids); @@ -3171,6 +3235,7 @@ void xclaimCommand(client *c) { mstime_t deliverytime = -1; /* -1 means IDLE/TIME options not given. */ int force = 0; int justid = 0; + int modified = 0; if (o) { if (checkType(c, o, OBJ_STREAM)) return; /* Type error. */ @@ -3267,6 +3332,7 @@ void xclaimCommand(client *c) { streamConsumer *consumer = streamLookupConsumer(group, objectGetVal(c->argv[3])); if (consumer == NULL) { consumer = streamCreateConsumer(group, objectGetVal(c->argv[3]), c->argv[1], c->db->id, SCC_DEFAULT); + modified = 1; } consumer->seen_time = commandTimeSnapshot(); @@ -3290,6 +3356,7 @@ void xclaimCommand(client *c) { streamPropagateXCLAIM(c, c->argv[1], group, c->argv[2], c->argv[j], nack); propagate_last_id = 0; /* Will be propagated by XCLAIM itself. */ server.dirty++; + modified = 1; /* Release the NACK */ raxRemove(group->pel, buf, sizeof(buf), NULL); raxRemove(nack->consumer->pel, buf, sizeof(buf), NULL); @@ -3355,12 +3422,15 @@ void xclaimCommand(client *c) { streamPropagateXCLAIM(c, c->argv[1], group, c->argv[2], c->argv[j], nack); propagate_last_id = 0; /* Will be propagated by XCLAIM itself. */ server.dirty++; + modified = 1; } } if (propagate_last_id) { streamPropagateGroupID(c, c->argv[1], group, c->argv[2]); server.dirty++; + modified = 1; } + if (modified) signalModifiedKey(c, c->db, c->argv[1]); setDeferredArrayLen(c, arraylenptr, arraylen); preventCommandPropagation(c); cleanup: @@ -3392,6 +3462,7 @@ void xautoclaimCommand(client *c) { streamID startid; int startex; int justid = 0; + int modified = 0; /* Parse idle/start/end/count arguments ASAP if needed, in order to report * syntax errors before any other error. */ @@ -3446,6 +3517,7 @@ void xautoclaimCommand(client *c) { streamConsumer *consumer = streamLookupConsumer(group, objectGetVal(c->argv[3])); if (consumer == NULL) { consumer = streamCreateConsumer(group, objectGetVal(c->argv[3]), c->argv[1], c->db->id, SCC_DEFAULT); + modified = 1; } consumer->seen_time = commandTimeSnapshot(); @@ -3476,6 +3548,7 @@ void xautoclaimCommand(client *c) { streamPropagateXCLAIM(c, c->argv[1], group, c->argv[2], idstr, nack); decrRefCount(idstr); server.dirty++; + modified = 1; /* Clear this entry from the PEL, it no longer exists */ raxRemove(group->pel, ri.key, ri.key_len, NULL); raxRemove(nack->consumer->pel, ri.key, ri.key_len, NULL); @@ -3526,8 +3599,11 @@ void xautoclaimCommand(client *c) { streamPropagateXCLAIM(c, c->argv[1], group, c->argv[2], idstr, nack); decrRefCount(idstr); server.dirty++; + modified = 1; } + if (modified) signalModifiedKey(c, c->db, c->argv[1]); + /* We need to return the next entry as a cursor for the next XAUTOCLAIM call */ raxNext(&ri); @@ -3551,48 +3627,294 @@ void xautoclaimCommand(client *c) { preventCommandPropagation(c); } +/* PEL handling modes shared by XDELEX & XACKDEL. */ +typedef enum { + PELMODE_KEEPREF = 0, + PELMODE_DELREF, + PELMODE_ACKED +} streamPELMode; + +/* Command variant for xdelGenericCommand. */ +typedef enum { + XDEL_CMD, /* XDEL ... */ + XDELEX_CMD, /* XDELEX [KEEPREF|DELREF|ACKED] IDS ... */ + XACKDEL_CMD, /* XACKDEL [KEEPREF|DELREF|ACKED] IDS ... */ +} xdelVariant; + /* XDEL [ ... ] + * XDELEX [KEEPREF | DELREF | ACKED] IDS num [ ... ] + * XACKDEL [KEEPREF | DELREF | ACKED] IDS num [ ... ] * - * Removes the specified entries from the stream. Returns the number - * of items actually deleted, that may be different from the number - * of IDs passed in case certain IDs do not exist. */ -void xdelCommand(client *c) { - robj *o; + * Unified implementation of XDEL, XDELEX and XACKDEL. + * + * XDEL removes stream entries unconditionally. + * XDELEX is XDEL with PEL-awareness across all consumer groups. + * XACKDEL is XDELEX scoped to a target consumer group: it acknowledges + * entries in the target group first, then consults remaining groups. */ +static void xdelGenericCommand(client *c, xdelVariant variant) { + bool has_group = (variant == XACKDEL_CMD); + bool has_pelmode = (variant != XDEL_CMD); + bool array_reply = (variant != XDEL_CMD); + + /* --- Argument parsing ------------------------------------------------ */ + streamCG *group = NULL; + streamPELMode mode = PELMODE_KEEPREF; + robj *o = lookupKeyWrite(c->db, c->argv[1]); + int argi = 2; - if ((o = lookupKeyWriteOrReply(c, c->argv[1], shared.czero)) == NULL || checkType(c, o, OBJ_STREAM)) return; + if (o && checkType(c, o, OBJ_STREAM)) return; /* Type error. */ + + if (has_group) { + /* The group name is a positional argument: always consume it, even + * when the key is missing (the lookup simply yields a NULL group). */ + if (o) { + group = streamLookupCG(objectGetVal(o), objectGetVal(c->argv[argi])); + } + argi++; /* past group */ + } + + size_t id_count; + if (!has_pelmode) { + /* XDEL has no IDS token, so the remaining args is the id count. */ + id_count = c->argc - argi; + } else { + /* Parse optional PEL mode: [KEEPREF | DELREF | ACKED] */ + if (strcasecmp(objectGetVal(c->argv[argi]), "KEEPREF") == 0) { + argi++; + } else if (strcasecmp(objectGetVal(c->argv[argi]), "DELREF") == 0) { + argi++; + mode = PELMODE_DELREF; + } else if (strcasecmp(objectGetVal(c->argv[argi]), "ACKED") == 0) { + argi++; + mode = PELMODE_ACKED; + } + + /* Expect IDS token. */ + if (strcasecmp(objectGetVal(c->argv[argi]), "IDS") != 0) { + addReplyErrorObject(c, shared.syntaxerr); + return; + } + argi++; /* past IDS */ + + /* Parse and validate numids: must be a positive integer. */ + long long ll; + if (getLongLongFromObject(c->argv[argi], &ll) != C_OK || ll <= 0) { + addReplyError(c, "Number of IDs must be a positive integer"); + return; + } + argi++; /* past numids */ + + /* Validate numids matches remaining arg count. */ + if (ll != c->argc - argi) { + addReplyErrorObject(c, shared.syntaxerr); + return; + } + id_count = (size_t)ll; + } + + /* --- Missing key / group early exit ---------------------------------- */ + if (o == NULL || (has_group && group == NULL)) { + if (array_reply) { + addReplyArrayLen(c, id_count); + for (size_t i = 0; i < id_count; i++) addReplyLongLong(c, -1); + } else { + addReply(c, shared.czero); + } + return; + } stream *s = objectGetVal(o); + /* --- Allocate working arrays ----------------------------------------- * + * Each variant only declares static buffers for the arrays it actually + * uses. Unused pointers are NULL so accidental access crashes rather + * than silently touching an unrelated stack buffer. For large id_count + * the heap path also skips allocations the variant does not need. */ + streamID static_ids[STREAMID_STATIC_VECTOR_LEN]; + streamID *ids = static_ids; + + streamID static_del_ids[STREAMID_STATIC_VECTOR_LEN]; + streamID *del_ids = static_del_ids; + int del_count = 0; + + int static_resps[STREAMID_STATIC_VECTOR_LEN]; + int *resps = array_reply ? static_resps : NULL; + + unsigned char static_acked_flags[STREAMID_STATIC_VECTOR_LEN]; + unsigned char *acked_flags = has_group ? static_acked_flags : NULL; + + unsigned char static_exists[STREAMID_STATIC_VECTOR_LEN]; + unsigned char *exists = (mode == PELMODE_ACKED) ? static_exists : NULL; + + unsigned char static_cleared[STREAMID_STATIC_VECTOR_LEN]; + unsigned char *cleared = (mode == PELMODE_DELREF || mode == PELMODE_ACKED) ? static_cleared : NULL; + + streamID static_ack_ids[STREAMID_STATIC_VECTOR_LEN]; + streamID *ack_ids = (has_group || mode == PELMODE_DELREF || mode == PELMODE_ACKED) ? static_ack_ids : NULL; + + if (id_count > STREAMID_STATIC_VECTOR_LEN) { + ids = zmalloc(sizeof(streamID) * id_count); + del_ids = zmalloc(sizeof(streamID) * id_count); + if (resps) resps = zmalloc(sizeof(int) * id_count); + if (acked_flags) acked_flags = zmalloc(sizeof(unsigned char) * id_count); + if (exists) exists = zmalloc(sizeof(unsigned char) * id_count); + if (cleared) cleared = zmalloc(sizeof(unsigned char) * id_count); + if (ack_ids) ack_ids = zmalloc(sizeof(streamID) * id_count); + } + /* We need to sanity check the IDs passed to start. Even if not * a big issue, it is not great that the command is only partially * executed because at some point an invalid ID is parsed. */ - streamID static_ids[STREAMID_STATIC_VECTOR_LEN]; - streamID *ids = static_ids; - int id_count = c->argc - 2; - if (id_count > STREAMID_STATIC_VECTOR_LEN) ids = zmalloc(sizeof(streamID) * id_count); - for (int j = 2; j < c->argc; j++) { - if (streamParseStrictIDOrReply(c, c->argv[j], &ids[j - 2], 0, NULL) != C_OK) goto cleanup; + for (size_t j = 0; j < id_count; j++) { + if (streamParseStrictIDOrReply(c, c->argv[argi + j], &ids[j], 0, NULL) != C_OK) goto cleanup; + if (array_reply) resps[j] = 1; } - /* Actually apply the command. */ + int acked = 0; int deleted = 0; - int first_entry = 0; - for (int j = 2; j < c->argc; j++) { - streamID *id = &ids[j - 2]; - if (streamDeleteItem(s, id)) { - /* We want to know if the first entry in the stream was deleted - * so we can later set the new one. */ - if (streamCompareID(id, &s->first_id) == 0) { - first_entry = 1; + bool first_entry = 0; + if (acked_flags) memset(acked_flags, 0, id_count); + + /* --- KEEPREF fast path ----------------------------------------------- * + * When the mode is KEEPREF: with a target group we gate deletion on the + * entry being pending in that group's PEL; without a group we delete + * unconditionally (the original XDEL / XDELEX KEEPREF behaviour). */ + if (mode == PELMODE_KEEPREF) { + for (size_t j = 0; j < id_count; j++) { + streamID *id = &ids[j]; + + if (group) { + /* XACKDEL KEEPREF: only delete if pending in target group. */ + if (!streamDeletePELEntry(group->pel, id)) { + if (array_reply) resps[j] = -1; + continue; + } + acked++; + acked_flags[j] = 1; } - /* Update the stream's maximal tombstone if needed. */ - if (streamCompareID(id, &s->max_deleted_entry_id) > 0) { - s->max_deleted_entry_id = *id; + + if (streamDeleteItem(s, id)) { + deleted++; + del_ids[del_count++] = *id; + if (streamCompareID(id, &s->first_id) == 0) first_entry = 1; + if (streamCompareID(id, &s->max_deleted_entry_id) > 0) s->max_deleted_entry_id = *id; + } else if (array_reply) { + /* Entry does not exist in the stream. */ + resps[j] = -1; } - deleted++; - }; + } + goto sync; + } + + /* --- Phase 1 (XACKDEL only): target-group PEL scan ------------------- * + * If the entry isn't pending in the target group we mark it -1 and skip + * it in Phase 2. XDELEX has no target group so this phase is skipped. */ + if (group) { + for (size_t j = 0; j < id_count; j++) { + if (streamDeletePELEntry(group->pel, &ids[j])) { + acked++; + acked_flags[j] = 1; + /* resps[j] stays 1: eligible for deletion. */ + } else { + resps[j] = -1; + } + } + } + + /* --- Phase 2: iterate consumer groups -------------------------------- * + * XDELEX iterates all groups uniformly; XACKDEL skips the target group + * (handled in Phase 1). The `if (cg == group) continue` naturally + * never fires when group is NULL (XDELEX). */ + if ((mode == PELMODE_DELREF || mode == PELMODE_ACKED) && s->cgroups != NULL) { + memset(cleared, 0, id_count); + + /* Determine stream message existence upfront for ACKED mode. */ + if (mode == PELMODE_ACKED) { + for (size_t j = 0; j < id_count; j++) { + exists[j] = streamEntryExists(s, &ids[j]); + } + } + + raxIterator ri_cgroups; + raxStart(&ri_cgroups, s->cgroups); + raxSeek(&ri_cgroups, "^", NULL, 0); + while (raxNext(&ri_cgroups)) { + streamCG *cg = ri_cgroups.data; + if (cg == group) continue; /* Target group handled in Phase 1. */ + + for (size_t j = 0; j < id_count; j++) { + if (resps[j] != 1) continue; /* Already finalized. */ + if (mode == PELMODE_ACKED && resps[j] == 2) continue; + + streamID *id = &ids[j]; + + if (mode == PELMODE_DELREF) { + if (streamDeletePELEntry(cg->pel, id)) { + acked++; + cleared[j] = 1; + } + } else { + /* ACKED: check PEL before consulting cg->last_id. */ + unsigned char buf[sizeof(streamID)]; + streamEncodeID(buf, id); + void *result; + if (raxFind(cg->pel, buf, sizeof(buf), &result)) { + resps[j] = 2; + } else if (exists[j] && + streamCompareID(id, &cg->last_id) > 0) { + resps[j] = 2; + } + } + } + + if (mode == PELMODE_DELREF) { + int ack_count = 0; + for (size_t j = 0; j < id_count; j++) { + if (cleared[j]) { + ack_ids[ack_count++] = ids[j]; + cleared[j] = 0; + } + } + if (ack_count) { + robj *groupname = createStringObject((char *)ri_cgroups.key, ri_cgroups.key_len); + streamPropagateAckIDs(c, c->argv[1], groupname, ack_ids, ack_count); + decrRefCount(groupname); + } + } + } + raxStop(&ri_cgroups); + + /* ACKED without a target group: entries that don't exist and that no + * group references return "not found". When a target group exists + * Phase 1 already marked non-pending entries as -1. */ + if (mode == PELMODE_ACKED && !group) { + for (size_t j = 0; j < id_count; j++) { + if (resps[j] == 1 && !exists[j]) resps[j] = -1; + } + } + } + + /* --- Deletion phase -------------------------------------------------- * + * Delete entries whose status is still 1 (eligible). */ + for (size_t j = 0; j < id_count; j++) { + if (resps[j] == 1) { + streamID *id = &ids[j]; + if (streamDeleteItem(s, id)) { + deleted++; + del_ids[del_count++] = *id; + if (streamCompareID(id, &s->first_id) == 0) first_entry = 1; + if (streamCompareID(id, &s->max_deleted_entry_id) > 0) s->max_deleted_entry_id = *id; + } else if (!acked_flags || !acked_flags[j]) { + /* Entry doesn't exist and was never pending in the target + * group — genuinely not found. When acked_flags[j] is set + * the target-group PEL was successfully cleared in Phase 1, + * so the entry being already gone is fine (status stays 1). */ + resps[j] = -1; + } + } } - /* Update the stream's first ID. */ +sync: + /* --- Stream bookkeeping & signalling --------------------------------- */ if (deleted) { if (s->length == 0) { s->first_id.ms = 0; @@ -3601,16 +3923,61 @@ void xdelCommand(client *c) { streamGetEdgeID(s, 1, 1, &s->first_id); } } - - /* Propagate the write if needed. */ - if (deleted) { + if (deleted || acked) { signalModifiedKey(c, c->db, c->argv[1]); - notifyKeyspaceEvent(NOTIFY_STREAM, "xdel", c->argv[1], c->db->id); - server.dirty += deleted; + server.dirty += deleted + acked; } - addReplyLongLong(c, deleted); + if (deleted) notifyKeyspaceEvent(NOTIFY_STREAM, "xdel", c->argv[1], c->db->id); + + /* --- Propagation ----------------------------------------------------- * + * XDELEX/XACKDEL are rewritten as XACK + XDEL primitives so that + * pre-9.2 replicas can apply them. XDEL propagates as itself. */ + if (has_pelmode) { + preventCommandPropagation(c); + + /* Target-group acknowledgements (XACKDEL only). */ + if (group) { + int ack_count = 0; + for (size_t j = 0; j < id_count; j++) { + if (acked_flags[j]) ack_ids[ack_count++] = ids[j]; + } + streamPropagateAckIDs(c, c->argv[1], c->argv[2], ack_ids, ack_count); + } + + streamPropagateDelIDs(c, c->argv[1], del_ids, del_count); + } + + /* --- Reply ----------------------------------------------------------- */ + if (array_reply) { + addReplyArrayLen(c, id_count); + for (size_t j = 0; j < id_count; j++) addReplyLongLong(c, resps[j]); + } else { + addReplyLongLong(c, deleted); + } + cleanup: if (ids != static_ids) zfree(ids); + if (resps != static_resps) zfree(resps); + if (acked_flags != static_acked_flags) zfree(acked_flags); + if (exists != static_exists) zfree(exists); + if (cleared != static_cleared) zfree(cleared); + if (ack_ids != static_ack_ids) zfree(ack_ids); + if (del_ids != static_del_ids) zfree(del_ids); +} + +/* XDEL [ ... ] */ +void xdelCommand(client *c) { + xdelGenericCommand(c, XDEL_CMD); +} + +/* XDELEX [KEEPREF | DELREF | ACKED] IDS num [ ... ] */ +void xdelexCommand(client *c) { + xdelGenericCommand(c, XDELEX_CMD); +} + +/* XACKDEL [KEEPREF | DELREF | ACKED] IDS num [ ... ] */ +void xackdelCommand(client *c) { + xdelGenericCommand(c, XACKDEL_CMD); } /* General form: XTRIM [... options ...] @@ -3998,7 +4365,7 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, uint64_t *va unsigned char *p, *next; /* Validate the listpack structure (header + all entries). */ - if (!lpValidateIntegrity(lp, size, NULL, NULL)) return 0; + if (!lpValidateIntegrity(lp, size, NULL, NULL, 0)) return 0; next = p = lpValidateFirst(lp); if (!lpValidateNext(lp, &next, size)) return 0; diff --git a/src/t_string.c b/src/t_string.c index 8d7e1649f..8f9a874f5 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -98,7 +98,7 @@ void setGenericCommand(client *c, robj *existing_value = lookupKeyWrite(c->db, key); found = existing_value != NULL; - /* Handle the IFEQ conditional check */ + /* Handle the IFEQ or IFNE conditional check */ if (flags & ARGS_SET_IFEQ && found) { if (!(flags & ARGS_SET_GET) && checkType(c, existing_value, OBJ_STRING)) { goto cleanup; @@ -115,6 +115,17 @@ void setGenericCommand(client *c, addReply(c, abort_reply ? abort_reply : shared.null[c->resp]); } goto cleanup; + } else if (flags & ARGS_SET_IFNE && found) { + if (!(flags & ARGS_SET_GET) && checkType(c, existing_value, OBJ_STRING)) { + goto cleanup; + } + + if (compareStringObjects(existing_value, comparison) == 0) { + if (!(flags & ARGS_SET_GET)) { + addReply(c, abort_reply ? abort_reply : shared.null[c->resp]); + } + goto cleanup; + } } if ((flags & ARGS_SET_NX && found) || (flags & ARGS_SET_XX && !found)) { @@ -254,7 +265,7 @@ void setCommand(client *c) { int unit = UNIT_SECONDS; int flags = ARGS_NO_FLAGS; - if (parseExtendedCommandArgumentsOrReply(c, COMMAND_SET, 3, c->argc, &flags, &unit, NULL, &expire, &comparison) != C_OK) { + if (parseExtendedCommandArgumentsOrReply(c, COMMAND_SET, 3, c->argc, &flags, &unit, NULL, &expire, &comparison, NULL) != C_OK) { return; } @@ -342,7 +353,7 @@ void getexCommand(client *c) { int unit = UNIT_SECONDS; int flags = ARGS_NO_FLAGS; - if (parseExtendedCommandArgumentsOrReply(c, COMMAND_GET, 2, c->argc, &flags, &unit, NULL, &expire, NULL) != C_OK) { + if (parseExtendedCommandArgumentsOrReply(c, COMMAND_GET, 2, c->argc, &flags, &unit, NULL, &expire, NULL, NULL) != C_OK) { return; } @@ -624,7 +635,7 @@ void msetexCommand(client *c) { return; } if (parseExtendedCommandArgumentsOrReply(c, COMMAND_MSET, (int)args_start_idx, c->argc, - &flags, &unit, &expire_idx, &expire, NULL) != C_OK) { + &flags, &unit, &expire_idx, &expire, NULL, NULL) != C_OK) { return; } @@ -788,6 +799,171 @@ void incrbyfloatCommand(client *c) { rewriteClientCommandArgument(c, 3, shared.keepttl); } +void increxCommand(client *c) { + robj *expire = NULL; + robj *incr_obj = NULL; /* value token for BYINT/BYFLOAT, if present */ + int unit = UNIT_SECONDS; + int flags = ARGS_NO_FLAGS; + long long incr_ll = 1; + long double incr_ld = 1.0L; + int use_float = 0; + + if (parseExtendedCommandArgumentsOrReply(c, COMMAND_INCREX, 2, c->argc, &flags, &unit, NULL, &expire, NULL, &incr_obj) != C_OK) { + return; + } + + long long value_ll = 0, oldvalue_ll = 0, applied_ll = 0; + long double value_ld = 0, oldvalue_ld = 0, applied_ld = 0; + long long milliseconds = 0; + robj *o, *new; + + if (expire && + getExpireMillisecondsOrReply(c, expire, flags, unit, &milliseconds) != C_OK) { + return; + } + + if (flags & ARGS_BYINT) { + if (getLongLongFromObjectOrReply(c, incr_obj, &incr_ll, "Increment is not an integer or out of range") != C_OK) { + return; + } + } else if (flags & ARGS_BYFLOAT) { + if (getLongDoubleFromObjectOrReply(c, incr_obj, &incr_ld, "Increment is not a valid float") != C_OK) { + return; + } + use_float = 1; + } + + o = lookupKeyWrite(c->db, c->argv[1]); + + if (o) { + if (checkType(c, o, OBJ_STRING)) return; + if (use_float) { + if (getLongDoubleFromObjectOrReply(c, o, &oldvalue_ld, NULL) != C_OK) return; + } else { + if (getLongLongFromObjectOrReply(c, o, &oldvalue_ll, NULL) != C_OK) return; + } + } + + if ((flags & ARGS_SET_NX) && o != NULL) { + if (use_float) { + addReplyArrayLen(c, 2); + addReplyHumanLongDouble(c, oldvalue_ld); + addReplyHumanLongDouble(c, 0); + } else { + addReplyArrayLen(c, 2); + addReplyLongLong(c, oldvalue_ll); + addReplyLongLong(c, 0); + } + return; + } + if ((flags & ARGS_SET_XX) && o == NULL) { + /* A non-existent key is treated as zero by the INCR family, and a + * declined operation reports the current value with a zero delta. */ + addReplyArrayLen(c, 2); + if (use_float) { + addReplyHumanLongDouble(c, 0); + addReplyHumanLongDouble(c, 0); + } else { + addReplyLongLong(c, 0); + addReplyLongLong(c, 0); + } + return; + } + + if (use_float) { + if (isinf(incr_ld)) { + addReplyError(c, "BYFLOAT increment cannot be Infinity"); + return; + } + if (isinf(oldvalue_ld)) { + addReplyError(c, "value cannot be Infinity"); + return; + } + value_ld = oldvalue_ld + incr_ld; + if (isnan(value_ld)) { + addReplyError(c, "Increment is not a valid float"); + return; + } + if (isinf(value_ld)) { + addReplyArrayLen(c, 2); + addReplyHumanLongDouble(c, oldvalue_ld); + addReplyHumanLongDouble(c, 0); + return; + } + /* Float accuracy may cause applied to differ from requested. */ + applied_ld = value_ld - oldvalue_ld; + } else { + value_ll = oldvalue_ll; + if ((incr_ll < 0 && value_ll < 0 && incr_ll < (LLONG_MIN - value_ll)) || + (incr_ll > 0 && value_ll > 0 && incr_ll > (LLONG_MAX - value_ll))) { + addReplyArrayLen(c, 2); + addReplyLongLong(c, value_ll); + addReplyLongLong(c, 0); + return; + } + value_ll += incr_ll; + applied_ll = value_ll - oldvalue_ll; + } + + /* If the `milliseconds` have expired, then we don't need to set it into the + * database, and then wait for the active expire to delete it, it is wasteful. + * If the key already exists, delete it. */ + if (expire && checkAlreadyExpired(milliseconds)) { + if (o) deleteExpiredKeyFromOverwriteAndPropagate(c, c->argv[1]); + addReplyArrayLen(c, 2); + if (use_float) { + addReplyHumanLongDouble(c, value_ld); + addReplyHumanLongDouble(c, applied_ld); + } else { + addReplyLongLong(c, value_ll); + addReplyLongLong(c, applied_ll); + } + return; + } + + if (!use_float && o && o->refcount == 1 && objectGetEncoding(o) == OBJ_ENCODING_INT && + value_ll >= LONG_MIN && value_ll <= LONG_MAX) { + new = o; + objectSetVal(o, (void *)((long)value_ll)); + } else { + new = use_float ? createStringObjectFromLongDouble(value_ld, 1) + : createStringObjectFromLongLongForValue(value_ll); + if (o) { + dbReplaceValue(c->db, c->argv[1], &new); + } else { + dbAdd(c->db, c->argv[1], &new); + } + } + + signalModifiedKey(c, c->db, c->argv[1]); + notifyKeyspaceEvent(NOTIFY_STRING, use_float ? "incrbyfloat" : "incrby", c->argv[1], c->db->id); + server.dirty++; + + if (expire) { + new = setExpire(c, c->db, c->argv[1], milliseconds); + robj *milliseconds_obj = createStringObjectFromLongLong(milliseconds); + rewriteClientCommandVector(c, 5, shared.set, c->argv[1], new, shared.pxat, milliseconds_obj); + decrRefCount(milliseconds_obj); + notifyKeyspaceEvent(NOTIFY_GENERIC, "expire", c->argv[1], c->db->id); + } else if (use_float) { + /* BYFLOAT with no expire still needs rewriting to SET for + * deterministic replication - reuse `new`, the exact object + * that was stored, rather than re-deriving the string from + * value_ld a second time (which risks formatting drift + * between what the master stored and what it propagates). */ + rewriteClientCommandVector(c, 4, shared.set, c->argv[1], new, shared.keepttl); + } + + addReplyArrayLen(c, 2); + if (use_float) { + addReplyHumanLongDouble(c, value_ld); + addReplyHumanLongDouble(c, applied_ld); + } else { + addReplyLongLong(c, value_ll); + addReplyLongLong(c, applied_ll); + } +} + void appendCommand(client *c) { size_t totlen; robj *o, *append; diff --git a/src/t_zset.c b/src/t_zset.c index 05b1d1412..1da970cd0 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -3118,35 +3118,91 @@ void zcardCommand(client *c) { addReplyLongLong(c, zsetLength(zobj)); } +/* Adds the member's score as a reply to the client. */ +static void zscoreReply(client *c, robj *zobj, robj *member) { + double score; + + if (zsetScore(zobj, objectGetVal(member), &score) == C_ERR) { + addReplyNull(c); + } else { + addReplyDouble(c, score); + } +} + void zscoreCommand(client *c) { robj *key = c->argv[1]; robj *zobj; - double score; if ((zobj = lookupKeyReadOrReply(c, key, shared.null[c->resp])) == NULL || checkType(c, zobj, OBJ_ZSET)) return; - if (zsetScore(zobj, objectGetVal(c->argv[2]), &score) == C_ERR) { - addReplyNull(c); - } else { - addReplyDouble(c, score); + zscoreReply(c, zobj, c->argv[2]); +} + +#define ZMSCORE_FIND_BATCH_SIZE 16 +static_assert(ZMSCORE_FIND_BATCH_SIZE <= HASHTABLE_FIND_BATCH_MAX_SIZE, + "ZMSCORE batch size exceeds hashtable batch lookup limit"); + +static void zmscoreReplyWithHashtable(client *c, hashtable *ht, robj **members, size_t count) { + const void *keys[ZMSCORE_FIND_BATCH_SIZE]; + void *found_entries[ZMSCORE_FIND_BATCH_SIZE]; + while (count) { + size_t batch = count > ZMSCORE_FIND_BATCH_SIZE ? ZMSCORE_FIND_BATCH_SIZE : count; + + /* The same SDS may appear more than once, so only mark it once. */ + for (size_t i = 0; i < batch; i++) { + sds member = objectGetVal(members[i]); + if (!zsetIsLookupKey(member)) zsetMarkLookupKey(member); + keys[i] = member; + } + + uint32_t result = hashtableFindBatch(ht, (int)batch, keys, found_entries); + + /* Unmark each SDS once; later duplicates are already unmarked. */ + for (size_t i = 0; i < batch; i++) { + sds member = objectGetVal(members[i]); + if (zsetIsLookupKey(member)) zsetUnmarkLookupKey(member); + } + + for (size_t i = 0; i < batch; i++) { + if ((result >> i) & 1) { + OrderedIndexItem *node = found_entries[i]; + addReplyDouble(c, orderedIndexItemGetScore(node)); + } else { + addReplyNull(c); + } + } + + members += batch; + count -= batch; } } void zmscoreCommand(client *c) { robj *key = c->argv[1]; robj *zobj; - double score; - zobj = lookupKeyRead(c->db, key); - if (checkType(c, zobj, OBJ_ZSET)) return; - addReplyArrayLen(c, c->argc - 2); - for (int j = 2; j < c->argc; j++) { - /* Treat a missing set the same way as an empty set */ - if (zobj == NULL || zsetScore(zobj, objectGetVal(c->argv[j]), &score) == C_ERR) { + zobj = lookupKeyRead(c->db, key); + if (zobj == NULL) { + addReplyArrayLen(c, c->argc - 2); + for (int j = 2; j < c->argc; j++) { addReplyNull(c); - } else { - addReplyDouble(c, score); } + return; + } + if (checkType(c, zobj, OBJ_ZSET)) return; + + size_t count = c->argc - 2; + addReplyArrayLen(c, count); + + /* Prefer hashtable batch lookup to improve performance. */ + if (zobj->encoding == OBJ_ENCODING_BTREE && count > 1) { + zset *zs = objectGetVal(zobj); + zmscoreReplyWithHashtable(c, zs->ht, c->argv + 2, count); + return; + } + + for (size_t i = 0; i < count; i++) { + zscoreReply(c, zobj, c->argv[i + 2]); } } diff --git a/src/throttle.c b/src/throttle.c new file mode 100644 index 000000000..20d4b54c6 --- /dev/null +++ b/src/throttle.c @@ -0,0 +1,414 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "server.h" +#include "throttle.h" +#include "throttle_token_bucket.h" +#include "stat_calc.h" +#include "hashtable.h" +#include "monotonic.h" + +#include + +static const int MAX_WAIT_TIME_MS = 100; /* max ms before rescheduling timer */ +static const uint64_t MAX_UNTHROTTLE_PROCESSING_TIME_MS = 10; /* max ms spent unthrottling per timer fire */ +static const double THROTTLE_OPS_PER_SEC_GUARDRAIL = 0.1; /* report when rate stays below this TPS */ +static const int TPS_WINDOW_SEC = 5; /* rolling window for incoming TPS measurement */ +static const double EPSILON = 0.0001; /* values below this are treated as zero */ +static const double TOKENS_BURST_RATE_SEC = 0.1; /* burst capacity in seconds of sustained rate */ +static const double MIN_ADJUST_AFTER_DISABLE = 100.0; /* initial rate when recovering from halted state */ + +static hashtable *metrics_table = NULL; /* Maps throtter type name to metricsEntry*. */ +static list *throttler_list = NULL; /* all currently registered throttlers */ +static long long total_throttled_commands; /* framework-level cumulative throttled-command counter */ + +typedef struct metricsEntry { + sds throttler_type; + int num_clients_throttled; + long long num_commands_throttled; + tpsCalculator *incoming_tps; +} metricsEntry; + +typedef struct throttler { + bool cleanup; /* true once deregistered; freed when its queue drains */ + throttleCriteriaProc *criteria_proc; /* callback defining throttling criteria */ + long long time_event_id; /* timer event id for throttlerTimeProc */ + void *priv_data; /* private data for use by the criteria_proc */ + tokenBucket *bucket; /* token bucket: 1 token = 1 operation */ + list *client_queue; /* clients currently queued for throttling */ + listNode *ln; /* my node in throttler_list */ + monotime rate_below_guardrail_since; /* timestamp when rate dropped below guardrail, or 0 */ + metricsEntry *metrics; /* reference to the named metrics object */ +} throttler; + +/* Metrics hashtable callbacks. */ +static const void *metricsGetKey(const void *entry) { + return ((metricsEntry *)entry)->throttler_type; +} + +static void metricsDestructor(void *entry) { + metricsEntry *m = entry; + sdsfree(m->throttler_type); + tpsCalculator_free(m->incoming_tps); + zfree(m); +} + +static hashtableType metricsHashtableType = { + .entryGetKey = metricsGetKey, + .hashFunction = dictSdsHash, + .keyCompare = dictSdsKeyCompare, + .entryDestructor = metricsDestructor, +}; + +/* dictType for metricsEntry* to earliest below-guardrail monotime, used for + * producing INFO output to report earliest guardrail per metrics type. */ +static dictType guardrailAggDictType = { + .entryGetKey = dictEntryGetKey, + .entryDestructor = zfree, +}; + +static metricsEntry *findMetrics(const char *name) { + sds key = sdsnew(name); + metricsEntry *found; + if (hashtableFind(metrics_table, key, (void **)&found)) { + sdsfree(key); + return found; + } + metricsEntry *m = zcalloc(sizeof(metricsEntry)); + m->throttler_type = key; + m->incoming_tps = tpsCalculator_create(TPS_WINDOW_SEC); + hashtableAdd(metrics_table, m); + return m; +} + +/* Compute how long to wait before the next token becomes available. */ +static int waitTimeMs(throttler *t) { + double ms = tokenBucket_msUntilAvailable(t->bucket, 1.0); + if (ms < 0 || ms >= MAX_WAIT_TIME_MS) return MAX_WAIT_TIME_MS; + return (int)ceil(ms); +} + +/* Release throttler resources. Only called when client queue is fully drained. */ +static void freeThrottler(throttler *t) { + serverAssert(listLength(t->client_queue) == 0); + serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); + serverAssert(t->ln != NULL); + listDelNode(throttler_list, t->ln); + listRelease(t->client_queue); + tokenBucket_free(t->bucket); + /* metrics is shared and do not free here */ + zfree(t); +} + +/* Remove a throttled client from its throttler's queue and clear its throttle state. */ +static void dequeueThrottledClient(client *c) { + serverAssert(c->flag.throttled); + c->flag.throttled = 0; + throttler *t = c->throttler; + serverAssert(t != NULL); + + listDelNode(t->client_queue, c->throttle_node); + t->metrics->num_clients_throttled--; + + c->throttler = NULL; + c->throttle_node = NULL; + c->throttle_start = 0; +} + +static void consumeOtherThrottlers(client *c, throttler *except) { + listNode *ln; + listIter li; + listRewind(throttler_list, &li); + while ((ln = listNext(&li))) { + throttler *t = ln->value; + if (t->cleanup || t == except) continue; + if (t->criteria_proc(c, t->priv_data)) tokenBucket_tryConsume(t->bucket, 1.0, true); + } +} + +/* Timer event handler: releases queued clients at the token bucket rate. + * Processes clients until tokens are exhausted or time budget is spent. */ +static long long throttlerTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData) { + UNUSED(eventLoop); + UNUSED(id); + + throttler *t = (throttler *)clientData; + + monotime work_start; + elapsedStart(&work_start); + + while (listLength(t->client_queue) > 0 && + elapsedMs(work_start) < MAX_UNTHROTTLE_PROCESSING_TIME_MS && + tokenBucket_tryConsume(t->bucket, 1.0, false)) { + client *c = listNodeValue(listFirst(t->client_queue)); + dequeueThrottledClient(c); + if (c->flag.throttle_multi) { + c->flag.throttle_multi = 0; + consumeOtherThrottlers(c, t); + } + serverAssert(c->argc > 0 && c->flag.pending_command && !c->flag.throttled); + queueClientForReprocessing(c); // Read handler will be installed during reprocessing. + } + + if (listLength(t->client_queue) == 0) { + t->time_event_id = AE_DELETED_EVENT_ID; + /* This throttler is drained and ready to be freed. */ + if (t->cleanup) freeThrottler(t); + return AE_NOMORE; + } + return waitTimeMs(t); +} + +static void throttlerAddClient(throttler *t, client *c) { + serverAssert(c->throttler == NULL); + serverAssert(!c->flag.throttled); + elapsedStart(&c->throttle_start); + c->flag.throttled = 1; + listAddNodeTail(t->client_queue, c); + + if (c->conn) connSetReadHandler(c->conn, NULL); + + t->metrics->num_clients_throttled++; + t->metrics->num_commands_throttled++; + total_throttled_commands++; + c->throttler = t; + c->throttle_node = listLast(t->client_queue); + + if (listLength(t->client_queue) == 1) { + serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); + t->time_event_id = aeCreateTimeEvent(server.el, + waitTimeMs(t), + throttlerTimeProc, + t, NULL); + } +} + +/* === Public API === */ + +void throttle_init(void) { + if (throttler_list == NULL) { + throttler_list = listCreate(); + } + if (metrics_table == NULL) { + metrics_table = hashtableCreate(&metricsHashtableType); + } +} + +/* In most cases, each throttler should have its own independent metrics_name. When the same + * throttler is instantiated multiple times (with different priv_data), they may share a single + * metrics object by using the same name. This allows statistics to be aggregated across related + * throttler instances. */ +throttler *throttle_register(throttleCriteriaProc *criteria_proc, + void *priv_data, + const char *metrics_name) { + serverAssert(criteria_proc != NULL); + serverAssert(metrics_name != NULL); + + throttler *t = zmalloc(sizeof(throttler)); + t->cleanup = false; + t->criteria_proc = criteria_proc; + t->time_event_id = AE_DELETED_EVENT_ID; + t->priv_data = priv_data; + t->bucket = tokenBucket_create(THROTTLE_UNLIMITED_RATE, TOKENS_BURST_RATE_SEC); + t->metrics = findMetrics(metrics_name); + t->client_queue = listCreate(); + t->rate_below_guardrail_since = 0; + listAddNodeTail(throttler_list, t); + t->ln = listLast(throttler_list); + throttle_setRate(t, THROTTLE_UNLIMITED_RATE); + serverLog(LL_DEBUG, "Throttler registered: type=%s", t->metrics->throttler_type); + return t; +} + +void throttle_deregister(throttler *t) { + serverAssert(t != NULL); + serverLog(LL_DEBUG, "Throttler deregistered: type=%s", t->metrics->throttler_type); + + if (listLength(t->client_queue) == 0) { + freeThrottler(t); + } else { + t->cleanup = true; + throttle_setRate(t, THROTTLE_UNLIMITED_RATE); + } +} + +void throttle_setRate(throttler *t, double ops_per_sec) { + serverAssert(ops_per_sec >= 0); + + if (ops_per_sec < EPSILON) { + ops_per_sec = 0; + } else if (ops_per_sec > THROTTLE_UNLIMITED_RATE) { + ops_per_sec = THROTTLE_UNLIMITED_RATE; + } + tokenBucket_setRate(t->bucket, ops_per_sec); + + if (ops_per_sec <= THROTTLE_OPS_PER_SEC_GUARDRAIL) { + if (t->rate_below_guardrail_since == 0) { + elapsedStart(&t->rate_below_guardrail_since); + } + } else { + t->rate_below_guardrail_since = 0; + } +} + +double throttle_adjustRate(throttler *t, double multiplier) { + serverAssert(multiplier >= 0.0 && multiplier <= 3.0); + double current = tokenBucket_getRate(t->bucket); + + /* No change needed if already unlimited and trying to increase. */ + if (multiplier >= 1.0 && current == THROTTLE_UNLIMITED_RATE) return current; + + double new_rate; + + if (multiplier < 1.0) { + /* Decrease: apply the multiplier to the current rate. If the result still exceeds the + * measured incoming TPS, reduce it directly to that rate. */ + new_rate = current * multiplier; + double incoming = tpsCalculator_averageTps(t->metrics->incoming_tps); + /* If there is no incoming rate, it's possible that the tps calculator hasn't been populated with + * data yet. Otherwise, if there's actually no incoming traffic, it doesn't matter if the + * rate is adjusted. */ + if (incoming > EPSILON && new_rate > incoming) new_rate = incoming; + } else if (current < EPSILON) { + /* Coming back from halted: jump to a sensible starting rate. */ + new_rate = MIN_ADJUST_AFTER_DISABLE; + } else { + /* Increase: proportional with minimum step of 1 ops/sec. */ + double delta = current * (multiplier - 1.0); + if (delta < 1.0) delta = 1.0; + new_rate = current + delta; + } + + if (new_rate != current) throttle_setRate(t, new_rate); + return tokenBucket_getRate(t->bucket); +} + +void throttle_removeClient(client *c) { + if (!c->flag.throttled) return; + + throttler *t = c->throttler; + dequeueThrottledClient(c); + + if (listLength(t->client_queue) == 0) { + serverAssert(t->time_event_id != AE_DELETED_EVENT_ID); + aeDeleteTimeEvent(server.el, t->time_event_id); + t->time_event_id = AE_DELETED_EVENT_ID; + if (t->cleanup) freeThrottler(t); + } +} + +bool throttle_throttleClientIfNeeded(client *c) { + /* Skip internal clients and clients already checked for this command. + * Prevents re-throttling after unblocking. */ + if (!c->conn || c->flag.throttle_checked) return false; + c->flag.throttle_checked = 1; + + if (throttler_list == NULL || listLength(throttler_list) == 0) return false; + + bool need_throttle = false; + int match_count = 0; + /* Strictest throttler is the applicable throttler with the lowest rate. + * It is the most restrictive throttler the client needs to throttle at. + */ + throttler *strictest = NULL; + listNode *ln; + listIter li; + listRewind(throttler_list, &li); + while ((ln = listNext(&li))) { + throttler *t = ln->value; + if (t->cleanup) continue; + + if (t->criteria_proc(c, t->priv_data)) { + match_count++; + tpsCalculator_record(t->metrics->incoming_tps, 1); + if (strictest == NULL || tokenBucket_getRate(t->bucket) < tokenBucket_getRate(strictest->bucket)) strictest = t; + } + } + + if (strictest != NULL) { + if (listLength(strictest->client_queue) == 0 && + tokenBucket_tryConsume(strictest->bucket, 1.0, false)) { + /* token available, consume and let command proceed. */ + if (match_count > 1) consumeOtherThrottlers(c, strictest); + } else { + /* no token available, defer the command. */ + if (match_count > 1) c->flag.throttle_multi = 1; + throttlerAddClient(strictest, c); + need_throttle = true; + } + } + + return need_throttle; +} + +/* === INFO metrics output === */ +void throttle_getMetrics(const char *metrics_name, throttleMetrics *metrics) { + metricsEntry *m = findMetrics(metrics_name); + + metrics->num_clients_throttled = m->num_clients_throttled; + metrics->num_commands_throttled = m->num_commands_throttled; + metrics->incoming_tps = tpsCalculator_averageTps(m->incoming_tps); + metrics->ops_per_sec = 0.0; + metrics->oldest_client_delay_us = 0; + + /* Aggregate ops_per_sec and oldest_client from all throttlers sharing this metrics. */ + listNode *ln; + listIter li; + listRewind(throttler_list, &li); + while ((ln = listNext(&li))) { + throttler *t = ln->value; + if (t->metrics != m || t->cleanup) continue; + metrics->ops_per_sec += tokenBucket_getRate(t->bucket); + if (listLength(t->client_queue) > 0) { + client *oldest = listNodeValue(listFirst(t->client_queue)); + long delay_us = elapsedUs(oldest->throttle_start); + metrics->oldest_client_delay_us = MAX(metrics->oldest_client_delay_us, delay_us); + } + } +} + +sds throttle_sdscatInfoMetrics(sds info) { + info = sdscatprintf(info, "total_throttled_commands:%lld\r\n", total_throttled_commands); + + /* Report the longest-below-guardrail throttler per metrics type. */ + dict *guardrail_agg = NULL; + listNode *ln; + listIter li; + listRewind(throttler_list, &li); + while ((ln = listNext(&li))) { + throttler *t = ln->value; + if (t->cleanup || t->rate_below_guardrail_since == 0) continue; + + if (guardrail_agg == NULL) guardrail_agg = dictCreate(&guardrailAggDictType); + dictEntry *existing; + dictEntry *de = dictAddRaw(guardrail_agg, t->metrics, &existing); + if (de != NULL) { + /* First seen for this type. */ + dictSetUnsignedIntegerVal(de, t->rate_below_guardrail_since); + } else if (t->rate_below_guardrail_since < dictGetUnsignedIntegerVal(existing)) { + /* Keep the earliest start. */ + dictSetUnsignedIntegerVal(existing, t->rate_below_guardrail_since); + } + } + + if (guardrail_agg != NULL) { + dictIterator it; + dictInitIterator(&it, guardrail_agg); + dictEntry *de; + while ((de = dictNext(&it)) != NULL) { + metricsEntry *m = dictGetKey(de); + int secs = elapsedSec((monotime)dictGetUnsignedIntegerVal(de)); + info = sdscatprintf(info, "throttle_%s_guardrail_secs:%d\r\n", m->throttler_type, secs); + } + dictRelease(guardrail_agg); + } + return info; +} + +long throttle_getGuardrailSecs(throttler *t) { + if (t == NULL || t->rate_below_guardrail_since == 0) return 0; + return (long)elapsedSec(t->rate_below_guardrail_since); +} diff --git a/src/throttle.h b/src/throttle.h new file mode 100644 index 000000000..8876c027c --- /dev/null +++ b/src/throttle.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * A generic client throttling framework using a token bucket algorithm. + * + * Plug-in evaluators register throttlers that control the rate at which client commands are + * processed. When a client's command matches a throttler's criteria, the client is queued and + * its commands are released at the configured rate. + * + * Design: + * Multiple throttlers can be registered simultaneously. When a client matches more than one, + * the most restrictive rate applies. Throttled clients have their read handler removed and + * are released via timer events at the configured rate. Throttling occurs in processCommand() + * before command execution. Once throttled, the client's command is deferred until tokens become + * available. + */ + +#ifndef THROTTLE_H +#define THROTTLE_H + +#include "sds.h" +#include +typedef struct client client; +typedef struct throttler throttler; + +static const double THROTTLE_UNLIMITED_RATE = 10000000.0; + +/* A throttleCriteriaProc checks a client's current command and decides if it meets the criteria + * for throttling. Returns true if the client meets the throttling criteria. + * + * The criteria proc should base decisions only on the state of the client, not considering + * the question of the current requirements for throttling. If this returns true, the client + * MIGHT be throttled. + * + * priv_data - a private data structure provided during throttle_register. It can provide + * anything needed by the criteria proc, or NULL if unneeded. */ +typedef bool throttleCriteriaProc(client *c, void *priv_data); + +/* Metrics for a throttler or group of related throttlers. The metrics name allows the metrics to + * persist even after the throttler(s) is deregistered. Metrics collection will continue (under the + * same name) if/when the throttler is registered again. + * + * Note: Multiple related throttlers can share the same metrics by using the same metrics_name. + * A typical use case is multiple instantiations of the same throttler with different private data. */ +typedef struct { + int num_clients_throttled; /* the backlog of currently throttled (queued) clients */ + long long num_commands_throttled; /* total number of commands throttled through this metrics group */ + double ops_per_sec; /* the current throttling rate (summed across related throttlers) */ + double incoming_tps; /* average incoming TPS over a 5-second rolling window */ + long oldest_client_delay_us; /* delay in microseconds for the oldest throttled client */ +} throttleMetrics; + +/* Initialize the throttling framework. Must be called once at startup before any + * throttler is registered. Idempotent: safe to call more than once. */ +void throttle_init(void); + +/* Register a new throttler. + * criteria_proc - identifies clients whose commands meet the criteria for throttling + * priv_data - private data for passing to the criteria_proc (may be NULL) + * metrics_name - a string used to identify a shared metrics group + * + * Returns the registered throttler. */ +throttler *throttle_register(throttleCriteriaProc *criteria_proc, + void *priv_data, + const char *metrics_name); + +/* Deregisters the throttler such that: + * - No new clients will be throttled by this throttler. + * - Existing queued clients will be drained at unlimited rate until the queue is empty. */ +void throttle_deregister(throttler *t); + +/* Set the absolute throttling rate for the given throttler. + * ops_per_sec - target rate in operations per second (must be >= 0) + * + * The rate is clamped: values below EPSILON are treated as 0, + * and values above THROTTLE_UNLIMITED_RATE are capped at that ceiling. */ +void throttle_setRate(throttler *t, double ops_per_sec); + +/* A smart adjustment to the throttling rate. The multiplier is applied to the current rate, + * with consideration for the actual incoming traffic rate. + * multiplier - applied to current rate to determine new rate (range 0.0 .. 3.0) + * + * If multiplier >= 1.0: increase the rate. If currently halted (rate ~0), jump to a + * starting rate; otherwise, increase proportionally with a minimum + * step of 1 ops/sec. + * If multiplier < 1.0: decrease the rate proportionally. If the rate is far above the current + * incoming rate, immediately adjusts down to the incoming rate. + * + * Returns the actual rate set after clamping and adjustment. + * + * Usage guidance: + * 1. Size each step to your call frequency: the more often you call this, the smaller + * each step should be. The driving metrics are smoothed and update slowly, so a large + * step applied at high frequency overshoots and causes hysteresis. + * 2. Prefer a small constant step, as a constant multiplicative step already tapers in + * absolute terms as the rate nears the target. */ +double throttle_adjustRate(throttler *t, double multiplier); + +/* Removes the client from the throttle queue. */ +void throttle_removeClient(client *c); + +/* Check if the client's current command should be throttled. Called at the beginning of + * processCommand(). If any registered throttler's criteria matches, the client is queued + * and the most restrictive throttle rate applies. + * + * Returns true if the client has been throttled. + * Returns false if the client may proceed normally. + * + * Note: Even if a client matches throttling criteria, it might not be queued if tokens + * are available. Throttling is checked before blocking, so a throttled + * command cannot be blocked. Once a client is passed to this function, it will not be + * throttled again for the same command after unblocking. */ +bool throttle_throttleClientIfNeeded(client *c); + +/* Get the metrics associated with a given metrics name. + * The caller provides the metrics structure. */ +void throttle_getMetrics(const char *metrics_name, throttleMetrics *metrics); + +/* Append framework-level throttle metrics to the INFO output string. + * Plug-in specific metrics are reported by their own sdscatInfoMetrics functions. */ +sds throttle_sdscatInfoMetrics(sds info); + +/* Get the number of seconds the throttler's rate has been below the guardrail. + * Returns 0 if the rate is above the guardrail or the throttler is not active. */ +long throttle_getGuardrailSecs(throttler *t); + +#endif diff --git a/src/throttle_repl.c b/src/throttle_repl.c new file mode 100644 index 000000000..92db4d2cf --- /dev/null +++ b/src/throttle_repl.c @@ -0,0 +1,217 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "server.h" +#include "throttle_repl.h" +#include "throttle.h" +#include "stat_calc.h" + +/* Configuration instance. */ +struct throttleReplConfig throttleRepl_config; + +/* A 2-second window gives 20 data points at 100ms serverCron. Sufficient for a good + * measurement, while remaining short enough for throttling adjustments every 100ms. */ +static const int COB_TREND_WINDOW_SECS = 2; +static const double RATE_INCREASE_MULTIPLIER = 1.05; +static const double RATE_DECREASE_MULTIPLIER = 0.95; +static const int STEADY_STATE_CONVERGENCE_SECS = 30; /* projection horizon for COB extrapolation */ +static const char *const METRICS_NAME = "repl_throttle"; /* shared metrics group name */ + +/* Metrics for INFO output and operational visibility. */ +typedef struct { + bool is_throttler_active; + double current_throttle_rate; /* only valid if throttler is active */ + unsigned long throttle_activation_events; /* cumulative times throttler has been activated */ + unsigned long throttle_more_events; /* cumulative times we throttled more */ + unsigned long throttle_less_events; /* cumulative times we throttled less */ +} throttleReplMetrics; + +static throttleReplMetrics metrics = {0}; +static throttler *repl_throttler = NULL; + +/* --- Internal helpers --- */ + +static bool isThrottlerActive(void) { + return (repl_throttler != NULL); +} + +/* Criteria: throttle commands that generate replication traffic. */ +static bool criteriaProc(client *c, void *priv_data) { + UNUSED(priv_data); + if (c->cmd->flags & (CMD_WRITE | CMD_MAY_REPLICATE)) return true; + return false; +} + +static void installThrottler(void) { + serverAssert(!isThrottlerActive()); + repl_throttler = throttle_register(criteriaProc, NULL, METRICS_NAME); + serverAssert(repl_throttler != NULL); + metrics.is_throttler_active = true; + metrics.current_throttle_rate = THROTTLE_UNLIMITED_RATE; + metrics.throttle_activation_events++; +} + +static void uninstallThrottler(void) { + serverAssert(isThrottlerActive()); + throttle_deregister(repl_throttler); + repl_throttler = NULL; + metrics.is_throttler_active = false; + metrics.current_throttle_rate = THROTTLE_UNLIMITED_RATE; +} + +/* Apply a rate change based on the evaluator's decision. Installs the throttler on first + * reduce request and removes it when rate reaches UNLIMITED. */ +static void adjustThrottleRate(bool reduce_traffic_rate) { + if (isThrottlerActive()) { + double rate; + if (reduce_traffic_rate) { + rate = throttle_adjustRate(repl_throttler, RATE_DECREASE_MULTIPLIER); + metrics.throttle_more_events++; + } else { + rate = throttle_adjustRate(repl_throttler, RATE_INCREASE_MULTIPLIER); + metrics.throttle_less_events++; + if (rate >= THROTTLE_UNLIMITED_RATE) uninstallThrottler(); + } + metrics.current_throttle_rate = rate; + } else { + /* Installing the throttler starts measurement of current traffic rate. + * Once the measurement is stable, rate adjustments will be meaningful. */ + if (reduce_traffic_rate) installThrottler(); + } +} + +static int64_t getReplicaSteadyStateCobTargetSize(void) { + int64_t limit = server.client_obuf_limits[CLIENT_TYPE_REPLICA].soft_limit_bytes; + if (limit == 0) limit = server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes; + + int64_t cob_target = limit / 2; /* Target is half the limit. */ + + return cob_target; +} + +/* Steady-state throttling targets the replica with the largest COB to ensure all replicas + * maintain sync. Throttling begins at 25% of the configured soft limit (half the target COB size). + * The short-term COB trend is used to project when COB will intersect the target within the + * convergence window. This will result in a convergence to the desired target, rather than + * overshooting the target. */ +static bool evaluateSteadyStateThrottle(client *c, int64_t cob_size) { + int64_t cob_target = getReplicaSteadyStateCobTargetSize(); + int64_t throttle_threshold = cob_target / 2; + + if (cob_size < throttle_threshold) return false; + + /* Using the full window for COB trend shows greater hysteresis than using only the final + * datapoints. The short trend results in more jittery rate adjustments, but this is good + * as the up/down/up/down... type adjustments result in a smoother traffic rate than + * up/up/up/down/down/down... */ + double short_trend = trendCalculator_changePerSecShortTerm(c->cob_trend); + int64_t extrapolated = cob_size + (int64_t)(short_trend * STEADY_STATE_CONVERGENCE_SECS); + + return (extrapolated > cob_target); +} + +/* --- Public API --- */ + +/* Determines whether a replica should be temporarily exempted from the soft client output + * buffer limit. While the steady-state throttle is converging, exempting the soft limit + * prevents a premature disconnect and allows the throttler to reduce the replica's buffer + * back below target. */ +bool throttleRepl_isClientExemptFromCobLimits(client *c) { + if (!throttleRepl_config.repl_throttling_enabled || !isThrottlerActive()) return false; + if (!iAmPrimary()) return false; + if (getClientType(c) != CLIENT_TYPE_REPLICA) return false; + + /* Throttle is actively working, protect this replica from COB + * disconnect if its COB is above target. */ + int64_t client_cob_size = (int64_t)getClientOutputBufferMemoryUsage(c); + /* There's no need to protect the replica if it's already using less than the target size. */ + if (client_cob_size < getReplicaSteadyStateCobTargetSize()) return false; + + /* Don't exempt if the server is over maxmemory. + * When eviction is already running, we can't afford to let replica output buffers grow further. */ + if (server.maxmemory && getMaxmemoryState(NULL, NULL, NULL, NULL) == C_ERR) return false; + + /* Don't protect if throttle has been working too long without success. */ + time_t elapsed = server.unixtime - c->obuf_soft_limit_reached_time; + if (elapsed > 4 * STEADY_STATE_CONVERGENCE_SECS) return false; + /* Otherwise, allow the replica to exceed the soft limit, giving the throttler time to correct. */ + return true; +} + +/* Called from serverCron every 100ms. Evaluates the replica with the largest COB and + * adjusts throttling as needed. */ +void throttleRepl_adjustThrottling(void) { + /* Tear down and stop if we're no longer the primary (e.g. after failover), replication + * throttling was disabled, no COB limit is configured, or the last replica disconnected. */ + if (!iAmPrimary() || !throttleRepl_config.repl_throttling_enabled || + getReplicaSteadyStateCobTargetSize() <= 0 || listLength(server.replicas) == 0) { + if (isThrottlerActive()) uninstallThrottler(); + return; + } + + bool reduce_traffic_rate = false; + client *measured_steady_state_replica = NULL; + uint64_t largest_steady_state_cob = 0; + + /* Scan replicas, find steady-state replica with largest COB. */ + listIter li; + listNode *ln; + listRewind(server.replicas, &li); + while ((ln = listNext(&li)) != NULL) { + client *c = ln->value; + if (!c->repl_data || c->repl_data->repl_state != REPLICA_STATE_ONLINE) continue; + + unsigned long cob_size = getClientOutputBufferMemoryUsage(c); + + if (c->cob_trend == NULL) c->cob_trend = trendCalculator_create(COB_TREND_WINDOW_SECS); + trendCalculator_recordMetric(c->cob_trend, cob_size); + + /* The COB size contains some overhead. Treat it as zero until we reach a minimum. */ + if (cob_size <= PROTO_REPLY_CHUNK_BYTES) cob_size = 0; + + if (measured_steady_state_replica == NULL || cob_size > largest_steady_state_cob) { + measured_steady_state_replica = c; + largest_steady_state_cob = cob_size; + } + } + + if (measured_steady_state_replica != NULL) { + reduce_traffic_rate = evaluateSteadyStateThrottle(measured_steady_state_replica, largest_steady_state_cob); + } + + adjustThrottleRate(reduce_traffic_rate); +} + +sds throttleRepl_sdscatInfoMetrics(sds info) { + throttleMetrics throttle_metrics; + throttle_getMetrics(METRICS_NAME, &throttle_metrics); + info = sdscatprintf(info, + "repl_throttle_rate:%.2f\r\n" + "repl_throttle_activation_events:%lu\r\n" + "repl_throttle_below_guardrail_secs:%ld\r\n" + "repl_throttle_total_commands:%lld\r\n", + metrics.is_throttler_active ? metrics.current_throttle_rate : -1.0, + metrics.throttle_activation_events, + isThrottlerActive() ? throttle_getGuardrailSecs(repl_throttler) : 0L, + throttle_metrics.num_commands_throttled); + + return info; +} + +/* Verbose debug metrics. */ +sds throttleRepl_sdscatInfoDebugMetrics(sds info) { + throttleMetrics throttle_metrics; + throttle_getMetrics(METRICS_NAME, &throttle_metrics); + info = sdscatprintf(info, + "repl_throttle_more_events:%lu\r\n" + "repl_throttle_less_events:%lu\r\n" + "repl_throttle_current_clients:%d\r\n", + metrics.throttle_more_events, + metrics.throttle_less_events, + throttle_metrics.num_clients_throttled); + + return info; +} diff --git a/src/throttle_repl.h b/src/throttle_repl.h new file mode 100644 index 000000000..5c96fb9a1 --- /dev/null +++ b/src/throttle_repl.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * A replication throttler plug-in for the generic throttler (throttle.h). + * + * Throttles client traffic on the primary to establish and maintain healthy replica + * connections. It monitors replica COB (Client Output Buffer) growth and reduces the + * command processing rate when needed. + * + * Steady-state evaluator (normal replication): + * Throttles when the projected COB exceeds the target within the convergence window. + * Rate increases automatically once COB stabilizes. + */ + +#ifndef THROTTLE_REPL_H +#define THROTTLE_REPL_H + +#include "sds.h" +struct throttleReplConfig { + int repl_throttling_enabled; +}; +extern struct throttleReplConfig throttleRepl_config; + +/* Returns true if the client should be exempt from COB disconnect limits because throttling + * is actively working to stabilize the replica. */ +bool throttleRepl_isClientExemptFromCobLimits(client *c); + +/* Determine throttling needs and adjust rate. Called from serverCron every 100ms. */ +void throttleRepl_adjustThrottling(void); + +/* Append replication throttle metrics to the INFO output string. */ +sds throttleRepl_sdscatInfoMetrics(sds info); + +/* Append verbose debug replication throttle metrics to the INFO output string. */ +sds throttleRepl_sdscatInfoDebugMetrics(sds info); + +#endif diff --git a/src/throttle_token_bucket.c b/src/throttle_token_bucket.c new file mode 100644 index 000000000..afa38077c --- /dev/null +++ b/src/throttle_token_bucket.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "throttle_token_bucket.h" +#include "monotonic.h" +#include "zmalloc.h" + +struct tokenBucket { + double tokens_per_sec; // Rate at which tokens are added to the bucket (tokens per second) + double max_burst_time_secs; // Maximum time for which tokens can accumulate in the bucket + double token_count; // Current number of tokens in the bucket (can be negative if force-consumed) + monotime last_time_check; // Last time the bucket was replenished (in microseconds) +}; + +#define BUCKET_EPSILON 0.0001 + +/* Bucket capacity scales with rate: higher rates allow larger bursts. + * The +2 guarantees the bucket can always hold at least 2 tokens, preventing + * permanent starvation at very low rates where rate * burst_time < 1. + * Returns 0 when rate is effectively zero. */ +static double getBucketSize(tokenBucket *bucket) { + return (bucket->tokens_per_sec < BUCKET_EPSILON) ? 0.0 + : 2.0 + bucket->tokens_per_sec * bucket->max_burst_time_secs; +} + +/* Clamp token count to valid range [-bucket_size, bucket_size]. */ +static void trimTokenBucket(tokenBucket *bucket) { + double bucket_size = getBucketSize(bucket); + if (bucket->token_count > bucket_size) bucket->token_count = bucket_size; + if (bucket->token_count < -bucket_size) bucket->token_count = -bucket_size; +} + +static void replenishTokenBucket(tokenBucket *bucket) { + monotime now = getMonotonicUs(); + uint64_t delta_us = now - bucket->last_time_check; + double tokens_to_add = delta_us * bucket->tokens_per_sec / 1000000.0; + bucket->token_count += tokens_to_add; + trimTokenBucket(bucket); + bucket->last_time_check = now; +} + +tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs) { + tokenBucket *bucket = zmalloc(sizeof(tokenBucket)); + bucket->tokens_per_sec = tokens_per_sec; + bucket->max_burst_time_secs = max_burst_time_secs; + bucket->token_count = getBucketSize(bucket); + bucket->last_time_check = getMonotonicUs(); + return bucket; +} + +void tokenBucket_free(tokenBucket *bucket) { + zfree(bucket); +} + +double tokenBucket_getRate(tokenBucket *bucket) { + return bucket->tokens_per_sec; +} + +void tokenBucket_setRate(tokenBucket *bucket, double new_rate) { + replenishTokenBucket(bucket); + bucket->tokens_per_sec = new_rate; + trimTokenBucket(bucket); +} + +bool tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consume) { + replenishTokenBucket(bucket); + if (!force_consume && bucket->token_count < tokens) return false; + bucket->token_count -= tokens; + trimTokenBucket(bucket); /* bound debt at -bucket_size so recovery time stays bounded */ + return true; +} + +double tokenBucket_msUntilAvailable(tokenBucket *bucket, double target_tokens) { + replenishTokenBucket(bucket); + if (bucket->token_count >= target_tokens) return 0.0; + /* Rates below BUCKET_EPSILON give zero capacity, so tokens never accumulate. */ + if (bucket->tokens_per_sec < BUCKET_EPSILON) return -1.0; /* halted -- never available */ + double needed = target_tokens - bucket->token_count; + return needed * 1000.0 / bucket->tokens_per_sec; +} diff --git a/src/throttle_token_bucket.h b/src/throttle_token_bucket.h new file mode 100644 index 000000000..dd1bee8e8 --- /dev/null +++ b/src/throttle_token_bucket.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * The Token Bucket Algorithm is a traffic control method where tokens are added to a bucket at a fixed rate (up to a + * maximum capacity), and tokens can be requested from the bucket as needed (if tokens are available). + * + * Terminology: + * Token: A permission unit required to perform some metered work; the caller will only perform the work if tokens are available. + * Bucket: A logical storage that holds tokens until they are used. + * + * Working: + * 1. Tokens are added to the bucket at a constant rate and stored up to the maximum capacity. + * 2. When a caller needs to perform work, an attempt is made to get one or more tokens. + * 3. If enough tokens are available, the required number of tokens is removed from the bucket, and the caller may proceed with the intended work. + * 4. If tokens are unavailable, the caller must wait until sufficient tokens are available. + */ + +#ifndef THROTTLE_TOKEN_BUCKET_H +#define THROTTLE_TOKEN_BUCKET_H + +#include + +typedef struct tokenBucket tokenBucket; + +/* Create a token bucket that starts full. + * max_burst_time_secs controls how many seconds of idle accumulation are + * allowed before the bucket is considered full. A larger value permits + * bigger bursts after idle periods. */ +tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs); + +/* Free a token bucket and its resources. */ +void tokenBucket_free(tokenBucket *bucket); + +/* Return the current refill rate in tokens per second. */ +double tokenBucket_getRate(tokenBucket *bucket); + +/* Update the refill rate. Tokens are clamped to the new bucket capacity. */ +void tokenBucket_setRate(tokenBucket *bucket, double new_rate); + +/* Attempt to consume tokens. Returns true if tokens were deducted. + * force_consume=false: only deducts if enough tokens are available. + * force_consume=true: always deducts (may drive count negative). */ +bool tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consume); + +/* Estimate milliseconds until the requested tokens become available. + * Returns 0 if already available, or -1 if rate is 0 (never reached). */ +double tokenBucket_msUntilAvailable(tokenBucket *bucket, double tokens); + +#endif diff --git a/src/tls.c b/src/tls.c index e443ce4d0..6e5797c7c 100644 --- a/src/tls.c +++ b/src/tls.c @@ -40,6 +40,7 @@ ((USE_OPENSSL == 2 /* BUILD_MODULE */) && \ (defined(BUILD_TLS_MODULE) && BUILD_TLS_MODULE == 2))) +#include #include #include #include @@ -56,7 +57,6 @@ #include #include #include -#include #include #include @@ -399,10 +399,21 @@ static int tlsUpdateCertInfoFromDir(const char *path, long long *expiry, sds *se } static void tlsRefreshServerCertInfo(void) { + /* Cycle through both certificates to get the correct info for each */ if (!(server.tls_port || server.tls_replication || server.tls_cluster) || !valkey_tls_ctx || + SSL_CTX_set_current_cert(valkey_tls_ctx, SSL_CERT_SET_FIRST) != 1 || tlsUpdateCertInfoFromCtx(valkey_tls_ctx, &server.tls_server_cert_expire_time, &server.tls_server_cert_serial) == C_ERR) { tlsClearCertInfo(&server.tls_server_cert_expire_time, &server.tls_server_cert_serial); } + if (SSL_CTX_set_current_cert(valkey_tls_ctx, SSL_CERT_SET_NEXT) != 1 || + tlsUpdateCertInfoFromCtx(valkey_tls_ctx, &server.tls_server_alt_cert_expire_time, &server.tls_server_alt_cert_serial) == C_ERR) { + tlsClearCertInfo(&server.tls_server_alt_cert_expire_time, &server.tls_server_alt_cert_serial); + } + if (SSL_CTX_set_current_cert(valkey_tls_ctx, SSL_CERT_SET_FIRST) != 1) { + serverLog(LL_WARNING, "Certificate unset during refresh, clearing all server certificate info"); + tlsClearCertInfo(&server.tls_server_cert_expire_time, &server.tls_server_cert_serial); + tlsClearCertInfo(&server.tls_server_alt_cert_expire_time, &server.tls_server_alt_cert_serial); + } } static void tlsRefreshClientCertInfo(void) { @@ -509,6 +520,7 @@ static bool loadCaCertDir(SSL_CTX *ctx, const char *ca_cert_dir) { return false; } + int loaded = 0; while ((entry = readdir(dir)) != NULL) { if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue; @@ -531,10 +543,17 @@ static bool loadCaCertDir(SSL_CTX *ctx, const char *ca_cert_dir) { ERR_clear_error(); } X509_free(cert); + loaded++; } } closedir(dir); + + if (loaded == 0) { + serverLog(LL_WARNING, "No CA certificates loaded from directory: %s", ca_cert_dir); + return false; + } + return true; } @@ -564,9 +583,14 @@ static SSL_CTX *createSSLContext(serverTLSContextConfig *ctx_config, int protoco const char *cert_file = client ? ctx_config->client_cert_file : ctx_config->cert_file; const char *key_file = client ? ctx_config->client_key_file : ctx_config->key_file; const char *key_file_pass = client ? ctx_config->client_key_file_pass : ctx_config->key_file_pass; + + const char *alt_cert_file = client ? NULL : ctx_config->alt_cert_file; + const char *alt_key_file = client ? NULL : ctx_config->alt_key_file; + const char *alt_key_file_pass = client ? NULL : ctx_config->alt_key_file_pass; char errbuf[256]; SSL_CTX *ctx = NULL; - + EVP_PKEY *primary_pkey = NULL; + EVP_PKEY *alt_pkey = NULL; ctx = SSL_CTX_new(SSLv23_method()); if (!ctx) goto error; @@ -606,11 +630,49 @@ static SSL_CTX *createSSLContext(serverTLSContextConfig *ctx_config, int protoco goto error; } + if (alt_cert_file) { + primary_pkey = X509_get_pubkey(SSL_CTX_get0_certificate(ctx)); + if (!primary_pkey) { + serverLog(LL_WARNING, "Could not get public key from primary certificate"); + goto error; + } + + if (SSL_CTX_use_certificate_chain_file(ctx, alt_cert_file) <= 0) { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + serverLog(LL_WARNING, "Failed to load certificate: %s: %s", alt_cert_file, errbuf); + goto error; + } + + if (!isCertValid(SSL_CTX_get0_certificate(ctx))) { + serverLog(LL_WARNING, "Alternate server TLS certificate is invalid. Aborting TLS configuration."); + goto error; + } + + alt_pkey = X509_get_pubkey(SSL_CTX_get0_certificate(ctx)); + if (!alt_pkey) { + serverLog(LL_WARNING, "Could not get public key from alternate certificate"); + goto error; + } + + if (EVP_PKEY_base_id(primary_pkey) == EVP_PKEY_base_id(alt_pkey)) { + serverLog(LL_WARNING, "Primary and alternate certificates must use different key algorithms"); + goto error; + } + } + if (SSL_CTX_use_PrivateKey_file(ctx, key_file, SSL_FILETYPE_PEM) <= 0) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); serverLog(LL_WARNING, "Failed to load private key: %s: %s", key_file, errbuf); goto error; } + if (alt_key_file) { + SSL_CTX_set_default_passwd_cb_userdata(ctx, (void *)alt_key_file_pass); + if (SSL_CTX_use_PrivateKey_file(ctx, alt_key_file, SSL_FILETYPE_PEM) <= 0) { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + serverLog(LL_WARNING, "Failed to load private key: %s: %s", alt_key_file, errbuf); + goto error; + } + } if (ctx_config->ca_cert_file || ctx_config->ca_cert_dir) { if (SSL_CTX_load_verify_locations(ctx, ctx_config->ca_cert_file, ctx_config->ca_cert_dir) <= 0) { @@ -642,9 +704,13 @@ static SSL_CTX *createSSLContext(serverTLSContextConfig *ctx_config, int protoco } #endif + EVP_PKEY_free(primary_pkey); + EVP_PKEY_free(alt_pkey); return ctx; error: + EVP_PKEY_free(primary_pkey); + EVP_PKEY_free(alt_pkey); if (ctx) SSL_CTX_free(ctx); return NULL; } @@ -669,6 +735,16 @@ static int tlsCreateContexts(serverTLSContextConfig *ctx_config, SSL_CTX **out_c goto error; } + if (ctx_config->alt_cert_file && !ctx_config->alt_key_file) { + serverLog(LL_WARNING, "tls-alt-cert-file provided without a key"); + goto error; + } + + if (ctx_config->alt_key_file && !ctx_config->alt_cert_file) { + serverLog(LL_WARNING, "tls-alt-key-file provided without a certificate"); + goto error; + } + if (((server.tls_auth_clients != TLS_CLIENT_AUTH_NO) || server.tls_cluster || server.tls_replication) && !ctx_config->ca_cert_file && !ctx_config->ca_cert_dir) { serverLog(LL_WARNING, "Either tls-ca-cert-file or tls-ca-cert-dir must be specified when tls-cluster, " @@ -781,6 +857,8 @@ static int tlsCreateContexts(serverTLSContextConfig *ctx_config, SSL_CTX **out_c typedef struct { unsigned char cert_fingerprint[EVP_MAX_MD_SIZE]; unsigned int cert_fingerprint_len; + unsigned char alt_cert_fingerprint[EVP_MAX_MD_SIZE]; + unsigned int alt_cert_fingerprint_len; unsigned char client_cert_fingerprint[EVP_MAX_MD_SIZE]; unsigned int client_cert_fingerprint_len; unsigned char ca_cert_fingerprint[EVP_MAX_MD_SIZE]; @@ -838,6 +916,7 @@ static void captureMetadata(serverTLSContextConfig *ctx_config, tlsMaterialsMeta /* Certificate files: fingerprint-based detection */ getCertFingerprint(ctx_config->cert_file, metadata->cert_fingerprint, &metadata->cert_fingerprint_len); + getCertFingerprint(ctx_config->alt_cert_file, metadata->alt_cert_fingerprint, &metadata->alt_cert_fingerprint_len); getCertFingerprint(ctx_config->client_cert_file, metadata->client_cert_fingerprint, &metadata->client_cert_fingerprint_len); getCertFingerprint(ctx_config->ca_cert_file, metadata->ca_cert_fingerprint, &metadata->ca_cert_fingerprint_len); @@ -865,6 +944,11 @@ static int metadataChanged(const tlsMaterialsMetadata *old, const tlsMaterialsMe return 1; } + if (old->alt_cert_fingerprint_len != new->alt_cert_fingerprint_len || + (new->alt_cert_fingerprint_len > 0 && memcmp(old->alt_cert_fingerprint, new->alt_cert_fingerprint, new->alt_cert_fingerprint_len) != 0)) { + return 1; + } + if (old->client_cert_fingerprint_len != new->client_cert_fingerprint_len || (new->client_cert_fingerprint_len > 0 && memcmp(old->client_cert_fingerprint, new->client_cert_fingerprint, new->client_cert_fingerprint_len) != 0)) { return 1; @@ -1198,14 +1282,16 @@ static int updateStateAfterSSLIO(tls_connection *conn, int ret_value, int update } static void registerSSLEvent(tls_connection *conn) { + int priority_flag = connGetAEPriorityFlag(&conn->c); int mask = aeGetFileEvents(server.el, conn->c.fd); + bool priority_changed = ((mask & AE_HIGH_PRIORITY) != (priority_flag & AE_HIGH_PRIORITY)); if (conn->flags & TLS_CONN_FLAG_WRITE_WANT_READ) { if (mask & AE_WRITABLE) aeDeleteFileEvent(server.el, conn->c.fd, AE_WRITABLE); - if (!(mask & AE_READABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE, tlsEventHandler, conn); + if (!(mask & AE_READABLE) || priority_changed) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE | priority_flag, tlsEventHandler, conn); } else if (conn->flags & TLS_CONN_FLAG_READ_WANT_WRITE) { if (mask & AE_READABLE) aeDeleteFileEvent(server.el, conn->c.fd, AE_READABLE); - if (!(mask & AE_WRITABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE, tlsEventHandler, conn); + if (!(mask & AE_WRITABLE) || priority_changed) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE | priority_flag, tlsEventHandler, conn); } else { serverAssert(0); } @@ -1247,16 +1333,16 @@ void updateSSLPendingFlag(tls_connection *conn) { static void updateSSLEvent(tls_connection *conn) { if (conn->flags & TLS_CONN_FLAG_POSTPONE_UPDATE_STATE) return; + int priority_flag = connGetAEPriorityFlag(&conn->c); int mask = aeGetFileEvents(server.el, conn->c.fd); + bool priority_changed = ((mask & AE_HIGH_PRIORITY) != (priority_flag & AE_HIGH_PRIORITY)); int need_read = conn->c.read_handler || (conn->c.write_handler && (conn->flags & TLS_CONN_FLAG_WRITE_WANT_READ)); int need_write = conn->c.write_handler || (conn->c.read_handler && (conn->flags & TLS_CONN_FLAG_READ_WANT_WRITE)); - if (need_read && !(mask & AE_READABLE)) - aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE, tlsEventHandler, conn); + if (need_read && (!(mask & AE_READABLE) || priority_changed)) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE | priority_flag, tlsEventHandler, conn); if (!need_read && (mask & AE_READABLE)) aeDeleteFileEvent(server.el, conn->c.fd, AE_READABLE); - if (need_write && !(mask & AE_WRITABLE)) - aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE, tlsEventHandler, conn); + if (need_write && (!(mask & AE_WRITABLE) || priority_changed)) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE | priority_flag, tlsEventHandler, conn); if (!need_write && (mask & AE_WRITABLE)) aeDeleteFileEvent(server.el, conn->c.fd, AE_WRITABLE); } @@ -1290,8 +1376,13 @@ static void updateSSLState(connection *conn_) { updatePendingData(conn); } -static int getCertSubjectFieldByName(X509 *cert, const char *field, char *out, size_t outlen) { - if (!cert || !field || !out || outlen == 0) return 0; +/* Return the named field of cert's subject, or NULL if it is absent or empty. + * Caller frees. + * + * sds rather than a C string, so the caller sees what the CA signed even when + * the value contains a NUL. */ +static sds getCertSubjectFieldByName(X509 *cert, const char *field) { + if (!cert || !field) return NULL; int nid = -1; @@ -1301,35 +1392,32 @@ static int getCertSubjectFieldByName(X509 *cert, const char *field, char *out, s nid = NID_organizationName; /* Add more mappings here as needed */ - if (nid == -1) return 0; + if (nid == -1) return NULL; #if OPENSSL_VERSION_NUMBER >= 0x30000000L const X509_NAME *subject = X509_get_subject_name(cert); #else X509_NAME *subject = X509_get_subject_name(cert); #endif - if (!subject) return 0; + if (!subject) return NULL; - /* X509_NAME_get_text_by_NID is deprecated in OpenSSL 4.0 */ + /* Not X509_NAME_get_text_by_NID(): it NUL terminates into a caller buffer, + * hiding an embedded NUL and truncating a long value. Also deprecated in + * OpenSSL 4.0. */ int idx = X509_NAME_get_index_by_NID(subject, nid, -1); - if (idx < 0) return 0; + if (idx < 0) return NULL; const X509_NAME_ENTRY *entry = X509_NAME_get_entry(subject, idx); - if (!entry) return 0; + if (!entry) return NULL; const ASN1_STRING *data = X509_NAME_ENTRY_get_data(entry); - if (!data) return 0; + if (!data) return NULL; const unsigned char *str = ASN1_STRING_get0_data(data); - int len = ASN1_STRING_length(data); - if (!str || len <= 0) return 0; - - /* Copy to output buffer, ensuring null termination */ - size_t copy_len = (size_t)len < outlen - 1 ? (size_t)len : outlen - 1; - memcpy(out, str, copy_len); - out[copy_len] = '\0'; + int str_len = ASN1_STRING_length(data); + if (!str || str_len <= 0) return NULL; - return 1; + return sdsnewlen(str, str_len); } /* Extract URI from Subject Alternative Name extension and return the first @@ -1403,16 +1491,28 @@ user *tlsGetPeerUser(connection *conn_, sds *cert_username) { break; case TLS_CLIENT_FIELD_CN: { - char field_value[256]; - if (getCertSubjectFieldByName(cert, "CN", field_value, sizeof(field_value))) { - if (cert_username) *cert_username = sdsnew(field_value); - result = ACLGetUserByName(field_value, strlen(field_value)); - if (!result || !(result->flags & USER_FLAG_ENABLED)) { - serverLog(LL_VERBOSE, "TLS: No matching user found for certificate CN '%s'", field_value); - result = NULL; - } - } else { + sds cn = getCertSubjectFieldByName(cert, "CN"); + if (!cn) { serverLog(LL_DEBUG, "TLS: Failed to extract CN in certificate subject"); + break; + } + + /* Compared over the whole CN, so "CN=admin\0attacker" does not match the + * user "admin". */ + result = ACLGetUserByName(cn, sdslen(cn)); + if (!result || !(result->flags & USER_FLAG_ENABLED)) { + sds repr = server.hide_user_data_from_log ? NULL : sdscatrepr(sdsempty(), cn, sdslen(cn)); + serverLog(LL_VERBOSE, "TLS: No matching user found for certificate CN %s", + repr ? repr : "*redacted*"); + sdsfree(repr); + result = NULL; + } + + /* Hand over the CN even when it does not match, so it reaches the ACL log. */ + if (cert_username) { + *cert_username = cn; + } else { + sdsfree(cn); } break; } @@ -2022,7 +2122,7 @@ static ConnectionType CT_TLS = { /* Miscellaneous */ .connIntegrityChecked = connTLSIsIntegrityChecked, - + .is_closing = connTcpSocketIsClosing, }; int RedisRegisterConnectionTypeTLS(void) { @@ -2059,6 +2159,7 @@ static void tlsClearCACertInfo(void) { static void tlsClearAllCertInfo(void) { tlsClearCertInfo(&server.tls_server_cert_expire_time, &server.tls_server_cert_serial); + tlsClearCertInfo(&server.tls_server_alt_cert_expire_time, &server.tls_server_alt_cert_serial); tlsClearCertInfo(&server.tls_client_cert_expire_time, &server.tls_client_cert_serial); tlsClearCACertInfo(); } @@ -2085,7 +2186,7 @@ int ValkeyModule_OnLoad(void *ctx, ValkeyModuleString **argv, int argc) { return VALKEYMODULE_ERR; } - ValkeyModule_SetModuleOptions(ctx, VALKEYMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD | VALKEYMODULE_OPTIONS_HANDLE_ATOMIC_SLOT_MIGRATION); + ValkeyModule_SetModuleOptions(ctx, VALKEYMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD | VALKEYMODULE_OPTIONS_HANDLE_ATOMIC_SLOT_MIGRATION | VALKEYMODULE_OPTIONS_HANDLE_FORKLESS); if (connTypeRegister(&CT_TLS) != C_OK) return VALKEYMODULE_ERR; diff --git a/src/tracking.c b/src/tracking.c index 00f030bd9..f81d4a722 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -423,6 +423,14 @@ void trackingInvalidateKey(client *c, robj *keyobj, int bcast) { raxRemove(TrackingTable, (unsigned char *)key, keylen, NULL); } +/* Whether any tracking (client-side-caching) key invalidation is pending + * flush. A tiny accessor over server.tracking_pending_keys rather than + * having callers reach into the list themselves, so this file stays the + * sole owner of how pending invalidations are tracked. */ +bool trackingHasPendingKeyInvalidations(void) { + return listLength(server.tracking_pending_keys) > 0; +} + void trackingHandlePendingKeyInvalidations(void) { if (!listLength(server.tracking_pending_keys)) return; diff --git a/src/unit/CMakeLists.txt b/src/unit/CMakeLists.txt index ca12c8be0..ccac65b25 100644 --- a/src/unit/CMakeLists.txt +++ b/src/unit/CMakeLists.txt @@ -168,7 +168,7 @@ add_custom_target(test-unit [ -n \"$large_memory\" ] && TEST_ARGS=\"$TEST_ARGS --large-memory\"; \ [ -n \"$valgrind\" ] && TEST_ARGS=\"$TEST_ARGS --valgrind\"; \ [ -n \"$seed\" ] && TEST_ARGS=\"$TEST_ARGS --seed $seed\"; \ - python3 ${CMAKE_SOURCE_DIR}/deps/gtest-parallel/gtest_parallel.py $ --gtest_filter=\"$UNIT_TEST_PATTERN\"* -- $TEST_ARGS" + python3 ${CMAKE_SOURCE_DIR}/deps/gtest-parallel/gtest_parallel.py $ --timeout_per_test 300 --gtest_filter=\"$UNIT_TEST_PATTERN\"* -- $TEST_ARGS" DEPENDS valkey-unit-gtests WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Running tests with gtest-parallel" diff --git a/src/unit/Makefile b/src/unit/Makefile index 0896a35ea..1e274e0cc 100644 --- a/src/unit/Makefile +++ b/src/unit/Makefile @@ -246,7 +246,7 @@ all: valkey-unit-gtests .PHONY: test-unit TEST_ARGS = $(if $(accurate),--accurate) $(if $(large_memory),--large-memory) $(if $(valgrind),--valgrind) $(if $(seed),--seed $(seed)) test-unit: valkey-unit-gtests - python3 ../../deps/gtest-parallel/gtest_parallel.py ./valkey-unit-gtests --gtest_filter=$(UNIT_TEST_PATTERN)* -- $(TEST_ARGS) + python3 ../../deps/gtest-parallel/gtest_parallel.py ./valkey-unit-gtests --timeout_per_test 300 --gtest_filter=$(UNIT_TEST_PATTERN)* -- $(TEST_ARGS) @printf '\033[32mAll UNIT TESTS PASSED!\033[0m\n' .PHONY: valgrind diff --git a/src/unit/custom_matchers.hpp b/src/unit/custom_matchers.hpp index 874d62543..d98c75b03 100644 --- a/src/unit/custom_matchers.hpp +++ b/src/unit/custom_matchers.hpp @@ -13,7 +13,11 @@ MATCHER_P(robjEqualsStr, str, "robj string matcher") { assert(arg->type == OBJ_STRING); assert(sdsEncodedObject(arg)); - return strcmp(static_cast(objectGetVal(arg)), str) == 0; + + if (strcmp(static_cast(objectGetVal(arg)), str) == 0) return true; + + *result_listener << "robj(\"" << (char *)objectGetVal(arg) << "\") doesn't match \"" << str << "\""; + return false; } #endif // _CUSTOM_MATCHERS_HPP_ diff --git a/src/unit/fake_connection.hpp b/src/unit/fake_connection.hpp new file mode 100644 index 000000000..707101d69 --- /dev/null +++ b/src/unit/fake_connection.hpp @@ -0,0 +1,213 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * A fake connection for unit tests. + * + * Provides a `connection` that never touches a socket: writes land in an + * in-memory buffer, reads are served from a caller-supplied byte array, and + * handler/state calls are recorded so tests can assert on them. + * + * Everything here is `inline` rather than `static inline` on purpose: the unit + * tests build with -Wall -Wextra -Werror, and a `static` helper that a given + * translation unit happens not to use would trip -Wunused-function. + */ + +#ifndef FAKE_CONNECTION_HPP +#define FAKE_CONNECTION_HPP + +#include +#include +#include + +extern "C" { +#include "connection.h" +#include "zmalloc.h" +} + +typedef struct fakeConnection { + connection conn; + + /* Write sink. Writes are clamped to buf_size; set error to fail them with + * EAGAIN, or fail_write for a hard error that also moves the state. */ + int error; + int fail_write; + char *buffer; + size_t buf_size; + size_t written; + + /* Read source. Once read_data is drained, the next read reports EAGAIN, + * or EOF / a hard error if the corresponding flag is set. A hard error + * moves the connection out of CONN_STATE_CONNECTED, which is how callers + * tell a real error from EAGAIN. */ + unsigned char *read_data; + size_t read_len; + size_t read_pos; + int eof; + int fail_read; + + /* Recorded calls, for tests that assert on connection lifecycle. */ + int close_calls; + int postpone_state; + int update_calls; +} fakeConnection; + +inline int fakeConnGetType(void) { + return CONN_TYPE_SOCKET; +} + +inline int fakeConnWrite(connection *conn, const void *data, size_t size) { + fakeConnection *fc = (fakeConnection *)conn; + if (fc->fail_write) { + conn->state = CONN_STATE_ERROR; + return -1; + } + if (fc->error) return -1; + + size_t to_write = size; + if (fc->written + to_write > fc->buf_size) { + to_write = fc->buf_size - fc->written; + } + memcpy(fc->buffer + fc->written, data, to_write); + fc->written += to_write; + return (int)to_write; +} + +inline int fakeConnWritev(connection *conn, const struct iovec *iov, int iovcnt) { + fakeConnection *fc = (fakeConnection *)conn; + if (fc->error) return -1; + + size_t total = 0; + for (int i = 0; i < iovcnt; i++) { + size_t to_write = iov[i].iov_len; + if (fc->written + to_write > fc->buf_size) { + to_write = fc->buf_size - fc->written; + } + if (to_write == 0) break; + + memcpy(fc->buffer + fc->written, iov[i].iov_base, to_write); + fc->written += to_write; + total += to_write; + } + return (int)total; +} + +inline int fakeConnRead(connection *conn, void *buf, size_t len) { + fakeConnection *fc = (fakeConnection *)conn; + if (fc->error) return -1; + if (fc->read_pos >= fc->read_len) { + if (fc->eof) return 0; + if (fc->fail_read) { + conn->state = CONN_STATE_ERROR; + return -1; + } + errno = EAGAIN; + return -1; + } + size_t avail = fc->read_len - fc->read_pos; + size_t n = (len < avail) ? len : avail; + memcpy(buf, fc->read_data + fc->read_pos, n); + fc->read_pos += n; + return (int)n; +} + +inline int fakeConnSetWriteHandler(connection *conn, ConnectionCallbackFunc handler, int barrier) { + UNUSED(barrier); + conn->write_handler = handler; + return C_OK; +} + +inline int fakeConnSetReadHandler(connection *conn, ConnectionCallbackFunc handler) { + conn->read_handler = handler; + return C_OK; +} + +inline void fakeConnPostponeUpdateState(connection *conn, int val) { + ((fakeConnection *)conn)->postpone_state = val; +} + +/* Mimics TLSHandleAcceptResult(): runs and clears conn_handler once the + * handshake is no longer in progress. */ +inline void fakeConnUpdateState(connection *conn) { + ((fakeConnection *)conn)->update_calls++; + if (conn->state == CONN_STATE_ACCEPTING) return; + ConnectionCallbackFunc handler = conn->conn_handler; + if (handler) { + conn->conn_handler = NULL; + handler(conn); + } +} + +inline void fakeConnClose(connection *conn) { + ((fakeConnection *)conn)->close_calls++; + conn->state = CONN_STATE_CLOSED; +} + +/* Complete the "handshake" immediately. Set error on the fake connection first + * to simulate a failed accept instead. */ +inline int fakeConnAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + if (((fakeConnection *)conn)->error) { + conn->state = CONN_STATE_ERROR; + return C_ERR; + } + conn->state = CONN_STATE_CONNECTED; + if (accept_handler) accept_handler(conn); + return C_OK; +} + +/* The one ConnectionType shared by every fake connection. A function-local + * static keeps a single instance without each fixture having to initialize it + * from SetUpTestSuite(). Fields are assigned by name because designated + * initializers need C++20. */ +inline ConnectionType *fakeConnType(void) { + static ConnectionType ct; + static bool initialized = false; + if (!initialized) { + memset(&ct, 0, sizeof(ct)); + ct.get_type = fakeConnGetType; + ct.close = fakeConnClose; + ct.accept = fakeConnAccept; + ct.write = fakeConnWrite; + ct.writev = fakeConnWritev; + ct.read = fakeConnRead; + ct.set_write_handler = fakeConnSetWriteHandler; + ct.set_read_handler = fakeConnSetReadHandler; + ct.postpone_update_state = fakeConnPostponeUpdateState; + ct.update_state = fakeConnUpdateState; + initialized = true; + } + return &ct; +} + +/* Create a fake connection. If write_cap is non-zero a write buffer of that + * size is allocated, so tests that only read can pass 0. */ +inline fakeConnection *connCreateFake(size_t write_cap = 0) { + fakeConnection *fc = (fakeConnection *)zcalloc(sizeof(fakeConnection)); + fc->conn.type = fakeConnType(); + fc->conn.fd = -1; + fc->conn.iovcnt = IOV_MAX; + if (write_cap > 0) { + fc->buffer = (char *)zmalloc(write_cap); + fc->buf_size = write_cap; + } + return fc; +} + +/* Point the connection's read side at a copy of the given bytes. */ +inline void fakeConnSetReadData(fakeConnection *fc, const void *data, size_t len) { + zfree(fc->read_data); + fc->read_data = (unsigned char *)zmalloc(len); + memcpy(fc->read_data, data, len); + fc->read_len = len; + fc->read_pos = 0; +} + +inline void connFreeFake(fakeConnection *fc) { + if (fc == NULL) return; + zfree(fc->buffer); + zfree(fc->read_data); + zfree(fc); +} + +#endif /* FAKE_CONNECTION_HPP */ diff --git a/src/unit/test_anet_subnet.cpp b/src/unit/test_anet_subnet.cpp new file mode 100644 index 000000000..4f43d4662 --- /dev/null +++ b/src/unit/test_anet_subnet.cpp @@ -0,0 +1,178 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "anet.h" +#include "server.h" +#include "zmalloc.h" +} + +class AnetSubnetTest : public ::testing::Test {}; + +TEST_F(AnetSubnetTest, ParseSubnetIpv4) { + anetSubnet subnet; + char err[ANET_ERR_LEN] = {0}; + + /* Valid IPv4 subnets */ + EXPECT_EQ(anetParseSubnet(err, "192.168.1.0/24", &subnet), ANET_OK); + EXPECT_EQ(subnet.family, AF_INET); + EXPECT_EQ(subnet.prefix_len, 24); + uint32_t expected_ip = 0; + inet_pton(AF_INET, "192.168.1.0", &expected_ip); + EXPECT_EQ(subnet.addr.ipv4.s_addr, expected_ip); + + EXPECT_EQ(anetParseSubnet(err, "10.0.0.0/8", &subnet), ANET_OK); + EXPECT_EQ(subnet.family, AF_INET); + EXPECT_EQ(subnet.prefix_len, 8); + + EXPECT_EQ(anetParseSubnet(err, "192.168.1.5/32", &subnet), ANET_OK); + EXPECT_EQ(subnet.family, AF_INET); + EXPECT_EQ(subnet.prefix_len, 32); + + EXPECT_EQ(anetParseSubnet(err, "0.0.0.0/0", &subnet), ANET_OK); + EXPECT_EQ(subnet.family, AF_INET); + EXPECT_EQ(subnet.prefix_len, 0); + + /* Valid raw IPv4 (no slash) */ + EXPECT_EQ(anetParseSubnet(err, "192.168.1.1", &subnet), ANET_OK); + EXPECT_EQ(subnet.family, AF_INET); + EXPECT_EQ(subnet.prefix_len, 32); + inet_pton(AF_INET, "192.168.1.1", &expected_ip); + EXPECT_EQ(subnet.addr.ipv4.s_addr, expected_ip); + + /* Invalid IPv4 subnets */ + EXPECT_EQ(anetParseSubnet(err, "1.2.3.4/999", &subnet), ANET_ERR); + EXPECT_EQ(anetParseSubnet(err, "invalid/24", &subnet), ANET_ERR); + EXPECT_EQ(anetParseSubnet(err, "1.2.3.4.5/24", &subnet), ANET_ERR); + EXPECT_EQ(anetParseSubnet(err, "/24", &subnet), ANET_ERR); + EXPECT_EQ(anetParseSubnet(err, "192.168.1.0/", &subnet), ANET_ERR); + EXPECT_EQ(anetParseSubnet(err, "192.168.1.0/-1", &subnet), ANET_ERR); + EXPECT_EQ(anetParseSubnet(err, "192.168.1.0/33", &subnet), ANET_ERR); + EXPECT_EQ(anetParseSubnet(err, "1.2.3.4/24a", &subnet), ANET_ERR); +} + +TEST_F(AnetSubnetTest, ParseSubnetIpv6) { + anetSubnet subnet; + char err[ANET_ERR_LEN] = {0}; + + /* Valid IPv6 subnets */ + EXPECT_EQ(anetParseSubnet(err, "2001:db8::/32", &subnet), ANET_OK); + EXPECT_EQ(subnet.family, AF_INET6); + EXPECT_EQ(subnet.prefix_len, 32); + struct in6_addr expected_ip; + inet_pton(AF_INET6, "2001:db8::", &expected_ip); + EXPECT_EQ(memcmp(&subnet.addr.ipv6, &expected_ip, sizeof(struct in6_addr)), 0); + + EXPECT_EQ(anetParseSubnet(err, "::1/128", &subnet), ANET_OK); + EXPECT_EQ(subnet.family, AF_INET6); + EXPECT_EQ(subnet.prefix_len, 128); + + EXPECT_EQ(anetParseSubnet(err, "::/0", &subnet), ANET_OK); + EXPECT_EQ(subnet.family, AF_INET6); + EXPECT_EQ(subnet.prefix_len, 0); + + /* Valid raw IPv6 (no slash) */ + EXPECT_EQ(anetParseSubnet(err, "2001:db8::1", &subnet), ANET_OK); + EXPECT_EQ(subnet.family, AF_INET6); + EXPECT_EQ(subnet.prefix_len, 128); + inet_pton(AF_INET6, "2001:db8::1", &expected_ip); + EXPECT_EQ(memcmp(&subnet.addr.ipv6, &expected_ip, sizeof(struct in6_addr)), 0); + + /* Invalid IPv6 subnets */ + EXPECT_EQ(anetParseSubnet(err, "2001:db8::/129", &subnet), ANET_ERR); + EXPECT_EQ(anetParseSubnet(err, "invalid/64", &subnet), ANET_ERR); + EXPECT_EQ(anetParseSubnet(err, "2001:db8::/-1", &subnet), ANET_ERR); +} + +TEST_F(AnetSubnetTest, MatchIpSubnetIpv4) { + anetSubnet subnets[3]; + ASSERT_EQ(anetParseSubnet(NULL, "192.168.1.0/24", &subnets[0]), ANET_OK); + ASSERT_EQ(anetParseSubnet(NULL, "10.0.0.0/8", &subnets[1]), ANET_OK); + ASSERT_EQ(anetParseSubnet(NULL, "172.16.0.0/12", &subnets[2]), ANET_OK); + + /* Matches */ + EXPECT_EQ(anetMatchIpSubnet("192.168.1.5", subnets, 3), 1); + EXPECT_EQ(anetMatchIpSubnet("10.254.0.1", subnets, 3), 1); + EXPECT_EQ(anetMatchIpSubnet("172.16.10.20", subnets, 3), 1); + EXPECT_EQ(anetMatchIpSubnet("172.31.255.254", subnets, 3), 1); + + /* Mismatches */ + EXPECT_EQ(anetMatchIpSubnet("192.168.2.5", subnets, 3), 0); + EXPECT_EQ(anetMatchIpSubnet("11.0.0.1", subnets, 3), 0); + EXPECT_EQ(anetMatchIpSubnet("172.32.0.1", subnets, 3), 0); + EXPECT_EQ(anetMatchIpSubnet("invalid-ip", subnets, 3), 0); +} + +TEST_F(AnetSubnetTest, MatchIpSubnetIpv6) { + anetSubnet subnets[3]; + ASSERT_EQ(anetParseSubnet(NULL, "2001:db8::/32", &subnets[0]), ANET_OK); + ASSERT_EQ(anetParseSubnet(NULL, "::1/128", &subnets[1]), ANET_OK); + ASSERT_EQ(anetParseSubnet(NULL, "fe80::/10", &subnets[2]), ANET_OK); + + /* Matches */ + EXPECT_EQ(anetMatchIpSubnet("2001:db8:abcd::1", subnets, 3), 1); + EXPECT_EQ(anetMatchIpSubnet("::1", subnets, 3), 1); + EXPECT_EQ(anetMatchIpSubnet("fe80::1ff:fe23:4567:890a", subnets, 3), 1); + + /* Mismatches */ + EXPECT_EQ(anetMatchIpSubnet("2001:db9::1", subnets, 3), 0); + EXPECT_EQ(anetMatchIpSubnet("::2", subnets, 3), 0); + EXPECT_EQ(anetMatchIpSubnet("fec0::1", subnets, 3), 0); + EXPECT_EQ(anetMatchIpSubnet("invalid-ip", subnets, 3), 0); +} + +TEST_F(AnetSubnetTest, MatchIpSubnetEdgeCases) { + anetSubnet subnets[2]; + ASSERT_EQ(anetParseSubnet(NULL, "0.0.0.0/0", &subnets[0]), ANET_OK); + ASSERT_EQ(anetParseSubnet(NULL, "::/0", &subnets[1]), ANET_OK); + + /* Any IPv4 matches 0.0.0.0/0 */ + EXPECT_EQ(anetMatchIpSubnet("192.168.1.1", &subnets[0], 1), 1); + EXPECT_EQ(anetMatchIpSubnet("8.8.8.8", &subnets[0], 1), 1); + EXPECT_EQ(anetMatchIpSubnet("::1", &subnets[0], 1), 0); + + /* Any IPv6 matches ::/0 */ + EXPECT_EQ(anetMatchIpSubnet("2001:db8::1", &subnets[1], 1), 1); + EXPECT_EQ(anetMatchIpSubnet("::1", &subnets[1], 1), 1); + EXPECT_EQ(anetMatchIpSubnet("192.168.1.1", &subnets[1], 1), 0); + + /* NULL IP (non-IP transports like UNIX domain sockets) safely returns 0 */ + EXPECT_EQ(anetMatchIpSubnet(NULL, subnets, 2), 0); + EXPECT_EQ(anetMatchIpSubnet("192.168.1.1", NULL, 0), 0); + EXPECT_EQ(anetMatchIpSubnet(NULL, NULL, 0), 0); +} + +TEST_F(AnetSubnetTest, Ipv4MappedIpv6DualStack) { + anetSubnet subnet; + ASSERT_EQ(anetParseSubnet(NULL, "192.168.1.0/24", &subnet), ANET_OK); + + /* IPv4-mapped IPv6 address ::ffff:192.168.1.5 matches IPv4 subnet */ + EXPECT_EQ(anetMatchIpSubnet("::ffff:192.168.1.5", &subnet, 1), 1); + EXPECT_EQ(anetMatchIpSubnet("::ffff:192.168.2.5", &subnet, 1), 0); +} + +TEST_F(AnetSubnetTest, ValidateAndUpdatePrioritySubnets) { + const char *err = NULL; + EXPECT_EQ(validatePrioritySubnets("192.168.1.0/24, 10.0.0.0/8", &err), C_OK); + EXPECT_EQ(err, nullptr); + + EXPECT_EQ(validatePrioritySubnets("192.168.1.0/24, invalid-ip", &err), C_ERR); + EXPECT_NE(err, nullptr); + + /* Test updating global server configuration */ + EXPECT_EQ(updatePrioritySubnets("192.168.1.0/24 10.0.0.0/8"), C_OK); + EXPECT_EQ(server.priority_subnets_count, 2); + ASSERT_NE(server.priority_subnets_array, nullptr); + EXPECT_EQ(anetMatchIpSubnet("192.168.1.100", server.priority_subnets_array, server.priority_subnets_count), 1); + EXPECT_EQ(anetMatchIpSubnet("172.16.1.1", server.priority_subnets_array, server.priority_subnets_count), 0); + + /* Clear */ + EXPECT_EQ(updatePrioritySubnets(NULL), C_OK); + EXPECT_EQ(server.priority_subnets_count, 0); + EXPECT_EQ(server.priority_subnets_array, nullptr); +} diff --git a/src/unit/test_bgiteration.cpp b/src/unit/test_bgiteration.cpp new file mode 100644 index 000000000..784f080a8 --- /dev/null +++ b/src/unit/test_bgiteration.cpp @@ -0,0 +1,3172 @@ +/* + * Copyright Valkey Contributors. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" +#include + +using namespace ::testing; + +extern "C" { +#include "bgiteration.h" +#include "module.h" +#include "server.h" +#include "stdlib.h" +extern hashtableType commandSetType; +extern dictType keylistDictType; +void bgIteration_feedIterators(void); +void createSharedObjects(void); +void hashtableDump(hashtable *ht); +void bgIteration_unitTestDisableCloning(void); +void bgIteration_unitTestEnableCloning(int item_bytes, int pool_bytes); +static size_t mockHashtableScan(hashtable *ht, size_t cursor, hashtableScanFunction fn, void *privdata); +size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid); +} + + +// The private data is a pointer to arbitrary data. This value is used just to +// test that the correct value is passed through. +#define PRIVDATA reinterpret_cast(12345) + +typedef int32_t bgIterationEntryMetadata; // opaque 4 bytes +static_assert(sizeof(bgIterationEntryMetadata) == BGITERATION_ENTRY_METADATA_SIZE); + +// A bgIteration cleanup function used for testing. +static int cleanupCount; +static bool cleanupTerminated; +static void iteratorCleanupFn(bool terminated, void *privdata) { + EXPECT_EQ(privdata, PRIVDATA); + cleanupCount++; + cleanupTerminated = terminated; +} + +// A bgIteration repldone function used for testing. +static int replDoneConfirmed; +static bool iteratorRepldoneFn(void *privdata) { + EXPECT_EQ(privdata, PRIVDATA); + replDoneConfirmed++; + return true; +} + +// A more complicated repldone function that can delay the replcation done condition. +static int replDoneRejected; +static bool iteratorRepldoneFnNotBeingReadyInitially(void *privdata) { + EXPECT_EQ(privdata, PRIVDATA); + // This is to test the behavior when Repl Done function is not ready to be executed. + if (replDoneRejected == 0) { + replDoneRejected++; + return false; + } + replDoneConfirmed++; + return true; +} + + +/* This mock for hashtableScan will return the items in lexical order. It assumes that the entries + * are robjs containing an sds string for the key. The key is expected to begin with a capital + * letter [A-Z]. The caller passes 0 as the cursor to start the iteration. The returned cursor + * value will indicate the prior letter returned (1=A, ...). After entries starting with 'Z' have + * been returned, the cursor of 0 will indicate that the scan is complete. Note that all entries + * starting with the same letter will be returned in a single call. */ +static size_t mockHashtableScan(hashtable *ht, size_t cursor, hashtableScanFunction fn, void *privdata) { + // Just in case, if it's not one of our hashtables, use the unmocked function + bool our_ht = (server.db[0]->keys && ht == kvstoreGetHashtable(server.db[0]->keys, 0)) || + (server.db[1]->keys && ht == kvstoreGetHashtable(server.db[1]->keys, 0)); + if (!our_ht) return __real_hashtableScan(ht, cursor, fn, privdata); + + // Collect all entries from the hashtable + std::vector entries; + hashtableIterator *iter = hashtableCreateIterator(ht, 0); + dbEntry *entry; + while (hashtableNext(iter, (void **)&entry)) { + char first = objectGetKey(entry)[0]; + assert(first >= 'A' && first <= 'Z'); + entries.push_back(entry); + } + hashtableReleaseIterator(iter); + + // Sort by key lexicographically + std::sort(entries.begin(), entries.end(), [](dbEntry *a, dbEntry *b) { + return strcmp(objectGetKey(a), objectGetKey(b)) < 0; + }); + + // cursor 0 means start at 'A', otherwise start after the cursor letter + char startLetter = (char)('A' + cursor); + + // Find the first letter to emit + char emitLetter = 0; + for (dbEntry *e : entries) { + char first = objectGetKey(e)[0]; + if (first >= startLetter) { + emitLetter = first; + break; + } + } + + if (emitLetter == 0) return 0; + + // Call fn for all entries starting with emitLetter + for (dbEntry *e : entries) { + char first = objectGetKey(e)[0]; + if (first == emitLetter) fn(privdata, (void *)e); + } + + size_t nextCursor = (size_t)(emitLetter - 'A' + 1); + return (nextCursor > 25) ? 0 : nextCursor; +} + + +static bool mockHashtableScanHasPassedKey(hashtable *ht, const void *key, size_t cursor) { + // If it's one of our tables, use the mock logic + bool itsOurs = false; + if (server.db[0]->keys && ht == kvstoreGetHashtable(server.db[0]->keys, 0)) itsOurs = true; + if (server.db[1]->keys && ht == kvstoreGetHashtable(server.db[1]->keys, 0)) itsOurs = true; + + // Mock logic uses a lexicographic cursor + if (itsOurs) return ((const char *)key)[0] < (char)('A' + cursor); + + // Otherwise, use the real logic for other hashtables + return __real_hashtableScanHasPassedKey(ht, key, cursor); +} + + +static const char *logfile = ""; + +/* Most of the bgIteration unit tests are based on a CMD instance with 2 DBs. There are 8 keys in + * each DB. The hashtableScan function is mocked to return the keys in a predictable order. + * + * There are a number of helper functions to simulate certain key modification actions within our + * test configuration. Note that this is isolated from the actual call to processCommand. + * + * Because most of bgIteration is based on an ordered processing of keys, it doesn't matter if we + * are simulating CMD or CME, full scan, or slot-based. The majority of tests are independent of + * these concerns. + * + * However, there are some tests which are are unique to these configurations and use a specialized + * derived class to handle the differences. We do not want to duplicate all of the tests for + * the different configurations, but we do want to ensure that each configuration works properly. + * - bgIterationTestCluster - handles tests unique to full scan in cluster mode + * - bgIterationTestClusterSlots - handles tests unique to cluster slot-based iteration */ +class BgIterationTest : public ::testing::Test { + protected: + static const int DB_COUNT = 2; + static const int ITEMS_PER_DB = 8; + + private: + /* With the mock hashtableScan, we get keys in a predictable order. DB0 works with buckets + * containing groups of keys (which hashtableScan returns in a single call). DB1 returns + * each key individually, as more separate buckets. Convention (for test readability) is + * that keys beginning [A-M] would be in DB0 and keys beginning [N-Z] in DB1. Letters are + * intentionally skipped to allow for possible insertions. */ + const char *keys[DB_COUNT][ITEMS_PER_DB] = {{"B0", "B1", "B2", "E0", "E1", "H0", "H1", "H2"}, + {"N0", "O0", "Q0", "R0", "T0", "U0", "W0", "Y0"}}; + + protected: + static const int TOTAL_ITEMS = DB_COUNT * ITEMS_PER_DB; + static const int LAST_ITEM = TOTAL_ITEMS - 1; + + MockValkey mock; + RealValkey real; + client *c = nullptr; // for general use in the tests (with common cleanup) + robj **orig_argv = nullptr; // Used when simulating multi + int orig_argc = 0; // Used when simulating multi + + + struct serverCommand dummy_cmd = {0}; + + // Helper functions for accessing the keys. We can access by db(0..1) and seq(0..7) + // or by item number (0..15). + // NOTE: These virtual functions can be overridden in subclasses which may have different item layout. + virtual const char *getKeyAtDbSeq(int db, int seq) { + assert(db < DB_COUNT); + assert(seq < ITEMS_PER_DB); + return keys[db][seq]; + } + + virtual int getDbFromItemNum(int itemNum) { + assert(itemNum < DB_COUNT * ITEMS_PER_DB); + return itemNum / ITEMS_PER_DB; + } + + virtual int getSeqFromItemNum(int itemNum) { + assert(itemNum < DB_COUNT * ITEMS_PER_DB); + return itemNum % ITEMS_PER_DB; + } + + const char *keyStr(int itemNum) { + return getKeyAtDbSeq(getDbFromItemNum(itemNum), getSeqFromItemNum(itemNum)); + } + + int itemNumFromKey(const char *key) { + for (int itemNum = 0; itemNum < DB_COUNT * ITEMS_PER_DB; itemNum++) { + if (strcmp(key, keyStr(itemNum)) == 0) return itemNum; + } + return -1; + } + + + // Do some general initialization before starting the suite. Normally, the tests are run in + // isolation - and this isn't much different than SetUp(). But if running the + // entire test suite together (just manually running the test executable), this gets called + // only once. + static void SetUpTestSuite() { + monotonicInit(); + + bzero(&server, sizeof(server)); + server.hz = 100; + server.logfile = const_cast(logfile); + createSharedObjects(); + + moduleInitModulesSystem(); + + server.commands = hashtableCreate(&commandSetType); + server.orig_commands = hashtableCreate(&commandSetType); + populateCommandTable(); + } + + + static void TearDownTestSuite() { + hashtableRelease(server.commands); + hashtableRelease(server.orig_commands); + } + + + void initializeServerDb(int dbid, int slot_count_bits = 0) { + server.db[dbid] = static_cast(zcalloc(sizeof(serverDb))); + server.db[dbid]->id = dbid; + server.db[dbid]->keys = kvstoreCreate(&kvstoreKeysHashtableType, slot_count_bits, 0); + server.db[dbid]->expires = kvstoreCreate(&kvstoreExpiresHashtableType, slot_count_bits, 0); + server.db[dbid]->watched_keys = dictCreate(&keylistDictType); + } + + + robj *createStringObjectFromCString(const char *s) { + return createStringObject(s, strlen(s)); + } + + + void addKeyToDb(int dbid, const char *key, const char *val) { + robj *key_obj = createStringObjectFromCString(key); + robj *val_obj = createStringObjectFromCString(val); + dbAdd(server.db[dbid], key_obj, &val_obj); + decrRefCount(key_obj); + } + + + virtual void setupDatabase() { + /* For these unit tests, a standard database is constructed. But we will use our own + * mocked scan function to ensure a consistent iteration order */ + + server.dbnum = DB_COUNT; + server.cluster_enabled = false; + server.db = static_cast(zcalloc(sizeof(serverDb *) * server.dbnum)); + + for (int dbid = 0; dbid < server.dbnum; dbid++) { + initializeServerDb(dbid); + for (int keynum = 0; keynum < ITEMS_PER_DB; keynum++) { + addKeyToDb(dbid, keys[dbid][keynum], keys[dbid][keynum]); + } + } + + EXPECT_CALL(mock, hashtableScan(_, _, _, _)) + .WillRepeatedly(Invoke(mockHashtableScan)); + EXPECT_CALL(mock, hashtableScanHasPassedKey(_, _, _)) + .WillRepeatedly(Invoke(mockHashtableScanHasPassedKey)); + + if (0) debugPrintBucketInfo(); + } + + + void SetUp() override { + server.main_thread_id = pthread_self(); + server.forkless_infrastructure_enabled = 1; + objectSetMetadataSize(BGITERATION_ENTRY_METADATA_SIZE); + + bgIteration_unitTestDisableCloning(); + + setupDatabase(); + + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).WillRepeatedly(Return(0)); + bgIteration_init(); + + cleanupCount = 0; + replDoneConfirmed = 0; + replDoneRejected = 0; + + // By default, do nothing for these + EXPECT_CALL(mock, blockClientInUseOnKeys(_, _, _)).WillRepeatedly(Return()); + EXPECT_CALL(mock, unblockClientsInUseOnKey(_)).WillRepeatedly(Return()); + + // By default, expect no permission issues + EXPECT_CALL(mock, ACLCheckAllUserCommandPerm(_, _, _, _, _, _)) + .WillRepeatedly(Return(ACL_OK)); + } + + + void TearDown() override { + bgIteration_feedIterators(); // process returning stuff before deleting DB + bgIteration_feedIterators(); // in case an iterator was closed there might be more + for (int i = 0; i < server.dbnum; i++) { + if (server.db[i]->keys) kvstoreRelease(server.db[i]->keys); + if (server.db[i]->expires) kvstoreRelease(server.db[i]->expires); + dictRelease(server.db[i]->watched_keys); + zfree(server.db[i]); + } + zfree(server.db); + + if (c != NULL) freeTestClient(c); + EXPECT_EQ(server.in_call, 0); // make sure tests are handling this properly + } + + + // Deletes an item from the DB (often at the start of a test) - but does NOT notify + // bgIteration. bgIteration_keyDelete() should be explicitly called where needed. + void simpleDelItem(int itemNum) { + int db = getDbFromItemNum(itemNum); + + sds delKey = sdsnew(keyStr(itemNum)); + int rc = kvstoreHashtableDelete(server.db[db]->keys, 0, delKey); + ASSERT_EQ(rc, 1); + sdsfree(delKey); + } + + + // Find the actual dbEntry object by itemNum + dbEntry *getItem(int itemNum) { + int db = getDbFromItemNum(itemNum); + sds key = sdsnew(keyStr(itemNum)); + dbEntry *de = dbFind(server.db[db], key); + sdsfree(key); + return de; + } + + + // The test expects that the next item read will be BGITERATOR_ITEM_COMPLETE + void expectReadComplete(bgIterator *iter) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + EXPECT_EQ(item->type, BGITERATOR_ITEM_COMPLETE); + bgIteratorClose(iter); + + int oldCleanupCount = cleanupCount; + bgIteration_feedIterators(); + EXPECT_EQ(cleanupCount, oldCleanupCount + 1); + } + + + // The test is cleaning up and isn't validating the remaining cleanup + void expectAnythingCleanup(bgIterator *iter) { + while (true) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + if ((item->type == BGITERATOR_ITEM_COMPLETE || + item->type == BGITERATOR_ITEM_TERMINATED)) { + bgIteratorClose(iter); + break; + } + } + bgIteration_feedIterators(); // Recognize the closed iterator + EXPECT_EQ(cleanupCount, 1); + } + + + void expectDictEntryMetadataMatch(dbEntry *de1, dbEntry *de2) { + bgIterationEntryMetadata *dm1 = static_cast(objectGetMetadata(de1)); + bgIterationEntryMetadata *dm2 = static_cast(objectGetMetadata(de2)); + + EXPECT_NE(dm1, nullptr); + EXPECT_NE(dm2, nullptr); + EXPECT_EQ(*dm1, *dm2); + } + + + // Useful when debugging new tests. It reads/prints all remaining items then crashes. + void cleanupIteratorDebugPrint(bgIterator *iter) { + bool done = false; + printf("[DEBUG] Printing bgIterator '%s' items:\n", bgIteratorName(iter)); + while (!done) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + switch (item->type) { + case BGITERATOR_ITEM_DBENTRY: { + auto obj = item->u.dbe.de; + const char *keyStr = objectGetKey(obj); + printf("Entry: %s -> %s [itemNum: %i]\n", + keyStr, + static_cast(objectGetVal(obj)), + itemNumFromKey(keyStr)); + break; + } + case BGITERATOR_ITEM_REPLICATION: + printf("Repl: DB=%d : ", item->dbid); + for (int i = 0; i < item->u.repl.argc; i++) + printf("%s ", static_cast(objectGetVal(item->u.repl.argv[i]))); + printf("\n"); + break; + case BGITERATOR_ITEM_COMPLETE: + case BGITERATOR_ITEM_TERMINATED: + bgIteratorClose(iter); + done = true; + break; + default: + printf("unhandled: %d\n", item->type); + } + } + bgIteration_feedIterators(); // Recognize the closed iterator + ASSERT_TRUE(false); // Halt the test here + } + + + // Make a copy of the metadata + void *cloneMetadata(dbEntry *de) { + int size = objectGetMetadataSize(de); + void *metadata = zmalloc(size); + memcpy(metadata, objectGetMetadata(de), size); + return metadata; + } + + + // Compare a previous metadata copy to an existing entry + void compareAndFreeClonedMetadata(dbEntry *de, void *metadata) { + EXPECT_EQ(memcmp(objectGetMetadata(de), metadata, objectGetMetadataSize(de)), 0); + zfree(metadata); + } + + + // The test expects the next item will be a specific key + // The item value is verified against the default unless provided as a parameter. + void expectReadKey(bgIterator *iter, int itemNum, const char *value = nullptr) { + int db = getDbFromItemNum(itemNum); + + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_DBENTRY); + EXPECT_EQ(item->dbid, db); + EXPECT_FALSE(item->u.dbe.is_cloned); + EXPECT_STREQ(objectGetKey(item->u.dbe.de), keyStr(itemNum)); + if (value) { + EXPECT_THAT(item->u.dbe.de, robjEqualsStr(value)); + } else { + EXPECT_THAT(item->u.dbe.de, robjEqualsStr(keyStr(itemNum))); + } + } + + + // The test expects the next item will be a specific key amd that the item is cloned. + // Metadata is tested (to make sure the clone includes the proper metadata). + // The item value is verified against the default unless provided as a parameter. + void expectReadClonedKey(bgIterator *iter, int itemNum, void *metadata, const char *value = nullptr) { + int db = getDbFromItemNum(itemNum); + + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_DBENTRY); + EXPECT_EQ(item->dbid, db); + EXPECT_TRUE(item->u.dbe.is_cloned); + compareAndFreeClonedMetadata(item->u.dbe.de, metadata); + EXPECT_STREQ(objectGetKey(item->u.dbe.de), keyStr(itemNum)); + if (value) { + EXPECT_THAT(item->u.dbe.de, robjEqualsStr(value)); + } else { + EXPECT_THAT(item->u.dbe.de, robjEqualsStr(keyStr(itemNum))); + } + } + + + // Test expects the next key, but specified by key name, not itemNum. + void expectReadDbKeyValue(bgIterator *iter, int db, const char *key, const char *value) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_DBENTRY); + EXPECT_EQ(item->dbid, db); + EXPECT_STREQ(objectGetKey(item->u.dbe.de), key); + EXPECT_THAT(item->u.dbe.de, robjEqualsStr(value)); + } + + + // Test expect to read a sequence of key items + void expectReadKeySequence(bgIterator *iter, int startItem, int endItem) { + for (int i = startItem; i <= endItem; i++) expectReadKey(iter, i); + } + + + // Just like expectReadKey, but also tests that a previous item is becoming unblocked. + void expectReadKeyWithUnblock(bgIterator *iter, int itemNum, int unblockItem, const char *value = nullptr) { + bool blocked = true; + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(unblockItem)))) + .WillOnce(Assign(&blocked, false)); + expectReadKey(iter, itemNum, value); + EXPECT_FALSE(blocked); + } + + + // Test expects to read a replication item matching the command help by client 'c' + void expectReadReplication(bgIterator *iter, client *c) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + EXPECT_EQ(item->dbid, c->db->id); + EXPECT_EQ(item->u.repl.cmd, c->cmd); + EXPECT_EQ(item->u.repl.argc, c->argc); + for (int i = 0; i < c->argc; i++) { + EXPECT_STREQ(static_cast(objectGetVal(item->u.repl.argv[i])), + static_cast(objectGetVal(c->argv[i]))); + } + } + + + // We expect to read a MULTI command which should have been inserted. + void expectReadMultiReplication(bgIterator *iter) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + EXPECT_EQ(item->u.repl.cmd, lookupCommandByCString("multi")); + } + + + // We expect to read an EXEC command which should have been inserted. + void expectReadExecReplication(bgIterator *iter) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + EXPECT_EQ(item->u.repl.cmd, lookupCommandByCString("exec")); + } + + + // Expecting that a DEL command should have been replicated. + void expectReadReplicationDel(bgIterator *iter, int itemNum) { + int db = getDbFromItemNum(itemNum); + + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + EXPECT_EQ(item->dbid, db); + EXPECT_EQ(item->u.repl.cmd, lookupCommandByCString("DEL")); + EXPECT_EQ(item->u.repl.argc, 2); + EXPECT_THAT(item->u.repl.argv[0], robjEqualsStr("DEL")); + EXPECT_THAT(item->u.repl.argv[1], robjEqualsStr(keyStr(itemNum))); + } + + + // Expecting that a special SWAPDB item has been inserted. + void expectReadSwapDB(bgIterator *iter, int db1, int db2) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_SWAPDB); + EXPECT_EQ(item->dbid, db1); + EXPECT_EQ(item->u.dbid2, db2); + } + + + // Expecting that a special FLUSHDB item has been inserted. + void expectReadFlushDB(bgIterator *iter, int db, bool withReplication = false) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_FLUSHDB); + EXPECT_EQ(item->dbid, db); + + if (withReplication) { + item = bgIteratorRead(iter); + bgIteration_feedIterators(); + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + EXPECT_EQ(item->dbid, db); + EXPECT_EQ(item->u.repl.cmd, lookupCommandByCString("FLUSHDB")); + EXPECT_EQ(item->u.repl.argc, 1); + EXPECT_THAT(item->u.repl.argv[0], robjEqualsStr("flushdb")); + } + } + + + static void debugPrintBucketInfoCb(void *privdata, void *entry) { + UNUSED(privdata); + dbEntry *de = (dbEntry *)entry; + printf("--- %s\n", objectGetKey(de)); + } + + void debugPrintBucketInfo() { + printf("*******DEBUG*******\n"); + for (int db = 0; db < server.dbnum; db++) { + int num_ht = kvstoreNumHashtables(server.db[db]->keys); + for (int slot = 0; slot < num_ht; slot++) { + hashtable *ht = kvstoreGetHashtable(server.db[db]->keys, slot); + if (!ht) continue; + + printf("DB: %d, slot: %d\n", db, slot); + size_t cursor = 0; + do { + cursor = hashtableScan(ht, cursor, debugPrintBucketInfoCb, NULL); + printf("-----------\n"); + } while (cursor != 0); + } + } + ASSERT_TRUE(false); + } + + + // Creates a client with a write command (SET) for the given itemNum + client *getWriteClient(int itemNum, const char *value) { + int db = getDbFromItemNum(itemNum); + + client *c = static_cast(zcalloc(sizeof(client))); + + c->cmd = lookupCommandByCString("set"); + c->db = server.db[db]; + + c->argc = 3; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(c->cmd->fullname); + c->argv[1] = createStringObjectFromCString(keyStr(itemNum)); + c->argv[2] = createStringObjectFromCString(value); + + return c; + } + + + // Create a client with a write command that touches multiple keys + client *getWriteMultiKeysClient(const char *cmdName, + int dstItemNum, + const std::vector &srcItemsNum) { + assert(!srcItemsNum.empty()); + + const int db = getDbFromItemNum(dstItemNum); + std::for_each(srcItemsNum.cbegin(), srcItemsNum.cend(), [&db, this](int srcItemNum) { + assert(db == getDbFromItemNum(srcItemNum)); + }); + + client *c = static_cast(zcalloc(sizeof(client))); + + c->cmd = lookupCommandByCString(cmdName); + assert(c->cmd != nullptr); + c->db = server.db[db]; + + c->argc = 2 + srcItemsNum.size(); + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(c->cmd->fullname); + c->argv[1] = createStringObjectFromCString(keyStr(dstItemNum)); + for (unsigned int i = 0; i < srcItemsNum.size(); i++) { + c->argv[2 + i] = createStringObjectFromCString(keyStr(srcItemsNum[i])); + } + + return c; + } + + + client *getWrite2KeysClient(const char *cmdName, int dstItemNum, int srcItemNum) { + return getWriteMultiKeysClient(cmdName, dstItemNum, {srcItemNum}); + } + + + client *getWrite3KeysClient(const char *cmdName, int dstItemNum, int src1ItemNum, int src2ItemNum) { + return getWriteMultiKeysClient(cmdName, dstItemNum, {src1ItemNum, src2ItemNum}); + } + + + // Create a client with a MULTI/EXEC block. + // This parses a series of commands separated by ';' + // Example: getMultiClient("SET A0 xxx; SELECT 1; SET A1 xxx; SET B1 xxx") + client *getMultiClient(const char *commands, int dbid = 0) { + char *commandsCopy = zstrdup(commands); // a mutable copy + char *commandStr, *commandStrSave; + char *token, *tokenSave; + + client *c = static_cast(zcalloc(sizeof(client))); + c->db = server.db[dbid]; + initClientMultiState(c); + c->flag.multi = 1; + c->mstate->cmd_flags |= CMD_WRITE; + + commandStr = strtok_r(commandsCopy, ";", &commandStrSave); + while (commandStr != NULL) { + token = strtok_r(commandStr, " ", &tokenSave); + c->cmd = lookupCommandByCString(token); + + c->argv = static_cast(zcalloc(sizeof(robj *) * 5)); // command + 4 args + + for (int i = 0; token != NULL; i++) { + c->argv[i] = createStringObjectFromCString(token); + c->argc = i + 1; + token = strtok_r(NULL, " ", &tokenSave); + } + + queueMultiCommand(c, 0); + freeClientArgv(c); + + commandStr = strtok_r(NULL, ";", &commandStrSave); + } + + c->cmd = lookupCommandByCString("exec"); + c->argc = 1; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString("EXEC"); + + zfree(commandsCopy); + return c; + } + + + // Initially, a MULTI client is set up to execute the EXEC command (which examines the + // contents of the multi/exec block). This function advances the client to begin executing + // the individual commands within the multi/exec block. + void advanceMultiClientToCommand(client *c, int cmdNum) { + assert(cmdNum >= 0 && cmdNum < c->mstate->count); + if (cmdNum == 0) { + // Save off the EXEC + orig_argc = c->argc; + orig_argv = c->argv; + } + c->argc = c->mstate->commands[cmdNum].argc; + c->argv = c->mstate->commands[cmdNum].argv; + c->argv_len = c->mstate->commands[cmdNum].argv_len; + c->cmd = c->realcmd = c->mstate->commands[cmdNum].cmd; + } + + + // A client with a fictional command: + // SETGET + // - writes a value to the first key (making this CMD_WRITE | CMD_WRITE_FIRSTKEY_ONLY) + // - reads a second key + client *getSetGetClient(int itemNum1, const char *value1, int itemNum2) { + // Fictional command which writes to 1st key and reads the 2nd + int db = getDbFromItemNum(itemNum1); + assert(db == getDbFromItemNum(itemNum2)); // (this would be a testcase error) + + client *c = static_cast(zcalloc(sizeof(client))); + struct serverCommand *cmd = static_cast(zcalloc(sizeof(struct serverCommand))); + + cmd->fullname = sdsnew("SETGET"); + cmd->arity = 4; + cmd->flags = CMD_WRITE | CMD_WRITE_FIRSTKEY_ONLY; + + cmd->legacy_range_key_spec.begin_search_type = KSPEC_BS_INDEX; + cmd->legacy_range_key_spec.bs.index.pos = 1; // firstkey + cmd->legacy_range_key_spec.fk.range.lastkey = -1; + cmd->legacy_range_key_spec.fk.range.keystep = 2; + + c->cmd = cmd; + c->db = server.db[db]; + + c->argc = 4; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(cmd->fullname); + c->argv[1] = createStringObjectFromCString(keyStr(itemNum1)); + c->argv[2] = createStringObjectFromCString(value1); + c->argv[3] = createStringObjectFromCString(keyStr(itemNum2)); + + return c; + } + + + // Client with a fictional write command with no keys specified + client *getNoKeysWriteClient() { + // Fictional command which is marked WRITE, but has no keys. + client *c = static_cast(zcalloc(sizeof(client))); + struct serverCommand *cmd = static_cast(zcalloc(sizeof(struct serverCommand))); + + cmd->fullname = sdsnew("NOKEYSWRITE"); + cmd->arity = 1; + cmd->flags = CMD_WRITE; + + cmd->legacy_range_key_spec.begin_search_type = KSPEC_BS_INVALID; // No keys + + c->cmd = cmd; + c->db = server.db[0]; + + c->argc = 1; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(cmd->fullname); + + return c; + } + + + void freeClientArgv(client *c) { + for (int i = 0; i < c->argc; i++) decrRefCount(c->argv[i]); + zfree(c->argv); + c->argv = NULL; + c->argc = 0; + } + + + // During testing, we create some fake commands. This checks if the command is real or fake. + // A fake command is dynamically allocated and can be freed. Real commands are static. + bool isRealValkeyCommand(struct serverCommand *cmd) { + return lookupCommandByCString(cmd->declared_name); + } + + + void freeTestClient(client *c) { + // If the current command references one of the multi commands, set it back to the EXEC + if (c->mstate != NULL) { + for (int i = 0; i < c->mstate->count; i++) { + if (c->argv == c->mstate->commands[i].argv) { + c->argc = orig_argc; + c->argv = orig_argv; + orig_argc = 0; + orig_argv = nullptr; + break; + } + } + } + freeClientMultiState(c); + freeClientArgv(c); + + if (!isRealValkeyCommand(c->cmd)) { + sdsfree(c->cmd->fullname); + zfree(c->cmd); + } + + zfree(c); + } + + + // Simulate what happens when a write command is blocked + void simulateBlockedWrite(client *c, int expectedNumberBlockedKeys = 1) { + EXPECT_CALL(mock, blockClientInUseOnKeys(c, expectedNumberBlockedKeys, _)).Times(1); + bool blocked = bgIteration_blockClientIfRequired(c); + EXPECT_TRUE(blocked); + } + + + // Simulate what happens when a write command isn't blocked + void simulateUnblockedWrite_inCall(client *c) { + EXPECT_CALL(mock, blockClientInUseOnKeys(c, _, _)).Times(0); + bool blocked = bgIteration_blockClientIfRequired(c); + EXPECT_FALSE(blocked); + server.in_call++; + } + + + // Simulates what happens when a write command (SET) actually executes. This requires a + // scenario where we would NOT be blocked on the write. It actually alters the value of + // the key and updates the metadata. + void simulateUnblockedWriteWithModification(client *c) { + simulateUnblockedWrite_inCall(c); + + // Fake execution of the command - touch the iterator_epoch counter and swap the value + // We need to duplicate the value because setKey() can reallocate it. + robj *value = dupStringObject(c->argv[2]); + setKey(c, c->db, c->argv[1], &value, SETKEY_ADD_OR_UPDATE); + + // Let's make sure that setKey updated the iteration epoch (as it should have) + dbEntry *de = dbFind(c->db, static_cast(objectGetVal(c->argv[1]))); + bgIterationEntryMetadata *md = static_cast(objectGetMetadata(de)); + bgIterationEntryMetadata md_after_setkey = *md; + // Now update the md again, and it should still match + bgIteration_dbEntryModified(de); + EXPECT_EQ(md, objectGetMetadata(de)); // the md location shouldn't have changed + EXPECT_EQ(md_after_setkey, *md); // the md value should still be the same + + bgIteration_handleCommandReplication(c->db->id, c->cmd, c->argc, c->argv); + server.in_call--; + } + + + // Simulate what happens when a write command is NOT blocked, because the key can be cloned + // and expedited. This requires a scenario where we would normally need to block the + // client so that bgIteration can process the item. + void simulateClonedWriteWithModification(bgIterator *it, client *c) { + bgIteratorStatus status; + bgIteratorGetStatus(it, &status); + unsigned long initialClones = status.dbentry_clones_queued; + + // Client should not get blocked + simulateUnblockedWriteWithModification(c); + + // Ensure that cloning took place + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, (initialClones + 1)); + + // Ensure that the real item isn't inuse (because we cloned it instead) + dbEntry *de = dbFind(c->db, static_cast(objectGetVal(c->argv[1]))); + ASSERT_FALSE(bgIteration_isEntryInuse(de)); + } + + + // Simulate the expiration (active expiration) of a key. This is independent of command execution. + void simulateExpiration(int itemNum) { + ASSERT_NE(getItem(itemNum), nullptr); // Should be there before expire + + // Send bgIteration the DEL + int db = getDbFromItemNum(itemNum); + robj *argv[2]; + argv[0] = createStringObjectFromCString("DEL"); + argv[1] = createStringObjectFromCString(keyStr(itemNum)); + serverCommand *cmd = lookupCommandByCString("DEL"); + // KeyDelete should be called before the deletion occurs + bgIteration_keyDelete(db, static_cast(objectGetVal(argv[1]))); + + simpleDelItem(itemNum); // Simulate the actual del + + // Replication happens after the deletion occurs + ASSERT_EQ(server.in_call, 0); // test sanity check + bgIteration_handleCommandReplication(db, cmd, 2, argv); + decrRefCount(argv[0]); + decrRefCount(argv[1]); + + EXPECT_EQ(getItem(itemNum), nullptr); + } + + + // Simulates an expiration, but validates behavior for an item inuse by bgIteration. + void simulateExpirationOfInuse(int itemNum) { + // An inuse item will have a refcount > 1. BgIteration should have incremented the + // refcount while it is inuse. + dbEntry *de = getItem(itemNum); + ASSERT_NE(de, nullptr); // Should be there before expire + EXPECT_TRUE(bgIteration_isEntryInuse(de)); + EXPECT_EQ(de->refcount, 2u); + + simulateExpiration(itemNum); + + // At this point, the item is removed from the DB, but still exists, and the refcount + // has been reduced to 1. This allows a background thread to continue using the item. + EXPECT_EQ(de->refcount, 1u); + } + + + // Simulates an expiration, but the item is a future item which will be expedited. + void simulateExpirationWithExpedite(int itemNum) { + // An inuse item will have a refcount > 1. BgIteration should have incremented the + // refcount while it is inuse. + dbEntry *de = getItem(itemNum); + ASSERT_NE(de, nullptr); // Should be there before expire + EXPECT_FALSE(bgIteration_isEntryInuse(de)); // Not yet inuse + EXPECT_EQ(de->refcount, 1u); + + simulateExpiration(itemNum); + + // At this point, the item is removed from the DB, but still exists, and the refcount + // has been reduced to 1. This allows a background thread to continue using the item. + EXPECT_TRUE(bgIteration_isEntryInuse(de)); // It's inuse now + EXPECT_EQ(getItem(itemNum), nullptr); // but it's not in the DB anymore + EXPECT_EQ(de->refcount, 1u); + } + + + // Simulate execution of a SWAPDB command + void simulateSwapDB(int dbid0, int dbid1) { + char dbStr[2] = {0}; + + client *c = static_cast(zcalloc(sizeof(client))); + + c->cmd = lookupCommandByCString("swapdb"); + c->db = server.db[0]; + + c->argc = 3; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(c->cmd->fullname); + dbStr[0] = '0' + dbid0; + c->argv[1] = createStringObjectFromCString(dbStr); + dbStr[0] = '0' + dbid1; + c->argv[2] = createStringObjectFromCString(dbStr); + + simulateUnblockedWrite_inCall(c); // SWAPDB should never block + + // The real SWAP does more than this, but this is enough for unit tests + serverDb *aux = server.db[dbid0]; + server.db[dbid0] = server.db[dbid1]; + server.db[dbid1] = aux; + + bgIteration_handleCommandReplication(0, c->cmd, c->argc, c->argv); + server.in_call--; + + freeTestClient(c); + } + + + // Simulate execution of a FLUSHDB or FLUSHALL command + void simulateFlushDB(int db, int anInUseItem = -1) { + client *c = static_cast(zcalloc(sizeof(client))); + + if (db == -1) { + c->cmd = lookupCommandByCString("flushall"); + c->db = server.db[0]; + } else { + c->cmd = lookupCommandByCString("flushdb"); + c->db = server.db[db]; + } + + c->argc = 1; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(c->cmd->fullname); + + dbEntry *de_in_use; + if (anInUseItem >= 0) { + de_in_use = getItem(anInUseItem); + EXPECT_EQ(de_in_use->refcount, 2u); + } + + simulateUnblockedWrite_inCall(c); // FLUSHDB should never block + + // The real FLUSH does more than this, but this is enough for unit tests + + // Now flush the items + for (int d = 0; d < server.dbnum; d++) { + if (db == -1 || db == d) { + kvstoreRelease(server.db[d]->keys); + server.db[d]->keys = NULL; + } + } + + if (anInUseItem >= 0) { + EXPECT_EQ(de_in_use->refcount, 1u); + } + + // and replicate + + bgIteration_handleCommandReplication(0, c->cmd, c->argc, c->argv); + server.in_call--; + + freeTestClient(c); + } +}; + + +TEST_F(BgIterationTest, dbIsOK) { + // Just run the setup/teardown code to make sure the DB is OK. +} + + +///////////////////////////////////////////////////// +// Simple Full-scan iterator tests +///////////////////////////////////////////////////// + +// A simple full scan that just checks basic flow. +TEST_F(BgIterationTest, createAndCleanup) { + bgIterator *it = bgIteratorCreateFullScanIter("simple", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + EXPECT_EQ(bgIteratorFind("simple"), it); + EXPECT_STREQ(bgIteratorName(it), "simple"); + + bgIteratorStatus status; + bgIteratorGetStatus(it, &status); + + EXPECT_EQ(status.dbentries_queued, 0u); + EXPECT_EQ(status.dbentries_processed, 0u); + EXPECT_EQ(status.replication_queued, 0u); + EXPECT_EQ(status.replication_processed, 0u); + EXPECT_EQ(status.swapdb_queued, 0u); + EXPECT_EQ(status.swapdb_processed, 0u); + EXPECT_EQ(status.flushdb_queued, 0u); + EXPECT_EQ(status.flushdb_processed, 0u); + + EXPECT_EQ(status.queue_length, 0u); + EXPECT_GT(status.queue_length_target, 0u); + + EXPECT_LT(status.runtime_ms, 5u); + EXPECT_EQ(status.current_item_ms, 0u); + + expectAnythingCleanup(it); + + EXPECT_EQ(bgIteratorFind("simple"), nullptr); +} + + +// Close client before reading anything +TEST_F(BgIterationTest, testClientCloseBeforeRead) { + bgIterator *it = bgIteratorCreateFullScanIter("simple", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + bgIteration_feedIterators(); + + bgIteratorClose(it); // Immediately close before reading + + bgIteration_feedIterators(); // Recognize the closed iterator + + // Check that the cleanup callback was executed properly + EXPECT_EQ(cleanupCount, 1); + EXPECT_TRUE(cleanupTerminated); +} + + +// Test that the full scan hits each item in the expected sequence. +TEST_F(BgIterationTest, orderedIteration) { + bgIterator *it = bgIteratorCreateFullScanIter("simple", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, LAST_ITEM); + + // Quick status check. At this point, the final item hasn't been returned yet. + bgIteratorStatus status; + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentries_queued, static_cast(TOTAL_ITEMS)); + EXPECT_EQ(status.dbentries_processed, static_cast(TOTAL_ITEMS) - 1); + + expectReadComplete(it); // Returns the final item, and reads the completion item + + // Check that the cleanup callback was executed properly + EXPECT_EQ(cleanupCount, 1); + EXPECT_FALSE(cleanupTerminated); +} + + +// Test that two simultaneous iterations work properly. +TEST_F(BgIterationTest, twoOrderedIterations) { + bgIterator *it1 = bgIteratorCreateFullScanIter("simple1", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + bgIterator *it2 = bgIteratorCreateFullScanIter("simple2", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + EXPECT_EQ(bgIteratorFind("simple1"), it1); + EXPECT_EQ(bgIteratorFind("simple2"), it2); + + int it1Count = 0; + int it2Count = 0; + while (it1Count < TOTAL_ITEMS || it2Count < TOTAL_ITEMS) { + // Randomly read from either iterator + if ((rand() % 2) == 0) { + if (it1Count < TOTAL_ITEMS) expectReadKey(it1, it1Count++); + } else { + if (it2Count < TOTAL_ITEMS) expectReadKey(it2, it2Count++); + } + } + + // Nothing left but to read the final completions + expectReadComplete(it1); + EXPECT_EQ(cleanupCount, 1); + EXPECT_FALSE(cleanupTerminated); + expectReadComplete(it2); + EXPECT_EQ(cleanupCount, 2); + EXPECT_FALSE(cleanupTerminated); +} + + +///////////////////////////////////////////////////// +// MODIFY A FUTURE ITEM +// The next tests validate the basic pattern when a key, not yet iterated, is modified. +// Each variation of iteration flags is tested. +// Note that these tests execute without cloning (cloning is tested elsewhere). +///////////////////////////////////////////////////// + +// Modify a future item, without replication or consistency. +// Our expectation for this case is that the modification should proceed without blocking, the item +// shouldn't be expedited, and we will see the modified item once the iterator reaches it. +TEST_F(BgIterationTest, modFutureItem) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // Fake a modification to a later key so that we can see if it gets processed out of order. + c = getWriteClient(6, "xxx"); + + // We DONT expect the client to be blocked - not consistent + simulateUnblockedWriteWithModification(c); + + // Now continue reading, 1, 2, 3, 4, 5 + expectReadKeySequence(it, 1, 5); + + // Let's validate that key 6 shows the new value + expectReadKey(it, 6, "xxx"); + + // Continue... + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a future item, without replication but with consistency. (Like a SAVE operation) +// Our expectation for this case is that the modification SHOULD be blocked, as we have to save the +// the item in it's state before the modification. To reduce blocking time, the item should be +// moved to the head of the queue - there's no replication in this case, so out-of-order processing +// isn't a concern. +TEST_F(BgIterationTest, modFutureItem_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // Fake a modification to a later key so that we can see if it gets processed out of order. + c = getWriteClient(6, "xxx"); + // Since this is consistent, we will block the client, disallowing the write. + simulateBlockedWrite(c); + + // On a consistent iterator, the event is expedited in-front of items already in queue! + // Read key 6 out of order. + expectReadKey(it, 6); + + // Now, when we read key 1, key 6 is released back to Valkey, and the client will be unblocked. + expectReadKeyWithUnblock(it, 1, 6); + simulateUnblockedWriteWithModification(c); // Now the write can proceed + + // Continue... + expectReadKeySequence(it, 2, 5); + // 6 has already been processed + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a future item, with replication but without consistency. (Like a Forkless Full Sync operation) +// Our expectation for this case is that the modification should proceed without blocking, as the +// mode is inconsistent. We don't expect replication, as we haven't reached the item yet. We'll +// see the modified item later. +TEST_F(BgIterationTest, modFutureItem_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // Fake a modification to a later key so that we can see if it gets processed out of order. + c = getWriteClient(6, "xxx"); + + // We DONT expect the client to be blocked - not consistent + simulateUnblockedWriteWithModification(c); + + // NOTE: Since we haven't reached this item yet, and consistency is not required, there's no + // need to replicate this command. So everything should wrap up just fine - we will see + // the new value when we get to it. + + // Now continue reading, 1, 2, 3, 4, 5 + expectReadKeySequence(it, 1, 5); + + // Let's validate that key 6 shows the new value + expectReadKey(it, 6, "xxx"); + + // Continue... + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// MODIFY A CURRENT ITEM +// The next tests validate the basic pattern when a key, currently in use, is modified. +// Each variation of iteration flags is tested. +// Note that these tests execute without cloning (cloning is tested elsewhere). +///////////////////////////////////////////////////// + +// Modify a current item, without replication or consistency. +// Our expectation for this case is that the modification SHOULD be blocked, the item shouldn't +// be expedited (it's already in use). +TEST_F(BgIterationTest, modCurrentItem) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + c = getWriteClient(2, "xxx"); + + // Must be blocked since key is queued + simulateBlockedWrite(c); + + // Now continue reading + expectReadKey(it, 1); + expectReadKey(it, 2); + expectReadKeyWithUnblock(it, 3, 2); + simulateUnblockedWriteWithModification(c); // the actual write won't affect anything (past key, no replication) + + // Continue... + expectReadKeySequence(it, 4, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a current item, without replication but with consistency. (Like a SAVE operation) +// Our expectation for this case is that the modification SHOULD be blocked, the item shouldn't +// be expedited (it's already in use). +TEST_F(BgIterationTest, modCurrentItem_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + c = getWriteClient(2, "xxx"); + + // Must be blocked since key is queued + simulateBlockedWrite(c); + + // Now continue reading + expectReadKey(it, 1); + expectReadKey(it, 2); + expectReadKeyWithUnblock(it, 3, 2); + simulateUnblockedWriteWithModification(c); // the actual write won't affect anything (past key, no replication) + + // Continue... + expectReadKeySequence(it, 4, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a current item, with replication but without consistency. (Like a Forkless Full Sync operation) +// Our expectation for this case is that the modification SHOULD be blocked. After the key is processed, +// the write will proceed, and the replication will be sent. +TEST_F(BgIterationTest, modCurrentItem_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + c = getWriteClient(2, "xxx"); + + // Must be blocked since key is queued + simulateBlockedWrite(c); + + // Now continue reading + expectReadKey(it, 1); + expectReadKey(it, 2); + expectReadKeyWithUnblock(it, 3, 2); + simulateUnblockedWriteWithModification(c); // the actual write will cause replication + + expectReadKey(it, 4); // 4 got put in queue when 3 was read + + expectReadReplication(it, c); + + // Continue... + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// MODIFY A PAST ITEM +// The next tests validate the basic pattern when a key, not yet iterated on, is modified. +// Each variation of iteration flags is tested. +// Note that these tests execute without cloning (cloning is tested elsewhere). +///////////////////////////////////////////////////// + +// Modify a past item, without replication or consistency. +// Our expectation for this case is that the modification should proceed without blocking. +// No replication is generated and keys are processed similar to no modification. +TEST_F(BgIterationTest, modPastItem) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // This read returns key 0 (making it a past item) + expectReadKey(it, 1); + + // At this point, key 0 is returned. + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); + + // Continue... + expectReadKeySequence(it, 2, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a past item, without replication but with consistency. (Like a SAVE operation) +// Our expectation for this case is that the modification should proceed without blocking. +// No replication is generated and keys are processed similar to no modification. +TEST_F(BgIterationTest, modPastItem_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // This read returns key 0 (making it a past item) + expectReadKey(it, 1); + + // At this point, key 0 is returned. + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); + + // Continue... + expectReadKeySequence(it, 2, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a past item, with replication but without consistency. (Like a Forkless Full Sync operation) +// Our expectation for this case is that the modification should proceed without blocking. +// Replication will be sent. +TEST_F(BgIterationTest, modPastItem_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // This read returns key 0 (making it a past item) + expectReadKey(it, 1); + + // At this point, key 0 is returned. + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); + + // Key 2 was already in queue (same bucket as key 1). The replication will follow. + expectReadKey(it, 2); + expectReadReplication(it, c); + + // Continue... + expectReadKeySequence(it, 3, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// TESTS FOR ITEM CLONING +///////////////////////////////////////////////////// + +// In a consistent iteration, verify that a simple string is properly cloned, and that a write can +// occur without blocking. Validate the cloned item and metadata. +TEST_F(BgIterationTest, modFutureItem_start_CloneExpeditedItem) { + // Initialize cloning configurations. + bgIteration_unitTestEnableCloning(50, 100); + + bgIteratorStatus status; + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // Fake a modification to a later key so that we can see if it gets processed out of order. + c = getWriteClient(6, "xxx"); + + // Quick status check. At this point, no clones exist yet. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, 0u); + + // Since item 6 should be cloned, it will not block the client, allowing the write. + void *de6_md = cloneMetadata(getItem(6)); + // This doesn't block, queues a cloned item, and modifies the item (touching metadata) + simulateClonedWriteWithModification(it, c); + + // At this point, one clone is in the queue. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, 1u); + + // On a consistent iterator, the event is expedited in-front of items already in queue! + // Read key 6 (which is cloned) out of order. The value will still match the key. + expectReadClonedKey(it, 6, de6_md); // Also validates and frees the metadata + + // Quick status check. At this point, cloned items have not been marked as processed yet. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_processed, 0u); + + // Reading key 1 will release key 6, and the clone will finish processing. + expectReadKey(it, 1); + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_processed, 1u); + + // Now, when we read key 2 should not have an impact on number of processed clones. + expectReadKey(it, 2); + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_processed, 1u); + + // Continue... + expectReadKeySequence(it, 3, 5); + // 6 has already been processed + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +// Check that cloning for simple strings is respecting the size limits and pool size. On a +// consistent iteration, we expect to block or clone on all future keys. We validate that we can +// clone if the item is small enough and the cloning pool has more space left. +TEST_F(BgIterationTest, modFutureItem_start_LargeItemOrClonePoolFull) { + // Initialize cloning configurations to test the clone pool functionality first. + bgIteration_unitTestEnableCloning(50, 50); + + bgIteratorStatus status; + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // Fake a modification to a later key so that we can see if it gets processed out of order. + client *c6 = getWriteClient(6, "xxx"); + client *c7 = getWriteClient(7, "xxx"); + client *c8 = getWriteClient(8, "xxx"); + + // Quick status check. At this point, no clones exist yet. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, 0u); + + // Since item 6 should be cloned, it will not block the client, allowing the write. + void *de6_md = cloneMetadata(getItem(6)); + // This doesn't block, queues a cloned item, and modifies the item (touching metadata) + simulateClonedWriteWithModification(it, c6); + + // At this point, one clone is in the queue. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, 1u); + + // Now that cloning pool is full, item 7 will not be cloned and the client will be blocked. + simulateBlockedWrite(c7); + ASSERT_TRUE(bgIteration_isEntryInuse(getItem(7))); + + // There is still only one cloned item in the queue. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, 1u); + + // Now change cloning configurations to test that large items will not be cloned. We adjust + // the clone pool size to allow two items, but set the maximum item size to be smaller than + // the size of item 8. The clone pool size must be larger than the total size of the existing + // clones plus the maximum item clone size. + bgIteration_unitTestEnableCloning(1, 101); + + // This write will pass the clone pool check but fail the item size check, blocking the client. + simulateBlockedWrite(c8); + ASSERT_TRUE(bgIteration_isEntryInuse(getItem(8))); + + // On a consistent iterator, the expedited item in-front of items already in queue! + // Read key 6 out of order. + expectReadClonedKey(it, 6, de6_md); + + // Now, when we expect to read key 7, which was expedited, key 6 will be released back to Valkey + // and the clone will be deallocated here. + expectReadKey(it, 7); + + // Now, when we read key 8, which was expedited, key 7 is released back to Valkey, and the client + // will be unblocked. + // (actually, unblock is called after every key [just in case] - but functionally we only care + // about this one) + expectReadKeyWithUnblock(it, 8, 7); + simulateUnblockedWriteWithModification(c7); + + // Now, when we read key 1, key 8 is released back to Valkey, and the client will be unblocked. + expectReadKeyWithUnblock(it, 1, 8); + simulateUnblockedWriteWithModification(c8); + + // Since only one item was cloned, there should be one clone processed + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_processed, 1u); + + // Continue... + expectReadKeySequence(it, 2, 5); + // 6, 7, and 8 have already been processed + expectReadKeySequence(it, 9, LAST_ITEM); + expectReadComplete(it); + freeTestClient(c6); + freeTestClient(c7); + freeTestClient(c8); +} + + +///////////////////////////////////////////////////// +// TESTS RELATED TO MODIFICATION OF TWO ITEMS +// When 2 keys are modified, we need to ensure that both keys have been sent before we can send +// replication. This means that if replication is present, we may have to block/expedite for +// future keys, even in the inconsistent scenario. +///////////////////////////////////////////////////// + +// Replication enabled, but NOT consistent. In this case, if ANY of the keys have been iterated, +// ALL of the keys must be replicated so that the command can be processed properly on the replica. +TEST_F(BgIterationTest, modPastFutureItem_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + // In this test, we need a past and future key IN THE SAME DB (they're used in the same command). + // DB1 has lots of buckets. After reading item 9, + // 8 will be past, 10 will be in queue, 11-15 will be future. + expectReadKeySequence(it, 0, 9); + + // We're going to write to key 8 (past) and read from key 12 (future) + // Even though key 12 is for READ in this command, it must be expedited so that it exists before + // the associated replication is sent. + c = getSetGetClient(8, "xxx", 12); + simulateBlockedWrite(c); + + // Key 12 will be expedited, to the front, because there are no barrier items in the queue. + + expectReadKey(it, 12); // expedited + + expectReadKeyWithUnblock(it, 10, 12); // reading key 10 (was in queue already) unblocks 12 + + simulateUnblockedWriteWithModification(c); + + // Continue... + expectReadKey(it, 11); + expectReadReplication(it, c); + + expectReadKeySequence(it, 13, LAST_ITEM); + expectReadComplete(it); +} + +// Replication enabled, but NOT consistent. In this case, if ANY of the keys have been iterated, +// ALL of the keys must be replicated so that the command can be processed properly on the replica. +// With a past and future item, the future item will be expedited. But in this case, we will ensure +// that there's a barrier item (flushdb) in the queue preventing expedite to front of line. +TEST_F(BgIterationTest, modPastFutureItemBarrier_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + // In this test, we need a past and future key IN THE SAME DB (they're used in the same command). + // DB1 has lots of buckets. After reading item 9, + // 8 will be past, 10 will be in queue, 11-15 will be future. + expectReadKeySequence(it, 0, 9); + + // Insert a FLUSHDB (barrier item) into the queue. + simulateFlushDB(0); + + // We're going to write to key 8 (past) and read from key 12 (future) + // Even though key 12 is for READ in this command, it must be expedited so that it exists before + // the associated replication is sent. + c = getSetGetClient(8, "xxx", 12); + simulateBlockedWrite(c); + + // Key 12 will be expedited, BUT NOT TO THE FRONT - because the FLUSHDB item is a barrier item + + expectReadKey(it, 10); // was already in queue + expectReadFlushDB(it, 0, true); // and now the flush (with replication) + expectReadKey(it, 12); // and then the expedited key + + expectReadKeyWithUnblock(it, 11, 12); // reading key 11 unblocks 12 + + simulateUnblockedWriteWithModification(c); + + // Continue... + expectReadKey(it, 13); + expectReadReplication(it, c); + + expectReadKeySequence(it, 14, LAST_ITEM); + expectReadComplete(it); +} + +// Replication NOT enabled. A read-only key doesn't need to be expedited, even if other keys have +// been processed already. (This should work identically for both consistent/non-consistent. +TEST_F(BgIterationTest, modPastFutureItem_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter1", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // In this test, we need a past and future key IN THE SAME DB (they're used in the same command). + // DB1 has lots of buckets. After reading item 9, + // 8 will be past, 10 will be in queue, 11-15 will be future. + expectReadKeySequence(it, 0, 9); + + // We're going to write to key 8 (past) and read from key 12 (future) + // Since there's no replication, we don't have to worry about expediting 12. The write will + // proceed without blocking. + c = getSetGetClient(8, "xxx", 12); + simulateUnblockedWriteWithModification(c); + + // Key 12 will not be expedited. Remaining keys should be received in normal order. + expectReadKeySequence(it, 10, LAST_ITEM); + expectReadComplete(it); +} + + +TEST_F(BgIterationTest, modPastFutureItem) { + bgIterator *it = bgIteratorCreateFullScanIter("iter2", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + + // In this test, we need a past and future key IN THE SAME DB (they're used in the same command). + // DB1 has lots of buckets. After reading item 9, + // 8 will be past, 10 will be in queue, 11-15 will be future. + expectReadKeySequence(it, 0, 9); + + // We're going to write to key 8 (past) and read from key 12 (future) + // Since there's no replication, we don't have to worry about expediting 12. The write will + // proceed without blocking. + c = getSetGetClient(8, "xxx", 12); + simulateUnblockedWriteWithModification(c); + + // Key 9 will not be expedited. Remaining keys should be received in normal order. + expectReadKeySequence(it, 10, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// TESTS RELATED TO MISSING ITEMS +// Missing items are tricky. A missing item might be logically located in the past or future, in +// relation to the current iteration position. The command may (or may not) create the "missing" +// key. Some general considerations: +// * In a consistent iteration, a missing key didn't exist at the time of consistency, or it was +// already processed (saved) at the time of the deletion. If the missing key gets created, we +// must be sure to skip it if we later iterate over it. +// * In a non-consistent iteration with replication: +// * If the key location is already passed, the replication is sent, allowing the key to be +// created (or not) based on the replication. +// * If the key location is in the future, we can allow the command to proceed, without +// replication. If the key is created, we will process it when the iterator gets to it. +// +// We expect: +// no-repl, no-consist: past items are ignored - future items are processed when iterated +// no-repl, yes-consist: past items are ignored - future items are ignored +// yes-repl, no-consist: past item skipped, but replicated - future items are created by replication and skipped later +// yes-repl, yes-consist: past item skipped, but replicated - future items are processed when iterated +///////////////////////////////////////////////////// + +// no-repl, no-consist: creation of PAST item has no impact +TEST_F(BgIterationTest, missingPastItem) { + simpleDelItem(0); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 1); + expectReadKey(it, 2); + + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 3, LAST_ITEM); + expectReadComplete(it); +} + + +// no-repl, yes-consist: creation of PAST item has no impact +TEST_F(BgIterationTest, missingPastItem_start) { + simpleDelItem(0); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 1); + expectReadKey(it, 2); + + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 3, LAST_ITEM); + expectReadComplete(it); +} + + +// yes-repl, no-consist: creation of a PAST item will be replicated +TEST_F(BgIterationTest, missingPastItem_eventual) { + simpleDelItem(0); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 1); + expectReadKey(it, 2); + expectReadKey(it, 3); + + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); // replication will be added after item 4 (3,4 in same bucket) + + expectReadKey(it, 4); + + expectReadReplication(it, c); + + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + + +// no-repl, no-consist: creation of FUTURE item is seen when reached by the iteration. +TEST_F(BgIterationTest, missingFutureItem) { + // Using DB1 so we have lots of buckets + simpleDelItem(14); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + + const char *newValue = "xxx"; + c = getWriteClient(14, newValue); + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 1, 13); + + // We expect to see item 14. + // Note that for an inconsistent DB view, it is logically undefined if this value is seen (or not). + // But as implemented, we should see it and the test is helpful to understand if/when the + // functionality changes. + expectReadKey(it, 14, newValue); + + expectReadKey(it, LAST_ITEM); + expectReadComplete(it); +} + + +// no-repl, yes-consist: creation of FUTURE item is ignored by consistent iteration. +TEST_F(BgIterationTest, missingFutureItem_start) { + // Using DB1 so we have lots of buckets + simpleDelItem(14); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + + c = getWriteClient(14, "xxx"); + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 1, 13); + // Key 14 is missing - it didn't exist at start of consistent iteration + expectReadKey(it, LAST_ITEM); + expectReadComplete(it); +} + + +// yes-repl, no-consist: creation of FUTURE item is handled by the replication, and then the key is +// later skipped (treated like an early iteration case). +TEST_F(BgIterationTest, missingFutureItem_eventual) { + // Using DB1 so we have lots of buckets + simpleDelItem(14); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKey(it, 0); // Items 1 & 2 are in queue (same bucket) + + c = getWriteClient(14, "xxx"); + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 1, 2); + + expectReadReplication(it, c); // Here's the replication creating item 14 + + expectReadKeySequence(it, 3, 13); + // We expect item 14 to be skipped, because it was created by the earlier replication + expectReadKey(it, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// TESTS RELATED TO EXPIRATION +// Expiration can be tricky. When pre-evaluating a command with bgIteration_blockClientIfRequired, +// a key might exist, but be ready for expiration. Then, as the command executes, the key expires +// and gets deleted before the write operation. Consider SET K V. +// In the unexpired case, this appears to bgIteration as a single SET command (which replaces the value). +// In the expired case, bgIteration will receive a DEL followed by a SET. +// +// Another case is a READ command. A read command won't cause the client to be blocked. However, +// if the key is expired, this will cause a DEL. For consistent processing, this key might need to +// be expedited so that it can be processed before it gets deleted. In this case, the key is +// unlinked from the main Valkey dictionary, but the actual deletion is deferred. +///////////////////////////////////////////////////// + +TEST_F(BgIterationTest, expireKeys) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + expectReadKey(it, 1); + + // At this point, key 1 is active, key 2 is in queue. + + simulateExpiration(0); // Past - we no longer care + simulateExpirationOfInuse(2); // Current - it's inuse + simulateExpiration(5); // Future - we don't care (non-consistent) + + expectReadKeySequence(it, 2, 4); + // key 5 has been deleted + expectReadKeySequence(it, 6, LAST_ITEM); + expectReadComplete(it); +} + + +TEST_F(BgIterationTest, expireKeys_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + expectReadKey(it, 1); + + // At this point, key 1 is active, key 2 is in queue. + + simulateExpiration(0); // Past - we expect replication + simulateExpirationOfInuse(2); // Current - it's inuse, but we expect replication + simulateExpiration(5); // Future - we don't care (non-consistent) + + expectReadKey(it, 2); // this was already queued + + expectReadReplicationDel(it, 0); // Past item should replicate + expectReadReplicationDel(it, 2); // Current item should replicate + // Item 5 is a future item and doesn't need to replicate + + expectReadKeySequence(it, 3, 4); + // Item 5 has been deleted + expectReadKeySequence(it, 6, LAST_ITEM); + expectReadComplete(it); +} + + +TEST_F(BgIterationTest, expireKeys_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + expectReadKey(it, 1); + + // At this point, key 1 is active, key 2 is in queue. + + simulateExpiration(0); // Past - we no longer care + simulateExpirationOfInuse(2); // Current - we must defer + simulateExpirationWithExpedite(5); // Future - will become inuse and expedited for consistency + + expectReadKey(it, 5); // Expedited to front + + expectReadKeySequence(it, 2, 4); + // Item 5 has been deleted + expectReadKeySequence(it, 6, LAST_ITEM); + expectReadComplete(it); +} + + +// Special case during a non-consistent iteration with replication and expiration. +// 1. A future key is created (and processed by its replication) - considered early iterated +// 2. Later the key is expired and deleted during command processing (causes DEL to be sent) - no longer early iterated +// 3. The key is recreated as part of the command processing (and this command was replicated) - again early iterated +// 4. Finally, when we iterate to the key, it shouldn't be sent, because it was replicated in step 3. +TEST_F(BgIterationTest, expireKeys_eventual_FutureKeyCreatedThenExpiredDuringSet) { + simpleDelItem(8); // Start with a missing future item + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKey(it, 0); // Get the iterator started + + c = getWriteClient(8, "xxx"); + simulateUnblockedWriteWithModification(c); // Not blocked because this is a future key (but we expect repl) + + // Now do it again, but break out the steps so that we can simulate an expiration + simulateUnblockedWrite_inCall(c); // Shouldn't be blocked because this is a future key + + // Now, as the SET command tries to execute, simulate that the key is expired. + // First, the key should be physically removed and bgIteration_keyDelete called + bgIteration_keyDelete(getDbFromItemNum(8), static_cast(objectGetVal(c->argv[1]))); + simpleDelItem(8); // Simulate the actual del (after bgIteration_keyDelete called) + // Then the replication for the delete occurs + robj *argv[2]; + argv[0] = createStringObjectFromCString("DEL"); + argv[1] = c->argv[1]; + serverCommand *cmd = lookupCommandByCString("DEL"); + bgIteration_handleCommandReplication(getDbFromItemNum(8), cmd, 2, argv); + decrRefCount(argv[0]); + + // Now the SET will run, re-creating the item (which is still a future item) + // We need to duplicate the value because setKey() can reallocate it. + robj *value = dupStringObject(c->argv[2]); + setKey(c, c->db, c->argv[1], &(value), SETKEY_ADD_OR_UPDATE); + + // Finally, replication will be sent because this is creating a new key + bgIteration_handleCommandReplication(getDbFromItemNum(8), c->cmd, c->argc, c->argv); + server.in_call--; + + // Test that everything comes as expected + expectReadKeySequence(it, 1, 2); // All one bucket - queued after key 0 read + + expectReadReplication(it, c); // Repl from the first SET command + expectReadReplicationDel(it, 8); // This is the expected replication of the DEL from expire + expectReadReplication(it, c); // Repl from the second SET command (recreating deleted key) + + expectReadKeySequence(it, 3, 7); // continue with normal iteration + // KEY 8 SHOULD BE OMITTED - This was already replicated + expectReadKeySequence(it, 9, LAST_ITEM); + + expectReadComplete(it); +} + + +// In this test, a future key is expedited. Then it is expired by normal expiration processing. +// We expect to see replication of the delete, since it was early iterated. +TEST_F(BgIterationTest, expireKeys_eventual_ExpeditedKeyExpired) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); // Also queues 1 & 2 + + // This will be blocked, and key 7 expedited + c = getWrite2KeysClient("sunionstore", 0, 7); + simulateBlockedWrite(c, 2); // blocked on both 0 and 7 + + expectReadKeyWithUnblock(it, 7, 0); // 7 expedited to front (unblocks 0) + expectReadKeyWithUnblock(it, 1, 7); // 1 was already in queue (unblocks 7) + + simulateUnblockedWriteWithModification(c); + + // At this point, + // * item 2 is still in the queue + // * replication for the sunionstore is queued + // * item 7 is in an early iterated state + + // Now expire key 7. We expect we will see replication (since 7 has been expedited) + simulateExpiration(7); + + // Check queue... + expectReadKey(it, 2); + expectReadReplication(it, c); + expectReadReplicationDel(it, 7); + + // and the rest + expectReadKeySequence(it, 3, 6); + // 7 is missing (expired) + expectReadKeySequence(it, 8, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// THE REMAINING TESTS ARE GENERAL / UNCATEGORIZED +///////////////////////////////////////////////////// + +// Iteration can be terminated from the main thread or from the child client. +// This tests termination driven from the main thread. +TEST_F(BgIterationTest, earlyTerminationFromMain) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + + // At this point, keys 1 & 2 are in queue. A termination should release those keys. + bool blocked1 = true; + bool blocked2 = true; + // We expect no general unblocks, we account for each specific unblock below. + EXPECT_CALL(mock, unblockClientsInUseOnKey(_)).Times(0); + // We should expect to see unblock called for items 1 & 2, as they are released from the queue. + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(1)))) + .WillOnce(Assign(&blocked1, false)); + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(2)))) + .WillOnce(Assign(&blocked2, false)); + bgIteratorTerminate(it); // queues the items for release + EXPECT_TRUE(bgIteratorIsTerminating(it)); + bgIteration_feedIterators(); // actually performs the release + EXPECT_FALSE(blocked1); + EXPECT_FALSE(blocked2); + + bool blocked0 = true; + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(0)))) + .WillOnce(Assign(&blocked0, false)); + bgIteratorItem *item = bgIteratorRead(it); + EXPECT_FALSE(blocked0); + EXPECT_EQ(item->type, BGITERATOR_ITEM_TERMINATED); + + bgIteratorClose(it); // background thread completes the termination + + EXPECT_EQ(cleanupCount, 0); + bgIteration_feedIterators(); // main thread, cleans up iterator and calls cleanup function + EXPECT_EQ(cleanupCount, 1); + EXPECT_TRUE(cleanupTerminated); +} + + +// Iteration can be terminated from the main thread or from the child client. +// This tests termination driven from the child client (the background thread). +TEST_F(BgIterationTest, earlyTerminationFromChild) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + + // At this point, keys 1 & 2 are in queue. A termination should release those keys. + bgIteratorClose(it); // background thread initiates the termination + EXPECT_TRUE(bgIteratorIsTerminating(it)); + + bool blocked0 = true; + bool blocked1 = true; + bool blocked2 = true; + // Expecting no extra unblocks + EXPECT_CALL(mock, unblockClientsInUseOnKey(_)).Times(0); + // We expect item 0 (the in progress item) to be released + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(0)))) + .WillOnce(Assign(&blocked0, false)); + // We expect items 1-4 (the queued items) to be released + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(1)))) + .WillOnce(Assign(&blocked1, false)); + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(2)))) + .WillOnce(Assign(&blocked2, false)); + bgIteration_feedIterators(); + EXPECT_FALSE(blocked0); + EXPECT_FALSE(blocked1); + EXPECT_FALSE(blocked2); + EXPECT_EQ(cleanupCount, 1); + EXPECT_TRUE(cleanupTerminated); +} + + +// Edge case. Executing a command (like SUNIONSTORE) which REPLACES the first key and reads the +// second key. In this case, bgIteration will get notified of the key deletion during execution of +// SETUNIONSTORE. Given that both keys are in the future (not iterated yet), we'll allow the +// command to execute, unblocked. We won't replicate as we'll pick up the key when we get to it. +TEST_F(BgIterationTest, writeWith2Keys_eventual_keyDeletedDuringSetReplace) { + // Using DB1 so we have lots of buckets + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, 8); // 9 is in queue + + // Write command that has 2 keys. 1 existing key that we write to and 1 dependant future key. + c = getWrite2KeysClient("sunionstore", 12, 13); + + simulateUnblockedWrite_inCall(c); + + // Now the call to keyDelete happens + sds sdskey = sdsnew(keyStr(12)); + bgIteration_keyDelete(getDbFromItemNum(12), sdskey); + sdsfree(sdskey); + simpleDelItem(12); // So simulate the actual del + + // Now the write will run, re-creating the item (which is still a future item) + const char *const newValueStr = "new value"; + robj *newValueRobj = createStringObjectFromCString(newValueStr); + setKey(c, c->db, c->argv[1], &newValueRobj, SETKEY_ADD_OR_UPDATE); + + // Finally, we are letting bgIteration know that the write command was executed + bgIteration_handleCommandReplication(getDbFromItemNum(12), c->cmd, c->argc, c->argv); + server.in_call--; + + // Since the write command was not replicated, we expect all the keys to be read in the normal + // order from the dictionary. + expectReadKeySequence(it, 9, 11); + expectReadKey(it, 12, newValueStr); + expectReadKeySequence(it, 13, LAST_ITEM); + + expectReadComplete(it); +} + + +// Edge case. When we have a new key which is created by a command, AND replication is enabled, we +// expect that we will replicate the command rather than serializing the key/value later. As an +// example, consider SUNIONSTORE A B. We want to create A by replicating the command. We don't +// want to have to process A as a key later on. But in this case, we can't run the command until +// B has been sent. We expect the command to be blocked while we send B. +TEST_F(BgIterationTest, writeWith2Keys_eventual_setNewKey_DependantFuture) { + // Using DB1 so we have lots of buckets + simpleDelItem(12); // Deleting key 12 to then create it with a write command + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, 8); // 9 is in queue + + // Write command that has 2 keys. 1 new key and 1 dependant future key. + c = getWrite2KeysClient("sunionstore", 12, 13); + + // We are simulating a new key in the dict. This command should block on the dependant key. + // This adds key 13 in the queue since the command depends on it. + simulateBlockedWrite(c); + + // Key 13 is processed out of order since the write depends on it. It was expedited to the + // front because there are no barrier events in the queue. + expectReadKey(it, 13); + + // Key 9 was already in the queue. Reading key 9 will unblock key 13, allowing us to write. + expectReadKey(it, 9); + + // Now that key 13 was processed and released by the iterator, the write command can be executed. + simulateUnblockedWriteWithModification(c); + + // Key 10 was queued when we read key 9 + expectReadKey(it, 10); + + // The replication of the write command was enqueued after key 11 + expectReadReplication(it, c); + + expectReadKey(it, 11); + + // We shouldn't see key 12 - as that was processed via replication. + // We shouldn't see key 13 - as that was expedited earlier + + // Now resuming processing of dict entries + expectReadKeySequence(it, 14, LAST_ITEM); + + expectReadComplete(it); +} + + +// A new key is being created, but is dependent on another key which has already been processed. +// In this case, the command shouldn't be blocked. +TEST_F(BgIterationTest, writeWith2Keys_eventual_setNewKey_DependantPast) { + // Using DB1 so we have lots of buckets + simpleDelItem(12); // Deleting key 12 to then create it with a write command + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKeySequence(it, 0, 9); // 10 is in queue, done with 8 + + // Write command that has 2 keys. 1 new key and 1 dependant past key. + c = getWrite2KeysClient("sunionstore", 12, 8); + + // We are simulating a new key in the dict. + // This command should not block since the dependant key has already been processed. + simulateUnblockedWriteWithModification(c); + + // Key 10 was put in the queue before the write + expectReadKey(it, 10); + + expectReadReplication(it, c); + + expectReadKey(it, 11); + + // Key 12 should be missing - it was processed by replication + + expectReadKeySequence(it, 13, LAST_ITEM); + expectReadComplete(it); +} + + +// A new key is being created, and has dependencies on 2 other keys - one already processed, one not. +// In this case, the command should be blocked so that the future key can be sent first. +TEST_F(BgIterationTest, writeWith3Keys_eventual_setNewKey_1DependantPast1DependantFuture) { + // Using DB1 so we have lots of buckets + simpleDelItem(12); // Deleting key 12 to then create it with a write command + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKeySequence(it, 0, 9); // 8 has been returned, 9 is active, 10 is in queue + + // Write command that has 1 new key and 2 dependencies (past/future) + c = getWrite3KeysClient("sunionstore", 12, 8, 13); + + // The write should be blocked, so that item 13 can be processed. + simulateBlockedWrite(c); + + expectReadKey(it, 13); // 13 was expedited to the front (no barrier events in queue) + + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(13)))).Times(1); + expectReadKey(it, 10); // 10 was already in queue (releases 13) + + simulateUnblockedWriteWithModification(c); + + expectReadKey(it, 11); + + expectReadReplication(it, c); + + expectReadKeySequence(it, 14, LAST_ITEM); + expectReadComplete(it); +} + + +// Test an edge case with the same (future) key being repeated in the command, like: +// SUNIONSTORE A B B +// In this test, A is a previously handled key, and B is a future key. We expect the future key B to +// be expedited (once). +TEST_F(BgIterationTest, writeWith3Keys_eventual_repeatedKey_1DependantPast1RepeatedFuture) { + // Using DB1 so we have lots of buckets + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKeySequence(it, 0, 9); // We're done with 8, and 10 is in queue + + // Write command that has 3 keys. 1 past key and 1 repeated key in the future. + c = getWrite3KeysClient("sunionstore", 8, 12, 12); + + // This command should block because 12 needs to be expedited. + simulateBlockedWrite(c); + + expectReadKey(it, 12); // expedited to the front (no barrier events) + + expectReadKey(it, 10); // was already in queue, releases 12 (unblocking the command) + + // Now that key 12 was processed and released by the iterator, the write command can be executed. + simulateUnblockedWriteWithModification(c); + + expectReadKey(it, 11); // was already in queue since reading key 10 + + expectReadReplication(it, c); + + // Now resuming processing of dict entries. + expectReadKeySequence(it, 13, LAST_ITEM); + expectReadComplete(it); +} + + +/* Tests the replication of a write command that creates a new key and depends on a + * future key which is duplicated in the command. */ +TEST_F(BgIterationTest, writeWith3Keys_eventual_repeatedKey_1newKey1RepeatedFuture) { + simpleDelItem(3); // Deleting key 3 to then create it with a write command + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // At this point, keys 1 & 2 are in queue. + + // Write command that has 3 keys. 1 new key and 1 repeated key in the future. + c = getWrite3KeysClient("sunionstore", 3, 5, 5); + + // This command should block on key 5. + // This adds key 5 in the queue because: + // - the command depends on key 5 which hasn't been processed yet + // - the command creates a new key (key 3). + simulateBlockedWrite(c); + + // Key 5 is expedited to the front because there are no barrier events in queue + expectReadKey(it, 5); + + expectReadKey(it, 1); // was already in queue - releases the expedited key 5 + + // Now that key 5 was processed and released by the iterator, the write command can be executed. + simulateUnblockedWriteWithModification(c); + + expectReadKey(it, 2); // was already in queue + + expectReadReplication(it, c); + + // Now resuming processing of dict entries. + expectReadKey(it, 4); + // Key 5 was handled earlier + expectReadKeySequence(it, 6, LAST_ITEM); + expectReadComplete(it); +} + + +/* A command modifying an in-progress key, but dependent on a future (repeated) key. */ +TEST_F(BgIterationTest, writeWith3Keys_start_repeatedKey_1DependantPast1RepeatedFuture) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // At this point, keys 1 & 2 are in queue. + + // Write command that has 3 keys. 0 is in progress. 4 is still future. + // How BLPOP works exactly is not relevant to bgIterator, we just chose BLPOP because it's a + // multi-key command that (potentially) modifies all of its keys (ie is not CMD_WRITE_FIRSTKEY_ONLY). + c = getWriteMultiKeysClient("blpop", 0, {4, 4, 0}); + + // This command should block on 2 keys (0 and 4), since: + // - key 0 is in use by the iterator (still in the queue since it has not been processed by the consumer yet) + // - key 4 is in the future + // This adds key 4 in the queue since the command depends on it and it hasn't been processed yet. + simulateBlockedWrite(c, 2); + + // Key 4 is processed out of order since the write depends on it. + // Key 4 is processed before key 1 even though key 1 was already in the queue + // because key 4 was enqueued as a priority item with a no-replication iterator. + // Reading key 4 will release key 0 - releasing that lock on the command + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(0)))).Times(1); + expectReadKey(it, 4); // This unblocks key 0 + + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(4)))).Times(1); + expectReadKey(it, 1); // this was already in queue (releases key 4) + + // Now that keys 4 and 0 were processed and released by the iterator, the write command can be executed. + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 2, 3); + + // 4 is skipped because it was already expedited + + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + + +/* Test that creates a new key, repeating the future key in the command. */ +TEST_F(BgIterationTest, writeWith3Keys_repeatedKey_1repeatedNewKey) { + simpleDelItem(6); // Deleting key 6 to then create it with a write command + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + // Getting started + expectReadKeySequence(it, 0, 3); + // Now, 0,1,2 are in the past. 3 is being processed, and 4 is in queue. + + // Write command that has 3 keys. 1 new repeated key and 1 key in the past. + // How BLPOP works exactly is not relevant to bgIterator, we just chose BLPOP because it's a + // multi-key command that (potentially) modifies all of its keys (ie is not CMD_WRITE_FIRSTKEY_ONLY). + c = getWriteMultiKeysClient("blpop", 6, {0, 6, 0}); + + // The write command is not blocked since key 0 & 6 are not in use, and no consistency requirements + simulateUnblockedWriteWithModification(c); + + // Keys 2, 3 are next in the queue (it was put in the queue at the same time as key 1). + expectReadKeySequence(it, 4, 5); + + // There are no consistency requirements - so the new key should just be iterated. + // Key 6 is now in the dict with the value of key 0. + expectReadKey(it, 6, keyStr(0)); + + // Processing the rest of the dict entries. + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +/* In this test, the COPY command is copying from one DB to another. We will create the + * same key in both DBs. We make sure that the proper key is created via replication, and + * the proper key is created by iteration. */ +TEST_F(BgIterationTest, copyHandlesProperDb_eventual) { + // NOTE: Adding H0 to dict 1. Now there is a H0 in both dict 0 and dict 1. + addKeyToDb(1, "H0", "H0"); + + // The test: + // We will simulate (with DB0 selected): COPY B1 H0 DB 1 REPLACE + // This will overwrite DB1:H0 that was created above. + // Since DB0:B1 is already in queue, we need to expedite the target (DB1:H0) as well + // After DB1:H0 is "overwritten", it should be marked early iterate. + // We expect DB0:H0 to NOT be marked early iterate, and should get processed normally. + + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); // B0 + // At this point, keys 1(B1) & 2(B2) are in queue. + + // COPY B1 H0 DB 1 REPLACE + c = static_cast(zcalloc(sizeof(client))); + c->cmd = lookupCommandByCString("copy"); + c->db = server.db[0]; + c->argc = 6; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(c->cmd->fullname); + c->argv[1] = createStringObjectFromCString("B1"); + c->argv[2] = createStringObjectFromCString("H0"); + c->argv[3] = createStringObjectFromCString("DB"); + c->argv[4] = createStringObjectFromCString("1"); + c->argv[5] = createStringObjectFromCString("REPLACE"); + + // This should block on 2 keys. DB0:B1 is in queue. DB1:H0 needs to be expedited. + simulateBlockedWrite(c, 2); + + // With no barrier events in queue, DB1:H0 gets moved to the front + // Queue is now 0:B0 (in progress), 1:H0 (expedited to front), 0:B1 (was in queue), 0:B2 (was in queue) + expectReadDbKeyValue(it, 1, "H0", "H0"); + + expectReadKey(it, 1); // DB0:B1 (was already in queue) + expectReadKey(it, 2); // DB0:B2 (was already in queue) - releases B1, unblocking the command (queues key 3 & 4) + + simulateUnblockedWrite_inCall(c); // We shouldn't be blocked this time + + // Now, we'll simulate the actual activity of the COPY. DB1:H0 will be deleted in order to + // be overwritten. + sds sdskey = sdsnew("H0"); + bgIteration_keyDelete(1, sdskey); // bgIteration would be signaled about the deletion + sdsfree(sdskey); + // At this point the key would actually be deleted and recreated by COPY (no need to actually do this) + + // And finally the replication (this should queue replication) + bgIteration_handleCommandReplication(c->db->id, c->cmd, c->argc, c->argv); + server.in_call--; + + expectReadKey(it, 3); + expectReadKey(it, 4); // Queued along with key 3 + + expectReadReplication(it, c); // This is the new replication (creating DB1:H0) + + // The rest should be normal. We shouldn't see DB1:E0 as it was recreated by replication + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + + +// Check that termination with replication in queue works OK. +TEST_F(BgIterationTest, terminateWithReplication) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + expectReadKey(it, 1); // makes sure we are done with key 0 (don't want to block) + + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); // Should replicate + + bgIteratorTerminate(it); + + bgIteratorItem *item = bgIteratorRead(it); + ASSERT_EQ(item->type, BGITERATOR_ITEM_TERMINATED); + + bgIteratorClose(it); // background thread completes the termination + + bgIteration_feedIterators(); // main thread, cleans up iterator and calls cleanup function + EXPECT_EQ(cleanupCount, 1); + EXPECT_TRUE(cleanupTerminated); +} + + +// SWAPDB tests - Get ready for the mind-bend... + +/* In the non-consistent iterator (without replication), items are identified with the DBID at + * the time they are placed into the queue. The SWAPDB event signals the change to the + * iterating process - and this is properly sequenced with the DB info for each item. */ +TEST_F(BgIterationTest, swapDB) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + bgIteratorStatus status; + + expectReadKey(it, 0); + // Keys 1 & 2 are in queue + + simulateSwapDB(0, 1); // The swap event will be queued after item 2 + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 1u); + EXPECT_EQ(status.swapdb_processed, 0u); + + expectReadKey(it, 1); // These were already in queue, + expectReadKey(it, 2); // ... and the iteration client hasn't seen the swap yet + + expectReadSwapDB(it, 0, 1); + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 1u); + EXPECT_EQ(status.swapdb_processed, 0u); // still processing it... + + // Since we've seen the swap event, items now have the new DBID + + expectReadDbKeyValue(it, 1, keyStr(3), keyStr(3)); // item 3 should show in DB1 + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 1u); + EXPECT_EQ(status.swapdb_processed, 1u); // done processing the swapdb + + // Keys 4 is in the queue - let's swap back! + simulateSwapDB(1, 0); // The swap event will be queued after item 4 + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 2u); // 2nd one queued + EXPECT_EQ(status.swapdb_processed, 1u); + + expectReadDbKeyValue(it, 1, keyStr(4), keyStr(4)); // item 4 should still show in DB1 + + expectReadSwapDB(it, 1, 0); // Now the iterator knows about the 2nd swap + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 2u); + EXPECT_EQ(status.swapdb_processed, 1u); // still processing it... + + // Since we've seen the second swap, items should now show with their original DB + + expectReadKey(it, 5); + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 2u); + EXPECT_EQ(status.swapdb_processed, 2u); // done processing all swaps + + expectReadKeySequence(it, 6, LAST_ITEM); + expectReadComplete(it); +} + + +/* In the consistent iterator (without replication) all items are presented to the iterating + * process using the DBID at the time of the iterator creation. No changes are evident. + * Swap events are not presented to the iteration client. */ +TEST_F(BgIterationTest, swapDB_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // Keys 1 & 2 are in queue + + simulateSwapDB(0, 1); // The swap occurs, but the iterator sees no change + + expectReadKey(it, 1); + expectReadKey(it, 2); + expectReadKey(it, 3); + + // Heck, let's go crazy with those swaps... + for (int itemNum = 4; itemNum <= LAST_ITEM; itemNum++) { + simulateSwapDB(0, 1); + expectReadKey(it, itemNum); + } + + expectReadComplete(it); +} + + +/* In the non-consistent iterator WITH replication, items are identified with the DBID at the + * time they are placed into the queue. The SWAPDB event signals the change to the iterating + * process - and this is properly sequenced with the DB info for each item. */ +TEST_F(BgIterationTest, swapDB_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // Keys 1 & 2 are in queue + + simulateSwapDB(0, 1); // The swap event will be queued after item 2 + + expectReadKey(it, 1); // These were already in queue, + expectReadKey(it, 2); // ... and the iteration client hasn't seen the swap yet + + expectReadSwapDB(it, 0, 1); // We should see a SWAPDB event + bgIteratorItem *item = bgIteratorRead(it); // followed by the associated replication + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + bgIteration_feedIterators(); + + // Since we've seen the swap event, items now have the new DBID + expectReadDbKeyValue(it, 1, keyStr(3), keyStr(3)); // item 3 is now in DB1 + + // Key 4 is in the queue - let's swap back! + simulateSwapDB(1, 0); // The swap event will be queued after item 4 + + expectReadDbKeyValue(it, 1, keyStr(4), keyStr(4)); // Still appears as DB1 + + expectReadSwapDB(it, 1, 0); // Now the iterator knows about the 2nd swap + item = bgIteratorRead(it); + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + bgIteration_feedIterators(); + + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + +// There is no test for swapDB_YesReplication_YesConsistent because this configuration is not +// permitted with multiple DBs (not permitted with swaps). + + +// FLUSHDB & FLUSHALL Tests + +TEST_F(BgIterationTest, flushDB_flushAll) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + expectReadKey(it, 1); + + // key 1 is active in the iterator - this key won't be deallocated because of the refcount. + // keys 2 is in queue - but will be returned to Valkey before the flush. It is yanked + // back by Valkey and will not be seen by iterator. + simulateFlushDB(-1, 1); + + bgIteratorItem *item = bgIteratorRead(it); + ASSERT_EQ(item->type, BGITERATOR_ITEM_TERMINATED); + + bgIteratorClose(it); // background thread completes the termination + + bgIteration_feedIterators(); // main thread, cleans up iterator and calls cleanup function + EXPECT_EQ(cleanupCount, 1); + EXPECT_TRUE(cleanupTerminated); +} + +TEST_F(BgIterationTest, flushDB_flushOne) { + bgIterator *it1 = bgIteratorCreateFullScanIter("iter1", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + bgIterator *it2 = bgIteratorCreateFullScanIter("iter2", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + bgIteratorStatus status; + + // The test flushes DB0. This is half the data. Since <= half, a non-consistent iterator is + // allowed to proceed. But the consistent iterator will be terminated. + + expectReadKey(it1, 0); + expectReadKey(it2, 0); + expectReadKey(it1, 1); + expectReadKey(it2, 1); + + // key 1 is active in the iterator - this key won't be deallocated because of the refcount. + // keys 2 is in queue - but will be returned to Valkey before the flush. These are yanked + // back by Valkey and will not be seen by iterator. + simulateFlushDB(0, 1); + bgIteratorGetStatus(it1, &status); + EXPECT_EQ(status.flushdb_queued, 1u); + EXPECT_EQ(status.flushdb_processed, 0u); + + // Testing the non-consistent one continues... + // Everything already on the iterator queue should be preserved (deleted from the DB). + // Keys 2 is already queued (and preserved). + expectReadKey(it1, 2); + + // Read the flushdb item on iterator 1. + bgIteratorItem *item = bgIteratorRead(it1); + ASSERT_EQ(item->type, BGITERATOR_ITEM_FLUSHDB); + ASSERT_EQ(item->dbid, 0); + bgIteratorGetStatus(it1, &status); + EXPECT_EQ(status.flushdb_queued, 1u); + EXPECT_EQ(status.flushdb_processed, 0u); // still processing it + + // And iterator 1 keeps processing with the 2nd DB + expectReadKey(it1, ITEMS_PER_DB); + bgIteratorGetStatus(it1, &status); + EXPECT_EQ(status.flushdb_queued, 1u); + EXPECT_EQ(status.flushdb_processed, 1u); // done with all flushdb's + + expectReadKeySequence(it1, ITEMS_PER_DB + 1, LAST_ITEM); + expectReadComplete(it1); + EXPECT_EQ(cleanupCount, 1); + EXPECT_FALSE(cleanupTerminated); + + // But the consistent iterator should be terminated + item = bgIteratorRead(it2); + ASSERT_EQ(item->type, BGITERATOR_ITEM_TERMINATED); + bgIteratorClose(it2); // background thread completes the termination + bgIteration_feedIterators(); // main thread, cleans up iterator and calls cleanup function + EXPECT_EQ(cleanupCount, 2); + EXPECT_TRUE(cleanupTerminated); +} + + +/* A multi with one future and one past key must expedite and replicate. */ +TEST_F(BgIterationTest, multiTwoKeysFirstFuture) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKey(it, 0); // Causes keys 1 & 2 to be queued (same bucket) + expectReadKey(it, 1); // Causes key 0 to be released + + // Now, B0(0) is in the past. H0(5) is in the future. R0(11) [in DB1] is also future. + + /* For a non-consistent iteration, with replication... + * Normally, H0 (future) wouldn't need to expedite - we'd just modify it in place (without + * replication and iterate on it later. But, in this case, since it's wrapped in a multi, with + * B0 (past) - we need to expedite H0 so that the multi can all be handled in the same way. + * Key R0(11) [DB1] just makes thing a little trickier. */ + c = getMultiClient("SET B0 xxx; SET H0 xxx; SELECT 1; SET R0 xxx"); + + // The EXEC should block on 2 keys, because H0(5) & R0(11) should be expedited + simulateBlockedWrite(c, 2); + + // Since there were no barrier events in the queue, these 2 get moved to the front. + // Note - it would be logically OK if these 2 were reversed, but this is how the current algorithm works. + expectReadKey(it, 5); // Key 5 (H0) was expedited + expectReadKey(it, 11); // Key 11 (R0) was expedited + + expectReadKey(it, 2); // (was already in queue) + + // We don't need to actually simulate the multi. Just checking that the keys were expedited. + + // and clean up the rest... + expectReadKeySequence(it, 3, 4); + // Key 5 was already read above (expedited) + expectReadKeySequence(it, 6, 10); + // Key 11 was already read above (expedited) + expectReadKeySequence(it, 12, LAST_ITEM); + expectReadComplete(it); +} + +// Multi blocking on future items. Consistent. +TEST_F(BgIterationTest, multiBlocksOnFutureKey) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // Keys 1 & 2 are in queue + + // Since there's no replication, an expedited key will be moved to the front of the queue. + // Let's fake a modification to key 6 (H1) + // Dummy up a MULTI... + c = getMultiClient("SET H1 xxx"); + + // Since this is consistent, we will block the client, disallowing the write. + simulateBlockedWrite(c); + + // H1 (key 6) will be expedited to the front of the queue (because no replication) + expectReadKey(it, 6); + + // Now that we've read key 6, key 0 (B0) is passed and should not block + freeTestClient(c); + c = getMultiClient("SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + + // and clean up the rest... + expectReadKeySequence(it, 1, 5); + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +// Scenario. We have a multi that doesn't need to be replicated because all of the keys exist +// but are all future keys. Note that missing keys are considered already-iterated, so all +// must exist for this test. Then: +// - we delete a key +// - we re-create the deleted (future) key - normally this would be replicated +// - we access another (future) key - we don't expect to get blocked! +TEST_F(BgIterationTest, multiNotReplicatedButDelRecreateAccess) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // Keys 1 & 2 are in queue + + c = getMultiClient("DEL H1; SET H1 xxx; SET H2 yyy"); + // Now let's process the multi. Since H1 & H2 are both future (existing) items, we shouldn't + // block or replicate. + simulateUnblockedWrite_inCall(c); // the EXEC + + // Simulate the DEL H1 + server.in_exec = 1; // Simulate actual execution of the MULTI/EXEC + + advanceMultiClientToCommand(c, 0); // DEL H1 + EXPECT_CALL(mock, blockClientInUseOnKeys(c, _, _)).Times(0); + bool blocked = bgIteration_blockClientIfRequired(c); + EXPECT_FALSE(blocked); + + sds delKey = sdsnew(keyStr(6)); + bgIteration_keyDelete(0, delKey); + sdsfree(delKey); + simpleDelItem(6); // H1 + + bgIteration_handleCommandReplication(c->db->id, c->cmd, c->argc, c->argv); // shouldn't replicate + + // Simulate SET H1 - the key doesn't exist, and would normally replicate and mark early iterate, + // but this is in a transaction, and we are not replicating this transaction. + advanceMultiClientToCommand(c, 1); // SET H1 xxx + simulateUnblockedWriteWithModification(c); + + // Now write to another existing future key - this should work if we weren't confused by the DEL + advanceMultiClientToCommand(c, 2); // SET H2 yyy + simulateUnblockedWriteWithModification(c); + server.in_exec = 0; + server.in_call--; + + // Now we can continue iterating, and we should pick up keys 1... (and no replication!) + expectReadKeySequence(it, 1, 5); + expectReadKey(it, 6, "xxx"); + expectReadKey(it, 7, "yyy"); + expectReadKeySequence(it, 8, LAST_ITEM); + expectReadComplete(it); +} + + +// For this test, B0 is added into DB1 - so it exists in both DB 0 and 1. We will process it +// in DB0, but it will be unprocessed in DB1. See if we track SELECT properly. +TEST_F(BgIterationTest, multiHandlesSelectProperly) { + addKeyToDb(1, "B0", "B0"); + + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + // Read the 1st key - B0 in DB0. + expectReadKey(it, 0); + // Now, we are done with B0 in DB0, but not in DB1 + expectReadKey(it, 1); // Reads B1, and releases B0 in DB0 + + // These cases should NOT block... (they access B0 in DB0) + c = getMultiClient("SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 0; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 1; SELECT 0; SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + + // These cases SHOULD block... (they access B0 in DB1) + c = getMultiClient("SET B0 xxx"); + c->db = server.db[1]; + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SELECT 1; SET B0 xxx"); + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SELECT 1; SET B0 xxx; SELECT 0"); + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SELECT 0; SELECT 1; SET B0 xxx; SELECT 1"); + simulateBlockedWrite(c); + + expectAnythingCleanup(it); +} + +// For this test, B0 is added into DB1 - so it exists in both DB0 and DB1. We will process it +// in DB0, but it will be unprocessed in DB1. See if we track select properly - WHEN WE HAVE NO +// PERMISSION TO EXECUTE SELECT! +TEST_F(BgIterationTest, multiHandlesSelectNoPermissionProperly) { + addKeyToDb(1, "B0", "B0"); + + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + // Read the 1st key - B0 in DB0. + expectReadKey(it, 0); + // Now, we are done with B0 in DB0, but not in DB1 + expectReadKey(it, 1); // Reads B1, and releases B0 in DB0 + + // No permission for any commands (specifically select/swapdb) + EXPECT_CALL(mock, ACLCheckAllUserCommandPerm(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(ACL_DENIED_CMD)); + + // These cases should NOT block... (they access B0 in DB0) + // The SELECTs below are inconsequential - with/without select, same result. + c = getMultiClient("SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 0; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 1; SELECT 0; SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + + // These cases SHOULD block IF SELECT IS WORKING... (they access B0 in DB1) + c = getMultiClient("SET B0 xxx"); + c->db = server.db[1]; // already starting on DB1 + simulateBlockedWrite(c); // will block, no select + freeTestClient(c); + c = getMultiClient("SELECT 1; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (select fails) + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 1; SET B0 xxx; SELECT 0"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (select fails) + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 0; SELECT 1; SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (select fails) + server.in_call--; + + expectAnythingCleanup(it); +} + +// For this test, B0 is added into DB1 - so it exists in both DB0 and DB1. We will process it +// in DB0, but it will be unprocessed in DB1. See if we track SWAPDB properly. +TEST_F(BgIterationTest, multiHandlesSwapdbProperly) { + addKeyToDb(1, "B0", "B0"); + + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + // Read the 1st key - B0 in DB0. + expectReadKey(it, 0); + // Now, we are done with B0 in DB0, but not in DB1 + expectReadKey(it, 1); // Reads B1, and releases B0 in DB0 + + // These cases should NOT block... (they access B0 in DB0) + c = getMultiClient("SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SWAPDB 0 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SWAPDB 0 1; SWAPDB 0 1; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SWAPDB 0 1; SELECT 1; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + + // These cases SHOULD block... (they access B0 in DB1) + c = getMultiClient("SET B0 xxx"); + c->db = server.db[1]; + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SET B0 xxx; SWAPDB 0 1"); + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SELECT 0; SET B0 xxx; SWAPDB 0 1"); + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SWAPDB 1 0; SELECT 1; SET B0 xxx; SELECT 1"); + simulateBlockedWrite(c); + + expectAnythingCleanup(it); +} + +// For this test, B0 is added into DB1 - so it exists in both DB0 and DB1. We will process it +// in DB0, but it will be unprocessed in DB1. See if we track select properly - WHEN WE HAVE NO +// PERMISSION TO EXECUTE SWAPDB! +TEST_F(BgIterationTest, multiHandlesSwapdbNoPermissionProperly) { + addKeyToDb(1, "B0", "B0"); + + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + // Read the 1st key - B0 in DB0. + expectReadKey(it, 0); + // Now, we are done with B0 in DB0, but not in DB1 + expectReadKey(it, 1); // Reads B1, and releases B0 in DB0 + + // No permission for any commands (specifically select/swapdb) + EXPECT_CALL(mock, ACLCheckAllUserCommandPerm(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(ACL_DENIED_CMD)); + + // These cases should NOT block... (they access B0 in DB0) + // The SELECTs & SWAPDBs below are inconsequential - with/without select/swapdb, same result. + c = getMultiClient("SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SWAPDB 0 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SWAPDB 0 1; SWAPDB 0 1; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SWAPDB 0 1; SELECT 1; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + + // These cases SHOULD block IF SELECT/SWAPDB IS WORKING... (they access B0 in DB1) + c = getMultiClient("SET B0 xxx"); + c->db = server.db[1]; + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SET B0 xxx; SWAPDB 0 1"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (swapdb fails) + server.in_call--; + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SELECT 0; SET B0 xxx; SWAPDB 0 1"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (swapdb/select fails) + server.in_call--; + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SWAPDB 1 0; SELECT 1; SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (swapdb/select fails) + server.in_call--; + + expectAnythingCleanup(it); +} + + +static void *pthreadWait200msAndReadTwoKeys(void *arg) { + bgIterator *it = static_cast(arg); + + usleep(200000); + bgIteratorRead(it); + bgIteratorRead(it); + return nullptr; +} + +static void asyncWait200msAndReadTwoKeys(bgIterator *it) { + int rc; + pthread_attr_t attr; + pthread_t thread; + + rc = pthread_attr_init(&attr); + assert(rc == 0); + rc = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + assert(rc == 0); + + rc = pthread_create(&thread, &attr, pthreadWait200msAndReadTwoKeys, it); + assert(rc == 0); + + rc = pthread_attr_destroy(&attr); + assert(rc == 0); +} + +TEST_F(BgIterationTest, testLuaWithUndeclaredKey) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // If we fake a modification to key 3, we won't know if it's handled out of order. + // So we fake a modification to key 4 + c = getWriteClient(4, "xxx"); + c->flag.script = 1; + + // Now for a LUA script, we have already blocked (on the eval/evalsha) for any declared keys + // But here, we're about to modify an undeclared key. We can't actually block in the middle + // of the LUA script. So this will behave as unblocked, but incur a synchronous wait. + + // Key 4 will get expedited when we simulate the write. After reading key 4, key 1 will need + // to be read to return key 4 to Valkey, unblocking the synchronous wait. + asyncWait200msAndReadTwoKeys(it); + + monotime blockTimer; + elapsedStart(&blockTimer); + simulateUnblockedWrite_inCall(c); // Not blocked, but delays internally + server.in_call--; + // Must have delayed at least 150ms (some time may have passed before timer start) + EXPECT_GT(elapsedMs(blockTimer), 150u); + + // Continue... + expectReadKeySequence(it, 2, 3); + // 4 has already been processed + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + + +// Make sure that replication received while processing the last key is sent +TEST_F(BgIterationTest, replicationReceivedWhileProcessingLastKey) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, LAST_ITEM); + + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); // Wouldn't be blocked because done with key 0 + + expectReadReplication(it, c); // Replication happened while processing the last item, should be here. + + simulateUnblockedWriteWithModification(c); // This won't replicate because we are done processing + + expectReadComplete(it); // We expect to see the completion instead +} + +TEST_F(BgIterationTest, repldoneFunctionCalled) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, + iteratorRepldoneFn, iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, LAST_ITEM); + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); // Wouldn't be blocked because done with key 0 + + // Since in testing, we are only feeding one item at a time, and synchronously, we won't call + // the repldone function until after we release the last item. + EXPECT_EQ(replDoneConfirmed, 0); + expectReadReplication(it, c); // Replication happened while processing the last item, should be here. + EXPECT_EQ(replDoneConfirmed, 1); // Last key released, now done feeding replication + + simulateUnblockedWriteWithModification(c); // This won't replicate because we are done processing + + expectReadComplete(it); // We expect to see the completion instead +} + +TEST_F(BgIterationTest, repldoneFunctionCalledTwice) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, + iteratorRepldoneFnNotBeingReadyInitially, iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, LAST_ITEM); + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); // Wouldn't be blocked because done with key 0 + + // Won't signal replDone until we've released the final item (which happens when reading the replication) + EXPECT_EQ(replDoneRejected, 0); + EXPECT_EQ(replDoneConfirmed, 0); + expectReadReplication(it, c); // Releases the final item + EXPECT_EQ(replDoneRejected, 1); // replDone called once (and rejected by client) + EXPECT_EQ(replDoneConfirmed, 0); + simulateUnblockedWriteWithModification(c); // This will replicate (because replDone returned false) + + expectReadReplication(it, c); // ReplDone gets called again (and accepted this time) + EXPECT_EQ(replDoneConfirmed, 1); + + simulateUnblockedWriteWithModification(c); // This won't replicate because replication is done + + expectReadComplete(it); // We expect to see the completion instead +} + +// Check that the memory reported for replication is correct +TEST_F(BgIterationTest, checkReplicationByteCount) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, + iteratorRepldoneFn, iteratorCleanupFn, PRIVDATA); + c = getWriteClient(0, "xxx"); + size_t expectedReplicationSize = sizeof(bgIteratorItem); + for (int i = 0; i < c->argc; i++) { + expectedReplicationSize += objectComputeSize(NULL, c->argv[i], 0, 0); + } + + expectReadKey(it, 0); + expectReadKey(it, 1); // Releases and unblocks 0 + EXPECT_EQ(bgIteration_memoryInuseForReplication(), 0u); + + simulateUnblockedWriteWithModification(c); // Wouldn't be blocked because done with key 0 + EXPECT_EQ(bgIteration_memoryInuseForReplication(), expectedReplicationSize); + simulateUnblockedWriteWithModification(c); // and write again (2nd replication) + EXPECT_EQ(bgIteration_memoryInuseForReplication(), 2 * expectedReplicationSize); + + expectReadKey(it, 2); // Keys 0..2 all in same bucket + + expectReadReplication(it, c); + // After reading the 1st replication, it hasn't been returned yet (it's the active item) + EXPECT_EQ(bgIteration_memoryInuseForReplication(), 2 * expectedReplicationSize); + expectReadReplication(it, c); + // After reading the 2nd replication, the 1st has been returned + EXPECT_EQ(bgIteration_memoryInuseForReplication(), expectedReplicationSize); + + expectReadKey(it, 3); + // Now all replication has been returned/freed + EXPECT_EQ(bgIteration_memoryInuseForReplication(), 0u); + + expectReadKeySequence(it, 4, LAST_ITEM); + expectReadComplete(it); +} + +// Test that for an arbitrary write command having no keys, replication should occur. +TEST_F(BgIterationTest, checkNoKeysWriteIsReplicated) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + + c = getNoKeysWriteClient(); + simulateUnblockedWrite_inCall(c); + bgIteration_handleCommandReplication(c->db->id, c->cmd, c->argc, c->argv); + server.in_call--; + + expectReadKeySequence(it, 1, 2); // These were already in queue + + expectReadReplication(it, c); + + expectReadKeySequence(it, 3, LAST_ITEM); + expectReadComplete(it); +} diff --git a/src/unit/test_blocked.cpp b/src/unit/test_blocked.cpp new file mode 100644 index 000000000..5fa789034 --- /dev/null +++ b/src/unit/test_blocked.cpp @@ -0,0 +1,369 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" +extern "C" { +#include "server.h" +int getBlockInUseKeyCount(void); +void releaseBlockInUse(void); +} + +/* These unit tests were introduced at the time of inuse key blocking. Functions + * introduced earlier are tested only indirectly through integration tests. */ +class BlockedInuseTest : public ::testing::Test { + protected: + MockValkey mock; + RealValkey real; + static inline ConnectionType dummyConnType = {0}; + + static void SetUpTestSuite() { + memset(&server, 0, sizeof(valkeyServer)); + server.hz = CONFIG_DEFAULT_HZ; + dummyConnType.set_read_handler = dummySetReadHandler; + } + + static void TearDownTestSuite() { + releaseBlockInUse(); + } + + void SetUp() override { + server.unblocked_clients = listCreate(); + ASSERT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + ASSERT_EQ(getBlockInUseKeyCount(), 0); + } + + void TearDown() override { + ASSERT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + ASSERT_EQ(getBlockInUseKeyCount(), 0); + ASSERT_EQ(listLength(server.unblocked_clients), 0UL); + listRelease(server.unblocked_clients); + server.unblocked_clients = NULL; + } + + + static int dummySetReadHandler(connection *conn, ConnectionCallbackFunc func) { + conn->read_handler = func; + return C_OK; + } + + client *createFakeClient(int client_id) { + client *c = (client *)zcalloc(sizeof(client)); + c->id = client_id; + c->conn = (connection *)zcalloc(sizeof(connection)); + c->conn->type = &dummyConnType; + c->conn->read_handler = (ConnectionCallbackFunc)1; + c->flag.pending_command = 1; + return c; + } + + void freeFakeClient(client *c) { + freeClientBlockingState(c); + if (c->conn) zfree(c->conn); + zfree(c); + } + + void verifyClientBlockState(client *c, bool blocked, bool unblocked) { + EXPECT_EQ(c->flag.unblocked, unblocked); + EXPECT_EQ(c->flag.blocked && c->bstate->btype == BLOCKED_INUSE, blocked); + if (blocked || unblocked) { + EXPECT_EQ(c->conn->read_handler, nullptr); + } else { + EXPECT_NE(c->conn->read_handler, nullptr); + } + } +}; + +using BlockedInuseDeathTest = BlockedInuseTest; + + +TEST_F(BlockedInuseTest, blockInitialState) { + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + ASSERT_NE(server.unblocked_clients, nullptr); +} + +TEST_F(BlockedInuseTest, blockClientOnSingleKey) { + client *c = createFakeClient(1); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + // Block + blockClientInUseOnKeys(c, 1, keys); + verifyClientBlockState(c, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 1u); + EXPECT_EQ(getBlockInUseKeyCount(), 1); + EXPECT_EQ(key->refcount, 3u); + + // Unblock + unblockClientsInUseOnKey(key); + verifyClientBlockState(c, 0, 1); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(key->refcount, 1u); + EXPECT_EQ(listLength(server.unblocked_clients), 1UL); + EXPECT_EQ(listFirst(server.unblocked_clients)->value, c); + + // Process unblocked client in event loop + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + processUnblockedClients(); + verifyClientBlockState(c, 0, 0); + EXPECT_EQ(key->refcount, 1u); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseTest, blockClientClearsLeftoverTimeout) { + client *c = createFakeClient(1); + initClientBlockingState(c); + c->bstate->timeout = 1000; + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + blockClientInUseOnKeys(c, 1, keys); + EXPECT_EQ(c->bstate->timeout, 0); + + unblockClientsInUseOnKey(key); + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + processUnblockedClients(); + + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseTest, blockClientOnMultipleKeys) { + client *c = createFakeClient(1); + robj *key1 = createObject(OBJ_STRING, sdsnew("key1")); + robj *key2 = createObject(OBJ_STRING, sdsnew("key2")); + robj *keys[] = {key1, key2}; + + // Block + blockClientInUseOnKeys(c, 2, keys); + verifyClientBlockState(c, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 1u); + EXPECT_EQ(getBlockInUseKeyCount(), 2); + EXPECT_EQ(key1->refcount, 3u); + EXPECT_EQ(key2->refcount, 3u); + + // Unblock key1 + unblockClientsInUseOnKey(key1); + verifyClientBlockState(c, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 1u); + EXPECT_EQ(getBlockInUseKeyCount(), 1); + EXPECT_EQ(key1->refcount, 1u); + EXPECT_EQ(key2->refcount, 3u); + + // Unblock key2, client gets unblocked + unblockClientsInUseOnKey(key2); + verifyClientBlockState(c, 0, 1); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(key1->refcount, 1u); + EXPECT_EQ(key2->refcount, 1u); + EXPECT_EQ(listLength(server.unblocked_clients), 1UL); + EXPECT_EQ(listFirst(server.unblocked_clients)->value, c); + + // Process unblocked client in event loop + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + processUnblockedClients(); + verifyClientBlockState(c, 0, 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + + EXPECT_EQ(key1->refcount, 1u); + EXPECT_EQ(key2->refcount, 1u); + decrRefCount(key1); + decrRefCount(key2); + freeFakeClient(c); +} + +TEST_F(BlockedInuseTest, blockMultipleClientsOnSameKey) { + client *c1 = createFakeClient(1); + client *c2 = createFakeClient(2); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + // Block + blockClientInUseOnKeys(c1, 1, keys); + blockClientInUseOnKeys(c2, 1, keys); + verifyClientBlockState(c1, 1, 0); + verifyClientBlockState(c2, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 2u); + EXPECT_EQ(getBlockInUseKeyCount(), 1); + EXPECT_EQ(key->refcount, 4u); + + // Unblock + unblockClientsInUseOnKey(key); + verifyClientBlockState(c1, 0, 1); + verifyClientBlockState(c2, 0, 1); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(key->refcount, 1u); + EXPECT_EQ(listLength(server.unblocked_clients), 2UL); + + // Process client in event loop + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c1)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c1)).Times(1); + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c2)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c2)).Times(1); + processUnblockedClients(); + verifyClientBlockState(c1, 0, 0); + verifyClientBlockState(c2, 0, 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + + EXPECT_EQ(key->refcount, 1u); + decrRefCount(key); + freeFakeClient(c1); + freeFakeClient(c2); +} + +TEST_F(BlockedInuseTest, unblockBlockedClient) { + client *c = createFakeClient(1); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + // Block + blockClientInUseOnKeys(c, 1, keys); + verifyClientBlockState(c, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 1u); + EXPECT_EQ(getBlockInUseKeyCount(), 1); + EXPECT_EQ(key->refcount, 3u); + + // Unblock client, simulate freeClient + unblockClient(c, 0); + EXPECT_FALSE(c->flag.blocked); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + EXPECT_EQ(key->refcount, 1u); + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseTest, blockClientOnDuplicateKeys) { + client *c = createFakeClient(1); + robj *key1 = createObject(OBJ_STRING, sdsnew("foo")); + robj *key2 = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key1, key2}; + + // Block + blockClientInUseOnKeys(c, 2, keys); + verifyClientBlockState(c, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 1u); + EXPECT_EQ(getBlockInUseKeyCount(), 1); + EXPECT_EQ(key1->refcount, 3u); + EXPECT_EQ(key2->refcount, 1u); // Key is deduplicated, only blocked once + + // Unblock + unblockClientsInUseOnKey(key1); + verifyClientBlockState(c, 0, 1); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(listLength(server.unblocked_clients), 1UL); + EXPECT_EQ(listFirst(server.unblocked_clients)->value, c); + + // Process client in event loop + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + processUnblockedClients(); + verifyClientBlockState(c, 0, 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + EXPECT_EQ(key1->refcount, 1u); + EXPECT_EQ(key2->refcount, 1u); + decrRefCount(key1); + decrRefCount(key2); + freeFakeClient(c); +} + +TEST_F(BlockedInuseTest, unblockAllKeys) { + client *c1 = createFakeClient(1); + client *c2 = createFakeClient(2); + robj *key1 = createObject(OBJ_STRING, sdsnew("key1")); + robj *key2 = createObject(OBJ_STRING, sdsnew("key2")); + robj *keys1[] = {key1}; + robj *keys2[] = {key2}; + + // Block c1 on key1, c2 on key2 + blockClientInUseOnKeys(c1, 1, keys1); + blockClientInUseOnKeys(c2, 1, keys2); + verifyClientBlockState(c1, 1, 0); + verifyClientBlockState(c2, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 2u); + EXPECT_EQ(getBlockInUseKeyCount(), 2); + EXPECT_EQ(key1->refcount, 3u); + EXPECT_EQ(key2->refcount, 3u); + + // Unblock all + unblockClientsInUseOnAllKeys(); + verifyClientBlockState(c1, 0, 1); + verifyClientBlockState(c2, 0, 1); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(listLength(server.unblocked_clients), 2UL); + + // Process clients + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c1)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c1)).Times(1); + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c2)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c2)).Times(1); + processUnblockedClients(); + verifyClientBlockState(c1, 0, 0); + verifyClientBlockState(c2, 0, 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + EXPECT_EQ(key1->refcount, 1u); + EXPECT_EQ(key2->refcount, 1u); + decrRefCount(key1); + decrRefCount(key2); + freeFakeClient(c1); + freeFakeClient(c2); +} + +TEST_F(BlockedInuseDeathTest, blockingOnKeysReplicaClient) { + client *c = createFakeClient(1); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + c->flag.replica = 1; + EXPECT_DEATH(blockClientInUseOnKeys(c, 1, keys), ""); + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseDeathTest, blockingOnKeysNonStringType) { + client *c = createFakeClient(1); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + keys[0]->type = OBJ_LIST; + EXPECT_DEATH(blockClientInUseOnKeys(c, 1, keys), ""); + keys[0]->type = OBJ_STRING; + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseDeathTest, blockingOnKeysZeroKeys) { + client *c = createFakeClient(1); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + EXPECT_DEATH(blockClientInUseOnKeys(c, 0, keys), ""); + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseDeathTest, blockingOnKeysWithoutPendingCommand) { + client *c = createFakeClient(1); + c->flag.pending_command = 0; + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + EXPECT_DEATH(blockClientInUseOnKeys(c, 1, keys), ""); + decrRefCount(key); + freeFakeClient(c); +} diff --git a/src/unit/test_cluster_io_offload.cpp b/src/unit/test_cluster_io_offload.cpp new file mode 100644 index 000000000..40c129ae9 --- /dev/null +++ b/src/unit/test_cluster_io_offload.cpp @@ -0,0 +1,873 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" + +#include "fake_connection.hpp" + +#include +#include + +extern "C" { +#include "cluster.h" +#include "cluster_legacy.h" +#include "connhelpers.h" +#include "io_threads.h" +#include "server.h" + +clusterLink *createClusterLink(clusterNode *node); +int freeClusterLink(clusterLink *link); +void testOnlyFreeClusterLinkOnBufferLimitReached(clusterLink *link); +} + +/* Mirrors clusterMsgSendBlock, which is private to cluster_legacy.c. The + * layout must match exactly: the write job and its completion both read the + * message's own totlen and type out of the block. */ +typedef struct TestMsgBlock { + size_t totlen; + int refcount; + union { + clusterMsg msg; + clusterMsgLight msg_light; + } data[1]; +} TestMsgBlock; + +class ClusterIOOffloadTest : public ::testing::Test { + protected: + static const int MAX_OWNED = 64; + fakeConnection *owned_conns[MAX_OWNED]; + int owned_conns_count; + clusterLink *owned_links[MAX_OWNED]; + int owned_links_count; + + void SetUp() override { + owned_conns_count = 0; + owned_links_count = 0; + memset(&server, 0, sizeof(server)); + server.el = aeCreateEventLoop(1024); + server.io_threads_num = 2; + server.active_io_threads_num = 2; + testOnlyInitIOThreadQueues(); + server.cluster_link_msg_queue_limit_bytes = 1024; + server.logfile = zstrdup(""); + server.verbosity = LL_WARNING; + server.cluster = (clusterState *)zcalloc(sizeof(clusterState)); + } + + void TearDown() override { + for (int i = 0; i < owned_links_count; i++) { + if (owned_links[i]) freeClusterLink(owned_links[i]); + } + for (int i = 0; i < owned_conns_count; i++) { + connFreeFake(owned_conns[i]); + } + if (server.cluster) { + zfree(server.cluster); + server.cluster = NULL; + } + zfree(server.logfile); + server.logfile = NULL; + /* Every test must leave the cluster pending-response count balanced. + * A dispatch that returns without publishing a result would strand it + * forever and stall processIOThreadsResponses(). */ + EXPECT_EQ(testOnlyGetClusterIOPendingResponses(), 0u) << "leaked a cluster I/O pending response"; + testOnlyFreeIOThreadQueues(); + if (server.el) { + aeDeleteEventLoop(server.el); + server.el = NULL; + } + } + + /* A connection in the state cluster code expects post-accept: established, + * cluster-owned, one reference held. */ + fakeConnection *makeConn(ConnectionOwnerKind owner_kind = CONN_OWNER_CLUSTER_LINK) { + fakeConnection *fc = connCreateFake(4096); + fc->conn.state = CONN_STATE_CONNECTED; + fc->conn.refs = 1; + fc->conn.owner_kind = owner_kind; + owned_conns[owned_conns_count++] = fc; + return fc; + } + + /* As clusterAcceptHandler() leaves it: conn_handler installed before dispatch. */ + fakeConnection *makeAcceptConn() { + fakeConnection *fc = makeConn(CONN_OWNER_CLUSTER_LINK); + fc->conn.conn_handler = clusterConnAcceptHandler; + return fc; + } + + clusterLink *makeLink() { + clusterLink *link = createClusterLink(NULL); + fakeConnection *fc = makeConn(); + link->conn = &fc->conn; + connSetPrivateData(link->conn, link); + owned_links[owned_links_count++] = link; + return link; + } + + void trackLink(clusterLink *link) { + owned_links[owned_links_count++] = link; + } + + void releaseLinkOwnership(clusterLink *link) { + for (int i = 0; i < owned_links_count; i++) { + if (owned_links[i] == link) { + owned_links[i] = NULL; + break; + } + } + } + + /* Queue one message of msg_len wire bytes. The block is sized to hold the + * message, since the write job reads msg_len bytes starting at data[0]. */ + void enqueueFakeMsg(clusterLink *link, uint32_t msg_len = 64) { + size_t alloc = offsetof(TestMsgBlock, data) + msg_len; + if (alloc < sizeof(TestMsgBlock)) alloc = sizeof(TestMsgBlock); + TestMsgBlock *blk = (TestMsgBlock *)zcalloc(alloc); + blk->refcount = 1; + blk->totlen = alloc; + clusterMsg *msg = &blk->data[0].msg; + memcpy(msg->sig, "RCmb", 4); + msg->totlen = htonl(msg_len); + msg->ver = htons(CLUSTER_PROTO_VER); + msg->type = htons(CLUSTERMSG_TYPE_PING); + listAddNodeTail(link->send_msg_queue, blk); + link->send_msg_queue_mem += sizeof(listNode) + blk->totlen; + } + + /* A minimal well-formed cluster packet. The type is chosen so that + * clusterProcessPacket() accepts it from an unknown sender and only bumps + * the per-type received counter. */ + unsigned char *buildRawPacket(uint32_t totlen) { + unsigned char *raw = (unsigned char *)zcalloc(totlen); + clusterMsgHeader *hdr = (clusterMsgHeader *)(void *)raw; + memcpy(hdr->sig, "RCmb", 4); + hdr->totlen = htonl(totlen); + hdr->ver = htons(CLUSTER_PROTO_VER); + hdr->type = htons(CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK); + return raw; + } + + /* A packet of arbitrary length. FAILOVER_AUTH_ACK is length-checked exactly, + * so an unknown type is used instead: clusterIsValidPacket() accepts any + * totlen for one, and the stats index is guarded by CLUSTERMSG_TYPE_COUNT. */ + unsigned char *buildOversizedPacket(uint32_t totlen) { + unsigned char *raw = buildRawPacket(totlen); + ((clusterMsgHeader *)(void *)raw)->type = htons(CLUSTERMSG_TYPE_COUNT + 1); + return raw; + } + + /* Exactly one complete packet, nothing after it. */ + void seedOneCompletePacket(fakeConnection *fc) { + unsigned char *pkt = buildRawPacket(CLUSTERMSG_MIN_LEN); + fakeConnSetReadData(fc, pkt, CLUSTERMSG_MIN_LEN); + zfree(pkt); + } + + /* One complete packet followed by a header that fails signature validation, + * so framing reports a protocol error with a valid prefix in front of it. + * The garbage must be long enough for the framing step to inspect it as a + * header rather than treat it as a partial read. */ + void seedCompletePacketFollowedByGarbage(fakeConnection *fc) { + const size_t garbage_len = 64; + size_t len = CLUSTERMSG_MIN_LEN + garbage_len; + unsigned char *pkt = buildRawPacket(CLUSTERMSG_MIN_LEN); + unsigned char *buf = (unsigned char *)zcalloc(len); + memcpy(buf, pkt, CLUSTERMSG_MIN_LEN); + memset(buf + CLUSTERMSG_MIN_LEN, 'Z', garbage_len); + fakeConnSetReadData(fc, buf, len); + zfree(pkt); + zfree(buf); + } + + /* Feed the connection one complete packet plus a partial tail, so a read + * job frames exactly one packet. */ + void seedReadableSocket(fakeConnection *fc) { + unsigned char *pkt = buildRawPacket(CLUSTERMSG_MIN_LEN); + unsigned char *buf = (unsigned char *)zmalloc(CLUSTERMSG_MIN_LEN + 1); + memcpy(buf, pkt, CLUSTERMSG_MIN_LEN); + buf[CLUSTERMSG_MIN_LEN] = 'T'; + fakeConnSetReadData(fc, buf, CLUSTERMSG_MIN_LEN + 1); + zfree(pkt); + zfree(buf); + } + + /* Two complete packets plus a partial tail larger than RCVBUF_INIT_LEN. */ + size_t seedPacketsAndLargePartialTail(fakeConnection *fc) { + const uint32_t whole = CLUSTERMSG_MIN_LEN; + const size_t partial = RCVBUF_INIT_LEN + 512; + size_t len = whole * 2 + partial; + unsigned char *buf = (unsigned char *)zcalloc(len); + unsigned char *pkt = buildRawPacket(whole); + memcpy(buf, pkt, whole); + memcpy(buf + whole, pkt, whole); + /* A valid header whose packet has not fully arrived yet. */ + unsigned char *tail = buildRawPacket(whole); + memcpy(buf + whole * 2, tail, partial); + fakeConnSetReadData(fc, buf, len); + zfree(pkt); + zfree(tail); + zfree(buf); + return partial; + } + + /* Three complete packets plus a partial tail. The middle packet is larger + * than the first so that sliding it to the front is an overlapping copy. + * Each packet carries a distinct type, so a packet landing at the wrong + * offset shows up as a wrong per-type counter. */ + void seedThreePacketsAndTail(fakeConnection *fc) { + const uint32_t small = CLUSTERMSG_MIN_LEN; + const uint32_t large = CLUSTERMSG_MIN_LEN + 512; + size_t len = small + large + small + 1; + unsigned char *buf = (unsigned char *)zcalloc(len); + unsigned char *p1 = buildRawPacket(small); + unsigned char *p2 = buildOversizedPacket(large); + unsigned char *p3 = buildRawPacket(small); + ((clusterMsgHeader *)(void *)p3)->type = htons(CLUSTERMSG_TYPE_FAILOVER_AUTH_REQUEST); + memcpy(buf, p1, small); + memcpy(buf + small, p2, large); + memcpy(buf + small + large, p3, small); + buf[small + large + small] = 'T'; + fakeConnSetReadData(fc, buf, len); + zfree(p1); + zfree(p2); + zfree(p3); + zfree(buf); + } + + /* Run the worker side of a dispatched job inline, then let the main thread + * consume the completion exactly as the event loop would. */ + void runInlineWorkerAndDrain(void (*job)(clusterLink *), clusterLink *link) { + job(link); + processIOThreadsResponses(); + } +}; + +/* --- Read path -------------------------------------------------------- */ + +TEST_F(ClusterIOOffloadTest, ReadJobFramesCompletePrefixAndLeavesTail) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + seedReadableSocket(fc); + + link->io_read_state = CLUSTER_LINK_IO_PENDING; + clusterReadJob(link); + + EXPECT_EQ(link->io_complete_bytes, (size_t)CLUSTERMSG_MIN_LEN); + EXPECT_EQ(link->io_complete_packets, 1u); + /* The partial tail is read but deliberately not published. */ + EXPECT_EQ(link->rcvbuf_len, (size_t)CLUSTERMSG_MIN_LEN + 1); + link->io_read_state = CLUSTER_LINK_IO_IDLE; +} + +TEST_F(ClusterIOOffloadTest, ReadOffloadRoundTrip) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + seedReadableSocket(fc); + + ASSERT_EQ(trySendClusterReadToIOThreads(link), C_OK); + EXPECT_EQ(link->io_read_state, CLUSTER_LINK_IO_PENDING); + EXPECT_EQ(link->io_refs, 1); + EXPECT_EQ(testOnlyGetClusterIOPendingResponses(), 1u); + /* The counter tracks completions, so nothing is counted at dispatch. */ + EXPECT_EQ(server.stat_cluster_threaded_reads_processed, 0LL); + + runInlineWorkerAndDrain(clusterReadJob, link); + + EXPECT_EQ(server.cluster->stats_bus_messages_received[CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK], 1LL); + EXPECT_EQ(link->io_read_state, CLUSTER_LINK_IO_IDLE); + EXPECT_EQ(link->io_refs, 0); + EXPECT_EQ(link->io_complete_bytes, 0u); + EXPECT_EQ(link->io_complete_packets, 0u); + /* Only the unparsed tail is left, compacted to the front. */ + EXPECT_EQ(link->rcvbuf_len, 1u); + EXPECT_EQ(link->rcvbuf[0], 'T'); + EXPECT_EQ(server.stat_cluster_threaded_reads_processed, 1LL); + EXPECT_EQ(fc->postpone_state, 0); +} + +/* A peer that sends a valid packet and then hangs up must have that packet + * applied before the link is torn down. Same for a hard read error and for a + * malformed trailing header. All three drive the real worker so the result code + * comes from clusterReadJob() rather than being injected. */ +TEST_F(ClusterIOOffloadTest, ReadOffloadOnEofDrainsThenCloses) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + seedOneCompletePacket(fc); + fc->eof = 1; + + ASSERT_EQ(trySendClusterReadToIOThreads(link), C_OK); + runInlineWorkerAndDrain(clusterReadJob, link); + releaseLinkOwnership(link); + + EXPECT_GE(fc->close_calls, 1); + EXPECT_EQ(server.cluster->stats_bus_messages_received[CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK], 1LL); +} + +TEST_F(ClusterIOOffloadTest, ReadOffloadOnReadErrorDrainsThenCloses) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + seedOneCompletePacket(fc); + fc->fail_read = 1; + + ASSERT_EQ(trySendClusterReadToIOThreads(link), C_OK); + runInlineWorkerAndDrain(clusterReadJob, link); + releaseLinkOwnership(link); + + EXPECT_GE(fc->close_calls, 1); + EXPECT_EQ(server.cluster->stats_bus_messages_received[CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK], 1LL); +} + +TEST_F(ClusterIOOffloadTest, ReadOffloadOnProtocolErrorDrainsValidPrefixThenCloses) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + seedCompletePacketFollowedByGarbage(fc); + + ASSERT_EQ(trySendClusterReadToIOThreads(link), C_OK); + runInlineWorkerAndDrain(clusterReadJob, link); + releaseLinkOwnership(link); + + EXPECT_GE(fc->close_calls, 1); + EXPECT_EQ(server.cluster->stats_bus_messages_received[CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK], 1LL); +} + +/* --- Write path ------------------------------------------------------- */ + +TEST_F(ClusterIOOffloadTest, ReadOffloadDrainsMultiplePacketsAndCompactsTail) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + seedThreePacketsAndTail(fc); + + ASSERT_EQ(trySendClusterReadToIOThreads(link), C_OK); + runInlineWorkerAndDrain(clusterReadJob, link); + + /* Each packet must be seen exactly once, in its own right: a wrong slide + * offset would put some other packet's header at rcvbuf[0]. */ + EXPECT_EQ(server.cluster->stats_bus_messages_received[CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK], 1LL); + EXPECT_EQ(server.cluster->stats_bus_messages_received[CLUSTERMSG_TYPE_FAILOVER_AUTH_REQUEST], 1LL); + EXPECT_EQ(link->io_complete_bytes, 0u); + EXPECT_EQ(link->io_complete_packets, 0u); + /* Only the unparsed tail survives, compacted to the front. */ + EXPECT_EQ(link->rcvbuf_len, 1u); + EXPECT_EQ(link->rcvbuf[0], 'T'); +} + +/* A leftover partial packet must not pin rcvbuf at its high-water mark. */ +TEST_F(ClusterIOOffloadTest, ReadCompletionShrinksAroundPartialTail) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + /* Bigger than RCVBUF_INIT_LEN, as a real partial packet usually is. */ + size_t partial = seedPacketsAndLargePartialTail(fc); + ASSERT_GT(partial, (size_t)RCVBUF_INIT_LEN); + + ASSERT_EQ(trySendClusterReadToIOThreads(link), C_OK); + clusterReadJob(link); + size_t grown = link->rcvbuf_alloc; + ASSERT_GT(grown, partial + RCVBUF_INIT_LEN); + processIOThreadsResponses(); + + /* Both packets applied; the tail survives and the buffer shrank around it. */ + EXPECT_EQ(server.cluster->stats_bus_messages_received[CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK], 2LL); + EXPECT_EQ(link->rcvbuf_len, partial); + EXPECT_EQ(link->rcvbuf_alloc, partial + RCVBUF_INIT_LEN); + EXPECT_LT(link->rcvbuf_alloc, grown); + EXPECT_EQ(memcmp(link->rcvbuf, "RCmb", 4), 0); +} + +/* One job must not read an unbounded stream; it stops on the budget. */ +TEST_F(ClusterIOOffloadTest, ReadJobStopsAtReadBudget) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + + const size_t stream = (size_t)RCVBUF_MAX_PREALLOC * 2; + unsigned char *buf = (unsigned char *)zcalloc(stream); + memset(buf, 'x', stream); + fakeConnSetReadData(fc, buf, stream); + zfree(buf); + + ASSERT_EQ(trySendClusterReadToIOThreads(link), C_OK); + clusterReadJob(link); + + /* Stopped on the budget rather than draining the whole stream. */ + EXPECT_GE(fc->read_pos, (size_t)RCVBUF_MAX_PREALLOC); + EXPECT_LT(fc->read_pos, stream); + + /* Garbage bytes, so framing reports a bad header and the link is torn down. */ + EXPECT_EQ(link->io_result, CLUSTER_IO_BAD_HEADER); + processIOThreadsResponses(); + releaseLinkOwnership(link); +} + +TEST_F(ClusterIOOffloadTest, WriteDispatchSnapshotsBoundary) { + clusterLink *link = makeLink(); + enqueueFakeMsg(link); + enqueueFakeMsg(link); + + ASSERT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + + EXPECT_NE(link->io_last_send_block, (listNode *)NULL); + EXPECT_EQ(link->io_head_offset, 0u); + + /* Let the job run to completion rather than unwinding the state by hand. */ + runInlineWorkerAndDrain(clusterWriteJob, link); + EXPECT_EQ(link->io_write_state, CLUSTER_LINK_IO_IDLE); + EXPECT_EQ(link->io_refs, 0); +} + +TEST_F(ClusterIOOffloadTest, WriteOffloadRoundTripDrainsQueue) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + enqueueFakeMsg(link); + enqueueFakeMsg(link); + + ASSERT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + EXPECT_EQ(testOnlyGetClusterIOPendingResponses(), 1u); + EXPECT_EQ(server.stat_cluster_threaded_writes_processed, 0LL); + + runInlineWorkerAndDrain(clusterWriteJob, link); + + /* Both messages fit in the 4096-byte sink, so the queue drains fully and + * the write handler is uninstalled. */ + EXPECT_EQ(listLength(link->send_msg_queue), 0UL); + EXPECT_EQ(link->head_msg_send_offset, 0u); + EXPECT_EQ(link->io_write_state, CLUSTER_LINK_IO_IDLE); + EXPECT_EQ(link->io_refs, 0); + EXPECT_EQ(link->conn->write_handler, (ConnectionCallbackFunc)NULL); + EXPECT_EQ(server.stat_cluster_threaded_writes_processed, 1LL); + EXPECT_EQ(fc->postpone_state, 0); +} + +TEST_F(ClusterIOOffloadTest, WriteOffloadRoundTripPartialSendKeepsHandler) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + /* A sink smaller than the message forces a partial write. */ + fc->buf_size = 8; + enqueueFakeMsg(link); + + ASSERT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + runInlineWorkerAndDrain(clusterWriteJob, link); + + EXPECT_EQ(listLength(link->send_msg_queue), 1UL); + EXPECT_EQ(link->head_msg_send_offset, 8u); + EXPECT_EQ(link->io_write_state, CLUSTER_LINK_IO_IDLE); + /* More to send, so the write handler must stay armed. */ + EXPECT_NE(link->conn->write_handler, (ConnectionCallbackFunc)NULL); +} + +/* One job must not drain an arbitrarily large backlog: a worker is shared, so it + * stops at NET_MAX_WRITES_PER_EVENT and the rest goes out on the next event. */ +TEST_F(ClusterIOOffloadTest, WriteJobStopsAtWriteBudget) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + /* A sink far larger than the budget, so only the budget bounds the job. */ + zfree(fc->buffer); + fc->buf_size = NET_MAX_WRITES_PER_EVENT * 4; + fc->buffer = (char *)zmalloc(fc->buf_size); + + const uint32_t msg_len = 16 * 1024; + const int msgs = NET_MAX_WRITES_PER_EVENT / msg_len + 4; + for (int i = 0; i < msgs; i++) enqueueFakeMsg(link, msg_len); + + ASSERT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + runInlineWorkerAndDrain(clusterWriteJob, link); + + /* Stopped on the budget, so messages are left and the handler stays armed. */ + EXPECT_LT(fc->written, (size_t)NET_MAX_WRITES_PER_EVENT + msg_len); + EXPECT_GT(listLength(link->send_msg_queue), 0UL); + EXPECT_NE(link->conn->write_handler, (ConnectionCallbackFunc)NULL); + EXPECT_EQ(link->io_write_state, CLUSTER_LINK_IO_IDLE); + EXPECT_EQ(link->io_refs, 0); + + /* The next dispatch resumes from where it stopped and drains the rest. */ + ASSERT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + runInlineWorkerAndDrain(clusterWriteJob, link); + EXPECT_EQ(listLength(link->send_msg_queue), 0UL); + EXPECT_EQ(fc->written, (size_t)msg_len * msgs); +} + +/* A hard write error must tear the link down. A -1 with the connection still + * CONNECTED is EAGAIN instead, which the completion has to tell apart. */ +TEST_F(ClusterIOOffloadTest, WriteOffloadHardErrorClosesLink) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + enqueueFakeMsg(link); + fc->fail_write = 1; + + ASSERT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + runInlineWorkerAndDrain(clusterWriteJob, link); + releaseLinkOwnership(link); + + EXPECT_GE(fc->close_calls, 1); +} + +/* EAGAIN is not an error: the message stays queued and the link survives. */ +TEST_F(ClusterIOOffloadTest, WriteOffloadEagainKeepsLink) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + enqueueFakeMsg(link); + fc->error = 1; /* connWrite returns -1 with the state left CONNECTED. */ + + ASSERT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + runInlineWorkerAndDrain(clusterWriteJob, link); + + EXPECT_EQ(fc->close_calls, 0); + EXPECT_EQ(listLength(link->send_msg_queue), 1UL); + EXPECT_NE(link->conn->write_handler, (ConnectionCallbackFunc)NULL); +} + +TEST_F(ClusterIOOffloadTest, WriteCompletionPopsOnlyVisibleNodes) { + clusterLink *link = makeLink(); + enqueueFakeMsg(link); + enqueueFakeMsg(link); + + link->io_write_state = CLUSTER_LINK_IO_PENDING; + link->io_refs = 1; + link->io_nodes_sent = 1; + link->io_head_offset = 0; + link->io_result = CLUSTER_IO_OK; + + clusterHandleWriteCompletion(link); + + EXPECT_EQ(listLength(link->send_msg_queue), 1UL); +} + +TEST_F(ClusterIOOffloadTest, WriteCompletionPartialSendUpdatesHeadOffset) { + clusterLink *link = makeLink(); + enqueueFakeMsg(link); + + link->io_write_state = CLUSTER_LINK_IO_PENDING; + link->io_refs = 1; + link->io_nodes_sent = 0; + link->io_head_offset = 7; + link->io_result = CLUSTER_IO_OK; + + clusterHandleWriteCompletion(link); + + EXPECT_EQ(listLength(link->send_msg_queue), 1UL); + EXPECT_EQ(link->head_msg_send_offset, 7u); +} + +/* --- Dispatch is skipped while the connection is not established ------ */ + +TEST_F(ClusterIOOffloadTest, WriteDispatchSkippedWhileConnecting) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + fc->conn.state = CONN_STATE_CONNECTING; + enqueueFakeMsg(link); + + /* C_OK, so the caller does not fall back to a synchronous write that would + * fail the same way and tear the link down. */ + EXPECT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + + EXPECT_EQ(link->io_write_state, CLUSTER_LINK_IO_IDLE); + EXPECT_EQ(link->io_refs, 0); + EXPECT_EQ(link->conn->refs, 1); + EXPECT_EQ(fc->postpone_state, 0); + /* The message stays queued for the next dispatch. */ + EXPECT_EQ(listLength(link->send_msg_queue), 1UL); + /* Not a fallback: no I/O was attempted anywhere. */ + EXPECT_EQ(server.stat_cluster_io_main_thread_fallbacks, 0LL); +} + +TEST_F(ClusterIOOffloadTest, ReadDispatchSkippedWhileConnecting) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + fc->conn.state = CONN_STATE_CONNECTING; + + EXPECT_EQ(trySendClusterReadToIOThreads(link), C_OK); + + EXPECT_EQ(link->io_read_state, CLUSTER_LINK_IO_IDLE); + EXPECT_EQ(link->io_refs, 0); + EXPECT_EQ(link->conn->refs, 1); + EXPECT_EQ(fc->postpone_state, 0); + EXPECT_EQ(server.stat_cluster_io_main_thread_fallbacks, 0LL); +} + +TEST_F(ClusterIOOffloadTest, WriteDispatchSkippedWhileAccepting) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + fc->conn.state = CONN_STATE_ACCEPTING; + enqueueFakeMsg(link); + + EXPECT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + + EXPECT_EQ(link->io_write_state, CLUSTER_LINK_IO_IDLE); + EXPECT_EQ(listLength(link->send_msg_queue), 1UL); + EXPECT_EQ(server.stat_cluster_io_main_thread_fallbacks, 0LL); +} + +/* --- Fallback paths --------------------------------------------------- */ + +TEST_F(ClusterIOOffloadTest, ReadDispatchInboxFullUnwindsAndCountsFallback) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + testOnlyFillIOThreadInbox(); + + EXPECT_EQ(trySendClusterReadToIOThreads(link), C_ERR); + + EXPECT_EQ(link->io_read_state, CLUSTER_LINK_IO_IDLE); + EXPECT_EQ(link->io_refs, 0); + EXPECT_EQ(link->conn->refs, 1); + EXPECT_EQ(fc->postpone_state, 0); + EXPECT_EQ(server.stat_cluster_io_main_thread_fallbacks, 1LL); +} + +TEST_F(ClusterIOOffloadTest, WriteDispatchInboxFullUnwindsAndCountsFallback) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + enqueueFakeMsg(link); + enqueueFakeMsg(link); + link->head_msg_send_offset = 5; + testOnlyFillIOThreadInbox(); + + EXPECT_EQ(trySendClusterWriteToIOThreads(link), C_ERR); + + EXPECT_EQ(link->io_write_state, CLUSTER_LINK_IO_IDLE); + EXPECT_EQ(link->io_refs, 0); + EXPECT_EQ(link->io_last_send_block, (listNode *)NULL); + EXPECT_EQ(link->io_head_offset, 0u); + EXPECT_EQ(link->io_nodes_sent, 0); + /* The caller still owns the queue and its offset. */ + EXPECT_EQ(listLength(link->send_msg_queue), 2UL); + EXPECT_EQ(link->head_msg_send_offset, 5u); + EXPECT_EQ(link->conn->refs, 1); + EXPECT_EQ(fc->postpone_state, 0); + EXPECT_EQ(server.stat_cluster_io_main_thread_fallbacks, 1LL); +} + +TEST_F(ClusterIOOffloadTest, AcceptDispatchInboxFullUnwindsAndCountsFallback) { + fakeConnection *fc = makeConn(CONN_OWNER_CLUSTER_LINK); + fc->conn.flags |= CONN_FLAG_ALLOW_ACCEPT_OFFLOAD; + testOnlyFillIOThreadInbox(); + + EXPECT_EQ(trySendClusterAcceptToIOThreads(&fc->conn), C_ERR); + + EXPECT_EQ(fc->conn.flags & CONN_FLAG_ACCEPT_OFFLOAD_PENDING, 0); + EXPECT_EQ(fc->conn.refs, 1); + EXPECT_EQ(fc->postpone_state, 0); + EXPECT_EQ(server.stat_cluster_io_main_thread_fallbacks, 1LL); +} + +TEST_F(ClusterIOOffloadTest, PoolInactiveCountsFallback) { + clusterLink *link = makeLink(); + enqueueFakeMsg(link); + server.active_io_threads_num = 1; + + /* An established connection with the pool disabled must still report a + * fallback, i.e. the connecting-state guard did not swallow this path. */ + EXPECT_EQ(trySendClusterWriteToIOThreads(link), C_ERR); + EXPECT_EQ(trySendClusterReadToIOThreads(link), C_ERR); + + EXPECT_EQ(server.stat_cluster_io_main_thread_fallbacks, 2LL); + EXPECT_EQ(link->io_refs, 0); +} + +TEST_F(ClusterIOOffloadTest, DispatchDeferredWhileJobInFlight) { + clusterLink *link = makeLink(); + enqueueFakeMsg(link); + + ASSERT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + ASSERT_EQ(testOnlyGetClusterIOPendingResponses(), 1u); + + /* A second dispatch of either kind must not enqueue anything while a job is + * in flight, and must not push the caller to a synchronous retry. */ + EXPECT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + EXPECT_EQ(trySendClusterReadToIOThreads(link), C_OK); + EXPECT_EQ(testOnlyGetClusterIOPendingResponses(), 1u); + EXPECT_EQ(link->io_refs, 1); + EXPECT_EQ(server.stat_cluster_io_main_thread_fallbacks, 0LL); + + runInlineWorkerAndDrain(clusterWriteJob, link); +} + +/* --- Buffer limit and deferred teardown ------------------------------- */ + +/* 'cluster-link-sendbuf-limit' bounds the send queue only, so a link holding a + * large receive buffer must survive. */ +TEST_F(ClusterIOOffloadTest, BufferLimitIgnoresRcvbuf) { + clusterLink *link = makeLink(); + link->send_msg_queue_mem = 8; + link->rcvbuf_len = 4096; + server.cluster_link_msg_queue_limit_bytes = 64; + + testOnlyFreeClusterLinkOnBufferLimitReached(link); + + EXPECT_EQ(server.cluster->stat_cluster_links_buffer_limit_exceeded, 0ULL); +} + +TEST_F(ClusterIOOffloadTest, BufferLimitCountsSendQueue) { + clusterLink *link = makeLink(); + link->send_msg_queue_mem = 4096; + server.cluster_link_msg_queue_limit_bytes = 64; + + testOnlyFreeClusterLinkOnBufferLimitReached(link); + releaseLinkOwnership(link); + + EXPECT_EQ(server.cluster->stat_cluster_links_buffer_limit_exceeded, 1ULL); +} + +/* Without the fairness yield, a link whose send queue never drains re-claims the + * link on every iteration and inbound packets are never applied. */ +TEST_F(ClusterIOOffloadTest, BusySendQueueDoesNotStarveReads) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + fc->buf_size = 8; /* Smaller than the message, so the queue stays backlogged. */ + seedReadableSocket(fc); + enqueueFakeMsg(link); + + ASSERT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + ASSERT_EQ(link->io_write_state, CLUSTER_LINK_IO_PENDING); + + ASSERT_EQ(trySendClusterReadToIOThreads(link), C_OK); + ASSERT_EQ(link->io_read_state, CLUSTER_LINK_IO_IDLE); + ASSERT_EQ(link->io_read_deferred, 1); + + runInlineWorkerAndDrain(clusterWriteJob, link); + ASSERT_EQ(listLength(link->send_msg_queue), 1UL); + + /* The write yields one turn, so the read gets the link and applies the packet. */ + EXPECT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + EXPECT_EQ(link->io_write_state, CLUSTER_LINK_IO_IDLE); + EXPECT_EQ(link->io_read_deferred, 0); + + EXPECT_EQ(trySendClusterReadToIOThreads(link), C_OK); + EXPECT_EQ(link->io_read_state, CLUSTER_LINK_IO_PENDING); + runInlineWorkerAndDrain(clusterReadJob, link); + EXPECT_EQ(server.cluster->stats_bus_messages_received[CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK], 1LL); +} + +/* The yield is one-shot: with no read waiting, writes dispatch back to back. */ +TEST_F(ClusterIOOffloadTest, WriteDispatchNotYieldedWithoutDeferredRead) { + clusterLink *link = makeLink(); + enqueueFakeMsg(link); + + ASSERT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + runInlineWorkerAndDrain(clusterWriteJob, link); + + enqueueFakeMsg(link); + EXPECT_EQ(trySendClusterWriteToIOThreads(link), C_OK); + EXPECT_EQ(link->io_write_state, CLUSTER_LINK_IO_PENDING); + runInlineWorkerAndDrain(clusterWriteJob, link); +} + +TEST_F(ClusterIOOffloadTest, FreeClusterLinkDefersWhenIoRefOutstanding) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + ASSERT_EQ(trySendClusterReadToIOThreads(link), C_OK); + + int freed_now = freeClusterLink(link); + + EXPECT_EQ(freed_now, 0); + EXPECT_EQ(link->async_close, 1); + /* The connection must outlive the deferred free, since the worker is still + * using it. */ + EXPECT_EQ(fc->close_calls, 0); + + /* The pending completion drops the last reference and finalizes the free. */ + runInlineWorkerAndDrain(clusterReadJob, link); + releaseLinkOwnership(link); + EXPECT_GE(fc->close_calls, 1); +} + +TEST_F(ClusterIOOffloadTest, ReadCompletionFinalizesDeferredFree) { + clusterLink *link = makeLink(); + fakeConnection *fc = (fakeConnection *)link->conn; + link->async_close = 1; + link->io_read_state = CLUSTER_LINK_IO_PENDING; + link->io_refs = 1; + link->io_result = CLUSTER_IO_OK; + + clusterHandleReadCompletion(link); + releaseLinkOwnership(link); + + EXPECT_GE(fc->close_calls, 1); +} + +/* --- Accept path ------------------------------------------------------ */ + +TEST_F(ClusterIOOffloadTest, AcceptDispatchRequiresOffloadAllowedFlag) { + fakeConnection *fc = makeConn(CONN_OWNER_CLUSTER_LINK); + fc->conn.state = CONN_STATE_ACCEPTING; + + /* Plain TCP cluster accepts never set the flag, so they are not offloaded + * and are not counted as a fallback either. */ + EXPECT_EQ(trySendClusterAcceptToIOThreads(&fc->conn), C_ERR); + EXPECT_EQ(fc->conn.flags & CONN_FLAG_ACCEPT_OFFLOAD_PENDING, 0); + EXPECT_EQ(server.stat_cluster_io_main_thread_fallbacks, 0LL); +} + +TEST_F(ClusterIOOffloadTest, AcceptDispatchIsIdempotentWhilePending) { + fakeConnection *fc = makeAcceptConn(); + fc->conn.state = CONN_STATE_ACCEPTING; + fc->conn.flags |= CONN_FLAG_ALLOW_ACCEPT_OFFLOAD; + + ASSERT_EQ(trySendClusterAcceptToIOThreads(&fc->conn), C_OK); + int refs_after_first = fc->conn.refs; + ASSERT_EQ(testOnlyGetClusterIOPendingResponses(), 1u); + + /* TLS retries re-enter this path; only one job may be in flight. */ + EXPECT_EQ(trySendClusterAcceptToIOThreads(&fc->conn), C_OK); + EXPECT_EQ(fc->conn.refs, refs_after_first); + EXPECT_EQ(testOnlyGetClusterIOPendingResponses(), 1u); + + /* Drain through the real path so nothing is left referenced by the queue. */ + clusterAcceptJob(&fc->conn); + processIOThreadsResponses(); + trackLink((clusterLink *)connGetPrivateData(&fc->conn)); +} + +TEST_F(ClusterIOOffloadTest, AcceptOffloadRoundTripCreatesLink) { + fakeConnection *fc = makeAcceptConn(); + fc->conn.flags |= CONN_FLAG_ALLOW_ACCEPT_OFFLOAD; + + ASSERT_EQ(trySendClusterAcceptToIOThreads(&fc->conn), C_OK); + EXPECT_NE(fc->conn.flags & CONN_FLAG_ACCEPT_OFFLOAD_PENDING, 0); + EXPECT_EQ(server.stat_cluster_threaded_accepts_processed, 0LL); + + /* State ends CONNECTED, so applying the deferred state runs conn_handler. */ + clusterAcceptJob(&fc->conn); + processIOThreadsResponses(); + + EXPECT_EQ(fc->conn.flags & CONN_FLAG_ACCEPT_OFFLOAD_PENDING, 0); + EXPECT_EQ(fc->postpone_state, 0); + ASSERT_NE(connGetPrivateData(&fc->conn), (void *)NULL); + trackLink((clusterLink *)connGetPrivateData(&fc->conn)); + EXPECT_NE(fc->conn.read_handler, (ConnectionCallbackFunc)NULL); + EXPECT_EQ(server.stat_cluster_threaded_accepts_processed, 1LL); +} + +TEST_F(ClusterIOOffloadTest, AcceptCompletionAcceptingKeepsConnectionOpen) { + fakeConnection *fc = makeAcceptConn(); + fc->conn.flags |= CONN_FLAG_ACCEPT_OFFLOAD_PENDING; + fc->conn.state = CONN_STATE_ACCEPTING; + + clusterHandleAcceptCompletion(&fc->conn); + + /* Handshake still pending: conn_handler must not have run. */ + EXPECT_EQ(fc->close_calls, 0); + EXPECT_EQ(fc->conn.flags & CONN_FLAG_ACCEPT_OFFLOAD_PENDING, 0); + EXPECT_EQ(connGetPrivateData(&fc->conn), (void *)NULL); + EXPECT_EQ(fc->conn.conn_handler, clusterConnAcceptHandler); +} + +TEST_F(ClusterIOOffloadTest, AcceptCompletionConnectedCreatesLink) { + fakeConnection *fc = makeAcceptConn(); + fc->conn.flags |= CONN_FLAG_ACCEPT_OFFLOAD_PENDING; + + clusterHandleAcceptCompletion(&fc->conn); + + ASSERT_NE(connGetPrivateData(&fc->conn), (void *)NULL); + trackLink((clusterLink *)connGetPrivateData(&fc->conn)); + EXPECT_NE(fc->conn.read_handler, (ConnectionCallbackFunc)NULL); +} + +TEST_F(ClusterIOOffloadTest, AcceptCompletionAssertsPrivateDataStillNull) { + fakeConnection *fc = makeAcceptConn(); + fc->conn.flags |= CONN_FLAG_ACCEPT_OFFLOAD_PENDING; + connSetPrivateData(&fc->conn, (void *)0x1); + + EXPECT_DEATH(clusterHandleAcceptCompletion(&fc->conn), ""); +} diff --git a/src/unit/test_cmdflags.cpp b/src/unit/test_cmdflags.cpp new file mode 100644 index 000000000..6c324c9e7 --- /dev/null +++ b/src/unit/test_cmdflags.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "server.h" +} +extern hashtableType commandSetType; +extern hashtableType originalCommandSetType; + + +class CmdFlagsTest : public ::testing::Test { + protected: + void SetUp() override { + server.commands = hashtableCreate(&commandSetType); + server.orig_commands = hashtableCreate(&originalCommandSetType); + populateCommandTable(); + } +}; + + +TEST_F(CmdFlagsTest, TestWriteFirstkeyOnly) { + /* Each command with this flag is explicitly listed here to ensure: + * - new commands are not mistakenly detected as write firstkey only + * - commands which should be write firstkey only are detected. */ + const char *const writeFirstkeyCommands[] = { + "bitop", "geosearchstore", "pfmerge", "sdiffstore", "sinterstore", + "sunionstore", "zdiffstore", "zinterstore", "zrangestore", "zunionstore"}; + int expectedCount = sizeof(writeFirstkeyCommands) / sizeof(char *); + + int count = 0; + + hashtableIterator iter; + hashtableInitIterator(&iter, server.commands, 0); + struct serverCommand *c; + while (hashtableNext(&iter, (void **)&c)) { + if (c->flags & CMD_WRITE_FIRSTKEY_ONLY) { + count++; + bool found = false; + for (int i = 0; i < expectedCount; i++) { + if (strcmp(c->declared_name, writeFirstkeyCommands[i]) == 0) { + found = true; + break; + } + } + EXPECT_TRUE(found); + } + } + hashtableCleanupIterator(&iter); + + EXPECT_EQ(count, expectedCount); +} diff --git a/src/unit/test_compression.cpp b/src/unit/test_compression.cpp index 38e26f5ad..2a465dc00 100644 --- a/src/unit/test_compression.cpp +++ b/src/unit/test_compression.cpp @@ -160,7 +160,7 @@ TEST(CompressionTest, streamCompressorOutputBound) { for (size_t i = 0; i < sizeof(input_sizes) / sizeof(input_sizes[0]); i++) { for (size_t j = 0; j < sizeof(flush_modes) / sizeof(flush_modes[0]); j++) { streamCompressor compressor; - ASSERT_EQ(streamCompressorInit(&compressor, ALGO_LZ4, 0, false), C_OK); + ASSERT_EQ(streamCompressorInit(&compressor, ALGO_LZ4, 0, 0), C_OK); size_t bound = streamCompressorOutputBound(&compressor, input_sizes[i]); uint8_t *output = (uint8_t *)zmalloc(bound); @@ -484,7 +484,8 @@ static int encodeRoundTripPayload(testCompressionLayer layer, if (layer == TEST_COMPRESSION_LAYER_CODEC) { streamCompressor compressor; - if (streamCompressorInit(&compressor, ALGO_LZ4, 0, true) == C_ERR) return C_ERR; + if (streamCompressorInit(&compressor, ALGO_LZ4, 0, STREAM_CHECKSUM_BLOCK | STREAM_CHECKSUM_CONTENT) == C_ERR) + return C_ERR; size_t bound = streamCompressorOutputBound(&compressor, payload_len); sds output = sdsMakeRoomFor(sdsempty(), bound); @@ -1254,39 +1255,86 @@ TEST(CompressionTest, streamReaderFinishStopsAtFrameEndBeforeTrailingBytes) { dynamicBufFree(&db); } -TEST(CompressionTest, streamReaderRejectsTruncatedFrameTrailer) { +/* Classify damaged frames: clean EOF before the frame end is recoverable + * TRUNCATED (the replica retries the sync), a mutated frame body is CORRUPT + * (the replica aborts the load). The writer always enables the LZ4 block and + * content checksums, so corruption is deterministically detectable. Late + * damage is checked through both Read and Finish, the two loader paths. */ +TEST(CompressionTest, streamReaderClassifiesDamagedFrames) { const size_t payload_len = 256; uint8_t payload[payload_len]; for (size_t i = 0; i < payload_len; i++) { payload[i] = (uint8_t)(i & 0xFF); } - DynamicBuf db; - dynamicBufInit(&db); - streamWriter w; - ASSERT_EQ(streamWriterInit(&w, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); - ASSERT_EQ(streamWriterWrite(&w, payload, payload_len), C_OK); - ASSERT_EQ(streamWriterFinish(&w), C_OK); - streamWriterFree(&w); + sds encoded = NULL; + ASSERT_EQ(encodeRoundTripPayload(TEST_COMPRESSION_LAYER_STREAM, + payload, payload_len, &encoded), + C_OK); - ASSERT_GT(sdslen((const char *)db.data), (size_t)VCS_ENVELOPE_SIZE + 1); - MemReader mr = {}; - mr.data = db.data; - mr.len = sdslen((const char *)db.data) - 1; - mr.max_chunk = 7; - streamReaderConfig rcfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_MIN, false); - streamReader r; - ASSERT_EQ(streamReaderInit(&r, &rcfg, memReaderRead, &mr, NULL), C_OK); + const size_t frame_len = sdslen(encoded); + /* A mid-buffer flip lands beyond the VCS envelope and the (at most + * 19-byte) LZ4 frame header, in checksum-covered block data. */ + const size_t middle_offset = (VCS_ENVELOPE_SIZE + frame_len) / 2; + ASSERT_GT(middle_offset, (size_t)VCS_ENVELOPE_SIZE + 19); - uint8_t out[payload_len]; - ASSERT_EQ(streamReaderRead(&r, out, payload_len), (ssize_t)payload_len); - EXPECT_EQ(memcmp(out, payload, payload_len), 0); - ASSERT_LT(streamReaderRead(&r, out, 1), 0) << "EOF before frame end should be treated as corruption"; - ASSERT_EQ(r.error_kind, STREAM_READER_ERROR_CORRUPT) - << "truncated compressed frame should latch corruption, not I/O"; + struct { + const char *name; + size_t source_len; + size_t flip_offset; /* 0 => no flip */ + streamReaderErrorKind expected_error; + bool payload_readable; + bool finish_detects_damage; + bool eof_is_truncation; /* Retryable source: mid-frame EOF => TRUNCATED, else CORRUPT. */ + } cases[] = { + /* Retryable (socket-like) source: a clean mid-frame EOF is a recoverable short read. */ + {"EOF one byte into frame", VCS_ENVELOPE_SIZE + 1, 0, STREAM_READER_ERROR_TRUNCATED, false, false, true}, + {"EOF mid-frame", middle_offset, 0, STREAM_READER_ERROR_TRUNCATED, false, false, true}, + {"EOF one byte before frame end on read", frame_len - 1, 0, STREAM_READER_ERROR_TRUNCATED, true, false, true}, + {"EOF one byte before frame end on finish", frame_len - 1, 0, STREAM_READER_ERROR_TRUNCATED, true, true, true}, + /* Seekable (file-like) source: no more bytes are coming, so a clean mid-frame EOF is corruption. */ + {"EOF one byte into frame (file source)", VCS_ENVELOPE_SIZE + 1, 0, STREAM_READER_ERROR_CORRUPT, false, false, false}, + {"EOF mid-frame (file source)", middle_offset, 0, STREAM_READER_ERROR_CORRUPT, false, false, false}, + /* A flipped byte latches a codec error regardless of source type. */ + {"flipped byte mid-frame", frame_len, middle_offset, STREAM_READER_ERROR_CORRUPT, false, false, false}, + {"flipped last byte (content checksum)", frame_len, frame_len - 1, STREAM_READER_ERROR_CORRUPT, true, true, false}, + }; - streamReaderFree(&r); - dynamicBufFree(&db); + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + if (cases[i].flip_offset) encoded[cases[i].flip_offset] ^= 0xFF; + + MemReader mr = {}; + mr.data = (const uint8_t *)encoded; + mr.len = cases[i].source_len; + mr.max_chunk = 7; /* odd chunk size to exercise refill boundaries */ + streamReaderConfig rcfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_MIN, false); + rcfg.eof_mid_frame_is_truncation = cases[i].eof_is_truncation; + streamReader r; + ASSERT_EQ(streamReaderInit(&r, &rcfg, memReaderRead, &mr, NULL), C_OK) << cases[i].name; + + uint8_t out[payload_len]; + if (cases[i].payload_readable) { + ASSERT_EQ(streamReaderRead(&r, out, payload_len), (ssize_t)payload_len) << cases[i].name; + ASSERT_EQ(memcmp(out, payload, payload_len), 0) << cases[i].name; + if (cases[i].finish_detects_damage) { + ASSERT_EQ(streamReaderFinish(&r), C_ERR) << cases[i].name; + } else { + ASSERT_EQ(streamReaderRead(&r, out, 1), -1) << cases[i].name; + } + } else { + /* The error may surface on the first or a later read depending on + * how much the codec buffers before validating. */ + ssize_t n = streamReaderRead(&r, out, payload_len); + while (n > 0) n = streamReaderRead(&r, out, payload_len); + ASSERT_LT(n, 0) << cases[i].name; + } + ASSERT_EQ(r.error_kind, cases[i].expected_error) << cases[i].name; + + streamReaderFree(&r); + if (cases[i].flip_offset) encoded[cases[i].flip_offset] ^= 0xFF; + } + + sdsfree(encoded); } TEST(CompressionTest, streamWriterWriteAfterFinish) { @@ -1321,3 +1369,303 @@ TEST(CompressionTest, streamWriterWriteAfterFinish) { streamWriterFree(&t); dynamicBufFree(&db); } + +/* ===== Replication push reader and repl-frame policy ===== */ + +static void fillIncompressible(unsigned char *buf, size_t n, uint32_t seed) { + /* xorshift32 produces deterministic pseudo-random test data. */ + uint32_t x = seed; + for (size_t i = 0; i < n; i++) { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + buf[i] = (unsigned char)(x >> 24); + } +} + +typedef struct { + streamCompressor compressor; + sds output; +} ReplTestStream; + +static int initReplTestStream(ReplTestStream *stream) { + memset(stream, 0, sizeof(*stream)); + if (streamCompressorInit(&stream->compressor, ALGO_LZ4, 0, STREAM_CHECKSUM_BLOCK) != C_OK) return C_ERR; + stream->output = sdsempty(); + return C_OK; +} + +static void freeReplTestStream(ReplTestStream *stream) { + streamCompressorFree(&stream->compressor); + sdsfree(stream->output); +} + +/* Build a replication-kind VCS stream without depending on the networking adapter. */ +static int compressReplTestStream(ReplTestStream *stream, const void *buf, size_t len, compressFlushMode flush_mode) { + if (!stream->compressor.stream_started) { + uint8_t envelope[VCS_ENVELOPE_SIZE]; + if (vcsBuildEnvelope(envelope, stream->compressor.algo, VCS_STREAM_REPL) == C_ERR) return C_ERR; + stream->output = sdscatlen(stream->output, envelope, sizeof(envelope)); + } + size_t bound = streamCompressorOutputBound(&stream->compressor, len); + if (bound == 0) return C_ERR; + stream->output = sdsMakeRoomFor(stream->output, bound); + ssize_t compressed = streamCompressorFeed(&stream->compressor, + (uint8_t *)stream->output + sdslen(stream->output), + sdsavail(stream->output), (const uint8_t *)buf, len, flush_mode); + if (compressed < 0) return C_ERR; + sdsIncrLen(stream->output, (size_t)compressed); + return C_OK; +} + +/* LZ4 frame FLG byte (follows the 4-byte frame magic): bit 2 = content + * checksum present, bit 4 = block checksums present. Frozen wire format. + * replFrameOmitsContentChecksum and rdbFrameKeepsContentChecksum verify wire + * bytes deliberately: "content checksum off on a never-ending frame" has no + * black-box observable (it is CPU silently wasted hashing bytes that are never + * validated), so the frame header is the only place the policy can regress + * visibly. */ +#define LZ4F_FLG_CONTENT_CHECKSUM 0x04 +#define LZ4F_FLG_BLOCK_CHECKSUM 0x10 + +/* Assert the VCS envelope prefix at its documented offsets, so a future + * envelope-layout change fails here loudly instead of silently shifting the + * frame bytes the caller is about to inspect. Returns the offset of the + * LZ4 frame FLG byte. */ +static size_t assertVcsEnvelopeAnchor(const unsigned char *stream, uint8_t expected_kind) { + EXPECT_EQ(stream[0], VCS_MAGIC_0); + EXPECT_EQ(stream[1], VCS_MAGIC_1); + EXPECT_EQ(stream[2], VCS_MAGIC_2); + EXPECT_EQ(stream[VCS_OFFSET_VERSION], VCS_VERSION); + EXPECT_EQ(stream[VCS_OFFSET_CODEC], VCS_CODEC_LZ4); + EXPECT_EQ(stream[VCS_OFFSET_RESERVED], 0x00); + EXPECT_EQ(stream[VCS_OFFSET_STREAM_KIND], expected_kind); + /* LZ4 frame magic 0x184D2204 (little-endian) right after the envelope. */ + EXPECT_EQ(stream[VCS_ENVELOPE_SIZE + 0], 0x04); + EXPECT_EQ(stream[VCS_ENVELOPE_SIZE + 1], 0x22); + EXPECT_EQ(stream[VCS_ENVELOPE_SIZE + 2], 0x4D); + EXPECT_EQ(stream[VCS_ENVELOPE_SIZE + 3], 0x18); + return VCS_ENVELOPE_SIZE + 4; /* FLG byte offset. */ +} + +TEST(replCompression, replFrameOmitsContentChecksum) { + /* A repl frame never ends, so its content checksum would be computed on + * every byte but never emitted or validated. It must be off in the frame + * header while block checksums stay on. */ + ReplTestStream test_stream; + ASSERT_EQ(initReplTestStream(&test_stream), C_OK); + const char payload[] = "content-checksum-off-for-repl"; + ASSERT_EQ(compressReplTestStream(&test_stream, payload, sizeof(payload), COMPRESS_FLUSH_CONTINUE), C_OK); + ASSERT_EQ(compressReplTestStream(&test_stream, NULL, 0, COMPRESS_FLUSH_SYNC), C_OK); + const unsigned char *wire = (const unsigned char *)test_stream.output; + ASSERT_GE(sdslen(test_stream.output), (size_t)(VCS_ENVELOPE_SIZE + 5)); + size_t flg_offset = assertVcsEnvelopeAnchor(wire, VCS_STREAM_REPL); + unsigned char flg = wire[flg_offset]; + EXPECT_EQ(flg & LZ4F_FLG_CONTENT_CHECKSUM, 0x00); + EXPECT_EQ(flg & LZ4F_FLG_BLOCK_CHECKSUM, LZ4F_FLG_BLOCK_CHECKSUM); + + /* Round-trip: the reader learns checksum presence from the frame header, + * so it needs no matching configuration. */ + streamPushReader reader; + streamPushReaderInit(&reader, VCS_STREAM_REPL); + sds out = sdsempty(); + ASSERT_EQ(streamPushReaderFeed(&reader, test_stream.output, sdslen(test_stream.output), &out, 1024 * 1024), + STREAM_PUSH_READER_OK); + ASSERT_EQ(sdslen(out), sizeof(payload)); + EXPECT_EQ(memcmp(out, payload, sizeof(payload)), 0); + EXPECT_EQ(reader.state, STREAM_PUSH_READER_COMPRESSED); + + streamPushReaderFree(&reader); + sdsfree(out); + freeReplTestStream(&test_stream); +} + +TEST(replCompression, rdbFrameKeepsContentChecksum) { + /* Contrast: the default (RDB) stream kind finishes its frame, so the + * content checksum stays on. */ + streamWriter writer; + DynamicBuf out; + dynamicBufInit(&out); + ASSERT_EQ(streamWriterInit(&writer, ALGO_LZ4, true, emitToDynamicBuf, &out), C_OK); + ASSERT_EQ(streamWriterWrite(&writer, "rdb-bytes", 9), C_OK); + const unsigned char *stream = (const unsigned char *)out.data; + ASSERT_GE(sdslen((sds)out.data), (size_t)(VCS_ENVELOPE_SIZE + 5)); + size_t flg_offset = assertVcsEnvelopeAnchor(stream, VCS_STREAM_RDB); + unsigned char flg = stream[flg_offset]; + EXPECT_EQ(flg & LZ4F_FLG_CONTENT_CHECKSUM, LZ4F_FLG_CONTENT_CHECKSUM); + EXPECT_EQ(flg & LZ4F_FLG_BLOCK_CHECKSUM, LZ4F_FLG_BLOCK_CHECKSUM); + streamWriterFree(&writer); + dynamicBufFree(&out); +} + +TEST(replCompression, pushReaderFrameDoneOnLiveLink) { + ReplTestStream test_stream; + ASSERT_EQ(initReplTestStream(&test_stream), C_OK); + const char payload[] = "frame-done-on-live-link"; + ASSERT_EQ(compressReplTestStream(&test_stream, payload, sizeof(payload), COMPRESS_FLUSH_CONTINUE), C_OK); + /* Finish ends the frame; a live replication link must never see that. */ + ASSERT_EQ(compressReplTestStream(&test_stream, NULL, 0, COMPRESS_FLUSH_END), C_OK); + + streamPushReader reader; + streamPushReaderInit(&reader, VCS_STREAM_REPL); + sds out = sdsempty(); + EXPECT_EQ(streamPushReaderFeed(&reader, test_stream.output, sdslen(test_stream.output), &out, 1024 * 1024), + STREAM_PUSH_READER_FRAME_DONE); + ASSERT_EQ(sdslen(out), sizeof(payload)); + EXPECT_EQ(memcmp(out, payload, sizeof(payload)), 0); + + streamPushReaderFree(&reader); + sdsfree(out); + freeReplTestStream(&test_stream); +} + +TEST(replCompression, pushReaderOutputLimitIsResumable) { + ReplTestStream test_stream; + ASSERT_EQ(initReplTestStream(&test_stream), C_OK); + const size_t n = 64 * 1024; + unsigned char *buf = (unsigned char *)zmalloc(n); + fillIncompressible(buf, n, 0x12345678u); + ASSERT_EQ(compressReplTestStream(&test_stream, buf, n, COMPRESS_FLUSH_CONTINUE), C_OK); + ASSERT_EQ(compressReplTestStream(&test_stream, NULL, 0, COMPRESS_FLUSH_SYNC), C_OK); + + streamPushReader reader; + streamPushReaderInit(&reader, VCS_STREAM_REPL); + sds out = sdsempty(); + streamPushReaderResult result = + streamPushReaderFeed(&reader, test_stream.output, sdslen(test_stream.output), &out, 1024); + EXPECT_EQ(result, STREAM_PUSH_READER_NEED_OUTPUT); + EXPECT_EQ(sdslen(out), (size_t)1024); + + while (result != STREAM_PUSH_READER_OK) { + ASSERT_EQ(result, STREAM_PUSH_READER_NEED_OUTPUT); + result = streamPushReaderFeed(&reader, NULL, 0, &out, 1024); + } + ASSERT_EQ(sdslen(out), n); + EXPECT_EQ(memcmp(out, buf, n), 0); + + streamPushReaderFree(&reader); + sdsfree(out); + zfree(buf); + freeReplTestStream(&test_stream); +} + +TEST(replCompression, pushReaderEnvelopeSplitAcrossFeeds) { + ReplTestStream test_stream; + ASSERT_EQ(initReplTestStream(&test_stream), C_OK); + const size_t n = 10 * 1024; + unsigned char *payload = (unsigned char *)zmalloc(n); + memset(payload, 'A', n); + ASSERT_EQ(compressReplTestStream(&test_stream, payload, n, COMPRESS_FLUSH_CONTINUE), C_OK); + ASSERT_EQ(compressReplTestStream(&test_stream, NULL, 0, COMPRESS_FLUSH_SYNC), C_OK); + const unsigned char *stream = (const unsigned char *)test_stream.output; + const size_t stream_len = sdslen(test_stream.output); + ASSERT_GT(stream_len, (size_t)VCS_ENVELOPE_SIZE); + + streamPushReader reader; + streamPushReaderInit(&reader, VCS_STREAM_REPL); + const char prefix[] = "existing:"; + const size_t prefix_len = sizeof(prefix) - 1; + sds out = sdsnewlen(prefix, prefix_len); + + /* A partial envelope neither classifies the stream nor changes output. */ + ASSERT_EQ(streamPushReaderFeed(&reader, stream, 1, &out, 1024 * 1024), STREAM_PUSH_READER_OK); + EXPECT_EQ(sdslen(out), prefix_len); + ASSERT_EQ(streamPushReaderFeed(&reader, stream + 1, 2, &out, 1024 * 1024), STREAM_PUSH_READER_OK); + EXPECT_EQ(sdslen(out), prefix_len); + + /* Decoded bytes append after existing caller-owned output. */ + ASSERT_EQ(streamPushReaderFeed(&reader, stream + 3, stream_len - 3, &out, 1024 * 1024), + STREAM_PUSH_READER_OK); + ASSERT_EQ(sdslen(out), prefix_len + n); + EXPECT_EQ(memcmp(out, prefix, prefix_len), 0); + EXPECT_EQ(memcmp(out + prefix_len, payload, n), 0); + + streamPushReaderFree(&reader); + sdsfree(out); + zfree(payload); + freeReplTestStream(&test_stream); +} + +TEST(replCompression, pushReaderPassthroughReplaysPrefix) { + streamPushReader reader; + streamPushReaderInit(&reader, VCS_STREAM_REPL); + sds out = sdsempty(); + /* "V" alone could still open the VCS magic: buffered, nothing emitted. */ + ASSERT_EQ(streamPushReaderFeed(&reader, "V", 1, &out, 1024), STREAM_PUSH_READER_OK); + EXPECT_EQ(sdslen(out), (size_t)0); + EXPECT_EQ(reader.state, STREAM_PUSH_READER_PROBE); + /* "X" rules out the magic: the buffered "V" replays ahead of the new bytes. */ + ASSERT_EQ(streamPushReaderFeed(&reader, "XYZ", 3, &out, 1024), STREAM_PUSH_READER_OK); + EXPECT_EQ(reader.state, STREAM_PUSH_READER_PASSTHROUGH); + ASSERT_EQ(sdslen(out), (size_t)4); + EXPECT_EQ(memcmp(out, "VXYZ", 4), 0); + streamPushReaderFree(&reader); + sdsfree(out); +} + +TEST(replCompression, pushReaderRejectsWrongStreamKind) { + /* A valid RDB envelope must not activate a replication reader. */ + unsigned char envelope[VCS_ENVELOPE_SIZE]; + ASSERT_EQ(vcsBuildEnvelope(envelope, ALGO_LZ4, VCS_STREAM_RDB), C_OK); + + streamPushReader reader; + streamPushReaderInit(&reader, VCS_STREAM_REPL); + sds out = sdsempty(); + EXPECT_EQ(streamPushReaderFeed(&reader, envelope, sizeof(envelope), &out, 1024), STREAM_PUSH_READER_ERR); + EXPECT_EQ(sdslen(out), (size_t)0); + streamPushReaderFree(&reader); + sdsfree(out); +} + +TEST(replCompression, pushReaderErrOnCorruptPayload) { + unsigned char stream[VCS_ENVELOPE_SIZE + 64]; + ASSERT_EQ(vcsBuildEnvelope(stream, ALGO_LZ4, VCS_STREAM_REPL), C_OK); + memset(stream + VCS_ENVELOPE_SIZE, 0xFF, 64); + + streamPushReader reader; + streamPushReaderInit(&reader, VCS_STREAM_REPL); + sds out = sdsempty(); + EXPECT_EQ(streamPushReaderFeed(&reader, stream, sizeof(stream), &out, 1024 * 1024), + STREAM_PUSH_READER_ERR); + EXPECT_EQ(sdslen(out), (size_t)0); + + streamPushReaderFree(&reader); + sdsfree(out); +} + +TEST(replCompression, pushReaderDrainsBufferedOutputWithoutMoreInput) { + /* The writer emits 64KB LZ4 blocks while the reader offers 16KB of room + * per iteration, so LZ4F decodes a compressed block into its internal + * buffer and can report the block's input consumed with output still + * undelivered. Once input runs out the reader must keep draining with + * empty input; otherwise the tail is stranded inside the codec until + * later transport bytes arrive. The payload ends with a compressible run: + * a stored (incompressible) block streams straight to the caller's buffer + * and would not strand. */ + const size_t incompressible = 36 * 1024; + const size_t compressible = 64 * 1024; + const size_t n = incompressible + compressible; /* ~100KB: multiple blocks */ + unsigned char *payload = (unsigned char *)zmalloc(n); + fillIncompressible(payload, incompressible, 0xC0FFEE42u); + memset(payload + incompressible, 'A', compressible); + + ReplTestStream test_stream; + ASSERT_EQ(initReplTestStream(&test_stream), C_OK); + ASSERT_EQ(compressReplTestStream(&test_stream, payload, n, COMPRESS_FLUSH_CONTINUE), C_OK); + ASSERT_EQ(compressReplTestStream(&test_stream, NULL, 0, COMPRESS_FLUSH_SYNC), C_OK); /* frame stays open */ + + /* All compressed bytes in ONE call: no later input can push out whatever + * the codec buffered, so the feed itself must drain it. */ + streamPushReader reader; + streamPushReaderInit(&reader, VCS_STREAM_REPL); + sds out = sdsempty(); + ASSERT_EQ(streamPushReaderFeed(&reader, test_stream.output, sdslen(test_stream.output), &out, 4 * 1024 * 1024), + STREAM_PUSH_READER_OK); + ASSERT_EQ(sdslen(out), n); + EXPECT_EQ(memcmp(out, payload, n), 0); + + streamPushReaderFree(&reader); + sdsfree(out); + freeReplTestStream(&test_stream); + zfree(payload); +} diff --git a/src/unit/test_fbtree.cpp b/src/unit/test_fbtree.cpp index 57ac87b91..cf43554cc 100644 --- a/src/unit/test_fbtree.cpp +++ b/src/unit/test_fbtree.cpp @@ -3026,6 +3026,86 @@ TEST_F(FbtreeTest, DeleteRangeByRankThenInsert) { EXPECT_EQ(all.size(), 20u); } +/* A range delete that strips every sibling from an inner node leaves that node + * with a single child. If updateCommonPrefix() skips the recompute for such a + * node, it retains the prefix derived when the node still had >= 2 anchors, + * while innerNodeRefreshChildMeta() has just rewritten the surviving child-0 + * anchor. Child 0's key range extends BELOW its own high key, so the refreshed + * anchor can fall below the retained prefix, breaking the "every anchor starts + * with the node prefix" invariant. This test guards that path. + * + * Needs a >= 3-level tree, and the delete must (a) start inside child 0 of a + * depth-1 inner node, (b) extend past the end of that node's subtree, and + * (c) stop short of the last root child so the node is not collapsed away. + * Keys are equal length, so the tree shape depends only on the element count: + * the first pass learns the shape, the second builds the triggering content. */ +TEST_F(FbtreeTest, RangeDeleteLeavingSingleChildKeepsPrefixValid) { + const int N = TEST_THREE_LEVEL_ITEMS * 3; /* comfortably 3 levels */ + + /* Pass 1: uniform keys, purely to locate a suitable depth-1 inner node. */ + for (int i = 0; i < N; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "b%08d", i); + fbtreeInsert(fbt, createString(buf)); + } + ASSERT_FALSE(fbt->root->is_leaf); + ASSERT_GT(fbtreeHeight(fbt), 2UL) << "test needs a >= 3-level tree"; + + int flip = -1, del_start = -1, del_end = -1; + { + innerNode *root = (innerNode *)(void *)fbt->root; + size_t rank = 0; + for (int i = 0; i < root->header.num_items; i++) { + node *child = root->children[i]; + size_t subtree = root->child_sizes[i]; + /* Skip root child 0: its child 0 has no lower neighbour to dip into. */ + if (i >= 1 && !child->is_leaf && flip < 0) { + innerNode *d = (innerNode *)(void *)child; + if (d->header.num_items >= 2) { + size_t c0 = d->child_sizes[0]; + flip = (int)(rank + c0 / 2); /* inside child 0 of this node */ + del_start = (int)(rank + 1); /* keep >= 1 survivor in child 0 */ + del_end = (int)(rank + subtree); /* one past this subtree */ + } + } + rank += subtree; + } + } + ASSERT_GE(flip, 0) << "no suitable depth-1 inner node found"; + + /* Pass 2: rebuild so the key prefix flips 'a' -> 'b' at `flip`. Ordering is + * still by index, so the shape is identical to pass 1. */ + fbtreeFree(fbt); + fbt = fbtreeCreate(); + for (int i = 0; i < N; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "%c%08d", i < flip ? 'a' : 'b', i); + fbtreeInsert(fbt, createString(buf)); + } + expectValid(); + + unsigned long deleted = fbtreeDeleteRangeByRank(fbt, del_start, del_end, NULL, NULL); + EXPECT_EQ(deleted, (unsigned long)(del_end - del_start + 1)); + + char errmsg[256]; + ASSERT_TRUE(fbtreeDebugValidate(fbt, false, errmsg, sizeof(errmsg))) << errmsg; + + /* The surviving elements must still be exactly the expected set, in order. */ + fbtreeIterator it; + fbtreeInitIterator(&it, fbt); + const_sds pos; + for (int i = 0; i < N; i++) { + if (i >= del_start && i <= del_end) continue; + char buf[32]; + snprintf(buf, sizeof(buf), "%c%08d", i < flip ? 'a' : 'b', i); + sds expected = createString(buf); + ASSERT_NE(pos = fbtreeNext(&it), nullptr) << "tree exhausted at index " << i; + EXPECT_EQ(sdscmp(pos, expected), 0); + sdsfree(expected); + } + EXPECT_EQ(pos = fbtreeNext(&it), nullptr); +} + TEST_F(FbtreeTest, DeleteRangeByRankDeepTree) { /* Build a 3+ level tree */ const int N = TEST_THREE_LEVEL_ITEMS; @@ -3340,6 +3420,255 @@ TEST_F(FbtreeTest, DeleteRangeByValueNoMatch) { expectValid(); } +/* Regression test for a false-empty short-circuit in deleteRangeCore. + * + * The tree spans multiple leaves (NODE_SIZE=61 forces this with 300+ + * elements). After trimming the first leaf down to a single low member, a + * later range delete has BOTH boundaries land in leaves with no locally + * matching elements ("leaf-untouched") while the split node still has one + * or more middle children strictly between the boundary subtrees, wholly + * inside the deleted range and non-empty. The buggy guard only checked + * leaf-local untouched-ness and returned 0 (deleted nothing) even though + * those middle leaves' elements were still in range. */ +TEST_F(FbtreeTest, DeleteRangeByValueSkipsNoMiddleLeafFalseEmpty) { + /* Seed enough elements to force several leaves/levels: an empty-string + * low sentinel plus 300 zero-padded members "m0001".."m0300". */ + insert(""); + char buf[16]; + for (int i = 1; i <= 300; i++) { + snprintf(buf, sizeof(buf), "m%04d", i); + insert(buf); + } + EXPECT_EQ(fbtreeLength(fbt), 301u); + + /* Trim the first leaf down to just the empty-string member: removes + * [m0001, m0060] inclusive, leaving the low leaf with a single element + * and no members in the immediately following range. */ + sds trim_min = createString("m0001"); + sds trim_max = createString("m0060"); + EXPECT_EQ(fbtreeDeleteRangeByValue(fbt, trim_min, trim_max, 0, 0, NULL, NULL), 60u); + sdsfree(trim_min); + sdsfree(trim_max); + expectValid(); + EXPECT_EQ(fbtreeLength(fbt), 241u); + + /* Exclusive lower bound just past the trimmed leaf's remaining element, + * and an upper bound landing in the gap just after "m0121" (no stored + * element equals "m0121x"). Both boundary leaves are leaf-locally + * untouched, but middle leaves fully inside the range still exist. */ + sds range_min = createString(""); + sds range_max = createString("m0121x"); + unsigned long expected = fbtreeCountRangeByValue(fbt, range_min, range_max, 1, 0); + ASSERT_GT(expected, 0u) << "range must be non-empty for this regression to be meaningful"; + + unsigned long removed = fbtreeDeleteRangeByValue(fbt, range_min, range_max, 1, 0, NULL, NULL); + sdsfree(range_min); + sdsfree(range_max); + expectValid(); + + EXPECT_EQ(removed, expected) << "deleteRangeCore must not short-circuit to 0 when non-empty middle leaves lie between untouched boundary leaves"; +} + +/* Regression test for a DEEPER false-empty short-circuit in deleteRangeCore + * than the one covered above. The prior fix's guard (no_middle_child) only + * checks adjacency of the SPLIT NODE's own boundary children (shared_left_idx + * / shared_right_idx at split_depth). It says nothing about levels below the + * split: when the split happens at the root and the root has exactly two + * children (a common shape once a two-level tree overflows into a third + * level), those two children are trivially "adjacent" (li=0, ri=1) even + * though each child is itself a whole inner-node subtree with many leaves. + * If the min boundary's descent path picks a non-rightmost child at some + * level under root->children[li], the right-siblings at that level are + * middle subtrees fully inside the deleted range that the split-node-only + * guard cannot see (symmetric on the right side under root->children[ri]). + * + * 5000 zero-padded members force height 3: NODE_SIZE=61 fans out to at most + * 61 leaves per level-2 inner node (61*61=3721 items per full level-2 + * subtree), so 5000 items split across exactly two such level-2 subtrees + * under the root -- the shape this test needs. + * + * The chosen bounds land both boundaries in an untouched state at the LEAF + * level (exclusive bound resolves past/before the boundary leaf's members), + * which is exactly what makes the split-node guard's no_middle_child==true + * wrongly conclude the whole range is empty -- while leaves strictly between + * the min's leaf and the max's leaf, one level below the root, still hold + * thousands of in-range elements. */ +TEST_F(FbtreeTest, DeleteRangeByValueSkipsDeeperMiddleLeafFalseEmpty) { + /* 5000 zero-padded members "m00001".."m05000", forcing height 3. */ + char buf[16]; + for (int i = 1; i <= 5000; i++) { + snprintf(buf, sizeof(buf), "m%05d", i); + insert(buf); + } + EXPECT_EQ(fbtreeLength(fbt), 5000u); + EXPECT_EQ(fbtreeHeight(fbt), 3u); + + /* Exclusive bounds discovered by direct probing of this exact 5000-item + * tree shape: min excludes "m00061" (the last member of the leftmost + * leaf, landing start_idx just past that leaf -- leaf-untouched), max + * excludes "m02868" (the first member of some leaf under the root's + * second child, landing end_idx just before that leaf -- also + * leaf-untouched). Both boundary leaves are untouched, root's two + * children are (trivially) adjacent, yet 2806 elements strictly between + * them are in range. */ + sds range_min = createString("m00061"); + sds range_max = createString("m02868"); + unsigned long expected = fbtreeCountRangeByValue(fbt, range_min, range_max, 1, 1); + ASSERT_EQ(expected, 2806u) << "expected count must match the probed shape for this regression to be meaningful"; + + unsigned long removed = fbtreeDeleteRangeByValue(fbt, range_min, range_max, 1, 1, NULL, NULL); + sdsfree(range_min); + sdsfree(range_max); + expectValid(); + + EXPECT_EQ(removed, expected) << "deleteRangeCore must not short-circuit to 0 when non-empty middle " + "leaves lie one or more levels below the split node between " + "leaf-untouched boundaries"; +} + +/* Exclusive range bounds that land exactly on a real member sitting at a + * leaf's fill edge (the last member of the leaf below the bound, or the + * first member of the leaf above it) have repeatedly disagreed between + * fbtreeCountRangeByValue and fbtreeDeleteRangeByValue: excluding the + * actual last item of a leaf pushes the descent index past that leaf's + * item count, which upstream code has mistaken for "this side is out of + * range" even when whole leaves and subtrees strictly between the two + * boundaries remain in range. Synthetic between-member values do not + * reach this state -- they land at the next leaf's insertion point without + * ever aligning with a real fill edge -- so this sweep only uses bounds + * equal to real inserted members, concentrated at multiples of NODE_SIZE + * (61) where a leaf actually fills, plus a coarser stride for background + * coverage away from those edges. The score-comparison path is not swept + * here: probing it directly showed no equivalent alignment sensitivity, + * so it is omitted to keep the property scoped to the code path that is + * actually affected. */ +TEST_F(FbtreeTest, DeleteRangeMatchesCountAcrossExclusiveBounds) { + /* --- Tree A: 300 members, height 2. --- */ + { + const int kTotalA = 300; + const int leaf_edges_a[] = {61, 122, 183, 244}; + const int num_leaf_edges_a = 4; + const int stride_a = 23; + const int j_stride_a = 31; + const int j_extra_a[] = {122, 183, 244}; + const int num_j_extra_a = 3; + + int min_indexes[64]; + int num_mins = 0; + for (int k = 0; k < num_leaf_edges_a; k++) min_indexes[num_mins++] = leaf_edges_a[k]; + for (int i = stride_a; i < kTotalA; i += stride_a) { + int dup = 0; + for (int k = 0; k < num_mins; k++) { + if (min_indexes[k] == i) dup = 1; + } + if (!dup) min_indexes[num_mins++] = i; + } + + int max_indexes[64]; + int num_maxs = 0; + for (int j = j_stride_a; j < kTotalA; j += j_stride_a) max_indexes[num_maxs++] = j; + for (int k = 0; k < num_j_extra_a; k++) { + int dup = 0; + for (int m = 0; m < num_maxs; m++) { + if (max_indexes[m] == j_extra_a[k]) dup = 1; + } + if (!dup) max_indexes[num_maxs++] = j_extra_a[k]; + } + + int num_pairs_a = 0; + char buf[16]; + for (int mi = 0; mi < num_mins; mi++) { + for (int mj = 0; mj < num_maxs; mj++) { + int i = min_indexes[mi]; + int j = max_indexes[mj]; + if (i >= j) continue; + if (num_pairs_a >= 120) continue; + num_pairs_a++; + + fbtreeIndex *sweep_fbt = fbtreeCreate(); + for (int n = 1; n <= kTotalA; n++) { + snprintf(buf, sizeof(buf), "m%04d", n); + fbtreeInsert(sweep_fbt, createString(buf)); + } + + snprintf(buf, sizeof(buf), "m%04d", i); + sds range_min = createString(buf); + snprintf(buf, sizeof(buf), "m%04d", j); + sds range_max = createString(buf); + + unsigned long expected = fbtreeCountRangeByValue(sweep_fbt, range_min, range_max, 1, 1); + unsigned long removed = fbtreeDeleteRangeByValue(sweep_fbt, range_min, range_max, 1, 1, NULL, NULL); + EXPECT_EQ(removed, expected) << "tree A pair (i=" << i << ", j=" << j << ") disagreed"; + ASSERT_TRUE(fbtreeDebugValidate(sweep_fbt, false, NULL, 0)) << "tree A pair (i=" << i << ", j=" << j + << ") left an invalid tree"; + + sdsfree(range_min); + sdsfree(range_max); + fbtreeFree(sweep_fbt); + } + } + ASSERT_GT(num_pairs_a, 0) << "tree A sweep must exercise at least one pair"; + } + + /* --- Tree B: 5000 members, height 3. Must include the known failing + * pair (61, 2868). --- */ + { + const int leaf_edges_b[] = {61, 610, 1220, 2440, 3721}; + const int num_leaf_edges_b = 5; + const int stride_b = 173; + const int known_max_b = 2868; + + int min_indexes[64]; + int num_mins = 0; + for (int k = 0; k < num_leaf_edges_b; k++) min_indexes[num_mins++] = leaf_edges_b[k]; + + int max_indexes[64]; + int num_maxs = 0; + max_indexes[num_maxs++] = known_max_b; + for (int j = stride_b; j < 5000; j += stride_b) { + if (j == known_max_b) continue; + max_indexes[num_maxs++] = j; + } + + int num_pairs_b = 0; + char buf[16]; + int saw_known_pair = 0; + for (int mi = 0; mi < num_mins; mi++) { + for (int mj = 0; mj < num_maxs; mj++) { + int i = min_indexes[mi]; + int j = max_indexes[mj]; + if (i >= j) continue; + if (num_pairs_b >= 30) continue; + num_pairs_b++; + if (i == 61 && j == known_max_b) saw_known_pair = 1; + + fbtreeIndex *sweep_fbt = fbtreeCreate(); + for (int n = 1; n <= 5000; n++) { + snprintf(buf, sizeof(buf), "m%05d", n); + fbtreeInsert(sweep_fbt, createString(buf)); + } + + snprintf(buf, sizeof(buf), "m%05d", i); + sds range_min = createString(buf); + snprintf(buf, sizeof(buf), "m%05d", j); + sds range_max = createString(buf); + + unsigned long expected = fbtreeCountRangeByValue(sweep_fbt, range_min, range_max, 1, 1); + unsigned long removed = fbtreeDeleteRangeByValue(sweep_fbt, range_min, range_max, 1, 1, NULL, NULL); + EXPECT_EQ(removed, expected) << "tree B pair (i=" << i << ", j=" << j << ") disagreed"; + ASSERT_TRUE(fbtreeDebugValidate(sweep_fbt, false, NULL, 0)) << "tree B pair (i=" << i << ", j=" << j + << ") left an invalid tree"; + + sdsfree(range_min); + sdsfree(range_max); + fbtreeFree(sweep_fbt); + } + } + ASSERT_GT(num_pairs_b, 0) << "tree B sweep must exercise at least one pair"; + ASSERT_TRUE(saw_known_pair) << "tree B sweep must include the known failing pair (61, 2868)"; + } +} + TEST_F(FbtreeTest, DeleteRangeByValueExactMatch) { insert("aaa"); insert("bbb"); diff --git a/src/unit/test_listpack.cpp b/src/unit/test_listpack.cpp index ded99cc97..360ee2c8a 100644 --- a/src/unit/test_listpack.cpp +++ b/src/unit/test_listpack.cpp @@ -174,6 +174,51 @@ static int lpValidation(unsigned char *p, unsigned int head_count, void *userdat return ret; } +static unsigned char *createListWithMetadata(void) { + unsigned char *lp = lpNew(0); + + lp = lpAppend(lp, (unsigned char *)"field1", 6); + lp = lpAppend(lp, (unsigned char *)"value1", 6); + + /* field 1 expiry */ + unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; + uint64_t enclen; + lpEncodeIntegerGetType(1234567890, intenc, &enclen); + unsigned char *eofptr = lp + lpGetTotalBytes(lp) - 1; + lp = lpInsertMetadata(lp, intenc, enclen, eofptr, LP_BEFORE, NULL); + + lp = lpAppend(lp, (unsigned char *)"field2", 6); + lp = lpAppend(lp, (unsigned char *)"value2", 6); + + return lp; +} + +static unsigned char *createListWithAllMetadata(void) { + unsigned char *lp = lpNew(0); + + lp = lpAppend(lp, (unsigned char *)"field1", 6); + lp = lpAppend(lp, (unsigned char *)"value1", 6); + + /* field 1 expiry */ + unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; + uint64_t enclen; + lpEncodeIntegerGetType(1234567890, intenc, &enclen); + unsigned char *eofptr = lp + lpGetTotalBytes(lp) - 1; + lp = lpInsertMetadata(lp, intenc, enclen, eofptr, LP_BEFORE, NULL); + + lp = lpAppend(lp, (unsigned char *)"field2", 6); + lp = lpAppend(lp, (unsigned char *)"value2", 6); + + /* field 2 expiry */ + unsigned char intenc2[LP_MAX_INT_ENCODING_LEN]; + uint64_t enclen2; + lpEncodeIntegerGetType(1234567890, intenc2, &enclen2); + unsigned char *eofptr2 = lp + lpGetTotalBytes(lp) - 1; + lp = lpInsertMetadata(lp, intenc2, enclen2, eofptr2, LP_BEFORE, NULL); + + return lp; +} + class ListpackTest : public ::testing::Test { protected: void SetUp() override { @@ -482,7 +527,7 @@ TEST_F(ListpackTest, listpackBatchDelete) { lp = lpBatchDelete(lp, ps, 3); ASSERT_EQ(lpLength(lp), 1u); verifyEntry(lpFirst(lp), (unsigned char *)mixlist[2], strlen(mixlist[2])); - ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr), 1); + ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr, 0), 1); lpFree(lp); } @@ -940,13 +985,156 @@ TEST_F(ListpackTest, listpackLpFind) { lpFree(lp); } +TEST_F(ListpackTest, listpackLpFindWithMetadata) { + /* test lpFind with metadata fields */ + unsigned char *lp; + + lp = createListWithMetadata(); + + ASSERT_NE(lpFind(lp, lpFirst(lp), (unsigned char *)"field1", 6, 1), nullptr); + ASSERT_NE(lpFind(lp, lpFirst(lp), (unsigned char *)"field2", 6, 1), nullptr); + ASSERT_EQ(lpFind(lp, lpFirst(lp), (unsigned char *)"1234567890", 10, 0), nullptr); + lpFree(lp); +} + +TEST_F(ListpackTest, listpackLpFindWithAllMetadata) { + /* test lpFind with all fields having metadata */ + unsigned char *lp; + + lp = createListWithAllMetadata(); + + ASSERT_NE(lpFind(lp, lpFirst(lp), (unsigned char *)"field1", 6, 1), nullptr); + ASSERT_NE(lpFind(lp, lpFirst(lp), (unsigned char *)"field2", 6, 1), nullptr); + ASSERT_EQ(lpFind(lp, lpFirst(lp), (unsigned char *)"1234567890", 10, 0), nullptr); + lpFree(lp); +} + +TEST_F(ListpackTest, listpackMetadataInvisibleToIterators) { + /* Metadata entries are skipped by logical iteration and excluded from + * numele; they are reachable only through lpGetMetadata(). */ + unsigned char *lp = createListWithAllMetadata(); + + /* 2 fields + 2 values; the 2 metadata entries are not counted */ + ASSERT_EQ(lpLength(lp), 4u); + + unsigned char *p = lpFirst(lp); + int seen = 0; + while (p) { + ASSERT_EQ(lpIsMetadata(p), 0); + seen++; + p = lpNext(lp, p); + } + ASSERT_EQ(seen, 4); + + /* Values carry metadata, fields do not */ + unsigned char *field1 = lpFirst(lp); + unsigned char *value1 = lpNext(lp, field1); + ASSERT_EQ(lpGetMetadata(lp, field1), nullptr); + unsigned char *meta = lpGetMetadata(lp, value1); + ASSERT_NE(meta, nullptr); + ASSERT_EQ(lpGetMetadataValue(meta), 1234567890); + + /* Backward iteration skips metadata too */ + unsigned char *last = lpLast(lp); + ASSERT_EQ(lpIsMetadata(last), 0); + seen = 0; + while (last) { + seen++; + last = lpPrev(lp, last); + } + ASSERT_EQ(seen, 4); + lpFree(lp); +} + +TEST_F(ListpackTest, listpackMetadataDeletedWithElement) { + /* Deleting a pair also deletes the metadata trailing it */ + unsigned char *lp = createListWithMetadata(); + ASSERT_EQ(lpLength(lp), 4u); + + unsigned char *field1 = lpFirst(lp); + lp = lpDeleteRangeWithEntry(lp, &field1, 2); + ASSERT_EQ(lpLength(lp), 2u); + ASSERT_NE(lpFind(lp, lpFirst(lp), (unsigned char *)"field2", 6, 1), nullptr); + ASSERT_EQ(lpFind(lp, lpFirst(lp), (unsigned char *)"field1", 6, 1), nullptr); + ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr, 1), 1); + lpFree(lp); +} + +TEST_F(ListpackTest, listpackValidateIntegrityMetadataGate) { + /* Metadata is only valid when the caller allows it */ + unsigned char *lp = createListWithAllMetadata(); + ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr, 1), 1); + ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr, 0), 0); + lpFree(lp); +} + +TEST_F(ListpackTest, listpackLeadingMetadataHeader) { + /* A single tagged entry may lead the listpack as an aggregate header: + * reachable only via lpStart + lpIsMetadata, invisible to iterators and + * numele, and accepted by the validator when metadata is allowed. */ + unsigned char *lp = createListWithMetadata(); /* f1,v1(+meta),f2,v2 */ + ASSERT_EQ(lpIsMetadata(lpStart(lp)), 0); + + /* Prepend the aggregate header carrying the count (1). */ + unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; + uint64_t enclen; + lpEncodeIntegerGetType(1, intenc, &enclen); + lp = lpInsertMetadata(lp, intenc, enclen, lpFirst(lp), LP_BEFORE, NULL); + + unsigned char *head = lpStart(lp); + ASSERT_EQ(lpIsMetadata(head), 1); + ASSERT_EQ(lpGetMetadataValue(head), 1); + + /* Invisible to logical iteration and numele. */ + ASSERT_EQ(lpLength(lp), 4u); + verifyEntry(lpFirst(lp), (unsigned char *)"field1", 6); + int seen = 0; + for (unsigned char *p = lpFirst(lp); p; p = lpNext(lp, p)) { + ASSERT_EQ(lpIsMetadata(p), 0); + seen++; + } + ASSERT_EQ(seen, 4); + + /* Valid with metadata allowed, corrupt otherwise. */ + ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr, 1), 1); + ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr, 0), 0); + + /* In-place replace of the header value. */ + lpEncodeIntegerGetType(2, intenc, &enclen); + lp = lpInsertMetadata(lp, intenc, enclen, lpStart(lp), LP_REPLACE, NULL); + ASSERT_EQ(lpGetMetadataValue(lpStart(lp)), 2); + ASSERT_EQ(lpLength(lp), 4u); + + /* A second leading tagged entry is corruption. */ + lpEncodeIntegerGetType(7, intenc, &enclen); + lp = lpInsertMetadata(lp, intenc, enclen, lpStart(lp), LP_AFTER, NULL); + ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr, 1), 0); + lpFree(lp); +} + +TEST_F(ListpackTest, listpackLeadingMetadataDeletion) { + /* Deleting the header restores a plain listpack. */ + unsigned char *lp = createListWithMetadata(); + unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; + uint64_t enclen; + lpEncodeIntegerGetType(1, intenc, &enclen); + lp = lpInsertMetadata(lp, intenc, enclen, lpFirst(lp), LP_BEFORE, NULL); + ASSERT_EQ(lpIsMetadata(lpStart(lp)), 1); + + lp = lpRemoveMetadata(lp, lpStart(lp)); + ASSERT_EQ(lpIsMetadata(lpStart(lp)), 0); + ASSERT_EQ(lpLength(lp), 4u); + ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr, 1), 1); + lpFree(lp); +} + TEST_F(ListpackTest, listpackLpValidateIntegrity) { /* Test lpValidateIntegrity */ unsigned char *lp; lp = createList(); long count = 0; - ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), lpValidation, &count), 1); + ASSERT_EQ(lpValidateIntegrity(lp, lpBytes(lp), lpValidation, &count, 0), 1); lpFree(lp); } @@ -1131,7 +1319,7 @@ TEST_F(ListpackBenchmark, DISABLED_listpackBenchmarkLpValidateIntegrity) { /* Benchmark lpValidateIntegrity */ unsigned long long start = usec(); for (int i = 0; i < 2000; i++) { - lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr); + lpValidateIntegrity(lp, lpBytes(lp), nullptr, nullptr, 0); } printf("Done. usec=%lld\n", usec() - start); } diff --git a/src/unit/test_networking.cpp b/src/unit/test_networking.cpp index 963ec25b8..c092bf29c 100644 --- a/src/unit/test_networking.cpp +++ b/src/unit/test_networking.cpp @@ -6,6 +6,8 @@ #include "generated_wrappers.hpp" +#include "fake_connection.hpp" + #include #include #include @@ -82,72 +84,11 @@ void testOnlyTrimReplyUnusedTailSpace(client *c); void setDeferredReply(client *c, void *node, const char *s, size_t length); } -/* Fake structures and functions */ -typedef struct fakeConnection { - connection conn; - int error; - char *buffer; - size_t buf_size; - size_t written; -} fakeConnection; - -/* Fake connWrite function */ -static int fake_connWrite(connection *conn, const void *data, size_t size) { - fakeConnection *fake_conn = (fakeConnection *)conn; - if (fake_conn->error) return -1; - - size_t to_write = size; - if (fake_conn->written + to_write > fake_conn->buf_size) { - to_write = fake_conn->buf_size - fake_conn->written; - } - - memcpy(fake_conn->buffer + fake_conn->written, data, to_write); - fake_conn->written += to_write; - return (int)to_write; -} - -/* Fake connWritev function */ -static int fake_connWritev(connection *conn, const struct iovec *iov, int iovcnt) { - fakeConnection *fake_conn = (fakeConnection *)conn; - if (fake_conn->error) return -1; - - size_t total = 0; - for (int i = 0; i < iovcnt; i++) { - size_t to_write = iov[i].iov_len; - if (fake_conn->written + to_write > fake_conn->buf_size) { - to_write = fake_conn->buf_size - fake_conn->written; - } - if (to_write == 0) break; - - memcpy(fake_conn->buffer + fake_conn->written, iov[i].iov_base, to_write); - fake_conn->written += to_write; - total += to_write; - } - return (int)total; -} - -/* Fake connection type - initialized in SetUpTestSuite */ -static ConnectionType CT_Fake; - -static fakeConnection *connCreateFake(void) { - fakeConnection *conn = (fakeConnection *)(zcalloc(sizeof(fakeConnection))); - conn->conn.type = &CT_Fake; - conn->conn.fd = -1; - conn->conn.iovcnt = IOV_MAX; - return conn; -} - -/* Test fixture for networking tests - minimal fixture with no setup/teardown */ +/* Test fixture for networking tests - minimal fixture with no setup/teardown. + * The fake connection itself lives in fake_connection.hpp, shared with the + * other unit tests that need one. */ class NetworkingTest : public ::testing::Test { protected: - static void SetUpTestSuite() { - /* Initialize CT_Fake explicitly by field name to avoid dependency - * on field order (designated initializers require C++20). */ - memset(&CT_Fake, 0, sizeof(CT_Fake)); - CT_Fake.write = fake_connWrite; - CT_Fake.writev = fake_connWritev; - } - void SetUp() override { /* Initialize server fields that are accessed by networking functions */ server.commandlog[COMMANDLOG_TYPE_LARGE_REPLY].threshold = -1; /* Disable tracking */ @@ -167,9 +108,7 @@ TEST_F(NetworkingTest, TestWriteToReplica) { c->reply = listCreate(); /* Test 1: Single block write */ { - fakeConnection *fake_conn = connCreateFake(); - fake_conn->buffer = (char *)zmalloc(1024); - fake_conn->buf_size = 1024; + fakeConnection *fake_conn = connCreateFake(1024); c->conn = (connection *)fake_conn; /* Create replication buffer block */ @@ -193,19 +132,14 @@ TEST_F(NetworkingTest, TestWriteToReplica) { ASSERT_EQ((c->write_flags & WRITE_FLAGS_WRITE_ERROR), 0); /* Cleanup */ - zfree(fake_conn->buffer); - zfree(fake_conn); + connFreeFake(fake_conn); zfree(block); listEmpty(server.repl_buffer_blocks); } /* Test 2: Multiple blocks write */ { - fakeConnection *fake_conn = connCreateFake(); - fake_conn->error = 0; - fake_conn->written = 0; - fake_conn->buffer = (char *)zmalloc(1024); - fake_conn->buf_size = 1024; + fakeConnection *fake_conn = connCreateFake(1024); c->conn = (connection *)fake_conn; /* Create multiple replication buffer blocks */ @@ -236,8 +170,7 @@ TEST_F(NetworkingTest, TestWriteToReplica) { ASSERT_EQ((c->write_flags & WRITE_FLAGS_WRITE_ERROR), 0); /* Cleanup */ - zfree(fake_conn->buffer); - zfree(fake_conn); + connFreeFake(fake_conn); zfree(block1); zfree(block2); listEmpty(server.repl_buffer_blocks); @@ -245,11 +178,8 @@ TEST_F(NetworkingTest, TestWriteToReplica) { /* Test 3: Write error */ { - fakeConnection *fake_conn = connCreateFake(); + fakeConnection *fake_conn = connCreateFake(1024); fake_conn->error = 1; /* Simulate write error */ - fake_conn->buffer = (char *)zmalloc(1024); - fake_conn->buf_size = 1024; - fake_conn->written = 0; c->conn = (connection *)fake_conn; /* Create replication buffer block */ @@ -272,8 +202,7 @@ TEST_F(NetworkingTest, TestWriteToReplica) { /* Cleanup */ listEmpty(server.repl_buffer_blocks); - zfree(fake_conn->buffer); - zfree(fake_conn); + connFreeFake(fake_conn); zfree(block); c->repl_data->ref_repl_buf_node = nullptr; } diff --git a/src/unit/test_object.cpp b/src/unit/test_object.cpp index 054802d08..35b49705b 100644 --- a/src/unit/test_object.cpp +++ b/src/unit/test_object.cpp @@ -15,7 +15,48 @@ extern "C" { #include "server.h" } +/* Metadata test helpers */ +typedef struct objMetadata { + uint32_t meta_int; +} objMetadata; + class ObjectTest : public ::testing::Test { + protected: + robj *createKeyValueObject(const char *k, const char *v) { + sds key = sdsnew(k); + robj *obj = createStringObject(v, strlen(v)); + robj *obj_with_key = objectSetKeyAndExpire(obj, key, -1); + sdsfree(key); + return obj_with_key; + } + + void objectSetMetaInt(robj *o, uint32_t metadata_int) { + objMetadata *meta = (objMetadata *)objectGetMetadata(o); + meta->meta_int = metadata_int; + } + + uint32_t objectGetMetaInt(const robj *o) { + objMetadata *meta = (objMetadata *)objectGetMetadata(o); + return meta->meta_int; + } + + /* Find the largest value length that still embeds with the given key and expire. */ + int findMaxEmbeddableValueLen(const char *key, long long expire) { + sds k = key ? sdsnew(key) : NULL; + + int len; + for (len = 1; len <= 256; len++) { + robj *obj = createStringObject(NULL, len); + if (k) obj = objectSetKeyAndExpire(obj, k, expire); + bool isEmbedded = (obj->encoding == OBJ_ENCODING_EMBSTR); + decrRefCount(obj); + if (!isEmbedded) break; + } + EXPECT_LE(len, 256) << "no embedding limit found within the search range"; + + sdsfree(k); + return len - 1; + } }; TEST_F(ObjectTest, object_with_key) { @@ -57,88 +98,71 @@ TEST_F(ObjectTest, object_with_key) { } TEST_F(ObjectTest, embedded_string_with_key) { - /* key of length 32 - type 8 */ - sds key = sdsnew("k:123456789012345678901234567890"); - ASSERT_EQ(sdslen(key), 32u); - - /* 32B key and 79B value should be embedded within 128B. Contents: - * - 8B robj (no ptr) + 1B key header size - * - 3B key header + 32B key + 1B null terminator - * - 3B val header + 79B val + 1B null terminator - * because no pointers are stored, there is no difference for 32 bit builds*/ - const char *short_value = "1234567890123456789012345678901234567890123456789012345678901234567890123456789"; - ASSERT_EQ(strlen(short_value), 79u); - robj *short_val_obj = createStringObject(short_value, strlen(short_value)); - robj *embstr_obj = objectSetKeyAndExpire(short_val_obj, key, -1); + const char *key = "k:123456789012345678901234567890"; + int max_len = findMaxEmbeddableValueLen(key, -1); + ASSERT_GT(max_len, 0); + + /* Value at max length should embed. */ + sds k1 = sdsnew(key); + robj *embstr_obj = createStringObject(NULL, max_len); + embstr_obj = objectSetKeyAndExpire(embstr_obj, k1, -1); ASSERT_EQ(embstr_obj->encoding, (unsigned)OBJ_ENCODING_EMBSTR); - ASSERT_EQ(sdslen(objectGetKey(embstr_obj)), 32u); - ASSERT_EQ(sdscmp(objectGetKey(embstr_obj), key), 0); - ASSERT_EQ(sdslen((sds)objectGetVal(embstr_obj)), 79u); - ASSERT_EQ(strcmp((const char *)objectGetVal(embstr_obj), short_value), 0); - - /* value of length 80 cannot be embedded with other contents within 128B */ - const char *longer_value = "12345678901234567890123456789012345678901234567890123456789012345678901234567890"; - ASSERT_EQ(strlen(longer_value), 80u); - robj *longer_val_obj = createStringObject(longer_value, strlen(longer_value)); - robj *raw_obj = objectSetKeyAndExpire(longer_val_obj, key, -1); + ASSERT_EQ(sdslen((sds)objectGetVal(embstr_obj)), (size_t)max_len); + + /* One byte more should not embed. */ + sds k2 = sdsnew(key); + robj *raw_obj = createStringObject(NULL, max_len + 1); + raw_obj = objectSetKeyAndExpire(raw_obj, k2, -1); ASSERT_EQ(raw_obj->encoding, (unsigned)OBJ_ENCODING_RAW); - ASSERT_EQ(sdslen(objectGetKey(raw_obj)), 32u); - ASSERT_EQ(sdscmp(objectGetKey(raw_obj), key), 0); - ASSERT_EQ(sdslen((sds)objectGetVal(raw_obj)), 80u); - ASSERT_EQ(strcmp((const char *)objectGetVal(raw_obj), longer_value), 0); + ASSERT_EQ(sdslen((sds)objectGetVal(raw_obj)), (size_t)(max_len + 1)); - sdsfree(key); + sdsfree(k1); + sdsfree(k2); decrRefCount(embstr_obj); decrRefCount(raw_obj); } TEST_F(ObjectTest, embedded_string_with_key_and_expire) { - /* key of length 32 - type 8 */ - sds key = sdsnew("k:123456789012345678901234567890"); - ASSERT_EQ(sdslen(key), 32u); - - /* 32B key and 71B value should be embedded within 128B. Contents: - * - 8B robj (no ptr) + 8B expire + 1B key header size - * - 3B key header + 32B key + 1B null terminator - * - 3B val header + 71B val + 1B null terminator - * because no pointers are stored, there is no difference for 32 bit builds*/ - const char *short_value = "12345678901234567890123456789012345678901234567890123456789012345678901"; - ASSERT_EQ(strlen(short_value), 71u); - robj *short_val_obj = createStringObject(short_value, strlen(short_value)); - robj *embstr_obj = objectSetKeyAndExpire(short_val_obj, key, 128); + const char *key = "k:123456789012345678901234567890"; + int max_len = findMaxEmbeddableValueLen(key, 128); + ASSERT_GT(max_len, 0); + + /* Adding an expire reduces the available space for the value. */ + int max_len_no_expire = findMaxEmbeddableValueLen(key, -1); + ASSERT_LT(max_len, max_len_no_expire); + + /* Value at max length should embed. */ + sds k1 = sdsnew(key); + robj *embstr_obj = createStringObject(NULL, max_len); + embstr_obj = objectSetKeyAndExpire(embstr_obj, k1, 128); ASSERT_EQ(embstr_obj->encoding, (unsigned)OBJ_ENCODING_EMBSTR); - ASSERT_EQ(sdslen(objectGetKey(embstr_obj)), 32u); - ASSERT_EQ(sdscmp(objectGetKey(embstr_obj), key), 0); - ASSERT_EQ(sdslen((sds)objectGetVal(embstr_obj)), 71u); - ASSERT_EQ(strcmp((const char *)objectGetVal(embstr_obj), short_value), 0); - - /* value of length 72 cannot be embedded with other contents within 128B */ - const char *longer_value = "123456789012345678901234567890123456789012345678901234567890123456789012"; - ASSERT_EQ(strlen(longer_value), 72u); - robj *longer_val_obj = createStringObject(longer_value, strlen(longer_value)); - robj *raw_obj = objectSetKeyAndExpire(longer_val_obj, key, 128); + + /* One byte more should not embed. */ + sds k2 = sdsnew(key); + robj *raw_obj = createStringObject(NULL, max_len + 1); + raw_obj = objectSetKeyAndExpire(raw_obj, k2, 128); ASSERT_EQ(raw_obj->encoding, (unsigned)OBJ_ENCODING_RAW); - ASSERT_EQ(sdslen(objectGetKey(raw_obj)), 32u); - ASSERT_EQ(sdscmp(objectGetKey(raw_obj), key), 0); - ASSERT_EQ(sdslen((sds)objectGetVal(raw_obj)), 72u); - ASSERT_EQ(strcmp((const char *)objectGetVal(raw_obj), longer_value), 0); - sdsfree(key); + sdsfree(k1); + sdsfree(k2); decrRefCount(embstr_obj); decrRefCount(raw_obj); } TEST_F(ObjectTest, embedded_value) { - /* with only value there is only 12B overhead, so we can embed up to 52B. - * 8B robj (no ptr) + 3B val header + 52B val + 1B null terminator */ - const char *val = "v:12345678901234567890123456789012345678901234567890"; - ASSERT_EQ(strlen(val), 52u); - robj *embstr_obj = createStringObject(val, strlen(val)); + /* Value-only object (no key): find the largest value that embeds. */ + int max_len = findMaxEmbeddableValueLen(NULL, -1); + ASSERT_GT(max_len, 0); + + robj *embstr_obj = createStringObject(NULL, max_len); ASSERT_EQ(embstr_obj->encoding, (unsigned)OBJ_ENCODING_EMBSTR); - ASSERT_EQ(sdslen((sds)objectGetVal(embstr_obj)), 52u); - ASSERT_EQ(strcmp((const char *)objectGetVal(embstr_obj), val), 0); + ASSERT_EQ(sdslen((sds)objectGetVal(embstr_obj)), (size_t)max_len); + + robj *raw_obj = createStringObject(NULL, max_len + 1); + ASSERT_EQ(raw_obj->encoding, (unsigned)OBJ_ENCODING_RAW); decrRefCount(embstr_obj); + decrRefCount(raw_obj); } TEST_F(ObjectTest, unembed_value) { @@ -166,3 +190,107 @@ TEST_F(ObjectTest, unembed_value) { sdsfree(key); decrRefCount(obj); } + + +TEST_F(ObjectTest, metadata_disabled) { + robj *obj_with_key = createKeyValueObject("testkey", "value"); + + ASSERT_EQ(objectGetMetadata(obj_with_key), nullptr); + ASSERT_EQ(objectGetMetadataSize(obj_with_key), 0u); + + decrRefCount(obj_with_key); +} + +TEST_F(ObjectTest, metadata_without_key) { + objectSetMetadataSize(sizeof(objMetadata)); + + robj *obj_no_key = createStringObject("value_without_key", 17); + + ASSERT_EQ(objectGetMetadata(obj_no_key), nullptr); + ASSERT_EQ(objectGetMetadataSize(obj_no_key), 0u); + + decrRefCount(obj_no_key); +} + +TEST_F(ObjectTest, metadata_with_key) { + objectSetMetadataSize(sizeof(objMetadata)); + + robj *obj_with_key = createKeyValueObject("testkey", "value"); + + ASSERT_EQ(objectGetMetadataSize(obj_with_key), sizeof(objMetadata)); + + objMetadata *meta = (objMetadata *)objectGetMetadata(obj_with_key); + ASSERT_NE(meta, nullptr); + EXPECT_EQ(meta->meta_int, 0u); + + decrRefCount(obj_with_key); +} + +TEST_F(ObjectTest, metadata_read_write) { + objectSetMetadataSize(sizeof(objMetadata)); + + robj *obj_with_key = createKeyValueObject("mykey", "myvalue"); + + ASSERT_EQ(objectGetMetadataSize(obj_with_key), sizeof(objMetadata)); + + objectSetMetaInt(obj_with_key, 12345); + EXPECT_EQ(objectGetMetaInt(obj_with_key), 12345u); + + objectSetMetaInt(obj_with_key, 67890); + EXPECT_EQ(objectGetMetaInt(obj_with_key), 67890u); + + decrRefCount(obj_with_key); +} + +TEST_F(ObjectTest, metadata_multiple_objects) { + objectSetMetadataSize(sizeof(objMetadata)); + + robj *obj_with_key1 = createKeyValueObject("key1", "val1"); + robj *obj_with_key2 = createKeyValueObject("key2", "val2"); + robj *obj_with_key3 = createKeyValueObject("key3", "val3"); + + ASSERT_EQ(objectGetMetadataSize(obj_with_key1), sizeof(objMetadata)); + ASSERT_EQ(objectGetMetadataSize(obj_with_key2), sizeof(objMetadata)); + ASSERT_EQ(objectGetMetadataSize(obj_with_key3), sizeof(objMetadata)); + + objectSetMetaInt(obj_with_key1, 100); + objectSetMetaInt(obj_with_key2, 200); + objectSetMetaInt(obj_with_key3, 300); + + EXPECT_EQ(objectGetMetaInt(obj_with_key1), 100u); + EXPECT_EQ(objectGetMetaInt(obj_with_key2), 200u); + EXPECT_EQ(objectGetMetaInt(obj_with_key3), 300u); + + objectSetMetaInt(obj_with_key2, 999); + EXPECT_EQ(objectGetMetaInt(obj_with_key1), 100u); + EXPECT_EQ(objectGetMetaInt(obj_with_key2), 999u); + EXPECT_EQ(objectGetMetaInt(obj_with_key3), 300u); + + decrRefCount(obj_with_key1); + decrRefCount(obj_with_key2); + decrRefCount(obj_with_key3); +} + +TEST_F(ObjectTest, metadata_changes_embed_threshold) { + /* Find the max embeddable value length without metadata, then verify + * that enabling metadata reduces it (some previously-embeddable objects + * become RAW). */ + const char *key = "k:123456789012345678901234567890"; + int max_without = findMaxEmbeddableValueLen(key, -1); + ASSERT_GT(max_without, 0); + + objectSetMetadataSize(sizeof(objMetadata)); + int max_with = findMaxEmbeddableValueLen(key, -1); + + /* Metadata takes space, so the threshold must shrink. */ + ASSERT_LT(max_with, max_without); + + /* An object that just fit before should now be RAW. */ + sds k = sdsnew(key); + robj *obj = createStringObject(NULL, max_without); + obj = objectSetKeyAndExpire(obj, k, -1); + ASSERT_EQ(obj->encoding, (unsigned)OBJ_ENCODING_RAW); + + sdsfree(k); + decrRefCount(obj); +} diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp new file mode 100644 index 000000000..cdd8b6006 --- /dev/null +++ b/src/unit/test_socket_prioritization.cpp @@ -0,0 +1,921 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" + +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif + +#define _Static_assert static_assert +extern "C" { +#include "ae.h" +#include "connection.h" +#include "io_threads.h" +#include "monotonic.h" +#include "server.h" +} +#undef _Static_assert + + +/* Static tracking for preemption test callbacks */ +static int g_execution_order[1024]; +static int g_execution_count = 0; + +static monotime g_mock_time_offset = 0; +static monotime (*g_orig_getMonotonicUs)(void) = NULL; +static monotime mockGetMonotonicUs(void) { + return g_orig_getMonotonicUs() + g_mock_time_offset; +} + +static void setupMockClock(void) { + if (g_orig_getMonotonicUs != NULL) return; + g_orig_getMonotonicUs = getMonotonicUs; + getMonotonicUs = mockGetMonotonicUs; + g_mock_time_offset = 0; +} + +static void restoreMockClock(void) { + if (g_orig_getMonotonicUs) { + getMonotonicUs = g_orig_getMonotonicUs; + g_orig_getMonotonicUs = NULL; + } +} + +static void advanceMockTime(uint64_t us) { + g_mock_time_offset += us; +} + +static void testNormalEventCallback(aeEventLoop *el, int fd, void *privdata, int mask) { + UNUSED(el); + UNUSED(mask); + UNUSED(privdata); + if (g_execution_count < 1024) { + g_execution_order[g_execution_count++] = fd; + } + /* Read 1 byte from pipe to clear the event */ + char buf[1]; + if (read(fd, buf, 1) < 0) { + /* Ignore read error */ + } +} + +static void testQoSEventCallback(aeEventLoop *el, int fd, void *privdata, int mask) { + UNUSED(el); + UNUSED(mask); + UNUSED(privdata); + if (g_execution_count < 1024) { + /* Use 999 to identify QoS execution */ + g_execution_order[g_execution_count++] = 999; + } + char buf[1]; + if (read(fd, buf, 1) < 0) { + /* Ignore read error */ + } +} + +static int g_qos_pipe2_write_fd = -1; +static int g_normal_cb_count = 0; +static void testNormalEventCallbackPreemptCheck(aeEventLoop *el, int fd, void *privdata, int mask) { + testNormalEventCallback(el, fd, privdata, mask); + g_normal_cb_count++; + /* Sleep > 2000 us on 2nd normal event and trigger 2nd QoS pipe so the next iteration immediately preempts and polls QoS events */ + if (g_normal_cb_count == 2 && g_qos_pipe2_write_fd != -1) { + usleep(2500); + char c = 'x'; + if (write(g_qos_pipe2_write_fd, &c, 1) < 0) { + } + } +} + +static void testLevelTriggeredCallback(aeEventLoop *el, int fd, void *privdata, int mask) { + UNUSED(el); + UNUSED(mask); + if (privdata) { + int *serve_counts = (int *)privdata; + serve_counts[0]++; + } + if (g_execution_count < 1024) { + g_execution_order[g_execution_count++] = fd; + } + /* Read only 1 byte per execution to test level-triggered behavior (if more data in buffer, fd remains readable) */ + char buf[1]; + if (read(fd, buf, 1) < 0) { + /* Ignore read error */ + } +} + +static void dummyConnectionHandler(struct connection *conn) { + UNUSED(conn); +} + +class SocketPrioritizationTest : public ::testing::Test { + protected: + void SetUp() override { + monotonicInit(); + setupMockClock(); + /* Initialize minimal server fields needed to prevent crashes in logging/connection layer */ + server.logfile = strdup(""); + server.syslog_enabled = 0; + + /* Initialize connection types registry ONCE */ + static bool conn_types_initialized = false; + if (!conn_types_initialized) { + connTypeInitialize(); + conn_types_initialized = true; + } + + server.el = aeCreateEventLoop(1024); + if (server.el) { + aeActuateQoSEventLoopIfSupported(server.el, 2000, NULL); + } + g_execution_count = 0; + } + + void TearDown() override { + if (server.el) { + aeDeleteEventLoop(server.el); + server.el = NULL; + } + if (server.logfile) { + free(server.logfile); + server.logfile = NULL; + } + restoreMockClock(); + } +}; + +class SocketPrioritizationConnTest : public SocketPrioritizationTest, public ::testing::WithParamInterface {}; + +TEST_F(SocketPrioritizationTest, EventLoopDualInitialization) { + ASSERT_NE(server.el, (aeEventLoop *)NULL); + ASSERT_NE(server.el->priority_apidata, (aeApiState *)NULL); + EXPECT_NE(server.el->priority_fd, -1); + EXPECT_EQ(server.el->priority_events_preempt_check_interval_us, 2000ULL); + aeSetQoSPreemptCheckInterval(server.el, 5000ULL); + EXPECT_EQ(server.el->priority_events_preempt_check_interval_us, 5000ULL); + aeSetQoSPreemptCheckInterval(server.el, 2000ULL); + + /* Newly created standalone loop must have preemption disabled (0) by default */ + aeEventLoop *standalone = aeCreateEventLoop(64); + ASSERT_NE(standalone, (aeEventLoop *)NULL); + EXPECT_EQ(standalone->priority_apidata, (aeApiState *)NULL); + EXPECT_EQ(standalone->priority_fd, -1); + EXPECT_EQ(standalone->priority_events_preempt_check_interval_us, 0ULL); + aeDeleteEventLoop(standalone); +} + +TEST_F(SocketPrioritizationTest, DynamicPreemptionIntervalThreshold) { + /* Test getter and setter */ + aeSetQoSPreemptCheckInterval(server.el, 500ULL); + EXPECT_EQ(server.el->priority_events_preempt_check_interval_us, 500ULL); + + /* Test disabling preemption (interval = 0) */ + aeSetQoSPreemptCheckInterval(server.el, 0ULL); + EXPECT_EQ(server.el->priority_events_preempt_check_interval_us, 0ULL); + aeSetQoSPreemptCheckInterval(server.el, 2000ULL); +} + +TEST_P(SocketPrioritizationConnTest, ConnectionPriorityMetadataAndHelpers) { + ConnectionType *ct = connectionByType(GetParam()); + if (ct == NULL) return; + connection *conn = connCreate(ct); + ASSERT_NE(conn, (connection *)NULL); + + /* Default priority should be normal (false) */ + EXPECT_FALSE(connIsPriority(conn)); + + /* Update metadata when fd is not yet set (-1) */ + connSetPriority(conn, true); + EXPECT_TRUE(connIsPriority(conn)); + + connSetPriority(conn, false); + EXPECT_FALSE(connIsPriority(conn)); + + conn->state = CONN_STATE_NONE; + connClose(conn); +} + +TEST_P(SocketPrioritizationConnTest, DynamicPriorityUpdateOnActiveConnection) { + ConnectionType *ct = connectionByType(GetParam()); + if (ct == NULL) return; + int fds[2]; + int ret = pipe(fds); + ASSERT_EQ(ret, 0); + int read_fd = fds[0]; + int write_fd = fds[1]; + + connection *conn = connCreate(ct); + ASSERT_NE(conn, (connection *)NULL); + conn->fd = read_fd; + + /* Set a read handler while priority is normal */ + ret = connSetReadHandler(conn, dummyConnectionHandler); + EXPECT_EQ(ret, C_OK); + + /* Event should exist in normal mode */ + EXPECT_NE(aeGetFileEvents(server.el, read_fd) & AE_READABLE, 0); + EXPECT_EQ(aeGetFileEvents(server.el, read_fd) & AE_HIGH_PRIORITY, 0); + + /* Dynamically upgrade connection to high priority */ + EXPECT_EQ(connSetPriority(conn, true), C_OK); + EXPECT_TRUE(connIsPriority(conn)); + int events = aeGetFileEvents(server.el, read_fd); + EXPECT_NE(events & AE_READABLE, 0); + EXPECT_NE(events & AE_HIGH_PRIORITY, 0); + + /* Dynamically downgrade connection back to normal priority */ + EXPECT_EQ(connSetPriority(conn, false), C_OK); + EXPECT_FALSE(connIsPriority(conn)); + EXPECT_NE(aeGetFileEvents(server.el, read_fd) & AE_READABLE, 0); + EXPECT_EQ(aeGetFileEvents(server.el, read_fd) & AE_HIGH_PRIORITY, 0); + + conn->state = CONN_STATE_NONE; + connClose(conn); + close(write_fd); +} + +TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByQoSLoop) { + int normal_pipes[2][2]; + int qos_pipe[2]; + int i; + int ret; + + for (i = 0; i < 2; i++) { + ret = pipe(normal_pipes[i]); + ASSERT_EQ(ret, 0); + /* Set non-blocking */ + fcntl(normal_pipes[i][0], F_SETFL, O_NONBLOCK); + fcntl(normal_pipes[i][1], F_SETFL, O_NONBLOCK); + } + ret = pipe(qos_pipe); + ASSERT_EQ(ret, 0); + fcntl(qos_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe[1], F_SETFL, O_NONBLOCK); + + /* Register normal events on server.el */ + ret = aeCreateFileEvent(server.el, normal_pipes[0][0], AE_READABLE, testNormalEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + ret = aeCreateFileEvent(server.el, normal_pipes[1][0], AE_READABLE, testNormalEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + + /* Register QoS event on server.el with AE_HIGH_PRIORITY */ + ret = aeCreateFileEvent(server.el, qos_pipe[0], AE_READABLE | AE_HIGH_PRIORITY, testQoSEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + + /* Write 1 byte to all pipes so they fire */ + char c = 'x'; + if (write(normal_pipes[0][1], &c, 1) < 0) { + } + if (write(normal_pipes[1][1], &c, 1) < 0) { + } + if (write(qos_pipe[1], &c, 1) < 0) { + } + + /* Ensure more than AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US (2000 us) has elapsed for priority_events_last_poll */ + advanceMockTime(1000000); + + g_execution_count = 0; + /* Process events on main event loop. Preemption check will trigger + * polling of QoS channels and execute testQoSEventCallback before processing the normal events! */ + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); + + /* Verify that QoS event (id 999) executed first due to preemption */ + ASSERT_GE(g_execution_count, 1); + EXPECT_EQ(g_execution_order[0], 999); + + /* Cleanup events and file descriptors */ + aeDeleteFileEvent(server.el, normal_pipes[0][0], AE_READABLE); + aeDeleteFileEvent(server.el, normal_pipes[1][0], AE_READABLE); + aeDeleteFileEvent(server.el, qos_pipe[0], AE_READABLE); + + for (i = 0; i < 2; i++) { + close(normal_pipes[i][0]); + close(normal_pipes[i][1]); + } + close(qos_pipe[0]); + close(qos_pipe[1]); +} + +static void testQoSWriteCallback(struct connection *conn) { + if (g_execution_count < 1024) { + g_execution_order[g_execution_count++] = 999; + } + connSetWriteHandler(conn, NULL); +} + +TEST_P(SocketPrioritizationConnTest, WriteHandlerPriorityPreemption) { + ConnectionType *ct = connectionByType(GetParam()); + if (ct == NULL) return; + int normal_pipe[2]; + int qos_pipe[2]; + int ret; + + ret = pipe(normal_pipe); + ASSERT_EQ(ret, 0); + fcntl(normal_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(normal_pipe[1], F_SETFL, O_NONBLOCK); + + ret = pipe(qos_pipe); + ASSERT_EQ(ret, 0); + fcntl(qos_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe[1], F_SETFL, O_NONBLOCK); + + connection *qos_conn = connCreate(ct); + ASSERT_NE(qos_conn, (connection *)NULL); + qos_conn->state = CONN_STATE_CONNECTED; + qos_conn->fd = qos_pipe[1]; + connSetPriority(qos_conn, true); + + /* Register normal read event on server.el */ + ret = aeCreateFileEvent(server.el, normal_pipe[0], AE_READABLE, testNormalEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + + /* Register QoS write event on QoS multiplexer via connection wrapper */ + ret = connSetWriteHandler(qos_conn, testQoSWriteCallback); + EXPECT_EQ(ret, C_OK); + + char c = 'w'; + if (write(normal_pipe[1], &c, 1) < 0) { + } + + advanceMockTime(1000000); + g_execution_count = 0; + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); + + ASSERT_GE(g_execution_count, 1); + EXPECT_EQ(g_execution_order[0], 999); + + aeDeleteFileEvent(server.el, normal_pipe[0], AE_READABLE); + qos_conn->state = CONN_STATE_NONE; + connClose(qos_conn); + close(normal_pipe[0]); + close(normal_pipe[1]); + close(qos_pipe[0]); +} + +TEST_F(SocketPrioritizationTest, ImmediatePreemptionDuringNormalEventProcessing) { + /* Verify that preemption check immediately services QoS events on the next iteration when elapsed time threshold is crossed */ + int normal_pipes[5][2]; + int qos_pipe[2]; + int qos_pipe2[2]; + int i, ret; + + for (i = 0; i < 5; i++) { + ret = pipe(normal_pipes[i]); + ASSERT_EQ(ret, 0); + fcntl(normal_pipes[i][0], F_SETFL, O_NONBLOCK); + fcntl(normal_pipes[i][1], F_SETFL, O_NONBLOCK); + } + ret = pipe(qos_pipe); + ASSERT_EQ(ret, 0); + fcntl(qos_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe[1], F_SETFL, O_NONBLOCK); + + ret = pipe(qos_pipe2); + ASSERT_EQ(ret, 0); + fcntl(qos_pipe2[0], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe2[1], F_SETFL, O_NONBLOCK); + + for (i = 0; i < 5; i++) { + ret = aeCreateFileEvent(server.el, normal_pipes[i][0], AE_READABLE, testNormalEventCallbackPreemptCheck, NULL); + EXPECT_EQ(ret, AE_OK); + } + ret = aeCreateFileEvent(server.el, qos_pipe[0], AE_READABLE | AE_HIGH_PRIORITY, testQoSEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + ret = aeCreateFileEvent(server.el, qos_pipe2[0], AE_READABLE | AE_HIGH_PRIORITY, testQoSEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + + char c = 'z'; + for (i = 0; i < 5; i++) { + if (write(normal_pipes[i][1], &c, 1) < 0) { + } + } + if (write(qos_pipe[1], &c, 1) < 0) { + } + + g_qos_pipe2_write_fd = qos_pipe2[1]; + g_normal_cb_count = 0; + advanceMockTime(1000000); + g_execution_count = 0; + + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); + + ASSERT_EQ(g_execution_count, 7); + /* At start (pre-loop), initial QoS event processed */ + EXPECT_EQ(g_execution_order[0], 999); + /* 1st normal event */ + EXPECT_NE(g_execution_order[1], 999); + /* 2nd normal event (sleeps >2000us and writes to qos_pipe2) */ + EXPECT_NE(g_execution_order[2], 999); + /* 3rd, 4th, and 5th normal events execute before mask check boundary at end of iteration (j = 4) */ + EXPECT_NE(g_execution_order[3], 999); + EXPECT_NE(g_execution_order[4], 999); + EXPECT_NE(g_execution_order[5], 999); + /* Preemptive polling services qos_pipe2 at mask check boundary (end of j = 4 iteration) */ + EXPECT_EQ(g_execution_order[6], 999); + + g_qos_pipe2_write_fd = -1; + + for (i = 0; i < 5; i++) { + aeDeleteFileEvent(server.el, normal_pipes[i][0], AE_READABLE); + close(normal_pipes[i][0]); + close(normal_pipes[i][1]); + } + aeDeleteFileEvent(server.el, qos_pipe[0], AE_READABLE); + close(qos_pipe[0]); + close(qos_pipe[1]); + aeDeleteFileEvent(server.el, qos_pipe2[0], AE_READABLE); + close(qos_pipe2[0]); + close(qos_pipe2[1]); +} + +TEST_F(SocketPrioritizationTest, LevelTriggeredEventsFairnessAndOrdering) { + const int NUM_FDS = 20; + const int BYTES_PER_FD = 5; + int pipes[NUM_FDS][2]; + int counts[NUM_FDS]; + int i, ret; + + g_execution_count = 0; + for (i = 0; i < NUM_FDS; i++) { + counts[i] = 0; + ret = pipe(pipes[i]); + ASSERT_EQ(ret, 0); + fcntl(pipes[i][0], F_SETFL, O_NONBLOCK); + fcntl(pipes[i][1], F_SETFL, O_NONBLOCK); + + ret = aeCreateFileEvent(server.el, pipes[i][0], AE_READABLE | AE_HIGH_PRIORITY, testLevelTriggeredCallback, &counts[i]); + EXPECT_EQ(ret, AE_OK); + + char data[BYTES_PER_FD] = {'a', 'b', 'c', 'd', 'e'}; + if (write(pipes[i][1], data, BYTES_PER_FD) < 0) { + } + } + + /* Run iterations until all FDs drain their BYTES_PER_FD bytes */ + int max_iterations = NUM_FDS * BYTES_PER_FD + 50; + for (int iter = 0; iter < max_iterations; iter++) { + int total_served = 0; + for (i = 0; i < NUM_FDS; i++) total_served += counts[i]; + if (total_served >= NUM_FDS * BYTES_PER_FD) break; + + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); + } + + /* Verify all FDs were served exactly BYTES_PER_FD times without any level-triggered event being lost or starved */ + for (i = 0; i < NUM_FDS; i++) { + EXPECT_EQ(counts[i], BYTES_PER_FD); + aeDeleteFileEvent(server.el, pipes[i][0], AE_READABLE); + close(pipes[i][0]); + close(pipes[i][1]); + } +} + +TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPrevention) { + const int NUM_FDS = 20; + int pipes[NUM_FDS][2]; + int counts[NUM_FDS]; + int i, ret; + + g_execution_count = 0; + for (i = 0; i < NUM_FDS; i++) { + counts[i] = 0; + ret = pipe(pipes[i]); + ASSERT_EQ(ret, 0); + fcntl(pipes[i][0], F_SETFL, O_NONBLOCK); + fcntl(pipes[i][1], F_SETFL, O_NONBLOCK); + } + + /* 1. Register Head FDs (0..6) with 3 bytes (unprocessed at the head) and Tail FDs (14..19) with 5 bytes */ + for (i = 0; i <= 6; i++) { + ret = aeCreateFileEvent(server.el, pipes[i][0], AE_READABLE | AE_HIGH_PRIORITY, testLevelTriggeredCallback, &counts[i]); + EXPECT_EQ(ret, AE_OK); + char data[3] = {'h', 'e', 'a'}; + if (write(pipes[i][1], data, 3) < 0) { + } + } + for (i = 14; i <= 19; i++) { + ret = aeCreateFileEvent(server.el, pipes[i][0], AE_READABLE | AE_HIGH_PRIORITY, testLevelTriggeredCallback, &counts[i]); + EXPECT_EQ(ret, AE_OK); + char data[5] = {'t', 'a', 'i', 'l', 's'}; + if (write(pipes[i][1], data, 5) < 0) { + } + } + + /* Process batch 1: Head and Tail FDs each fire once (reading 1 byte due to level trigger) */ + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); + for (i = 0; i <= 6; i++) EXPECT_GE(counts[i], 1); + for (i = 14; i <= 19; i++) EXPECT_GE(counts[i], 1); + + /* 2. Register Middle FDs (7..13) with 3 bytes (new events in the middle while head/tail have remaining level-triggered data) */ + for (i = 7; i <= 13; i++) { + ret = aeCreateFileEvent(server.el, pipes[i][0], AE_READABLE | AE_HIGH_PRIORITY, testLevelTriggeredCallback, &counts[i]); + EXPECT_EQ(ret, AE_OK); + char data[3] = {'m', 'i', 'd'}; + if (write(pipes[i][1], data, 3) < 0) { + } + } + + /* Process remaining batches until all buffers across all groups are completely drained */ + for (int iter = 0; iter < 200; iter++) { + int total_served = 0; + for (i = 0; i < NUM_FDS; i++) total_served += counts[i]; + /* 7 head FDs x 3 + 7 middle FDs x 3 + 6 tail FDs x 5 = 21 + 21 + 30 = 72 total bytes */ + if (total_served >= 72) break; + + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); + } + + /* Verify every single FD across head (unprocessed), middle (new), and tail (level-triggered remaining data) was fully served */ + for (i = 0; i <= 6; i++) EXPECT_EQ(counts[i], 3); + for (i = 7; i <= 13; i++) EXPECT_EQ(counts[i], 3); + for (i = 14; i <= 19; i++) EXPECT_EQ(counts[i], 5); + + for (i = 0; i < NUM_FDS; i++) { + aeDeleteFileEvent(server.el, pipes[i][0], AE_READABLE); + close(pipes[i][0]); + close(pipes[i][1]); + } +} + +TEST_F(SocketPrioritizationTest, DualEventLoopResize) { + ASSERT_NE(server.el, (aeEventLoop *)NULL); + ASSERT_NE(server.el->priority_apidata, (aeApiState *)NULL); + EXPECT_EQ(aeGetSetSize(server.el), 1024); + + int ret = aeResizeSetSize(server.el, 2048); + EXPECT_EQ(ret, AE_OK); + EXPECT_EQ(aeGetSetSize(server.el), 2048); +} + +TEST_F(SocketPrioritizationTest, FallbackMaskSanitization) { + aeEventLoop *standalone_el = aeCreateEventLoop(64); + ASSERT_NE(standalone_el, (aeEventLoop *)NULL); + EXPECT_EQ(standalone_el->priority_apidata, (aeApiState *)NULL); + EXPECT_EQ(standalone_el->priority_fd, -1); + + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + + /* Register with AE_HIGH_PRIORITY on standalone event loop with no QoS state */ + int ret = aeCreateFileEvent(standalone_el, fds[0], AE_READABLE | AE_HIGH_PRIORITY, testNormalEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + + /* Mask should only have AE_READABLE, and NOT retain AE_HIGH_PRIORITY (0x8) */ + int events = aeGetFileEvents(standalone_el, fds[0]); + EXPECT_EQ(events, AE_READABLE); + + /* Delete the readable event */ + aeDeleteFileEvent(standalone_el, fds[0], AE_READABLE); + EXPECT_EQ(aeGetFileEvents(standalone_el, fds[0]), AE_NONE); + + close(fds[0]); + close(fds[1]); + aeDeleteEventLoop(standalone_el); +} + +TEST_F(SocketPrioritizationTest, PostponedStateDynamicPriorityUpdate) { + ConnectionType *ct = connectionByType(CONN_TYPE_SOCKET); + ASSERT_NE(ct, (ConnectionType *)NULL); + + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + + connection *conn = connCreate(ct); + ASSERT_NE(conn, (connection *)NULL); + conn->fd = fds[0]; + + int ret = connSetReadHandler(conn, dummyConnectionHandler); + EXPECT_EQ(ret, C_OK); + EXPECT_FALSE(connIsPriority(conn)); + + /* Set postponed state (e.g. while offloaded to IO threads) */ + conn->flags |= CONN_FLAG_POSTPONE_UPDATE_STATE; + + /* Upgrading priority while postponed must update priority field but defer event loop migration */ + ret = connSetPriority(conn, true); + EXPECT_EQ(ret, C_OK); + EXPECT_TRUE(connIsPriority(conn)); + + conn->flags &= ~CONN_FLAG_POSTPONE_UPDATE_STATE; + conn->state = CONN_STATE_NONE; + connClose(conn); + close(fds[1]); +} + +TEST_F(SocketPrioritizationTest, WriteBarrierPreservation) { + ConnectionType *ct = connectionByType(CONN_TYPE_SOCKET); + ASSERT_NE(ct, (ConnectionType *)NULL); + + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + + connection *conn = connCreate(ct); + ASSERT_NE(conn, (connection *)NULL); + conn->fd = fds[0]; + + int ret = connSetWriteHandlerWithBarrier(conn, dummyConnectionHandler, 1); + EXPECT_EQ(ret, C_OK); + + int events = aeGetFileEvents(server.el, fds[0]); + EXPECT_NE(events & AE_WRITABLE, 0); + + /* Upgrade priority: verify migration succeeds with barrier */ + ret = connSetPriority(conn, true); + EXPECT_EQ(ret, C_OK); + EXPECT_TRUE(connIsPriority(conn)); + + conn->state = CONN_STATE_NONE; + connClose(conn); + close(fds[1]); +} + +INSTANTIATE_TEST_SUITE_P( + ConnTypes, + SocketPrioritizationConnTest, + ::testing::Values(CONN_TYPE_SOCKET, CONN_TYPE_TLS)); + +#ifdef __linux__ +/* This test verifies the Linux kernel epoll starvation prevention behavior + * as discussed in Valkey Issue #3927 (https://github.com/valkey-io/valkey/issues/3927#issuecomment-4664882443). + * + * When epoll_wait is called with a maxevents limit smaller than the number of + * currently ready level-triggered FDs: + * 1. The kernel moves all ready FDs to a temporary transfer list (txlist). + * 2. It copies up to maxevents to userspace. + * 3. The copied FDs (still ready in LT mode) are re-queued to the tail of the ready list (rdllist). + * 4. The remaining unprocessed FDs in txlist are bulk re-spliced to the FRONT of rdllist (via ep_done_scan). + * + * This ensures that: + * - Unprocessed FDs are prioritized over re-queued FDs in the next call. + * - New events arriving later are not starved by persistent LT events, because + * eventually the older unreturned events are promoted to the front. + * + * We verify this by: + * 1. Registering 5 pipes (A, B, C, D, E). + * 2. Making A, B, C, D ready. + * 3. Calling epoll_wait(limit=3) -> returns 3 (e.g., A, B, C). D is left out. + * 4. Making E (new event) ready (appended to tail -> [D, A, B, C, E]). + * 5. Calling epoll_wait(limit=3) -> must return D (promoted to front), but NOT E (still at tail). + * 6. Calling epoll_wait(limit=3) -> must return E (now promoted to front because it was left in txlist). + */ +TEST(EpollFairnessTest, KernelBehavior) { + const int NUM_PIPES = 5; + int pipes[NUM_PIPES][2]; + int epfd = epoll_create1(0); + ASSERT_NE(epfd, -1); + + // 1. Registering 5 pipes (A, B, C, D, E) to epoll (Level-Triggered). + for (int i = 0; i < NUM_PIPES; i++) { + ASSERT_EQ(pipe(pipes[i]), 0); + fcntl(pipes[i][0], F_SETFL, O_NONBLOCK); + + struct epoll_event ee = {0}; + ee.events = EPOLLIN; + ee.data.fd = pipes[i][0]; + ASSERT_EQ(epoll_ctl(epfd, EPOLL_CTL_ADD, pipes[i][0], &ee), 0); + } + + char c = 'x'; + // 2. Making A, B, C, D ready. + for (int i = 0; i < 4; i++) { + ASSERT_EQ(write(pipes[i][1], &c, 1), 1); + } + + // 3. Calling epoll_wait(limit=3) -> returns 3 (e.g., A, B, C). D is left out. + // The unreturned one (D) is moved to the FRONT of rdllist. + // The returned ones (A, B, C) are re-queued to the tail. + // rdllist is now [D, A, B, C]. + struct epoll_event events[3]; + int ready = epoll_wait(epfd, events, 3, 0); + ASSERT_EQ(ready, 3); + + // Track which FD was left out in the first call (D). + int left_out_fd = -1; + for (int i = 0; i < 4; i++) { + int fd = pipes[i][0]; + int found = 0; + for (int j = 0; j < ready; j++) { + if (events[j].data.fd == fd) { + found = 1; + break; + } + } + if (!found) { + left_out_fd = fd; + break; + } + } + ASSERT_NE(left_out_fd, -1); + + // 4. Making E (new event) ready (appended to tail -> [D, A, B, C, E]). + ASSERT_EQ(write(pipes[4][1], &c, 1), 1); + + // 5. Calling epoll_wait(limit=3) -> must return D (promoted to front), but NOT E (still at tail). + // Kernel moves [D, A, B, C, E] to txlist. + // Copies D, A, B (limit 3 reached). Re-queues D, A, B. + // C, E are left in txlist and moved to the FRONT of rdllist -> [C, E, D, A, B]. + ready = epoll_wait(epfd, events, 3, 0); + ASSERT_EQ(ready, 3); + + // Verify that the left-out FD (D) WAS returned in this second call + // (proving it was moved to the front and not starved by A, B, C). + int found_left_out = 0; + for (int i = 0; i < ready; i++) { + if (events[i].data.fd == left_out_fd) { + found_left_out = 1; + break; + } + } + EXPECT_TRUE(found_left_out) << "Left-out FD was starved!"; + + // Verify that the new event (E, pipe 4) was NOT returned yet (it was at the tail). + for (int i = 0; i < ready; i++) { + EXPECT_NE(events[i].data.fd, pipes[4][0]) << "New event E should not be returned yet"; + } + + // 6. Calling epoll_wait(limit=3) -> must return E (now promoted to front because it was left in txlist). + // rdllist was [C, E, D, A, B] due to C, E being moved to the front. + // It should return C, E, and one of D, A, B. + ready = epoll_wait(epfd, events, 3, 0); + ASSERT_EQ(ready, 3); + + // Verify that the new event (E, pipe 4) IS returned now + // (proving it was moved to the front in the previous step and not starved). + int found_new_event = 0; + for (int i = 0; i < ready; i++) { + if (events[i].data.fd == pipes[4][0]) { + found_new_event = 1; + break; + } + } + EXPECT_TRUE(found_new_event) << "New event E was starved by persistent LT events!"; + + // Cleanup resources. + for (int i = 0; i < NUM_PIPES; i++) { + close(pipes[i][0]); + close(pipes[i][1]); + } + close(epfd); +} +#endif + +#ifdef HAVE_KQUEUE +#include +#include + +/* This test verifies the kqueue event delivery fairness behavior. + * kqueue generally processes events in FIFO order. When calling kevent + * with a limit smaller than the number of ready events: + * 1. The returned events are popped from the head of the active list. + * 2. Unreturned ready events remain at the head of the list. + * 3. Returned events (if still ready/level-triggered) are re-queued to the tail. + * + * This natural FIFO queuing prevents starvation of both unreturned and new events. + * + * We verify this by: + * 1. Registering 5 pipes (A, B, C, D, E) to kqueue (Level-Triggered). + * 2. Making A, B, C, D ready. + * 3. Calling kevent(limit=3) -> returns 3 (e.g., A, B, C). D is left out. + * 4. Making E (new event) ready (appended to tail -> [D, A, B, C, E]). + * 5. Calling kevent(limit=3) -> must return D (promoted to front), but NOT E (still at tail). + * 6. Calling kevent(limit=3) -> must return E (now promoted to front because it was left out). + */ +TEST(KqueueFairnessTest, KernelBehavior) { + const int NUM_PIPES = 5; + int pipes[NUM_PIPES][2]; + int kq = kqueue(); + ASSERT_NE(kq, -1); + + // 1. Registering 5 pipes (A, B, C, D, E) to kqueue (Level-Triggered). + for (int i = 0; i < NUM_PIPES; i++) { + ASSERT_EQ(pipe(pipes[i]), 0); + fcntl(pipes[i][0], F_SETFL, O_NONBLOCK); + + struct kevent ke; + EV_SET(&ke, pipes[i][0], EVFILT_READ, EV_ADD | EV_ENABLE, 0, 0, (void *)(intptr_t)pipes[i][0]); + ASSERT_NE(kevent(kq, &ke, 1, NULL, 0, NULL), -1); + } + + char c = 'x'; + // 2. Making A, B, C, D ready. + for (int i = 0; i < 4; i++) { + ASSERT_EQ(write(pipes[i][1], &c, 1), 1); + } + + // 3. Calling kevent(limit=3) -> returns 3 (e.g., A, B, C). D is left out. + struct kevent events[3]; + struct timespec timeout = {0, 0}; + int ready = kevent(kq, NULL, 0, events, 3, &timeout); + ASSERT_EQ(ready, 3); + + // Track which FD was left out in the first call (D). + int left_out_fd = -1; + for (int i = 0; i < 4; i++) { + int fd = pipes[i][0]; + int found = 0; + for (int j = 0; j < ready; j++) { + if ((int)events[j].ident == fd) { + found = 1; + break; + } + } + if (!found) { + left_out_fd = fd; + break; + } + } + ASSERT_NE(left_out_fd, -1); + + // 4. Making E (new event) ready (appended to tail -> [D, A, B, C, E]). + ASSERT_EQ(write(pipes[4][1], &c, 1), 1); + + // 5. Calling kevent(limit=3) -> must return D (promoted to front), but NOT E (still at tail). + ready = kevent(kq, NULL, 0, events, 3, &timeout); + ASSERT_EQ(ready, 3); + + // Verify that the left-out FD (D) WAS returned in this second call. + int found_left_out = 0; + for (int i = 0; i < ready; i++) { + if ((int)events[i].ident == left_out_fd) { + found_left_out = 1; + break; + } + } + EXPECT_TRUE(found_left_out) << "Left-out FD was starved!"; + + // Verify that the new event (E, pipe 4) was NOT returned yet (it was at the tail). + for (int i = 0; i < ready; i++) { + EXPECT_NE((int)events[i].ident, pipes[4][0]) << "New event E should not be returned yet"; + } + + // 6. Calling kevent(limit=3) -> must return E (now promoted to front because it was left out). + ready = kevent(kq, NULL, 0, events, 3, &timeout); + ASSERT_EQ(ready, 3); + + // Verify that the new event (E, pipe 4) IS returned now. + int found_new_event = 0; + for (int i = 0; i < ready; i++) { + if ((int)events[i].ident == pipes[4][0]) { + found_new_event = 1; + break; + } + } + EXPECT_TRUE(found_new_event) << "New event E was starved!"; + + // Cleanup resources. + for (int i = 0; i < NUM_PIPES; i++) { + close(pipes[i][0]); + close(pipes[i][1]); + } + close(kq); +} +#endif + +static void qosMetricTestCb(struct aeEventLoop *el, uint64_t duration_us) { + (void)el; + durationAddSample(EL_DURATION_TYPE_PRIORITY_EL, duration_us); +} + +static void qosTestFileProc(aeEventLoop *el, int fd, void *privdata, int mask) { + (void)el; + (void)privdata; + (void)mask; + char b; + if (read(fd, &b, 1) < 0) { + /* Ignore read error in test callback */ + } +} + +/* Test that QoS event loop duration callback correctly samples QoS metrics. */ +TEST_F(SocketPrioritizationTest, QoSEventLoopStatsMetrics) { + aeEventLoop *main_loop = aeCreateEventLoop(64); + ASSERT_NE(main_loop, (aeEventLoop *)NULL); + ASSERT_EQ(aeActuateQoSEventLoopIfSupported(main_loop, 2000, qosMetricTestCb), AE_OK); + + unsigned long long orig_cnt = server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].cnt; + unsigned long long orig_sum = server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].sum; + + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + ASSERT_EQ(aeCreateFileEvent(main_loop, fds[0], AE_READABLE | AE_HIGH_PRIORITY, qosTestFileProc, NULL), AE_OK); + + char b = 'x'; + ASSERT_EQ(write(fds[1], &b, 1), 1); + aeProcessEvents(main_loop, AE_DONT_WAIT | AE_ALL_EVENTS); + + EXPECT_GT(server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].cnt, orig_cnt); + EXPECT_GE(server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].sum, orig_sum); + + close(fds[0]); + close(fds[1]); + aeDeleteEventLoop(main_loop); +} diff --git a/src/unit/test_space_saving.cpp b/src/unit/test_space_saving.cpp new file mode 100644 index 000000000..8c19fbbe3 --- /dev/null +++ b/src/unit/test_space_saving.cpp @@ -0,0 +1,491 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * Unit tests for the Space-Saving frozen-window top-K used by hot-key + * detection. These cover the algorithmic properties that integration tests + * cannot pin down deterministically: the [count - error, count] band across + * evictions, the "frequency > N/K is tracked" guarantee, window freezing + * (including a double freeze after an idle gap), top-K selection when the + * capacity shrinks, and predicate-based removal across both windows. + * + * The clock is supplied by the caller, so time is fully deterministic here: no + * sleeping, no wall-clock dependency. + */ + +#include "generated_wrappers.hpp" + +#include +#include + +extern "C" { +#include "sds.h" +#include "space_saving.h" +} + +#define WINDOW_US 1000ULL /* 1ms windows keep the arithmetic obvious */ + +/* Record one observation of `name` in database `dbid`. The key is borrowed by + * the module (copied only if a slot is committed to it), so we free our copy. */ +static void recordName(spaceSavingManager *m, const char *name, int dbid) { + sds k = sdsnew(name); + recordSpaceSavingManagerSample(m, k, dbid); + sdsfree(k); +} + +/* Look up a key in the frozen window. Returns 1 and fills count/error when + * found, 0 otherwise. Out-params may be NULL. */ +static int frozenFind(spaceSavingManager *m, const char *name, int dbid, uint64_t *count, uint64_t *error) { + int n = spaceSavingManagerCount(m); + for (int i = 0; i < n; i++) { + sds key = NULL; + int db = 0; + uint64_t c = 0, e = 0; + spaceSavingManagerAt(m, i, &key, &db, &c, &e); + if (db == dbid && key != NULL && strcmp(key, name) == 0) { + if (count) *count = c; + if (error) *error = e; + return 1; + } + } + return 0; +} + +/* Freeze the live window by advancing exactly one window length. Returns the new + * "now". */ +static uint64_t freezeOnce(spaceSavingManager *m, uint64_t now_us) { + now_us += WINDOW_US; + spaceSavingManagerRotate(m, now_us); + return now_us; +} + +/* --------------------------------------------------------------------------- + * 1. The [count - error, count] band always contains the true count, including + * for entries that landed in a slot by evicting another. + * --------------------------------------------------------------------------*/ +TEST(SpaceSaving, ErrorBandContainsTrueCountAcrossEvictions) { + const int k = 3; + /* Six distinct keys into three slots forces repeated eviction. */ + const int nkeys = 6; + const char *names[nkeys] = {"k0", "k1", "k2", "k3", "k4", "k5"}; + const int true_counts[nkeys] = {10, 8, 6, 4, 2, 1}; + + spaceSavingManager *m = spaceSavingManagerCreate(k, WINDOW_US, 0); + ASSERT_NE(m, nullptr); + + /* Interleave the streams so evictions happen throughout, not just at the + * start: round r records every key whose true count is still >= r. */ + uint64_t total = 0; + for (int r = 1; r <= 10; r++) { + for (int i = 0; i < nkeys; i++) { + if (true_counts[i] >= r) { + recordName(m, names[i], 0); + total++; + } + } + } + ASSERT_EQ(total, 31u); /* 10+8+6+4+2+1 */ + + freezeOnce(m, 0); + EXPECT_EQ(spaceSavingManagerFrozenTotal(m), total); + /* Capacity is never exceeded. */ + ASSERT_LE(spaceSavingManagerCount(m), k); + ASSERT_GT(spaceSavingManagerCount(m), 0); + + int n = spaceSavingManagerCount(m); + for (int i = 0; i < n; i++) { + sds key = NULL; + int db = 0; + uint64_t count = 0, error = 0; + spaceSavingManagerAt(m, i, &key, &db, &count, &error); + ASSERT_NE(key, nullptr); + + int idx = -1; + for (int j = 0; j < nkeys; j++) + if (strcmp(key, names[j]) == 0) idx = j; + ASSERT_NE(idx, -1) << "frozen window reported an unknown key"; + + uint64_t truth = (uint64_t)true_counts[idx]; + /* The error can never exceed the count, and the true count must lie + * within [count - error, count]. */ + EXPECT_LE(error, count) << "key " << key; + EXPECT_LE(count - error, truth) << "key " << key; + EXPECT_GE(count, truth) << "key " << key; + } + + /* Keys that were never recorded are absent, and an unused db is empty. */ + EXPECT_EQ(frozenFind(m, "never-seen", 0, NULL, NULL), 0); + EXPECT_EQ(frozenFind(m, "k0", 7, NULL, NULL), 0); + + spaceSavingManagerRelease(m); +} + +/* --------------------------------------------------------------------------- + * 2. Any item whose frequency exceeds N/K is guaranteed to be tracked, even + * when the rest of the stream is a flood of one-hit keys competing for slots. + * --------------------------------------------------------------------------*/ +TEST(SpaceSaving, FrequencyAboveNOverKIsTracked) { + const int k = 4; + const int hot_hits = 40; + const int noise_hits = 60; /* 60 distinct keys, one hit each */ + const uint64_t n = (uint64_t)(hot_hits + noise_hits); + + /* The guarantee only applies when the frequency is above N/K. */ + ASSERT_GT((uint64_t)hot_hits, n / (uint64_t)k); + + spaceSavingManager *m = spaceSavingManagerCreate(k, WINDOW_US, 0); + ASSERT_NE(m, nullptr); + + /* Interleave: the hot key must survive continuous eviction pressure rather + * than simply being recorded last. */ + int noise_emitted = 0, hot_emitted = 0; + while (noise_emitted < noise_hits || hot_emitted < hot_hits) { + for (int i = 0; i < 3 && noise_emitted < noise_hits; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "noise:%d", noise_emitted++); + recordName(m, buf, 0); + } + for (int i = 0; i < 2 && hot_emitted < hot_hits; i++) { + recordName(m, "hot", 0); + hot_emitted++; + } + } + ASSERT_EQ(hot_emitted, hot_hits); + ASSERT_EQ(noise_emitted, noise_hits); + + freezeOnce(m, 0); + EXPECT_EQ(spaceSavingManagerFrozenTotal(m), n); + + uint64_t count = 0, error = 0; + ASSERT_EQ(frozenFind(m, "hot", 0, &count, &error), 1) << "a key above N/K must be tracked"; + /* Its band must still contain the true frequency. */ + EXPECT_LE(count - error, (uint64_t)hot_hits); + EXPECT_GE(count, (uint64_t)hot_hits); + + spaceSavingManagerRelease(m); +} + +/* --------------------------------------------------------------------------- + * 3. Rotation policy. A window is published only if it was closed within twice + * its configured length; past that its counts span too coarse an interval to + * label "the last window", so they are dropped — even when the window did + * receive traffic. A merely-late rotation keeps its counts. + * --------------------------------------------------------------------------*/ +TEST(SpaceSaving, LateRotationKeepsCountsUntilTheStalenessCutoff) { + spaceSavingManager *m = spaceSavingManagerCreate(8, WINDOW_US, 0); + ASSERT_NE(m, nullptr); + + for (int i = 0; i < 3; i++) recordName(m, "a", 0); + + /* Still inside the first window: nothing is readable yet. */ + spaceSavingManagerRotate(m, WINDOW_US - 1); + EXPECT_EQ(spaceSavingManagerCount(m), 0); + EXPECT_EQ(spaceSavingManagerFrozenTotal(m), 0u); + + /* Crossing the boundary freezes exactly the completed window. */ + spaceSavingManagerRotate(m, WINDOW_US); + ASSERT_EQ(spaceSavingManagerCount(m), 1); + uint64_t count = 0; + ASSERT_EQ(frozenFind(m, "a", 0, &count, NULL), 1); + EXPECT_EQ(count, 3u); + EXPECT_EQ(spaceSavingManagerFrozenTotal(m), 3u); + EXPECT_EQ(spaceSavingManagerFrozenDurationUs(m), WINDOW_US); + + /* A rotate that crosses no new boundary leaves the snapshot untouched. */ + spaceSavingManagerRotate(m, WINDOW_US + 1); + EXPECT_EQ(spaceSavingManagerCount(m), 1); + EXPECT_EQ(spaceSavingManagerFrozenTotal(m), 3u); + + /* A late rotation, still within the cutoff: the counts are kept and the + * reported duration includes the lag rather than the nominal length. */ + for (int i = 0; i < 5; i++) recordName(m, "b", 0); + uint64_t late = WINDOW_US + WINDOW_US + (WINDOW_US - 1); /* just under 2x */ + spaceSavingManagerRotate(m, late); + ASSERT_EQ(frozenFind(m, "b", 0, &count, NULL), 1) << "a late rotation must not lose its counts"; + EXPECT_EQ(count, 5u); + EXPECT_EQ(spaceSavingManagerFrozenDurationUs(m), 2 * WINDOW_US - 1); + + /* Past the cutoff the window is dropped, INCLUDING the traffic it saw: its + * span is too coarse to publish as one window. */ + for (int i = 0; i < 7; i++) recordName(m, "c", 0); + spaceSavingManagerRotate(m, late + 2 * WINDOW_US); + EXPECT_EQ(spaceSavingManagerCount(m), 0) << "an over-long window must be dropped, not reported"; + EXPECT_EQ(spaceSavingManagerFrozenTotal(m), 0u); + EXPECT_EQ(frozenFind(m, "c", 0, NULL, NULL), 0); + EXPECT_EQ(spaceSavingManagerFrozenDurationUs(m), 0u); + + /* A long stall behaves the same way, and measuring re-bases on the drop, so + * the very next window is a normal one. */ + for (int i = 0; i < 4; i++) recordName(m, "d", 0); + spaceSavingManagerRotate(m, 1000 * WINDOW_US); + EXPECT_EQ(spaceSavingManagerCount(m), 0); + for (int i = 0; i < 6; i++) recordName(m, "e", 0); + spaceSavingManagerRotate(m, 1001 * WINDOW_US); + ASSERT_EQ(spaceSavingManagerCount(m), 1); + ASSERT_EQ(frozenFind(m, "e", 0, &count, NULL), 1); + EXPECT_EQ(count, 6u); + EXPECT_EQ(spaceSavingManagerFrozenDurationUs(m), WINDOW_US); + + spaceSavingManagerRelease(m); +} + +/* A late rotation must not shorten the FOLLOWING window: boundaries are measured + * from when a window really started, so the lag does not propagate. */ +TEST(SpaceSaving, RotationLagDoesNotShortenTheNextWindow) { + spaceSavingManager *m = spaceSavingManagerCreate(8, WINDOW_US, 0); + ASSERT_NE(m, nullptr); + + const uint64_t lag_us = WINDOW_US / 2; + recordName(m, "a", 0); + spaceSavingManagerRotate(m, WINDOW_US + lag_us); /* late by half a window */ + ASSERT_EQ(spaceSavingManagerCount(m), 1); + + /* On a nominal grid the next window would already be due at 2 * WINDOW_US, + * i.e. only half a length after this one opened. Measuring from the real + * start keeps it open for a full length. */ + recordName(m, "b", 0); + spaceSavingManagerRotate(m, 2 * WINDOW_US); + EXPECT_EQ(frozenFind(m, "a", 0, NULL, NULL), 1) << "the next window must not be cut short by the lag"; + EXPECT_EQ(frozenFind(m, "b", 0, NULL, NULL), 0); + + /* It closes a full length after it actually opened. */ + spaceSavingManagerRotate(m, WINDOW_US + lag_us + WINDOW_US); + ASSERT_EQ(frozenFind(m, "b", 0, NULL, NULL), 1); + EXPECT_EQ(spaceSavingManagerFrozenDurationUs(m), WINDOW_US) << "no window may be shorter than configured"; + + spaceSavingManagerRelease(m); +} + +/* --------------------------------------------------------------------------- + * 4. Reconfiguring the capacity: shrinking keeps the highest-count entries of + * the frozen window (and drops the rest), growing keeps everything. + * --------------------------------------------------------------------------*/ +TEST(SpaceSaving, TopKSelectionWhenCapacityShrinks) { + spaceSavingManager *m = spaceSavingManagerCreate(5, WINDOW_US, 0); + ASSERT_NE(m, nullptr); + + /* Five keys in five slots: no eviction, so the counts are exact. */ + const int nkeys = 5; + const char *names[nkeys] = {"a", "b", "c", "d", "e"}; + const int hits[nkeys] = {5, 4, 3, 2, 1}; + for (int i = 0; i < nkeys; i++) + for (int h = 0; h < hits[i]; h++) recordName(m, names[i], 0); + + uint64_t now = freezeOnce(m, 0); + ASSERT_EQ(spaceSavingManagerCount(m), 5); + + /* Shrink to 2: the two hottest survive with their counts intact. */ + spaceSavingManagerReconfigure(m, 2, WINDOW_US, now); + ASSERT_EQ(spaceSavingManagerCount(m), 2); + uint64_t count = 0; + ASSERT_EQ(frozenFind(m, "a", 0, &count, NULL), 1); + EXPECT_EQ(count, 5u); + ASSERT_EQ(frozenFind(m, "b", 0, &count, NULL), 1); + EXPECT_EQ(count, 4u); + EXPECT_EQ(frozenFind(m, "c", 0, NULL, NULL), 0); + EXPECT_EQ(frozenFind(m, "d", 0, NULL, NULL), 0); + EXPECT_EQ(frozenFind(m, "e", 0, NULL, NULL), 0); + + /* The new capacity is honoured by the live window too: three distinct keys + * cannot all be tracked at K=2. */ + recordName(m, "x", 0); + recordName(m, "y", 0); + recordName(m, "z", 0); + now = freezeOnce(m, now); + EXPECT_EQ(spaceSavingManagerCount(m), 2); + + /* Growing preserves what is already tracked. */ + for (int i = 0; i < 3; i++) recordName(m, "p", 0); + recordName(m, "q", 0); + now = freezeOnce(m, now); + ASSERT_EQ(spaceSavingManagerCount(m), 2); + spaceSavingManagerReconfigure(m, 8, WINDOW_US, now); + EXPECT_EQ(spaceSavingManagerCount(m), 2) << "growing must not drop entries"; + ASSERT_EQ(frozenFind(m, "p", 0, &count, NULL), 1); + EXPECT_EQ(count, 3u); + + spaceSavingManagerRelease(m); +} + +/* --------------------------------------------------------------------------- + * 5. RemoveIf applies to BOTH the live and the frozen window, so invalidated + * entries neither show up now nor resurface on the next rotation. + * --------------------------------------------------------------------------*/ + +/* Predicate: drop everything in the database passed via `arg`. */ +static int dropDb(sds key, int dbid, void *arg) { + (void)key; + return dbid == *(int *)arg; +} + +/* Predicate: drop the single key named by `arg`, in any database. */ +static int dropNamed(sds key, int dbid, void *arg) { + (void)dbid; + return strcmp(key, (const char *)arg) == 0; +} + +TEST(SpaceSaving, RemoveIfPurgesLiveAndFrozenWindows) { + spaceSavingManager *m = spaceSavingManagerCreate(8, WINDOW_US, 0); + ASSERT_NE(m, nullptr); + + /* Frozen window: one entry in db 0, one in db 1. */ + recordName(m, "keep-frozen", 0); + recordName(m, "drop-frozen", 1); + uint64_t now = freezeOnce(m, 0); + ASSERT_EQ(spaceSavingManagerCount(m), 2); + + /* Live window: another pair, again split across the two databases. */ + recordName(m, "keep-live", 0); + recordName(m, "drop-live", 1); + + int victim_db = 1; + spaceSavingManagerRemoveIf(m, dropDb, &victim_db); + + /* The frozen window is purged immediately. */ + ASSERT_EQ(spaceSavingManagerCount(m), 1); + EXPECT_EQ(frozenFind(m, "keep-frozen", 0, NULL, NULL), 1); + EXPECT_EQ(frozenFind(m, "drop-frozen", 1, NULL, NULL), 0); + + /* Rotating promotes the live window: the dropped entry must not resurface, + * which proves the live window was purged as well. */ + now = freezeOnce(m, now); + ASSERT_EQ(spaceSavingManagerCount(m), 1); + EXPECT_EQ(frozenFind(m, "keep-live", 0, NULL, NULL), 1); + EXPECT_EQ(frozenFind(m, "drop-live", 1, NULL, NULL), 0); + + /* Removing by key name keeps the surrounding entries and their counts. */ + for (int i = 0; i < 4; i++) recordName(m, "target", 0); + for (int i = 0; i < 2; i++) recordName(m, "bystander", 0); + now = freezeOnce(m, now); + ASSERT_EQ(spaceSavingManagerCount(m), 2); + char victim[] = "target"; + spaceSavingManagerRemoveIf(m, dropNamed, victim); + ASSERT_EQ(spaceSavingManagerCount(m), 1); + uint64_t count = 0; + ASSERT_EQ(frozenFind(m, "bystander", 0, &count, NULL), 1); + EXPECT_EQ(count, 2u); + + /* A predicate matching nothing is a no-op; one matching everything empties + * both windows. */ + char absent[] = "no-such-key"; + spaceSavingManagerRemoveIf(m, dropNamed, absent); + EXPECT_EQ(spaceSavingManagerCount(m), 1); + char keeper[] = "bystander"; + spaceSavingManagerRemoveIf(m, dropNamed, keeper); + EXPECT_EQ(spaceSavingManagerCount(m), 0); + + spaceSavingManagerRelease(m); +} + +/* --------------------------------------------------------------------------- + * The per-window sampling percentage travels with the window it was recorded + * for, so a frozen window stays interpretable after the configuration changes. + * --------------------------------------------------------------------------*/ +TEST(SpaceSaving, FrozenWindowKeepsTheSamplingPercentageThatProducedIt) { + spaceSavingManager *m = spaceSavingManagerCreate(4, WINDOW_US, 0); + ASSERT_NE(m, nullptr); + + /* Nothing has been recorded or configured yet. */ + EXPECT_EQ(spaceSavingManagerFrozenSamplingPercentage(m), 0); + + spaceSavingManagerSetLiveSamplingPercentage(m, 100); + recordName(m, "a", 0); + uint64_t now = freezeOnce(m, 0); + EXPECT_EQ(spaceSavingManagerFrozenSamplingPercentage(m), 100); + + /* Sampling is lowered afterwards. The already-frozen window must still + * report the value its counts were gathered under. */ + spaceSavingManagerReconfigure(m, 4, WINDOW_US, now); + spaceSavingManagerSetLiveSamplingPercentage(m, 10); + EXPECT_EQ(spaceSavingManagerFrozenSamplingPercentage(m), 100) + << "the frozen window must keep its own sampling percentage"; + EXPECT_EQ(frozenFind(m, "a", 0, NULL, NULL), 1) << "reconfigure must keep the frozen window"; + + /* Once that window rotates out, the new percentage applies. */ + recordName(m, "b", 0); + freezeOnce(m, now); + EXPECT_EQ(spaceSavingManagerFrozenSamplingPercentage(m), 10); + + /* A full reset clears the percentage along with the data. */ + spaceSavingManagerReset(m, 0); + EXPECT_EQ(spaceSavingManagerFrozenSamplingPercentage(m), 0); + EXPECT_EQ(spaceSavingManagerCount(m), 0); + + spaceSavingManagerRelease(m); +} + +/* The configured sampling percentage must survive a window being dropped. The + * discard path resets both windows, and if that cleared the percentage the next + * frozen window would carry 0 and every rate derived from it would silently come + * back as zero until a config change re-set it. */ +TEST(SpaceSaving, SamplingPercentageSurvivesADroppedWindow) { + spaceSavingManager *m = spaceSavingManagerCreate(4, WINDOW_US, 0); + ASSERT_NE(m, nullptr); + + spaceSavingManagerSetLiveSamplingPercentage(m, 100); + recordName(m, "a", 0); + + /* Stall well past the cutoff, so the window is dropped rather than frozen. */ + spaceSavingManagerRotate(m, 10 * WINDOW_US); + ASSERT_EQ(spaceSavingManagerCount(m), 0); + + /* The next window still knows how its counts are being sampled. */ + recordName(m, "b", 0); + spaceSavingManagerRotate(m, 11 * WINDOW_US); + ASSERT_EQ(spaceSavingManagerCount(m), 1); + EXPECT_EQ(spaceSavingManagerFrozenSamplingPercentage(m), 100) << "a dropped window must not clear the config"; + + /* An explicit reset preserves it as well. */ + spaceSavingManagerReset(m, 11 * WINDOW_US); + recordName(m, "c", 0); + spaceSavingManagerRotate(m, 12 * WINDOW_US); + ASSERT_EQ(spaceSavingManagerCount(m), 1); + EXPECT_EQ(spaceSavingManagerFrozenSamplingPercentage(m), 100); + + spaceSavingManagerRelease(m); +} + +/* --------------------------------------------------------------------------- + * A window reports the interval it REALLY accumulated over, not its configured + * length. Rotation is timer-driven, so it runs at or after the nominal + * boundary; using the configured length as a rate denominator would + * systematically over-report by that lag. + * --------------------------------------------------------------------------*/ +TEST(SpaceSaving, FrozenDurationIsTheRealSpanIncludingRotationLag) { + spaceSavingManager *m = spaceSavingManagerCreate(8, WINDOW_US, 0); + ASSERT_NE(m, nullptr); + + /* No completed window yet. */ + EXPECT_EQ(spaceSavingManagerFrozenDurationUs(m), 0u); + + /* Rotation runs late: the boundary is at WINDOW_US but cron only gets to it + * half a window later, and the samples in between land in this window. */ + const uint64_t lag_us = WINDOW_US / 2; + for (int i = 0; i < 10; i++) recordName(m, "a", 0); + spaceSavingManagerRotate(m, WINDOW_US + lag_us); + ASSERT_EQ(spaceSavingManagerCount(m), 1); + EXPECT_EQ(spaceSavingManagerFrozenDurationUs(m), WINDOW_US + lag_us) + << "the frozen duration must include the rotation lag"; + + /* The next window starts when the rotation actually happened, not at the + * nominal grid position, so consecutive durations do not double-count the + * lag: closing the next window one length later spans exactly one length. */ + for (int i = 0; i < 4; i++) recordName(m, "b", 0); + spaceSavingManagerRotate(m, 2 * WINDOW_US + lag_us); + ASSERT_EQ(frozenFind(m, "b", 0, NULL, NULL), 1); + EXPECT_EQ(spaceSavingManagerFrozenDurationUs(m), WINDOW_US); + + /* Reset clears the recorded timing. */ + spaceSavingManagerReset(m, 5 * WINDOW_US); + EXPECT_EQ(spaceSavingManagerFrozenDurationUs(m), 0u); + for (int i = 0; i < 3; i++) recordName(m, "c", 0); + spaceSavingManagerRotate(m, 6 * WINDOW_US); + ASSERT_EQ(frozenFind(m, "c", 0, NULL, NULL), 1); + EXPECT_EQ(spaceSavingManagerFrozenDurationUs(m), WINDOW_US) + << "the window must be measured from the reset, not from creation"; + + spaceSavingManagerRelease(m); +} diff --git a/src/unit/test_stat_calc.cpp b/src/unit/test_stat_calc.cpp new file mode 100644 index 000000000..df8037529 --- /dev/null +++ b/src/unit/test_stat_calc.cpp @@ -0,0 +1,187 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Unit tests for stat_calc.h. + */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "stat_calc.h" +static monotime fakeGetMonotonicUs(void); +static monotime (*origGetMonotonicUs)(void); +} + +static const long ONE_SECOND_IN_MICROS = 1000000; + +static monotime fakeMonotimeUs; + +static monotime fakeGetMonotonicUs(void) { + return fakeMonotimeUs; +} + +class StatCalcTest : public ::testing::Test { + protected: + tpsCalculator *tps; + trendCalculator *trend; + + static void SetUpTestSuite() { + origGetMonotonicUs = getMonotonicUs; + getMonotonicUs = fakeGetMonotonicUs; + } + + static void TearDownTestSuite() { + getMonotonicUs = origGetMonotonicUs; + } + + void SetUp() override { + fakeMonotimeUs = 100; + tps = tpsCalculator_create(5); + trend = trendCalculator_create(5); + } + + void TearDown() override { + tpsCalculator_free(tps); + trendCalculator_free(trend); + } +}; + +/* ========================== TPS Calculator Tests ========================== */ + +TEST_F(StatCalcTest, TpsInitZero) { + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 0.0); +} + +TEST_F(StatCalcTest, TpsExtrapolateFromOneSecond) { + /* 1 second of data at 10 transactions, TPS should be 10 */ + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 10); + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 10.0); +} + +TEST_F(StatCalcTest, TpsInterpolateFromTenSeconds) { + /* 10 seconds of data at 10 transactions, TPS should be 1 */ + fakeMonotimeUs += 10 * ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 10); + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 1.0); +} + +TEST_F(StatCalcTest, TpsSuddenIncrease) { + /* Initialize at 10/sec */ + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 10); + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 10.0); + + /* Add 1 more second at 100/sec */ + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 100); + + /* Window: 4s at 10 TPS + 1s at 100 TPS = 140/5 = 28 TPS */ + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 28.0); +} + +TEST_F(StatCalcTest, TpsSuddenDecrease) { + /* Fill window at 100/sec */ + for (int i = 0; i < 5; i++) { + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 100); + } + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 100.0); + + /* One second at 0 */ + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 0); + + /* Window shifts: 4s at 100 TPS + 1s at 0 TPS = 400/5 = 80 */ + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 80.0); +} + +TEST_F(StatCalcTest, TpsMultipleRecordsInOneInterval) { + /* Two records before the update interval elapses accumulate (5 + 5); the + * next flush folds them in together as 10 transactions. */ + tpsCalculator_record(tps, 5); + tpsCalculator_record(tps, 5); + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 10.0); +} + +/* ======================== Trend Calculator Tests ========================== */ + +/* Trend calc updates once per window/DATA_POINTS. For a 5s window and 10 data + * points, that is 500ms per datapoint. */ +static const monotime TREND_INTERVAL = 5 * ONE_SECOND_IN_MICROS / 10; + +TEST_F(StatCalcTest, TrendInitZero) { + EXPECT_DOUBLE_EQ(trendCalculator_changePerSecShortTerm(trend), 0.0); +} + +TEST_F(StatCalcTest, TrendSingleDatapointFlat) { + /* A single datapoint cannot establish a slope, so the trend stays flat. */ + fakeMonotimeUs += TREND_INTERVAL; + trendCalculator_recordMetric(trend, 100); + EXPECT_DOUBLE_EQ(trendCalculator_changePerSecShortTerm(trend), 0.0); +} + +TEST_F(StatCalcTest, TrendTwoPoint) { + fakeMonotimeUs += TREND_INTERVAL; + trendCalculator_recordMetric(trend, 100); /* First point fills all 10 slots */ + + fakeMonotimeUs += TREND_INTERVAL; + trendCalculator_recordMetric(trend, 0); + + /* Now we have 9 points at 100 and 1 point at 0. + * Left average is 100. Right average is 400/5 = 80. + * Trend has decreased 20 over 2.5 seconds, or 8/sec. */ + /* The short-term view shows a decrease from 100 to 0 over 1/2 sec. */ + EXPECT_DOUBLE_EQ(trendCalculator_changePerSecShortTerm(trend), -200.0); + + fakeMonotimeUs += TREND_INTERVAL; + trendCalculator_recordMetric(trend, 0); + + /* Now we have 8 points at 100 and 2 points at 0. + * Left average is 100. Right average is 300/5 = 60. + * Trend has decreased 40 over 2.5 seconds, or 16/sec. */ + /* The short-term view shows no change (0 to 0). */ + EXPECT_DOUBLE_EQ(trendCalculator_changePerSecShortTerm(trend), 0.0); +} + +TEST_F(StatCalcTest, TrendIntervalGating) { + fakeMonotimeUs += TREND_INTERVAL; + trendCalculator_recordMetric(trend, 100); /* First point fills all 10 slots */ + + fakeMonotimeUs += TREND_INTERVAL - 1; /* Not at the collection interval yet */ + trendCalculator_recordMetric(trend, 0); + + /* The datapoint was not collected, so the trend should not have changed. */ + EXPECT_DOUBLE_EQ(trendCalculator_changePerSecShortTerm(trend), 0.0); +} + +TEST_F(StatCalcTest, TrendRising) { + /* Metric increases by 10 per 100ms. + * Batch averages: 20, 70, 120, ..., 470. + * olderAvg = (20+70+120+170+220)/5 = 120 + * newerAvg = (270+320+370+420+470)/5 = 370 + * trend = (370 - 120) / 2.5 = 100.0 + * Short-term: last two slots (420 -> 470) over 0.5s = 100.0 */ + for (int i = 0; i < 50; i++) { + fakeMonotimeUs += ONE_SECOND_IN_MICROS / 10; + trendCalculator_recordMetric(trend, (long)(i * 10)); + } + EXPECT_DOUBLE_EQ(trendCalculator_changePerSecShortTerm(trend), 100.0); +} + +TEST_F(StatCalcTest, TrendFalling) { + /* Metric decreases by 10 per 100ms. + * Batch averages: 480, 430, 380, ..., 30. + * olderAvg = (480+430+380+330+280)/5 = 380 + * newerAvg = (230+180+130+80+30)/5 = 130 + * trend = (130 - 380) / 2.5 = -100.0 + * Short-term: last two slots (80 -> 30) over 0.5s = -100.0 */ + for (int i = 0; i < 50; i++) { + fakeMonotimeUs += ONE_SECOND_IN_MICROS / 10; + trendCalculator_recordMetric(trend, (long)(500 - i * 10)); + } + EXPECT_DOUBLE_EQ(trendCalculator_changePerSecShortTerm(trend), -100.0); +} diff --git a/src/unit/test_throttle.cpp b/src/unit/test_throttle.cpp new file mode 100644 index 000000000..888fa0315 --- /dev/null +++ b/src/unit/test_throttle.cpp @@ -0,0 +1,639 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Unit tests for throttle.h. + */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "throttle.h" +static monotime fakeGetMonotonicUs(void); +static monotime (*origGetMonotonicUs)(void); + +static bool fakeWriteCriteria(client *c, void *priv_data) { + UNUSED(priv_data); + return c->cmd && (c->cmd->flags & CMD_WRITE); +} +} + +static monotime fakeMonotimeUs; + +static monotime fakeGetMonotonicUs(void) { + return fakeMonotimeUs; +} + +class ThrottleTest : public ::testing::Test { + protected: + MockValkey mock; + RealValkey real; + serverCommand get_cmd; + serverCommand set_cmd; + static inline ConnectionType dummyConnType = {0}; + + static void SetUpTestSuite() { + memset(&server, 0, sizeof(valkeyServer)); + server.hz = CONFIG_DEFAULT_HZ; + server.logfile = (char *)""; + dummyConnType.set_read_handler = dummySetReadHandler; + throttle_init(); + + origGetMonotonicUs = getMonotonicUs; + getMonotonicUs = fakeGetMonotonicUs; + } + + static void TearDownTestSuite() { + getMonotonicUs = origGetMonotonicUs; + } + + void SetUp() override { + fakeMonotimeUs = 100; + get_cmd = {0}; + get_cmd.fullname = (sds) "get"; + get_cmd.proc = getCommand; + get_cmd.flags = CMD_READONLY; + + set_cmd = {0}; + set_cmd.fullname = (sds) "set"; + set_cmd.proc = setCommand; + set_cmd.flags = CMD_WRITE; + } + + void TearDown() override { + } + + static int dummySetReadHandler(connection *conn, ConnectionCallbackFunc func) { + conn->read_handler = func; + return C_OK; + } + + /* A set_read_handler that always fails, to simulate a connection error. */ + static int failSetReadHandler(connection *conn, ConnectionCallbackFunc func) { + UNUSED(conn); + UNUSED(func); + return C_ERR; + } + + client *createFakeClient(int client_id, bool write_command) { + client *c = (client *)zcalloc(sizeof(client)); + c->id = client_id; + c->conn = (connection *)zcalloc(sizeof(connection)); + c->conn->type = &dummyConnType; + c->conn->read_handler = (ConnectionCallbackFunc)1; + c->flag.pending_command = 1; + c->argc = 1; + c->cmd = write_command ? &set_cmd : &get_cmd; + return c; + } + + void freeFakeClient(client *c) { + EXPECT_EQ(c->throttler, nullptr); + EXPECT_EQ(c->throttle_node, nullptr); + EXPECT_EQ(c->flag.throttled, 0ULL); + if (c->conn) zfree(c->conn); + zfree(c); + } + + bool clientIsThrottled(client *c) { + bool throttled = c->flag.throttled == 1; + if (throttled) { + EXPECT_EQ(c->conn->read_handler, nullptr); + EXPECT_NE(c->throttler, nullptr); + EXPECT_NE(c->throttle_node, nullptr); + EXPECT_EQ(c->flag.throttle_checked, 1ULL); + EXPECT_NE(c->throttle_start, 0ULL); + } else { + EXPECT_EQ(c->throttler, nullptr); + EXPECT_EQ(c->throttle_node, nullptr); + EXPECT_EQ(c->throttle_start, 0ULL); + EXPECT_EQ(c->flag.throttle_multi, 0ULL); + } + return throttled; + } + + void verifyThrottler(const char *metric_name, int clients_throttled, int cmds_throttled) { + throttleMetrics m; + throttle_getMetrics(metric_name, &m); + EXPECT_EQ(m.num_clients_throttled, clients_throttled); + EXPECT_EQ(m.num_commands_throttled, cmds_throttled); + } +}; + +using ThrottleDeathTest = ThrottleTest; + +/* ---- throttle_throttleClientIfNeeded tests ---- */ + +TEST_F(ThrottleTest, noThrottlerPassesThrough) { + client *c = createFakeClient(1, true); + /* No throttler registered yet, nothing to throttle. */ + EXPECT_FALSE(throttle_throttleClientIfNeeded(c)); + EXPECT_FALSE(clientIsThrottled(c)); + freeFakeClient(c); +} + +TEST_F(ThrottleTest, throttleHappyCase) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + client *c = createFakeClient(1, true); + throttle_setRate(t, 0.0); // This will empty the bucket + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c)); + EXPECT_TRUE(clientIsThrottled(c)); + + EXPECT_FALSE(throttle_throttleClientIfNeeded(c)); // We don't throttle client if it's already throttled + verifyThrottler("fake_throttler", 1, 1); + + /* Drain via timeProc */ + throttle_setRate(t, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + EXPECT_CALL(mock, queueClientForReprocessing(c)).Times(1); + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + + EXPECT_FALSE(clientIsThrottled(c)); + verifyThrottler("fake_throttler", 0, 1); + + throttle_deregister(t); + freeFakeClient(c); +} + +TEST_F(ThrottleTest, criteriaMismatchPassesThrough) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + client *c = createFakeClient(1, false); // client with read command + + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).Times(0); + EXPECT_FALSE(throttle_throttleClientIfNeeded(c)); + EXPECT_FALSE(clientIsThrottled(c)); + verifyThrottler("fake_throttler", 0, 0); + + throttle_deregister(t); + freeFakeClient(c); +} + +TEST_F(ThrottleTest, tokenAvailablePassesThrough) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); /* starts at UNLIMITED rate, full bucket */ + client *c = createFakeClient(1, true); + + /* Criteria matches, tokens are available, consume token and proceed. */ + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).Times(0); + EXPECT_FALSE(throttle_throttleClientIfNeeded(c)); + EXPECT_FALSE(clientIsThrottled(c)); + verifyThrottler("fake_throttler", 0, 0); + + throttle_deregister(t); + freeFakeClient(c); +} + +TEST_F(ThrottleTest, deregisteredThrottlerDrainsButDoesNotThrottleNewClients) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + /* Queue a client so the throttler cannot be freed on deregister. */ + client *queued = createFakeClient(1, true); + EXPECT_TRUE(throttle_throttleClientIfNeeded(queued)); + EXPECT_TRUE(clientIsThrottled(queued)); + + /* Deregister with a non-empty queue: the throttler stays alive in CLEANUP + * state (still draining the queued client) but must not throttle new clients. */ + throttle_deregister(t); + + /* A new matching write command passes through untouched. */ + client *fresh = createFakeClient(2, true); + EXPECT_FALSE(throttle_throttleClientIfNeeded(fresh)); + EXPECT_FALSE(clientIsThrottled(fresh)); + verifyThrottler("fake_throttler", 1, 1); + + /* Drain via timeProc — deregister already set rate to UNLIMITED. */ + fakeMonotimeUs += 1000000; + EXPECT_CALL(mock, queueClientForReprocessing(queued)).Times(1); + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + verifyThrottler("fake_throttler", 0, 1); + + freeFakeClient(queued); + freeFakeClient(fresh); +} + +TEST_F(ThrottleTest, strictestThrottlerThrottle) { + /* Two throttlers both match a write command. The strictest (lowest rate) + * wins: the client is queued under it and the multi-match flag is set so + * the other bucket is charged on release. */ + throttler *loose = throttle_register(fakeWriteCriteria, NULL, "loose"); /* UNLIMITED */ + throttler *strict = throttle_register(fakeWriteCriteria, NULL, "strict"); + throttle_setRate(strict, 0.0); /* no tokens -> strictest */ + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c = createFakeClient(1, true); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c)); + EXPECT_TRUE(clientIsThrottled(c)); + EXPECT_EQ(c->flag.throttle_multi, 1ULL); /* matched >1 throttler */ + + /* Queued under the strict throttler; the loose one is untouched. */ + verifyThrottler("strict", 1, 1); + verifyThrottler("loose", 0, 0); + + /* Drain via timeProc. */ + throttle_setRate(strict, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, false)).WillOnce(Return(true)); + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, true)).WillOnce(Return(true)); + EXPECT_CALL(mock, queueClientForReprocessing(c)).Times(1); + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_EQ(c->flag.throttle_multi, 0ULL); + + throttle_deregister(loose); + throttle_deregister(strict); + freeFakeClient(c); +} + +TEST_F(ThrottleTest, strictestThrottlerNonThrottle) { + /* Two throttlers both match, but the strictest still has a token, so the + * client passes through (not throttled). Because it matched >1 throttler, + * consumeOtherThrottlers also charges the other bucket on the pass path. */ + throttler *loose = throttle_register(fakeWriteCriteria, NULL, "loose"); /* UNLIMITED */ + throttler *strict = throttle_register(fakeWriteCriteria, NULL, "strict"); + throttle_setRate(strict, 1.0); + + client *c = createFakeClient(1, true); + + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, true)).WillOnce(Return(true)); // Force consume the loose bucket + EXPECT_FALSE(throttle_throttleClientIfNeeded(c)); // token available -> passes through + EXPECT_FALSE(clientIsThrottled(c)); + EXPECT_EQ(c->flag.throttle_multi, 0ULL); // multi flag is only set on the defer path */ + + /* Neither throttler queued the client. */ + verifyThrottler("loose", 0, 0); + verifyThrottler("strict", 0, 0); + + throttle_deregister(loose); + throttle_deregister(strict); + freeFakeClient(c); +} + +/* ---- throttler rate tests ---- */ + +TEST_F(ThrottleTest, adjustRatePolicyIncrease) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); /* starts UNLIMITED */ + + /* Increase while already UNLIMITED is a no-op. */ + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 2.0), THROTTLE_UNLIMITED_RATE); + + /* Recover from halted state jumps to the fixed restart rate (100 ops/sec). */ + throttle_setRate(t, 0.0); + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 2.0), 100.0); + + /* Normal increase: 100 * (2.0 - 1.0) = 200. */ + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 2.0), 200.0); + + /* Tiny multiplier still increases by the minimum step of 1 ops/sec. */ + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 1.00001), 201.0); + + throttle_deregister(t); +} + +TEST_F(ThrottleTest, adjustRatePolicyDecrease) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); /* starts UNLIMITED */ + + EXPECT_CALL(mock, tpsCalculator_averageTps(_)).WillRepeatedly(Return(500.0)); /* incoming TPS */ + + /* Decreasing a rate that is still above incoming snaps straight down to incoming. */ + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 0.95), 500.0); + + /* Once at/below incoming, a further decrease goes below it (real throttling): + * 500 * 0.8 = 400 < 500. */ + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 0.8), 400.0); + + /* With no measured incoming TPS the snap is disabled: 400 * 0.5 = 200. */ + EXPECT_CALL(mock, tpsCalculator_averageTps(_)).WillRepeatedly(Return(0.0)); + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 0.5), 200.0); + + throttle_deregister(t); +} + +TEST_F(ThrottleTest, setRate) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttleMetrics m; + + /* A normal rate is stored as-is. */ + throttle_setRate(t, 1234.0); + throttle_getMetrics("fake_throttler", &m); + EXPECT_DOUBLE_EQ(m.ops_per_sec, 1234.0); + + /* Above the unlimited ceiling is clamped down to THROTTLE_UNLIMITED_RATE. */ + throttle_setRate(t, THROTTLE_UNLIMITED_RATE * 2); + throttle_getMetrics("fake_throttler", &m); + EXPECT_DOUBLE_EQ(m.ops_per_sec, THROTTLE_UNLIMITED_RATE); + + /* Below epsilon collapses to zero. */ + throttle_setRate(t, 0.00001); + throttle_getMetrics("fake_throttler", &m); + EXPECT_DOUBLE_EQ(m.ops_per_sec, 0.0); + + throttle_deregister(t); +} + +TEST_F(ThrottleTest, guardrailSecsTracking) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + + /* 0.05 ops/sec, at or below the 0.1 ops/sec guardrail. */ + throttle_setRate(t, 0.05); + fakeMonotimeUs += 3 * 1000000; /* advance 3 seconds */ + EXPECT_EQ(throttle_getGuardrailSecs(t), 3L); + + /* Back above the guardrail resets the timer. */ + throttle_setRate(t, 1.0); /* above the 0.1 ops/sec guardrail */ + EXPECT_EQ(throttle_getGuardrailSecs(t), 0L); + + throttle_deregister(t); +} + +/* ---- metrics aggregation ---- */ + +TEST_F(ThrottleTest, metricsAggregateAcrossSharedName) { + /* Two throttlers sharing one metrics group ("shared") aggregate their metrics */ + throttler *a = throttle_register(fakeWriteCriteria, NULL, "shared"); + throttler *b = throttle_register(fakeWriteCriteria, NULL, "shared"); + + /* ops_per_sec is the SUM of both throttlers' rates. */ + throttle_setRate(a, 100.0); + throttle_setRate(b, 250.0); + throttleMetrics m; + throttle_getMetrics("shared", &m); + EXPECT_DOUBLE_EQ(m.ops_per_sec, 350.0); + + /* Empty both buckets. Both clients are writes matching both throttlers, so the + * strictest (a, rate 0) wins and both queue under a; b stays empty. */ + throttle_setRate(a, 0.0); + throttle_setRate(b, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c1 = createFakeClient(1, true); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c1)); /* throttle_start = 100 (fake clock) */ + fakeMonotimeUs += 5 * 1000000; /* +5s */ + client *c2 = createFakeClient(2, true); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c2)); /* throttle_start = 5,000,100 */ + + throttle_getMetrics("shared", &m); + /* Both clients increment the shared metrics group. */ + EXPECT_EQ(m.num_clients_throttled, 2); + EXPECT_EQ(m.num_commands_throttled, 2); + /* oldest_client_delay_us tracks the oldest queued client (c1, queued 5s ago). */ + EXPECT_EQ(m.oldest_client_delay_us, 5 * 1000000); + + /* Drain via timeProc. */ + throttle_setRate(a, THROTTLE_UNLIMITED_RATE); + throttle_setRate(b, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + /* Both clients have throttle_multi set (matched both throttlers), so each + * release will also consume the other throttler's bucket. */ + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, false)).Times(2).WillRepeatedly(Return(true)); + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, true)).Times(2).WillRepeatedly(Return(true)); + EXPECT_CALL(mock, queueClientForReprocessing(_)).Times(2); + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + + throttle_getMetrics("shared", &m); + verifyThrottler("shared", 0, 2); + EXPECT_EQ(m.oldest_client_delay_us, 0); + + throttle_deregister(a); + throttle_deregister(b); + freeFakeClient(c1); + freeFakeClient(c2); +} + +TEST_F(ThrottleTest, guardrailInfoReportsLongestPerType) { + /* Two throttlers share a metrics name, both below guardrail. INFO reports a single line + * per type, showing the longest-below-guardrail (earliest start) one. */ + throttler *a = throttle_register(fakeWriteCriteria, NULL, "shared"); + throttler *b = throttle_register(fakeWriteCriteria, NULL, "shared"); + + throttle_setRate(a, 0.05); + fakeMonotimeUs += 2000000; + throttle_setRate(b, 0.05); + fakeMonotimeUs += 3000000; + + sds info = throttle_sdscatInfoMetrics(sdsempty()); + + /* Only throttler a guardrail secs is reported. */ + EXPECT_NE(strstr(info, "throttle_shared_guardrail_secs:5\r\n"), nullptr); + EXPECT_EQ(strstr(info, "throttle_shared_guardrail_secs:3\r\n"), nullptr); + + sdsfree(info); + throttle_deregister(a); + throttle_deregister(b); +} + +/* ---- throttlerTimeProc tests ---- */ + +TEST_F(ThrottleTest, timeProcHappyCaseOneCall) { + /* When enough tokens are available, timeProc releases all queued clients in one call. */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c1 = createFakeClient(1, true); + client *c2 = createFakeClient(2, true); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c1)); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c2)); + + /* Set unlimited rate so both clients are released in one timeProc call. */ + throttle_setRate(t, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + + EXPECT_CALL(mock, queueClientForReprocessing(_)).Times(2); + + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + + EXPECT_FALSE(clientIsThrottled(c1)); + EXPECT_FALSE(clientIsThrottled(c2)); + verifyThrottler("fake_throttler", 0, 2); + + throttle_deregister(t); + freeFakeClient(c1); + freeFakeClient(c2); +} + +TEST_F(ThrottleTest, timeProcHappyCaseMultipleCall) { + /* When the timer fires but tokens run out before the queue is empty, it reschedules. */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c1 = createFakeClient(1, true); + client *c2 = createFakeClient(2, true); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c1)); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c2)); + + /* Set 1 ops/sec rate, so only 1 token available after refill. */ + throttle_setRate(t, 1.0); + fakeMonotimeUs += 1000000; + + /* Only the first client will be released. No aeDeleteTimeEvent since queue stays non-empty. */ + EXPECT_CALL(mock, queueClientForReprocessing(c1)).Times(1); + + long long ret = timeProc(server.el, 1, clientData); + /* Should return a positive wait time (reschedule). */ + EXPECT_EQ(ret, 100); + + /* c1 released, c2 still throttled. */ + EXPECT_FALSE(clientIsThrottled(c1)); + EXPECT_TRUE(clientIsThrottled(c2)); + verifyThrottler("fake_throttler", 1, 2); + + /* Drain c2 for cleanup. */ + fakeMonotimeUs += 1000000; /* advance 1s, 1 token available */ + EXPECT_CALL(mock, queueClientForReprocessing(c2)).Times(1); + ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_FALSE(clientIsThrottled(c2)); + verifyThrottler("fake_throttler", 0, 2); + + throttle_deregister(t); + freeFakeClient(c1); + freeFakeClient(c2); +} + +TEST_F(ThrottleTest, timeProcMultiThrottlerConsumesOtherBuckets) { + /* When a client matched multiple throttlers (throttle_multi flag), releasing it + * via the timer should also consume tokens from the other throttlers. */ + throttler *loose = throttle_register(fakeWriteCriteria, NULL, "share"); + throttler *strict = throttle_register(fakeWriteCriteria, NULL, "share"); + throttle_setRate(strict, 0.0); /* strict has no tokens and client queues here */ + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c = createFakeClient(1, true); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c)); + EXPECT_EQ(c->flag.throttle_multi, 1ULL); + + /* Now release: set strict to high rate. */ + throttle_setRate(strict, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + + /* The strict throttler's own bucket is consumed (force_consume=false) in the while loop, + * then consumeOtherThrottlers charges the loose throttler (force_consume=true). */ + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, false)).WillOnce(Return(true)); + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, true)).WillOnce(Return(true)); + EXPECT_CALL(mock, queueClientForReprocessing(c)).Times(1); + + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_FALSE(clientIsThrottled(c)); + verifyThrottler("share", 0, 1); + + throttle_deregister(loose); + throttle_deregister(strict); + freeFakeClient(c); +} + +TEST_F(ThrottleTest, timeProcCleanupThrottlerFreesOnDrain) { + /* A deregistered throttler (CLEANUP state) is freed when the timer drains it. */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c = createFakeClient(1, true); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c)); + + /* Deregister while client is still queued, enters CLEANUP state. + * deregister sets rate to THROTTLE_UNLIMITED_RATE internally. */ + throttle_deregister(t); + + fakeMonotimeUs += 1000000; + EXPECT_CALL(mock, queueClientForReprocessing(c)).Times(1); + + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_FALSE(clientIsThrottled(c)); + verifyThrottler("fake_throttler", 0, 1); + + freeFakeClient(c); +} + +/* ---- throttle_removeClient test ---- */ +TEST_F(ThrottleTest, removeClient) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).WillOnce(Return(1)); + + client *c1 = createFakeClient(1, true); + client *c2 = createFakeClient(2, true); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c1)); + EXPECT_TRUE(throttle_throttleClientIfNeeded(c2)); + EXPECT_TRUE(clientIsThrottled(c1)); + EXPECT_TRUE(clientIsThrottled(c2)); + verifyThrottler("fake_throttler", 2, 2); + + /* Remove the client from the throttler queue. */ + throttle_removeClient(c1); + EXPECT_FALSE(clientIsThrottled(c1)); + EXPECT_EQ(c1->conn->read_handler, nullptr); + verifyThrottler("fake_throttler", 1, 2); + + EXPECT_CALL(mock, aeDeleteTimeEvent(_, _)).WillOnce(Return(AE_OK)); + throttle_removeClient(c2); + EXPECT_FALSE(clientIsThrottled(c2)); + EXPECT_EQ(c2->conn->read_handler, nullptr); + verifyThrottler("fake_throttler", 0, 2); + + throttle_deregister(t); + freeFakeClient(c1); + freeFakeClient(c2); +} + +/* ---- Death tests ---- */ + +TEST_F(ThrottleDeathTest, deregisterThrottlerFail) { + /* deregister a NULL throttler */ + EXPECT_DEATH(throttle_deregister(NULL), ""); +} + +TEST_F(ThrottleDeathTest, setRateNegativeAsserts) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "neg_rate"); + EXPECT_DEATH(throttle_setRate(t, -1.0), ""); + throttle_deregister(t); +} + +TEST_F(ThrottleDeathTest, adjustRateOutOfRangeAsserts) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "bad_mult"); + EXPECT_DEATH(throttle_adjustRate(t, 3.5), ""); /* multiplier must be <= 3.0 */ + throttle_deregister(t); +} diff --git a/src/unit/test_throttle_repl.cpp b/src/unit/test_throttle_repl.cpp new file mode 100644 index 000000000..a94a7f13e --- /dev/null +++ b/src/unit/test_throttle_repl.cpp @@ -0,0 +1,315 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Unit tests for throttle_repl.h + */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "stat_calc.h" +#include "throttle_repl.h" +static monotime fakeGetMonotonicUs(void); +static monotime (*origGetMonotonicUs)(void); +} + +static monotime fakeMonotimeUs; + +static monotime fakeGetMonotonicUs(void) { + return fakeMonotimeUs; +} + +class ThrottleReplTest : public ::testing::Test { + protected: + MockValkey mock; + RealValkey real; + static const unsigned long long COB_LIMIT = 10 * 1024 * 1024; /* 10 MB */ + client *replica_steady = nullptr; + throttler *dummy_throttler = (throttler *)1; + + static void SetUpTestSuite() { + /* Server set up */ + memset(&server, 0, sizeof(valkeyServer)); + server.hz = CONFIG_DEFAULT_HZ; + server.replicas = listCreate(); + server.client_obuf_limits[CLIENT_TYPE_REPLICA].soft_limit_bytes = COB_LIMIT; + server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes = COB_LIMIT; + + /* throttle_repl set up */ + throttleRepl_config.repl_throttling_enabled = 1; + + /* monotonic set up */ + origGetMonotonicUs = getMonotonicUs; + getMonotonicUs = fakeGetMonotonicUs; + } + + static void TearDownTestSuite() { + getMonotonicUs = origGetMonotonicUs; + listRelease(server.replicas); + server.replicas = NULL; + } + + void SetUp() override { + replica_steady = createFakeReplicaClient(1); + replica_steady->repl_data->repl_state = REPLICA_STATE_ONLINE; + EXPECT_CALL(mock, throttle_getMetrics(_, _)).WillRepeatedly(SetArgPointee<1>(throttleMetrics{})); + EXPECT_CALL(mock, throttle_getGuardrailSecs(_)).WillRepeatedly(Return(0L)); + } + + void TearDown() override { + freeFakeReplicaClient(replica_steady); + replica_steady = NULL; + } + + client *createFakeReplicaClient(int client_id) { + client *c = (client *)zcalloc(sizeof(client)); + c->id = client_id; + c->flag.replica = 1; + c->repl_data = (ClientReplicationData *)zcalloc(sizeof(ClientReplicationData)); + listAddNodeTail(server.replicas, c); + return c; + } + + void freeFakeReplicaClient(client *c) { + ASSERT_TRUE(c->flag.throttled == 0); + ASSERT_TRUE(c->throttler == NULL); + ASSERT_TRUE(c->throttle_node == NULL); + if (c->cob_trend) trendCalculator_free(c->cob_trend); + if (c->repl_data) zfree(c->repl_data); + listNode *ln = listSearchKey(server.replicas, c); + if (ln) listDelNode(server.replicas, ln); + zfree(c); + } + + bool isReplThrottlerActive() { + return readMetric("repl_throttle_rate") >= 0.0; + } + + double getThrottlerRate() { + return readMetric("repl_throttle_rate"); + } + + bool verifyThrottleEvent(long activation_events, long more, long less) { + return (long)readMetric("repl_throttle_activation_events") == activation_events && + (long)readMetric("repl_throttle_more_events") == more && + (long)readMetric("repl_throttle_less_events") == less; + } + + private: + /* Snapshot the INFO output and return one field's numeric value (-1 if absent). */ + double readMetric(const char *key) { + sds info = throttleRepl_sdscatInfoMetrics(sdsempty()); + info = throttleRepl_sdscatInfoDebugMetrics(info); + char search_for[128]; + snprintf(search_for, sizeof(search_for), "%s:", key); + char *p = strstr(info, search_for); + double v = p ? strtod(p + strlen(search_for), NULL) : -1.0; + sdsfree(info); + return v; + } +}; + +TEST_F(ThrottleReplTest, NoReplicasNoThrottle) { + listEmpty(server.replicas); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); +} + +TEST_F(ThrottleReplTest, noCobLimitConfiguredNoThrottle) { + /* With neither a soft nor a hard COB limit configured, the cob target is 0. + * The adjustThrottling treats target 0 as "feature off": it never activates, even with + * a huge COB and increasing trend. */ + server.client_obuf_limits[CLIENT_TYPE_REPLICA].soft_limit_bytes = 0; + server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes = 0; + + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT * 1000)); + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT * 1000)); + + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); +} + +TEST_F(ThrottleReplTest, steadyStateNoThrottleCases) { + /* Test cases for steady-state replica that throttler will not enabled. */ + + /* For cob size < 1/4 soft limit, throttler should not be enabled regardless of trend. */ + /* cob size < 1/4 soft limit, trend is 0 */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 8)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + EXPECT_TRUE(replica_steady->cob_trend != NULL); + + /* cob size < 1/4 soft limit, huge trend: still no throttle (below threshold, so the trend is not considered) */ + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + + /* For cob size >= 1/4 soft limit, throttler should be enabled if the extrapolated cob size exceeds the cob target (1/2 soft limit). */ + /* cob size >= 1/4 soft limit but < 1/2 soft limit, trend is decreasing */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(-1.0)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + + /* cob size >= 1/4 soft limit but < 1/2 soft limit, trend is slowly increasing */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(1.0)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + + /* cob size >1/2 soft limit, trend is decreasing and extrapolated value below target */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 2 + 1)); + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(-1.0)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); +} + +TEST_F(ThrottleReplTest, steadyStateThrottleIncreasingTrend) { + /* Test case for steady-state replica above threshold (1/4 cob soft limit), + * for increasing trend, throttle could happen when the extrapolated value exceed the 1/2 cob soft limit. */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); + + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(isReplThrottlerActive()); + EXPECT_TRUE(verifyThrottleEvent(1, 0, 0)); // Throttler activated, no more/less events yets + + EXPECT_CALL(mock, throttle_adjustRate(_, 0.95)).WillOnce(Return(1.0)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(verifyThrottleEvent(1, 1, 0)); // Reduce traffic, throttle more traffic. + + EXPECT_CALL(mock, throttle_adjustRate(_, 0.95)).WillOnce(Return(1.0)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(verifyThrottleEvent(1, 2, 0)); // Reduce traffic, throttles more traffic. + + // Now mock traffic trend is slowed down more, throttler should be deregistered + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(1.0)); + EXPECT_CALL(mock, throttle_adjustRate(_, 1.05)).WillOnce(Return(10000000.0)); + EXPECT_CALL(mock, throttle_deregister(_)).Times(1); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + EXPECT_TRUE(verifyThrottleEvent(1, 2, 1)); +} + +TEST_F(ThrottleReplTest, steadyStateThrottleDecreasingTrend) { + /* Test case for steady-state replica above threshold (1/4 cob soft limit), + * for decreasing trend, throttle could happen when the extrapolated value exceed the 1/2 cob soft limit. */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 2 + 40)); + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(-1.0)); + + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(isReplThrottlerActive()); + EXPECT_TRUE(verifyThrottleEvent(1, 0, 0)); // Throttler activated, no more/less events yets + + EXPECT_CALL(mock, throttle_adjustRate(_, 0.95)).WillOnce(Return(1.0)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(verifyThrottleEvent(1, 1, 0)); // Reduce traffic, throttle more traffic. + + EXPECT_CALL(mock, throttle_adjustRate(_, 0.95)).WillOnce(Return(1.0)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(verifyThrottleEvent(1, 2, 0)); // Reduce traffic, throttles more traffic. + + // Now mock traffic trend is slowed down more, throttler should be deregistered + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(-2.0)); + EXPECT_CALL(mock, throttle_adjustRate(_, 1.05)).WillOnce(Return(10000000.0)); + EXPECT_CALL(mock, throttle_deregister(_)).Times(1); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + EXPECT_TRUE(verifyThrottleEvent(1, 2, 1)); +} + +TEST_F(ThrottleReplTest, steadyStateThrottleBasedOnLargestCob) { + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 8)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + + /* Add a second replica with a large COB. The scan tracks the largest + * COB, so the decision is driven by this replica and throttler activates. */ + client *dummy_replica = createFakeReplicaClient(2); + dummy_replica->repl_data->repl_state = REPLICA_STATE_ONLINE; + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(dummy_replica)).WillRepeatedly(Return(COB_LIMIT)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(isReplThrottlerActive()); + freeFakeReplicaClient(dummy_replica); +} + +TEST_F(ThrottleReplTest, disabledConfigNoNewThrottle) { + throttleRepl_config.repl_throttling_enabled = 0; + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); + + throttleRepl_adjustThrottling(); + + /* Should not activate when config disabled */ + EXPECT_FALSE(isReplThrottlerActive()); +} + +TEST_F(ThrottleReplTest, throttlerRemovedAfterFailover) { + /* Simulate active throttler then failover (become replica) */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(isReplThrottlerActive()); + + server.primary_host = (char *)"127.0.0.1"; /* now a replica */ + + EXPECT_CALL(mock, throttle_deregister(dummy_throttler)).Times(1); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); +} + +TEST_F(ThrottleReplTest, clientCobLimitsExempt) { + // Default should return false since throttler not active + EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + + /* Throttler now active */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 2 + 1)); + EXPECT_CALL(mock, trendCalculator_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(isReplThrottlerActive()); + + /* If throttler has been working too long, not exempt. */ + server.unixtime = 1000; + replica_steady->obuf_soft_limit_reached_time = server.unixtime - 200; // > 4 * STEADY_STATE_CONVERGENCE_SECS + EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + replica_steady->obuf_soft_limit_reached_time = server.unixtime - 100; // < 4 * STEADY_STATE_CONVERGENCE_SECS + EXPECT_TRUE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + + /* If available memory is exhausted, not exempt */ + server.maxmemory = 100; + EXPECT_CALL(mock, getMaxmemoryState(_, _, _, _)).WillRepeatedly(Return(C_ERR)); + EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + EXPECT_CALL(mock, getMaxmemoryState(_, _, _, _)).WillRepeatedly(Return(C_OK)); + EXPECT_TRUE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + + /* If the cob size is below the cob target, not exempt */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4)); + EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 2 + 1)); + EXPECT_TRUE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + + /* If it's not replica client, not exempt. */ + replica_steady->flag.replica = 0; + EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + replica_steady->flag.replica = 1; + EXPECT_TRUE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + + /* If I am not primary, not exempt. */ + server.primary_host = (char *)"127.0.0.1"; + EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + server.primary_host = NULL; + EXPECT_TRUE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + + /* If throttle repl disabled, not exempt. */ + throttleRepl_config.repl_throttling_enabled = 0; + EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); + throttleRepl_config.repl_throttling_enabled = 1; + EXPECT_TRUE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); +} diff --git a/src/unit/test_token_bucket.cpp b/src/unit/test_token_bucket.cpp new file mode 100644 index 000000000..62cfe67d0 --- /dev/null +++ b/src/unit/test_token_bucket.cpp @@ -0,0 +1,181 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Unit tests for throttle_token_bucket.h (token bucket algorithm). + */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "throttle_token_bucket.h" +static monotime fakeGetMonotonicUs(void); +static monotime (*origGetMonotonicUs)(void); +} + +static monotime fakeMonotimeUs; + +static monotime fakeGetMonotonicUs(void) { + return fakeMonotimeUs; +} + +class TokenBucketTest : public ::testing::Test { + protected: + tokenBucket *bucket; + + static void SetUpTestSuite() { + origGetMonotonicUs = getMonotonicUs; + getMonotonicUs = fakeGetMonotonicUs; + } + + static void TearDownTestSuite() { + getMonotonicUs = origGetMonotonicUs; + } + + void SetUp() override { + fakeMonotimeUs = 1000000; /* start at 1 second */ + bucket = tokenBucket_create(100.0, 0.1); /* 100 tokens/sec, 0.1s burst */ + } + + void TearDown() override { + tokenBucket_free(bucket); + } +}; + +TEST_F(TokenBucketTest, BucketCreation) { + /* Bucket starts full and can consume up to bucket capacity */ + EXPECT_DOUBLE_EQ(tokenBucket_getRate(bucket), 100.0); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 10.0), 0.0); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); +} + +TEST_F(TokenBucketTest, HaltedBucket) { + /* Set the rate to zero, this will also empty the bucket. */ + tokenBucket_setRate(bucket, 0.0); + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), -1.0); + + fakeMonotimeUs += 1000000; /* advance 1 second */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, true)); /* Force consume should work */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), -1.0); /* Never available */ + + /* Now set the rate back to a positive value */ + tokenBucket_setRate(bucket, 100.0); + fakeMonotimeUs += 1000000; /* advance 1 second, now the token bucket is refilled */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 11.0), 0.0); /* We should still have 11 tokens available */ +} + +TEST_F(TokenBucketTest, MsUntilAvailableReplenishes) { + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 12.0, false)); /* drain (size = 100*0.1+2 = 12) */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); /* empty right now */ + + fakeMonotimeUs += 1000000; /* advance 1s, bucket refills to full */ + + /* Bucket is full again */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 12.0), 0.0); + + /* Partial refill: drain again, advance only 5ms */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 12.0, false)); + fakeMonotimeUs += 5000; /* 0.5 token accrued */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 5.0); /* need 5ms more */ +} + +TEST_F(TokenBucketTest, ConsumeTokens_normal) { + /* Drain all tokens (bucket size = rate * burst_time + 2 = 100*0.1+2 = 12) */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 12.0, false)); + /* Now empty */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + + fakeMonotimeUs += 10000; // Advance 10ms -> 1 token replenished + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.1, false)); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); /* Now empty */ + + fakeMonotimeUs += 1000000; /* advance 1 second */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 12.1, false)); /* Cannot consume tokens over bucket capacity */ + for (int i = 0; i < 12; ++i) { + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 0.0); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); + } + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); /* Now empty */ +} + +TEST_F(TokenBucketTest, ConsumeTokens_force) { + /* Drain all tokens (bucket size = rate * burst_time + 2 = 100*0.1+2 = 12) */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 12.0, true)); + /* Now empty */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); + /* Force consume should work even when empty */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, true)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 20.0); /* Now we need to wait for 2 tokens to be available */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + + /* Force consume should work even when the token count is negative */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, true)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 30.0); /* Now we need to wait for 3 tokens to be available */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + + fakeMonotimeUs += 30000; + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.1, false)); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); + + /* Force consume can drop tokens below zero, but not below the minimum capacity (- bucket size)*/ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 100.0, true)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 130.0); + + fakeMonotimeUs += 130000; + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.1, false)); /* replenish -12+13=1; 1 < 1.1 */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); + + fakeMonotimeUs += 1000000; /* advance 1 second, refill bucket */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, true)); /* Force consume 1, 11 should be left*/ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 12.0), 10.0); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 11.0), 0.0); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 11.0, true)); + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); +} + +TEST_F(TokenBucketTest, SetRateChangesRate) { + tokenBucket_setRate(bucket, 200.0); + EXPECT_DOUBLE_EQ(tokenBucket_getRate(bucket), 200.0); + + fakeMonotimeUs += 1000000; /* replenish caps at the new 22 */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 22.0, false)); /* larger capacity is reachable */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); /* now empty */ + + fakeMonotimeUs += 1000000; + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); /* replenish to 22, 21 left */ + tokenBucket_setRate(bucket, 10.0); + EXPECT_DOUBLE_EQ(tokenBucket_getRate(bucket), 10.0); + /* only 3 remain after trim */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 21.0, false)); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 3.0, false)); + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); /* now empty */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 100.0); +} + +TEST_F(TokenBucketTest, SetRateSettlesElapsedAtOldRate) { + /* setRate must credit elapsed time at the OLD rate before switching. */ + tokenBucket *b = tokenBucket_create(1.0, 10.0); + EXPECT_TRUE(tokenBucket_tryConsume(b, 12.0, false)); /* drain to empty */ + EXPECT_FALSE(tokenBucket_tryConsume(b, 0.5, false)); + + fakeMonotimeUs += 5000000; /* 5 tokens should accrue */ + + tokenBucket_setRate(b, 1000.0); /* the 5 elapsed seconds belong to the OLD rate */ + + EXPECT_FALSE(tokenBucket_tryConsume(b, 6.0, false)); /* only 5 available, not thousands */ + EXPECT_TRUE(tokenBucket_tryConsume(b, 5.0, false)); + + /* Confirm the NEW rate now governs accrual: at 1000/s, 10ms yields ~10 tokens */ + fakeMonotimeUs += 10000; + EXPECT_TRUE(tokenBucket_tryConsume(b, 10.0, false)); + EXPECT_FALSE(tokenBucket_tryConsume(b, 0.1, false)); /* now empty */ + + tokenBucket_free(b); +} diff --git a/src/unit/test_util.cpp b/src/unit/test_util.cpp index 2348a4a05..2e3662eb0 100644 --- a/src/unit/test_util.cpp +++ b/src/unit/test_util.cpp @@ -195,7 +195,7 @@ TEST_F(UtilTest, TestLd2string) { long double v; int sz; - v = 0.0 / 0.0; + v = 0.0L / 0.0L; sz = ld2string(buf, sizeof(buf), v, LD_STR_AUTO); ASSERT_EQ(sz, 3); ASSERT_TRUE(!strcmp(buf, "nan")); @@ -312,10 +312,10 @@ TEST_F(UtilTest, TestReclaimFilePageCache) { #if defined(__linux__) struct statfs stats; - /* Check if /tmp is memory-backed (e.g., tmpfs) */ + /* fadvise(FADV_DONTNEED) has no effect on memory-backed filesystems */ if (statfs("/tmp", &stats) == 0) { - if (stats.f_type != TMPFS_MAGIC) { // Not tmpfs, use /tmp - GTEST_SKIP() << "Skipping test because /tmp is not tmpfs"; + if (stats.f_type == TMPFS_MAGIC) { + GTEST_SKIP() << "Skipping test because /tmp is tmpfs"; } } diff --git a/src/unit/test_vset.cpp b/src/unit/test_vset.cpp index 194153b04..3c97469d5 100644 --- a/src/unit/test_vset.cpp +++ b/src/unit/test_vset.cpp @@ -204,6 +204,34 @@ TEST_F(VsetTest, TestVsetAddAndIterate) { mockFreeEntry(e2); } +TEST_F(VsetTest, TestVsetGetSize) { + vset set; + vsetInit(&set); + + ASSERT_EQ(vsetSize(&set), (size_t)0); + + const long long expiry_time = 1000LL; + const size_t total_entries = 250; + + mock_entry **entries = (mock_entry **)zmalloc(sizeof(mock_entry *) * total_entries); + ASSERT_NE(entries, nullptr); + + for (size_t i = 0; i < total_entries; i++) { + char key_buf[32]; + snprintf(key_buf, sizeof(key_buf), "entry_%zu", i); + entries[i] = mockCreateEntry(key_buf, expiry_time); + ASSERT_TRUE(vsetAddEntry(&set, mockGetExpiry, entries[i])); + } + + ASSERT_FALSE(vsetIsEmpty(&set)); + + ASSERT_EQ(vsetSize(&set), total_entries); + + vsetRelease(&set); + for (size_t i = 0; i < total_entries; i++) mockFreeEntry(entries[i]); + zfree(entries); +} + /* Exercises vsetEstimatedEarliestExpiry() across every reachable bucket * encoding: NONE, SINGLE, VECTOR, and RAX (both a single time-bucket and * multiple time-buckets). diff --git a/src/unit/test_zmalloc.cpp b/src/unit/test_zmalloc.cpp index 8b987098b..77ccbfc6b 100644 --- a/src/unit/test_zmalloc.cpp +++ b/src/unit/test_zmalloc.cpp @@ -66,3 +66,30 @@ TEST_F(ZmallocTest, TestZmallocCacheAlignedAllocAndFree) { ASSERT_GE(usable_size, 123u); ASSERT_EQ(zmalloc_used_memory(), used_memory_before); } + +TEST_F(ZmallocTest, TestZmallocExternalUsedMemory) { + size_t used_memory_before = zmalloc_used_memory(); + size_t external_memory_before = zmalloc_used_external_memory(); + + ASSERT_EQ(zmalloc_increase_used_memory_external(123), 0); + ASSERT_EQ(zmalloc_used_external_memory(), external_memory_before + 123); + ASSERT_EQ(zmalloc_used_memory(), used_memory_before + 123); + + ASSERT_EQ(zmalloc_decrease_used_memory_external(123), 0); + ASSERT_EQ(zmalloc_used_external_memory(), external_memory_before); + ASSERT_EQ(zmalloc_used_memory(), used_memory_before); +} + +TEST_F(ZmallocTest, TestZmallocExternalUsedMemoryBounds) { + size_t external_memory_before = zmalloc_used_external_memory(); + + ASSERT_EQ(zmalloc_decrease_used_memory_external(external_memory_before + 1), -1); + if (external_memory_before == 0) { + ASSERT_EQ(zmalloc_increase_used_memory_external(1), 0); + ASSERT_EQ(zmalloc_increase_used_memory_external(SIZE_MAX), -1); + ASSERT_EQ(zmalloc_decrease_used_memory_external(1), 0); + } else { + ASSERT_EQ(zmalloc_increase_used_memory_external(SIZE_MAX - external_memory_before + 1), -1); + } + ASSERT_EQ(zmalloc_used_external_memory(), external_memory_before); +} diff --git a/src/unit/wrappers.h b/src/unit/wrappers.h index 72fcf381b..3d291392e 100644 --- a/src/unit/wrappers.h +++ b/src/unit/wrappers.h @@ -48,6 +48,9 @@ extern "C" { #include "ae.h" #include "compression.h" #include "server.h" +#include "stat_calc.h" +#include "throttle.h" +#include "throttle_token_bucket.h" /** * The list of wrapper methods defined. Each wrapper method must @@ -63,8 +66,38 @@ extern "C" { * Example: serverLog(int level, const char *fmt, ...) should NOT be mocked. */ long long __wrap_aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, aeTimeProc *proc, void *clientData, aeEventFinalizerProc *finalizerProc); +int __wrap_aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id); +size_t __wrap_getClientOutputBufferMemoryUsage(client *c); +int __wrap_getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level); +void __wrap_queueClientForReprocessing(client *c); +int __wrap_freeClient(client *c); ssize_t __wrap_streamDecompressorFeed(streamDecompressor *decompressor, uint8_t *output, size_t output_capacity, const uint8_t *input, size_t input_len, size_t *input_consumed); void __wrap_zmadvise_dontneed(void *ptr, size_t size_hint); +int __wrap_processPendingCommandAndInputBuffer(client *c); +void __wrap_beforeNextClient(client *c); + +void __wrap_blockClientInUseOnKeys(client *c, int nKeys, robj **keys); +void __wrap_unblockClientsInUseOnKey(robj *key); + +int __wrap_ACLCheckAllUserCommandPerm(user *u, struct serverCommand *cmd, robj **argv, int argc, int dbid, int *idxptr); + +size_t __wrap_hashtableScan(hashtable *ht, size_t cursor, hashtableScanFunction fn, void *privdata); +bool __wrap_hashtableScanHasPassedKey(hashtable *ht, const void *key, size_t cursor); + +/* Throttler mocks */ +throttler *__wrap_throttle_register(throttleCriteriaProc *criteria_proc, void *priv_data, const char *metrics_name); +void __wrap_throttle_deregister(throttler *t); +double __wrap_throttle_adjustRate(throttler *t, double multiplier); +void __wrap_throttle_getMetrics(const char *metrics_name, throttleMetrics *metrics); +long __wrap_throttle_getGuardrailSecs(throttler *t); + +/* Token bucket mocks */ +bool __wrap_tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consume); + +/* Statcalc mocks */ +double __wrap_tpsCalculator_averageTps(tpsCalculator *calc); +double __wrap_trendCalculator_changePerSecShortTerm(trendCalculator *calc); + #undef protected #undef _Bool #undef typename diff --git a/src/unix.c b/src/unix.c index e5db7cbb9..e2b4ec656 100644 --- a/src/unix.c +++ b/src/unix.c @@ -214,6 +214,7 @@ static ConnectionType CT_Unix = { /* Miscellaneous */ .connIntegrityChecked = NULL, + .is_closing = NULL, }; int RedisRegisterConnectionTypeUnix(void) { diff --git a/src/util.c b/src/util.c index b989a06c2..6df54cca9 100644 --- a/src/util.c +++ b/src/util.c @@ -1667,3 +1667,21 @@ uint64_t wangHash64(uint64_t hash) { hash = hash + (hash << 31); return hash; } + +/* Lock-free, per-thread Bernoulli sampler: advances a thread-local xorshift64* + * PRNG and returns 1 with probability `percentage`/100, using only a multiply + + * compare (no modulo, and no glibc rand() internal lock) so it stays cheap on + * hot paths that call it for every event. */ +int bernoulliSampleHit(int percentage) { + static __thread uint64_t s = 0; + if (s == 0) { + s = (uint64_t)(uintptr_t)&s ^ 0x9E3779B97F4A7C15ULL; /* seed from stack address; multiplier from Vigna's xorshift64* paper */ + s |= 1; /* xorshift64 needs a nonzero state */ + } + s ^= s >> 12; + s ^= s << 25; + s ^= s >> 27; + uint32_t r = (uint32_t)((s * 0x2545F4914F6CDD1DULL) >> 32); /* multiplier from Vigna's xorshift64* paper */ + /* r/2^32 < pct/100 <=> r*100 < pct*2^32 (no division). */ + return (uint64_t)r * 100 < ((uint64_t)percentage << 32); +} diff --git a/src/util.h b/src/util.h index c82773ed1..5e3706439 100644 --- a/src/util.h +++ b/src/util.h @@ -124,5 +124,6 @@ mstime_t mstime(void); void writePointerWithPadding(unsigned char *buf, const void *ptr); sds escapeJsonString(sds s, const char *p, size_t len); uint64_t wangHash64(uint64_t hash); +int bernoulliSampleHit(int percentage); #endif diff --git a/src/valkey-benchmark.c b/src/valkey-benchmark.c index 1d595ae45..356ddb462 100644 --- a/src/valkey-benchmark.c +++ b/src/valkey-benchmark.c @@ -215,6 +215,13 @@ typedef struct benchmarkThread { pthread_t thread; aeEventLoop *el; list *paused_clients; + /* Per-thread latency histograms: recording into one shared histogram + * with hdr_record_value_atomic makes every command from every thread + * contend on the same counter cache lines. Each thread records into + * its own histograms lock-free; thread 0 folds them for live display + * and the main thread folds them after join for the final report. */ + struct hdr_histogram *latency_histogram; + struct hdr_histogram *current_sec_latency_histogram; } benchmarkThread; /* Cluster. */ @@ -482,6 +489,48 @@ void initPlaceholders(const char *cmd, size_t cmd_len) { return; } +/* Batched per-thread accounting for config.requests_finished. A relaxed + * fetch_add per completed command from every thread turns the counter's + * cache line into a process-wide contention point at high rates (the same + * failure mode as the shared latency histogram). Each thread accumulates + * locally and publishes one fetch_add per REQUESTS_FINISHED_FLUSH_BATCH + * completions, so the line is written at ~rps/batch instead of ~rps; the + * per-command read below then almost always hits a Shared cached copy. + * + * The residue is flushed from each thread's showThroughput timer (so + * count-mode termination, detected from the global value, cannot stall on + * unpublished counts) and after aeMain() returns (so the final report is + * exact). Consequences of the batching: termination detection and the + * "stop recording latency at the end" gate can lag by up to + * batch * num_threads commands, and a warmup-boundary reset can carry over + * a residue of the same magnitude -- both are the same benign-race class + * as the surrounding relaxed counter resets. The warmup carry-over is an + * intentional, documented trade-off: with --threads and --warmup, completion + * accounting around the warmup boundary is exact only to within + * batch * num_threads (an exact boundary would reintroduce cross-thread + * coordination on the completion path -- the contention this design + * removes). Runs whose -n is small enough for this to matter are far below + * any meaningful measurement size; the limitation is stated in the --warmup + * help text. Single-threaded mode keeps the exact per-command path. */ +#define REQUESTS_FINISHED_FLUSH_BATCH 256 +static _Thread_local int pending_requests_finished = 0; + +static void flushRequestsFinished(void) { + if (pending_requests_finished > 0) { + atomic_fetch_add_explicit(&config.requests_finished, pending_requests_finished, memory_order_relaxed); + pending_requests_finished = 0; + } +} + +static int addRequestFinished(void) { + if (config.num_threads == 0) { + return atomic_fetch_add_explicit(&config.requests_finished, 1, memory_order_relaxed); + } + pending_requests_finished++; + if (pending_requests_finished >= REQUESTS_FINISHED_FLUSH_BATCH) flushRequestsFinished(); + return atomic_load_explicit(&config.requests_finished, memory_order_relaxed) + pending_requests_finished; +} + static void replacePlaceholder(const size_t *indices, const size_t count, char *cmd, _Atomic uint64_t *key_counter) { if (count == 0) return; @@ -789,7 +838,7 @@ static void readHandler(aeEventLoop *el, int fd, void *privdata, int mask) { } continue; } - int requests_finished = atomic_fetch_add_explicit(&config.requests_finished, 1, memory_order_relaxed); + int requests_finished = addRequestFinished(); if (!isBenchmarkFinished(requests_finished)) { if (config.num_threads == 0) { hdr_record_value(config.latency_histogram, // Histogram to record to @@ -801,14 +850,15 @@ static void readHandler(aeEventLoop *el, int fd, void *privdata, int mask) { ? (long)c->latency : CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE); // Value to record } else { - hdr_record_value_atomic(config.latency_histogram, // Histogram to record to - (long)c->latency <= CONFIG_LATENCY_HISTOGRAM_MAX_VALUE - ? (long)c->latency - : CONFIG_LATENCY_HISTOGRAM_MAX_VALUE); // Value to record - hdr_record_value_atomic(config.current_sec_latency_histogram, // Histogram to record to - (long)c->latency <= CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE - ? (long)c->latency - : CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE); // Value to record + benchmarkThread *thread = config.threads[c->thread_id]; + hdr_record_value(thread->latency_histogram, // Histogram to record to + (long)c->latency <= CONFIG_LATENCY_HISTOGRAM_MAX_VALUE + ? (long)c->latency + : CONFIG_LATENCY_HISTOGRAM_MAX_VALUE); // Value to record + hdr_record_value(thread->current_sec_latency_histogram, // Histogram to record to + (long)c->latency <= CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE + ? (long)c->latency + : CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE); // Value to record } } c->pending--; @@ -1295,6 +1345,13 @@ static void startBenchmarkThreads(void) { } } for (i = 0; i < config.num_threads; i++) pthread_join(config.threads[i]->thread, NULL); + /* All threads have stopped: fold the per-thread latency histograms into + * the global histogram for exact final reporting. Reset it first -- + * showThroughput may have populated it with approximate live merges. */ + hdr_reset(config.latency_histogram); + for (i = 0; i < config.num_threads; i++) { + hdr_add(config.latency_histogram, config.threads[i]->latency_histogram); + } } /* Benchmark a sequence of commands. The cmd is RESP encoded of length len and @@ -1364,12 +1421,22 @@ static benchmarkThread *createBenchmarkThread(int index) { thread->index = index; thread->el = aeCreateEventLoop(1024 * 10); thread->paused_clients = listCreate(); + hdr_init(CONFIG_LATENCY_HISTOGRAM_MIN_VALUE, // Minimum value + CONFIG_LATENCY_HISTOGRAM_MAX_VALUE, // Maximum value + config.precision, // Number of significant figures + &thread->latency_histogram); // Pointer to initialise + hdr_init(CONFIG_LATENCY_HISTOGRAM_MIN_VALUE, // Minimum value + CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE, // Maximum value + config.precision, // Number of significant figures + &thread->current_sec_latency_histogram); // Pointer to initialise aeCreateTimeEvent(thread->el, 1, showThroughput, (void *)thread, NULL); return thread; } static void freeBenchmarkThread(benchmarkThread *thread) { if (thread->el) aeDeleteEventLoop(thread->el); + if (thread->latency_histogram) hdr_close(thread->latency_histogram); + if (thread->current_sec_latency_histogram) hdr_close(thread->current_sec_latency_histogram); listRelease(thread->paused_clients); zfree(thread); } @@ -1387,6 +1454,9 @@ static void freeBenchmarkThreads(void) { static void *execBenchmarkThread(void *ptr) { benchmarkThread *thread = (benchmarkThread *)ptr; aeMain(thread->el); + /* Publish any batched completion residue: the final report reads + * config.requests_finished after all threads are joined. */ + flushRequestsFinished(); return NULL; } @@ -2039,7 +2109,10 @@ int parseOptions(int argc, char **argv) { " Run benchmark for specified number of seconds\n" " (mutually exclusive with -n)\n" " --warmup Run benchmark for specified warmup period before\n" - " recording data\n" + " recording data. With --threads, completed-request\n" + " accounting is batched per thread, so a small number\n" + " of pre-warmup completions may carry into the counted\n" + " total at the warmup boundary.\n" " -d Data size of SET/GET value in bytes (default 3)\n" " --dbnum SELECT the specified db number (default 0)\n" " -3 Start session in RESP3 protocol mode.\n" @@ -2095,6 +2168,9 @@ int parseOptions(int argc, char **argv) { " --rps Limit the total number of requests per second.\n" " Default 0 (no limit)\n" " --seed Set the seed for random number generator.\n" + " With --threads, per-thread streams are seeded\n" + " deterministically but their order of first use\n" + " depends on thread scheduling.\n" " Default seed is based on time.\n" " --num-functions \n" " Sets the number of functions present in the Lua lib that is\n" @@ -2142,6 +2218,9 @@ long long showThroughput(struct aeEventLoop *eventLoop, long long id, void *clie UNUSED(eventLoop); UNUSED(id); benchmarkThread *thread = (benchmarkThread *)clientData; + /* Publish this thread's batched completion count so global termination + * detection and the displayed totals stay fresh (see addRequestFinished). */ + flushRequestsFinished(); int requests_finished = atomic_load_explicit(&config.requests_finished, memory_order_relaxed); int previous_requests_finished = atomic_load_explicit(&config.previous_requests_finished, memory_order_relaxed); long long current_tick = mstime(); @@ -2162,6 +2241,13 @@ long long showThroughput(struct aeEventLoop *eventLoop, long long id, void *clie atomic_store_explicit(&config.requests_issued, 0, memory_order_relaxed); atomic_store_explicit(&config.previous_requests_finished, 0, memory_order_relaxed); hdr_reset(config.latency_histogram); + /* Per-thread histograms carry the recorded values now; clear them + * too. Concurrent recording may leak a warmup sample into the + * results -- same benign-race class as the counter resets above. */ + for (int t = 0; t < config.num_threads; t++) { + hdr_reset(config.threads[t]->latency_histogram); + hdr_reset(config.threads[t]->current_sec_latency_histogram); + } } } else if (isBenchmarkFinished(requests_finished)) { aeStop(eventLoop); @@ -2182,6 +2268,18 @@ long long showThroughput(struct aeEventLoop *eventLoop, long long id, void *clie fflush(stdout); return SHOW_THROUGHPUT_INTERVAL; } + if (config.num_threads) { + /* Use thread-0's own histograms for the live display line. This is + * race-free (showThroughput runs on thread-0's event loop, same thread + * that records into these histograms) and representative under uniform + * workloads. The final report folds all threads exactly after join. */ + benchmarkThread *t0 = config.threads[0]; + hdr_reset(config.latency_histogram); + hdr_add(config.latency_histogram, t0->latency_histogram); + hdr_reset(config.current_sec_latency_histogram); + hdr_add(config.current_sec_latency_histogram, t0->current_sec_latency_histogram); + hdr_reset(t0->current_sec_latency_histogram); + } const float dt = (float)(current_tick - config.start) / 1000.0; const float rps = (float)requests_finished / dt; const float instantaneous_dt = (float)(current_tick - config.previous_tick) / 1000.0; diff --git a/src/valkeymodule.h b/src/valkeymodule.h index 3f909e261..41b3a1674 100644 --- a/src/valkeymodule.h +++ b/src/valkeymodule.h @@ -342,10 +342,18 @@ typedef uint64_t ValkeyModuleTimerID; * slot migration must be used. */ #define VALKEYMODULE_OPTIONS_HANDLE_ATOMIC_SLOT_MIGRATION (1 << 5) +/* Declare that the module handles forkless operations. Opting in has a + * tradeoff: while a forkless operation is running, opening a key for write can + * return NULL if the key is currently in use, and the module must handle that + * NULL return. A module that registers a data type also declares that its RDB + * save callbacks are safe to run on a background thread. When any loaded module + * does not set this, forkless operations are blocked. */ +#define VALKEYMODULE_OPTIONS_HANDLE_FORKLESS (1 << 6) + /* Next option flag, must be updated when adding new module flags above! * This flag should not be used directly by the module. * Use ValkeyModule_GetModuleOptionsAll instead. */ -#define _VALKEYMODULE_OPTIONS_FLAGS_NEXT (1 << 6) +#define _VALKEYMODULE_OPTIONS_FLAGS_NEXT (1 << 7) /* Definitions for ValkeyModule_SetCommandInfo. */ @@ -1504,6 +1512,12 @@ typedef void (*ValkeyModuleScanKeyCB)(ValkeyModuleKey *key, ValkeyModuleString *field, ValkeyModuleString *value, void *privdata); +typedef void (*ValkeyModuleScanKeyRawBorrowedCB)(ValkeyModuleKey *key, + const char *field, + size_t field_len, + const char *value, + size_t value_len, + void *privdata); typedef ValkeyModuleString *(*ValkeyModuleConfigGetStringFunc)(const char *name, void *privdata); typedef long long (*ValkeyModuleConfigGetNumericFunc)(const char *name, void *privdata); typedef unsigned long long (*ValkeyModuleConfigGetUnsignedNumericFunc)(const char *name, void *privdata); @@ -1572,6 +1586,8 @@ VALKEYMODULE_API void (*ValkeyModule_Free)(void *ptr) VALKEYMODULE_ATTR; VALKEYMODULE_API void *(*ValkeyModule_Calloc)(size_t nmemb, size_t size)VALKEYMODULE_ATTR; VALKEYMODULE_API void *(*ValkeyModule_TryCalloc)(size_t nmemb, size_t size)VALKEYMODULE_ATTR; VALKEYMODULE_API char *(*ValkeyModule_Strdup)(const char *str)VALKEYMODULE_ATTR; +VALKEYMODULE_API int (*ValkeyModule_IncrExternalMemory)(size_t bytes) VALKEYMODULE_ATTR; +VALKEYMODULE_API int (*ValkeyModule_DecrExternalMemory)(size_t bytes) VALKEYMODULE_ATTR; VALKEYMODULE_API int (*ValkeyModule_GetApi)(const char *, void *) VALKEYMODULE_ATTR; VALKEYMODULE_API int (*ValkeyModule_CreateCommand)(ValkeyModuleCtx *ctx, const char *name, @@ -2041,6 +2057,10 @@ VALKEYMODULE_API int (*ValkeyModule_ScanKey)(ValkeyModuleKey *key, ValkeyModuleScanCursor *cursor, ValkeyModuleScanKeyCB fn, void *privdata) VALKEYMODULE_ATTR; +VALKEYMODULE_API int (*ValkeyModule_ScanKeyRawBorrowed)(ValkeyModuleKey *key, + ValkeyModuleScanCursor *cursor, + ValkeyModuleScanKeyRawBorrowedCB fn, + void *privdata) VALKEYMODULE_ATTR; VALKEYMODULE_API int (*ValkeyModule_GetContextFlagsAll)(void) VALKEYMODULE_ATTR; VALKEYMODULE_API int (*ValkeyModule_GetModuleOptionsAll)(void) VALKEYMODULE_ATTR; VALKEYMODULE_API int (*ValkeyModule_GetKeyspaceNotificationFlagsAll)(void) VALKEYMODULE_ATTR; @@ -2167,7 +2187,7 @@ VALKEYMODULE_API size_t (*ValkeyModule_MallocUsableSize)(void *ptr) VALKEYMODULE VALKEYMODULE_API size_t (*ValkeyModule_MallocSizeString)(ValkeyModuleString *str) VALKEYMODULE_ATTR; VALKEYMODULE_API size_t (*ValkeyModule_MallocSizeDict)(ValkeyModuleDict *dict) VALKEYMODULE_ATTR; VALKEYMODULE_API ValkeyModuleUser *(*ValkeyModule_CreateModuleUser)(const char *name)VALKEYMODULE_ATTR; -VALKEYMODULE_API void (*ValkeyModule_FreeModuleUser)(ValkeyModuleUser *user) VALKEYMODULE_ATTR; +VALKEYMODULE_API int (*ValkeyModule_FreeModuleUser)(ValkeyModuleUser *user) VALKEYMODULE_ATTR; VALKEYMODULE_API void (*ValkeyModule_SetContextUser)(ValkeyModuleCtx *ctx, const ValkeyModuleUser *user) VALKEYMODULE_ATTR; VALKEYMODULE_API int (*ValkeyModule_SetModuleUserACL)(ValkeyModuleUser *user, const char *acl) VALKEYMODULE_ATTR; @@ -2192,14 +2212,14 @@ VALKEYMODULE_API int (*ValkeyModule_ACLCheckPermissions)(ValkeyModuleUser *user, int argc, int dbid, ValkeyModuleACLLogEntryReason *denial_reason) VALKEYMODULE_ATTR; -VALKEYMODULE_API void (*ValkeyModule_ACLAddLogEntry)(ValkeyModuleCtx *ctx, - ValkeyModuleUser *user, - ValkeyModuleString *object, - ValkeyModuleACLLogEntryReason reason) VALKEYMODULE_ATTR; -VALKEYMODULE_API void (*ValkeyModule_ACLAddLogEntryByUserName)(ValkeyModuleCtx *ctx, - ValkeyModuleString *user, - ValkeyModuleString *object, - ValkeyModuleACLLogEntryReason reason) VALKEYMODULE_ATTR; +VALKEYMODULE_API int (*ValkeyModule_ACLAddLogEntry)(ValkeyModuleCtx *ctx, + ValkeyModuleUser *user, + ValkeyModuleString *object, + ValkeyModuleACLLogEntryReason reason) VALKEYMODULE_ATTR; +VALKEYMODULE_API int (*ValkeyModule_ACLAddLogEntryByUserName)(ValkeyModuleCtx *ctx, + ValkeyModuleString *user, + ValkeyModuleString *object, + ValkeyModuleACLLogEntryReason reason) VALKEYMODULE_ATTR; VALKEYMODULE_API int (*ValkeyModule_AuthenticateClientWithACLUser)(ValkeyModuleCtx *ctx, const char *name, size_t len, @@ -2342,6 +2362,8 @@ static int ValkeyModule_Init(ValkeyModuleCtx *ctx, const char *name, int ver, in VALKEYMODULE_GET_API(Realloc); VALKEYMODULE_GET_API(TryRealloc); VALKEYMODULE_GET_API(Strdup); + VALKEYMODULE_GET_API(IncrExternalMemory); + VALKEYMODULE_GET_API(DecrExternalMemory); VALKEYMODULE_GET_API(CreateCommand); VALKEYMODULE_GET_API(GetCommand); VALKEYMODULE_GET_API(CreateSubcommand); @@ -2591,6 +2613,7 @@ static int ValkeyModule_Init(ValkeyModuleCtx *ctx, const char *name, int ver, in VALKEYMODULE_GET_API(ScanCursorDestroy); VALKEYMODULE_GET_API(Scan); VALKEYMODULE_GET_API(ScanKey); + VALKEYMODULE_GET_API(ScanKeyRawBorrowed); VALKEYMODULE_GET_API(GetContextFlagsAll); VALKEYMODULE_GET_API(GetModuleOptionsAll); VALKEYMODULE_GET_API(GetKeyspaceNotificationFlagsAll); diff --git a/src/vset.c b/src/vset.c index 5200cc9a2..d5e0a9364 100644 --- a/src/vset.c +++ b/src/vset.c @@ -2266,6 +2266,74 @@ size_t vsetMemUsage(vset *set) { return 0; } +static inline size_t vsetBucketSize_NONE(vsetBucket *bucket) { + UNUSED(bucket); + return 0; +} + +static inline size_t vsetBucketSize_SINGLE(vsetBucket *bucket) { + UNUSED(bucket); + return 1; +} + +static inline size_t vsetBucketSize_VECTOR(vsetBucket *bucket) { + pVector *pv = vsetBucketVector(bucket); + assert(pv); + return pv->len; +} + +static inline size_t vsetBucketSize_HASHTABLE(vsetBucket *bucket) { + hashtable *ht = vsetBucketHashtable(bucket); + return hashtableSize(ht); +} + +static inline size_t vsetBucketSize_RAX(vsetBucket *bucket) { + rax *r = vsetBucketRax(bucket); + size_t numele = 0; + raxIterator it; + raxStart(&it, r); + assert(raxSeek(&it, "^", NULL, 0)); + while (raxNext(&it)) { + switch (vsetBucketType(it.data)) { + case VSET_BUCKET_NONE: + numele += vsetBucketSize_NONE(it.data); + break; + case VSET_BUCKET_SINGLE: + numele += vsetBucketSize_SINGLE(it.data); + break; + case VSET_BUCKET_VECTOR: + numele += vsetBucketSize_VECTOR(it.data); + break; + case VSET_BUCKET_HT: + numele += vsetBucketSize_HASHTABLE(it.data); + break; + default: + panic("Unknown bucket type encountered in vsetBucketSize_RAX"); + } + } + raxStop(&it); + return numele; +} + +size_t vsetSize(vset *set) { + int bucket_type = vsetBucketType(*set); + switch (bucket_type) { + case VSET_BUCKET_NONE: + return vsetBucketSize_NONE(*set); + case VSET_BUCKET_SINGLE: + return vsetBucketSize_SINGLE(*set); + case VSET_BUCKET_VECTOR: + return vsetBucketSize_VECTOR(*set); + case VSET_BUCKET_HT: + panic("Unsupported hashtable bucket type for vset"); + case VSET_BUCKET_RAX: + return vsetBucketSize_RAX(*set); + default: + panic("Unknown set type encountered in vsetSize"); + } + return 0; +} + /* Initializes a volatile set iterator. * * This function prepares the iterator for scanning a volatile set from the beginning. diff --git a/src/vset.h b/src/vset.h index 23270395a..afef32842 100644 --- a/src/vset.h +++ b/src/vset.h @@ -81,6 +81,7 @@ bool vsetAddEntry(vset *set, vsetGetExpiryFunc getExpiry, void *entry); bool vsetRemoveEntry(vset *set, vsetGetExpiryFunc getExpiry, void *entry); bool vsetUpdateEntry(vset *set, vsetGetExpiryFunc getExpiry, void *old_entry, void *new_entry, long long old_expiry, long long new_expiry); bool vsetIsEmpty(vset *set); +size_t vsetSize(vset *set); void vsetInitIterator(vset *set, vsetIterator *it); bool vsetNext(vsetIterator *it, void **entryptr); void vsetResetIterator(vsetIterator *it); diff --git a/src/zmalloc.c b/src/zmalloc.c index bb012c142..e084dc41d 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -114,6 +114,9 @@ static _Atomic size_t *used_memory_thread = &used_memory_thread_padded[PADDING_E static atomic_int total_active_threads = 0; /* This is a simple protection. It's used only if some modules create a lot of threads. */ static atomic_size_t used_memory_for_additional_threads = 0; +/* Memory tracked by modules outside the allocator. Writers are serialized by moduleGIL, + * but readers may sample it without the GIL. */ +static atomic_size_t used_memory_external = 0; /* Register the thread index in start_routine. */ static inline void zmalloc_register_thread_index(void) { @@ -526,9 +529,36 @@ size_t zmalloc_used_memory(void) { for (int i = 0; i < threads_num; i++) { um += used_memory_thread[i]; } + um += atomic_load_explicit(&used_memory_external, memory_order_relaxed); return um; } +size_t zmalloc_used_external_memory(void) { + return atomic_load_explicit(&used_memory_external, memory_order_relaxed); +} + +int zmalloc_increase_used_memory_external(size_t size) { + size_t current = atomic_load_explicit(&used_memory_external, memory_order_relaxed); + while (1) { + size_t next; + if (SIZE_MAX - current < size) return -1; + next = current + size; + if (atomic_compare_exchange_weak_explicit(&used_memory_external, ¤t, next, memory_order_relaxed, memory_order_relaxed)) + return 0; + } +} + +int zmalloc_decrease_used_memory_external(size_t size) { + size_t current = atomic_load_explicit(&used_memory_external, memory_order_relaxed); + while (1) { + size_t next; + if (current < size) return -1; + next = current - size; + if (atomic_compare_exchange_weak_explicit(&used_memory_external, ¤t, next, memory_order_relaxed, memory_order_relaxed)) + return 0; + } +} + void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) { zmalloc_oom_handler = oom_handler; } diff --git a/src/zmalloc.h b/src/zmalloc.h index fafd97894..a44ddfbc4 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -131,6 +131,9 @@ void *ztrycalloc_usable(size_t size, size_t *usable); void *ztryrealloc_usable(void *ptr, size_t size, size_t *usable); __attribute__((malloc)) char *zstrdup(const char *s); size_t zmalloc_used_memory(void); +size_t zmalloc_used_external_memory(void); +int zmalloc_increase_used_memory_external(size_t size); +int zmalloc_decrease_used_memory_external(size_t size); void zmalloc_set_oom_handler(void (*oom_handler)(size_t)); size_t zmalloc_get_rss(void); int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident, size_t *retained, size_t *muzzy); diff --git a/tests/assets/role.acl b/tests/assets/role.acl new file mode 100644 index 000000000..807b4a40f --- /dev/null +++ b/tests/assets/role.acl @@ -0,0 +1,7 @@ +role customer ~* &* +@all -@admin -@dangerous -@scripting +role viewer ~* &* +@read + +user alice on >alice role=customer +user bob on >bob role=viewer +user carol on >carol role=customer +eval +user default on nopass ~* &* +@all role=viewer diff --git a/tests/assets/test_cli_hint_suite.txt b/tests/assets/test_cli_hint_suite.txt index 4b27724f0..2490e19cd 100644 --- a/tests/assets/test_cli_hint_suite.txt +++ b/tests/assets/test_cli_hint_suite.txt @@ -10,6 +10,9 @@ "DECRBY xyz " "decrement" "DECRBY " "key decrement" +# Command with optional arg: INCREX +"INCREX xyz " "[NX|XX] [EX ex|PX px|EXAT unix-time-seconds|PXAT unix-time-milliseconds] [BYINT integer|BYFLOAT float]" + # Command with optional arg: LPOP key [count] "LPOP key " "[count]" "LPOP key 3 " "" @@ -68,17 +71,17 @@ "ZRANGE k 1 2 WITHSCORES " "[BYSCORE|BYLEX] [REV] [LIMIT offset count] [XX]" # Optional one-of args with parameters: SET key value [NX|XX] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL] -"SET key value " "[NX|XX|IFEQ comparison-value] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" -"SET key value EX" "[NX|XX|IFEQ comparison-value] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" -"SET key value EX " "seconds [NX|XX|IFEQ comparison-value] [GET]" -"SET key value EX 23 " "[NX|XX|IFEQ comparison-value] [GET]" -"SET key value EXAT" "[NX|XX|IFEQ comparison-value] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" -"SET key value EXAT " "unix-time-seconds [NX|XX|IFEQ comparison-value] [GET]" -"SET key value PX" "[NX|XX|IFEQ comparison-value] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" -"SET key value PX " "milliseconds [NX|XX|IFEQ comparison-value] [GET]" -"SET key value PXAT" "[NX|XX|IFEQ comparison-value] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" -"SET key value PXAT " "unix-time-milliseconds [NX|XX|IFEQ comparison-value] [GET]" -"SET key value KEEPTTL " "[NX|XX|IFEQ comparison-value] [GET]" +"SET key value " "[NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" +"SET key value EX" "[NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" +"SET key value EX " "seconds [NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET]" +"SET key value EX 23 " "[NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET]" +"SET key value EXAT" "[NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" +"SET key value EXAT " "unix-time-seconds [NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET]" +"SET key value PX" "[NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" +"SET key value PX " "milliseconds [NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET]" +"SET key value PXAT" "[NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" +"SET key value PXAT " "unix-time-milliseconds [NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET]" +"SET key value KEEPTTL " "[NX|XX|IFEQ comparison-value|IFNE comparison-not-equal] [GET]" "SET key value XX " "[GET] [EX seconds|PX milliseconds|EXAT unix-time-seconds|PXAT unix-time-milliseconds|KEEPTTL]" # If an input word can't be matched, stop hinting. diff --git a/tests/helpers/fake_dual_channel_primary.tcl b/tests/helpers/fake_dual_channel_primary.tcl new file mode 100644 index 000000000..090d842a3 --- /dev/null +++ b/tests/helpers/fake_dual_channel_primary.tcl @@ -0,0 +1,86 @@ +# A fake primary for testing errors while a dual-channel replica replays its +# buffered command stream. The main channel negotiates dual-channel sync; the +# RDB channel receives a valid size-framed RDB, while the main channel receives +# the supplied command-stream bytes before the RDB finishes loading. +# +# Usage: tclsh fake_dual_channel_primary.tcl PORT RDB_FILE STREAM_FILE + +set port [lindex $argv 0] +set rdb_file [lindex $argv 1] +set stream_file [lindex $argv 2] + +set fd [open $rdb_file r] +fconfigure $fd -translation binary +set rdb_payload [read $fd] +close $fd + +set fd [open $stream_file r] +fconfigure $fd -translation binary +set stream_payload [read $fd] +close $fd + +array set channel {} +set main_psync_count 0 +set stream_sent 0 +set done 0 + +proc send_bytes {sock bytes} { + puts -nonewline $sock $bytes + flush $sock +} + +proc send_rdb {sock} { + global rdb_payload + catch { + send_bytes $sock "\$[string length $rdb_payload]\r\n$rdb_payload" + } + catch {close $sock} +} + +proc read_command {sock} { + global channel main_psync_count stream_payload stream_sent server_socket done + + while {[gets $sock line] >= 0} { + set command [string toupper [string trim $line]] + if {$command eq "PING"} { + set channel($sock) main + send_bytes $sock "+PONG\r\n" + } elseif {$command eq "REPLCONF"} { + send_bytes $sock "+OK\r\n" + } elseif {$command eq "PSYNC"} { + set channel($sock) main + incr main_psync_count + if {$main_psync_count == 1} { + send_bytes $sock "+DUALCHANNELSYNC\r\n" + } else { + send_bytes $sock "+CONTINUE [string repeat 0 40]\r\n$stream_payload" + set stream_sent 1 + catch {close $server_socket} + } + } elseif {$command eq "SYNC"} { + set channel($sock) rdb + send_bytes $sock "\$ENDOFF:0 [string repeat 0 40] 0 1\r\n" + after 50 [list send_rdb $sock] + } + } + + if {[eof $sock]} { + set was_main [expr {[info exists channel($sock)] && $channel($sock) eq "main"}] + catch {close $sock} + catch {unset channel($sock)} + if {$was_main && $stream_sent} { + set done served + } + } +} + +proc accept {sock host port} { + global channel + fconfigure $sock -translation binary -blocking 0 -buffering none + set channel($sock) unknown + fileevent $sock readable [list read_command $sock] +} + +set server_socket [socket -server accept $port] +after 60000 set done timeout +vwait done diff --git a/tests/helpers/fake_primary.tcl b/tests/helpers/fake_primary.tcl new file mode 100644 index 000000000..6387d842f --- /dev/null +++ b/tests/helpers/fake_primary.tcl @@ -0,0 +1,69 @@ +# A single-shot fake primary for replication negative tests. Answers the +# replication handshake (PING -> +PONG, REPLCONF -> +OK each, PSYNC -> +# +FULLRESYNC), announces a bulk transfer of ANNOUNCE_SIZE bytes, sends the +# contents of PAYLOAD_FILE, then closes the connection and exits. When +# STREAM_FILE is provided, its contents are sent after the RDB and the +# connection stays open until the replica disconnects. +# +# Usage: tclsh fake_primary.tcl PORT PAYLOAD_FILE ANNOUNCE_SIZE ?STREAM_FILE? + +set port [lindex $argv 0] +set payload_file [lindex $argv 1] +set announce_size [lindex $argv 2] +set stream_file [lindex $argv 3] + +set fd [open $payload_file r] +fconfigure $fd -translation binary +set payload [read $fd] +close $fd + +set stream_payload "" +if {$stream_file ne ""} { + set fd [open $stream_file r] + fconfigure $fd -translation binary + set stream_payload [read $fd] + close $fd +} + +# The replica sends RESP-encoded commands. Reading line by line and replying +# once per command-name line keeps replies in step with pipelined commands; +# RESP framing lines (*N, $N) and argument lines fall through unmatched. +proc accept {sock host port} { + global payload announce_size stream_payload done + fconfigure $sock -translation binary -blocking 1 + set served 0 + catch { + while {[gets $sock line] >= 0} { + set cmd [string toupper [string trim $line]] + if {$cmd eq "PING"} { + puts -nonewline $sock "+PONG\r\n" + flush $sock + } elseif {$cmd eq "REPLCONF"} { + puts -nonewline $sock "+OK\r\n" + flush $sock + } elseif {$cmd eq "PSYNC"} { + puts -nonewline $sock "+FULLRESYNC [string repeat 0 40] 0\r\n" + puts -nonewline $sock "\$$announce_size\r\n" + puts -nonewline $sock $payload + flush $sock + if {$stream_payload ne ""} { + puts -nonewline $sock $stream_payload + flush $sock + while {![eof $sock]} { + read $sock 4096 + } + } + set served 1 + break + } + } + } + catch {close $sock} + # Port-probe connections come and go without a PSYNC; only a served + # transfer completes the single shot. + if {$served} {set done served} +} + +socket -server accept $port +after 60000 set done timeout +vwait done diff --git a/tests/integration/cross-version-replication.tcl b/tests/integration/cross-version-replication.tcl index 0e3de03de..48ee1d89c 100644 --- a/tests/integration/cross-version-replication.tcl +++ b/tests/integration/cross-version-replication.tcl @@ -87,4 +87,48 @@ start_server {tags {"repl needs:other-server external:skip"}} { assert_equal value1 [$old_replica hget hfe field1] } } + + test "XACKDEL replicates as equivalent pre-9.2 commands XACK/XDEL for backwards compatibility" { + if {[version_greater_or_equal $old_replica_version 9.2.0]} { + skip "Replica $old_replica_version must be before 9.2.0 for this test" + } + + r FLUSHALL + r XADD mystream 1-0 hello world + r XGROUP CREATE mystream grp1 0 + r XGROUP CREATE mystream grp2 0 + r XREADGROUP GROUP grp1 alice COUNT 1 STREAMS mystream > + r XREADGROUP GROUP grp2 bob COUNT 1 STREAMS mystream > + r XACKDEL mystream grp1 DELREF IDS 1 1-0 + start_server {start-other-server 1 config "minimal.conf"} { + set old_replica [srv 0 client] + $old_replica replicaof $primary_host $primary_port + wait_for_sync $old_replica 500 100 + assert_equal [llength [$old_replica XRANGE mystream - +]] 0 + assert_equal [llength [$old_replica XPENDING mystream grp1 - + 10]] 0 + assert_equal [llength [$old_replica XPENDING mystream grp2 - + 10]] 0 + } + } + + test "XDELEX replicates as equivalent pre-9.2 commands XACK/XDEL for backwards compatibility" { + if {[version_greater_or_equal $old_replica_version 9.2.0]} { + skip "Replica $old_replica_version must be before 9.2.0 for this test" + } + + r FLUSHALL + r XADD mystream 1-0 hello world + r XGROUP CREATE mystream grp1 0 + r XGROUP CREATE mystream grp2 0 + r XREADGROUP GROUP grp1 alice COUNT 1 STREAMS mystream > + r XREADGROUP GROUP grp2 bob COUNT 1 STREAMS mystream > + r XDELEX mystream DELREF IDS 1 1-0 + start_server {start-other-server 1 config "minimal.conf"} { + set old_replica [srv 0 client] + $old_replica replicaof $primary_host $primary_port + wait_for_sync $old_replica 500 100 + assert_equal [llength [$old_replica XRANGE mystream - +]] 0 + assert_equal [llength [$old_replica XPENDING mystream grp1 - + 10]] 0 + assert_equal [llength [$old_replica XPENDING mystream grp2 - + 10]] 0 + } + } } diff --git a/tests/integration/dual-channel-replication.tcl b/tests/integration/dual-channel-replication.tcl index 2674a9d74..fdd374b0f 100644 --- a/tests/integration/dual-channel-replication.tcl +++ b/tests/integration/dual-channel-replication.tcl @@ -827,19 +827,27 @@ start_server {tags {"dual-channel-replication external:skip"}} { resume_process $replica_pid set res [wait_for_log_messages -1 {"*Unable to partial resync with replica * for lack of backlog*"} $loglines 200 100] set loglines [lindex $res 1] + + # Waiting for the primary to enter the paused state, that is, make sure that bgsave is triggered. + wait_process_paused [srv -1 pid] + wait_for_log_messages 0 {"*Done loading RDB*"} $replica_loglines 5000 10 + $replica replicaof no one + # Resume the primary and make sure the sync is dropped. + resume_process [srv -1 pid] + $primary debug pause-after-fork 0 + wait_for_condition 500 1000 { + [s -1 rdb_bgsave_in_progress] eq 0 + } else { + fail "Primary should abort sync" + } } - # Waiting for the primary to enter the paused state, that is, make sure that bgsave is triggered. - wait_process_paused [srv -1 pid] - wait_for_log_messages 0 {"*Done loading RDB*"} $replica_loglines 5000 10 - $replica replicaof no one - # Resume the primary and make sure the sync is dropped. + # On failure the test skips its own resume_process, leaving the primary armed with + # pause-after-fork. It then stops itself when the fork lands and blocks the commands below, + # which have no read timeout. Resume, disarm, resume again in case the fork landed in + # between. No-ops when the test passed. resume_process [srv -1 pid] $primary debug pause-after-fork 0 - wait_for_condition 500 1000 { - [s -1 rdb_bgsave_in_progress] eq 0 - } else { - fail "Primary should abort sync" - } + resume_process [srv -1 pid] stop_write_load $load_handle0 stop_write_load $load_handle1 stop_write_load $load_handle2 @@ -1623,14 +1631,15 @@ start_server {tags {"dual-channel-replication external:skip"}} { $primary config set repl-diskless-sync-delay 0 $replica config set dual-channel-replication-enabled yes - # A hash with field-level TTLs (HEXPIRE) is hashtable-encoded with - # volatile fields, which can only be serialized in RDB version >= 80. + # A small hash with field-level TTLs (HEXPIRE) keeps the listpack + # encoding with tagged expiry metadata, which can only be serialized + # in RDB version >= 81 (or as HASH_2 triplets for RDB 80 targets). # The primary must learn the replica's version over the RDB connection # to pick a new enough RDB version; otherwise it falls back to RDB 11 # and the full sync fails with "Can't store key ... in RDB version 11". $primary hset myhash field1 value1 field2 value2 field3 value3 $primary hexpire myhash 3600 FIELDS 3 field1 field2 field3 - assert_encoding hashtable myhash + assert_encoding listpack myhash test "Dual channel full sync succeeds with hash field expiration data" { set sync_full [s 0 sync_full] diff --git a/tests/integration/rdb-compression.tcl b/tests/integration/rdb-compression.tcl index 4574abc21..d548a3905 100644 --- a/tests/integration/rdb-compression.tcl +++ b/tests/integration/rdb-compression.tcl @@ -349,6 +349,8 @@ start_server {tags {"rdb-compression repl external:skip"} overrides {save ""}} { $primary config set repl-diskless-sync-delay 0 $replica config set repl-diskless-load swapdb + # Keep the replica non-capable so this covers the cohort downgrade. + $replica config set rdbcompression no foreach diskless {no yes} { $replica replicaof no one diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index b312965e9..c74ea9209 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -224,96 +224,1187 @@ start_server_and_kill_it [list "dir" $server_path] { } } -start_server {} { - test {Test FLUSHALL aborts bgsave} { - r config set save "" - # 5000 keys with 1ms sleep per key should take 5 second - r config set rdb-key-save-delay 1000 - populate 5000 - assert_lessthan 999 [s rdb_changes_since_last_save] +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + foreach bgsave_type {"fork" "forkless"} { + test "Test FLUSHALL aborts bgsave $bgsave_type" { + # 5000 keys with 1ms sleep per key should take 5 second + r config set rdb-key-save-delay 1000 + populate 5000 + assert_lessthan 999 [s rdb_changes_since_last_save] + r config set bgsave-default-method $bgsave_type + r bgsave + assert_equal [s rdb_bgsave_in_progress] 1 + + # Verify we're testing the right save type while it's running + set expected_type [expr {$bgsave_type eq "forkless" ? "forkless" : "fork"}] + assert_equal [s rdb_current_bgsave_type] $expected_type + + # Use this opportunity to also test the "bad arg" reply. + assert_error {ERR*} {r flushall bad_arg} + assert_equal [r ping] "PONG" + + r flushall + # wait a second max (bgsave should take 5) + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 0 + } else { + fail "bgsave not aborted" + } + # verify that bgsave failed, by checking that the change counter is still high + assert_lessthan 999 [s rdb_changes_since_last_save] + # make sure the server is still writable + r set x xx + } + } + + foreach bgsave_type {"fork" "forkless"} { + test "bgsave $bgsave_type resets the change counter" { + r config set rdb-key-save-delay 0 + r config set bgsave-default-method $bgsave_type + r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 0 + } else { + fail "bgsave not done" + } + assert_equal [s rdb_changes_since_last_save] 0 + + # Verify we tested the right save type + set expected_type [expr {$bgsave_type eq "forkless" ? "forkless" : "fork"}] + assert_equal [s rdb_last_bgsave_type] $expected_type + } + } + + foreach bgsave_type {"fork" "forkless"} { + test "bgsave $bgsave_type metrics are correct after success" { + set saves_before [s rdb_saves] + populate 100 "" 16 + r config set bgsave-default-method $bgsave_type + r bgsave + waitForBgsave r + assert {[s rdb_saves] == $saves_before + 1} + assert {[s rdb_last_bgsave_time_sec] >= 0 && [s rdb_last_bgsave_time_sec] < 3600} + assert_equal [s rdb_last_bgsave_status] "ok" + assert_equal [s rdb_last_bgsave_type] $bgsave_type + assert {[s rdb_bgsave_in_progress] == 0} + assert {[s current_fork_perc] == 0} + assert {[s current_save_keys_processed] == 0} + assert {[s current_save_keys_total] == 0} + } + } + + foreach bgsave_type {"fork" "forkless"} { + test "bgsave $bgsave_type metrics are correct after failure" { + set saves_before [s rdb_saves] + populate 1000 "" 16 + r config set bgsave-default-method $bgsave_type + r config set rdb-key-save-delay 10000000 + if {$bgsave_type eq "forkless"} { + # Inject a failure to make the save fail: a directory whose name + # collides with the RDB file makes the final rename fail. We + # can't just kill -9 like fork-based bgsave since there is no + # child process. + set rdb_path [file join [lindex [r config get dir] 1] [lindex [r config get dbfilename] 1]] + file delete -force $rdb_path + file mkdir $rdb_path + } + r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "$bgsave_type bgsave didn't start" + } + if {$bgsave_type eq "fork"} { + set pid [get_child_pid 0] + catch {exec kill -9 $pid} + } + r config set rdb-key-save-delay 0 + waitForBgsave r + assert {[s rdb_last_bgsave_time_sec] >= 0 && [s rdb_last_bgsave_time_sec] < 3600} + assert_equal [s rdb_last_bgsave_status] "err" + assert_equal [s rdb_last_bgsave_type] $bgsave_type + assert {[s rdb_bgsave_in_progress] == 0} + assert {[s current_fork_perc] == 0} + assert {[s current_save_keys_processed] == 0} + assert {[s current_save_keys_total] == 0} + r config set rdb-key-save-delay 0 + if {$bgsave_type eq "forkless"} { + # Remove the directory so later saves in this server can succeed. + file delete [file join [lindex [r config get dir] 1] [lindex [r config get dbfilename] 1]] + } + } + } + + foreach bgsave_type {"fork" "forkless"} { + test "bgsave cancel aborts $bgsave_type save" { + # Generating RDB will take some 100 seconds + r config set rdb-key-save-delay 1000000 + populate 100 "" 16 + + r config set bgsave-default-method $bgsave_type + r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "bgsave did not start in time" + } + + # Verify we're testing the right save type + set expected_type [expr {$bgsave_type eq "forkless" ? "forkless" : "fork"}] + assert_equal [s rdb_current_bgsave_type] $expected_type + + if {$bgsave_type ne "forkless"} { + set fork_child_pid [get_child_pid 0] + } + + assert {[r bgsave cancel] eq {Background saving cancelled}} + + if {$bgsave_type ne "forkless"} { + set temp_rdb [file join [lindex [r config get dir] 1] temp-${fork_child_pid}.rdb] + # Temp rdb must be deleted + wait_for_condition 50 100 { + ![file exists $temp_rdb] + } else { + fail "bgsave temp file was not deleted after cancel" + } + } + + # Make sure no save is running and that bgsave return an error + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 0 + } else { + fail "bgsave is currently running" + } + assert_error "ERR Background saving is currently not in progress or scheduled" {r bgsave cancel} + } + } +} + +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "forkless bgsave contains expired keys from when save started" { + + # Set two keys that expire together + r set k1 v1 + r set k2 v2 + set curr_time [clock seconds] + r expireat k1 [expr {$curr_time + 2}] + r expireat k2 [expr {$curr_time + 2}] + + # Start slow forkless save + r config set rdb-key-save-delay 10000000 + r config set bgsave-default-method forkless r bgsave - assert_equal [s rdb_bgsave_in_progress] 1 - r flushall - # wait a second max (bgsave should take 5) wait_for_condition 50 100 { - [s rdb_bgsave_in_progress] == 0 + [s rdb_bgsave_in_progress] == 1 } else { - fail "bgsave not aborted" + fail "forkless bgsave did not start" } - # verify that bgsave failed, by checking that the change counter is still high - assert_lessthan 999 [s rdb_changes_since_last_save] - # make sure the server is still writable - r set x xx - } + + # Let both keys expire + after 3000 + + # Serialize k1 in the foreground by touching it + r set k1 v11 + + # Complete forkless save so k2 will be serialized in background + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Check both keys are in the RDB + set rdb_path [file join [lindex [r config get dir] 1] [lindex [r config get dbfilename] 1]] + set fd [open $rdb_path rb] + set rdb_content [read $fd] + close $fd + assert {[string first "k1" $rdb_content] != -1} + assert {[string first "k2" $rdb_content] != -1} + } {} {needs:debug} +} - test {bgsave resets the change counter} { +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "FLUSHDB during single-db forkless bgsave aborts the save without failing it" { + + # Populate database with complex dataset + createComplexDataset r 1000 + + # Get initial key count + set initial_keys [r dbsize] + assert {$initial_keys > 0} + + # Start forkless save with very slow save (high delay per key) + r config set rdb-key-save-delay 100000 + r config set bgsave-default-method forkless + r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless bgsave did not start" + } + + r flushdb + assert_equal [r dbsize] 0 + + # Speed up and wait for save to abort + # Note: Cancellation needs to be processed by background thread r config set rdb-key-save-delay 0 + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 0 + } else { + fail "forkless bgsave did not abort" + } + + # A flush aborts the save the way FLUSHALL aborts a fork save: it is not + # a failure, so writes stay allowed. + assert {[s rdb_last_bgsave_status] ne "err"} + r set k v + assert_equal [r get k] v + } {} {needs:debug} +} + +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "FLUSHDB during multi-db forkless bgsave aborts the save without failing it" { + + # Populate multiple databases + for {set i 0} {$i < 1000} {incr i} { + r set key$i val$i + } + r select 1 + for {set i 0} {$i < 1000} {incr i} { + r set key$i val$i + } + r select 0 + + # Start slow forkless save + r config set rdb-key-save-delay 100000 + r config set bgsave-default-method forkless r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless bgsave did not start" + } + + # Give forkless save time to start iterating + after 100 + + # FLUSHDB on db 1 while save is running - this should terminate the save + r select 1 + r flushdb + assert_equal [r dbsize] 0 + r select 0 + + # Resume save speed and wait for save to abort + r config set rdb-key-save-delay 0 wait_for_condition 50 100 { [s rdb_bgsave_in_progress] == 0 } else { - fail "bgsave not done" + fail "forkless bgsave did not abort" } - assert_equal [s rdb_changes_since_last_save] 0 - } + + # A flush aborts the save; it is not a failure, so writes stay allowed. + assert {[s rdb_last_bgsave_status] ne "err"} + r set k v + assert_equal [r get k] v + } {} {needs:debug} +} + +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "multiple databases modifications during forkless bgsave" { + + # Populate 5 databases with all data types + for {set db 0} {$db < 5} {incr db} { + r select $db + createComplexDatasetForVerification r 20 "db${db}_" + } + r select 0 + + # Start slow forkless save + r config set rdb-key-save-delay 10000 + r config set bgsave-default-method forkless + r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless bgsave did not start" + } + + # Modify keys in all databases while save is running + for {set db 0} {$db < 5} {incr db} { + r select $db + for {set i 0} {$i < 20} {incr i} { + r append db${db}_before_$i "value_after_$i" + r incr db${db}_int_$i + r set db${db}_after_$i "VALUE_AFTER_$i" + r lpush db${db}_lst_$i "LL2" "LL1" + r rpush db${db}_lst_$i "RR1" "RR2" + r sadd db${db}_set_$i "BB1" "BB2" + r zadd db${db}_zset_$i 5 "Z2" + r hset db${db}_hash_$i "H1" "c" + r pfadd db${db}_hll_$i "PF2" + r geoadd db${db}_geo_$i -122.1592 47.5976 "bellevue" + r xadd db${db}_stream_$i "*" "D1" "V2" + r xreadgroup GROUP db${db}_group_$i consumer_after_$i COUNT 1 STREAMS db${db}_stream_$i > + r bitfield db${db}_bits_$i SET u4 0 0 INCRBY u4 0 1 + r geosearchstore db${db}_geo_set_$i db${db}_geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi + r geosearchstore db${db}_geo_set_dist_$i db${db}_geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi ASC COUNT 10 STOREDIST + } + } + r select 0 + + # Verify modifications happened in live database + assert {[s rdb_changes_since_last_save] > 0} + for {set db 0} {$db < 5} {incr db} { + r select $db + for {set i 0} {$i < 20} {incr i} { + assert_equal [r get db${db}_before_$i] "value_before_${i}value_after_$i" + } + } + r select 0 + + # Speed up and complete save + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Verify save completed successfully + assert_equal [s rdb_last_bgsave_status] ok + + # Reload from RDB and verify ORIGINAL values are preserved + # (consistent snapshot should capture state at start of save) + catch {r debug reload nosave} + for {set db 0} {$db < 5} {incr db} { + r select $db + for {set i 0} {$i < 20} {incr i} { + # Strings: original value, not appended + assert_equal [r get db${db}_before_$i] "value_before_$i" + # Ints: original value, not incremented + assert_equal [r get db${db}_int_$i] [expr {42 + $i}] + # New keys should not exist + assert_equal [r exists db${db}_after_$i] 0 + # Lists: original 4 elements, not 8 + assert_equal [r llen db${db}_lst_$i] 4 + assert_equal [r lrange db${db}_lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + # Sets: original 2 members + assert_equal [r scard db${db}_set_$i] 2 + assert_equal [lsort [r smembers db${db}_set_$i]] [list "B1" "B2"] + # Sorted sets: original score + assert_equal [r zscore db${db}_zset_$i "Z2"] 2 + # Hashes: original value + assert_equal [r hget db${db}_hash_$i "H1"] "a" + # HLL: original count + assert_equal [r pfcount db${db}_hll_$i] 1 + # Geo: original 1 member + assert_equal [r zcard db${db}_geo_$i] 1 + assert_equal [r zcard db${db}_geo_set_$i] 1 + } + } + } {} {needs:debug} +} - test {bgsave cancel aborts save} { - r config set save "" - # Generating RDB will take some 100 seconds - r config set rdb-key-save-delay 1000000 - populate 100 "" 16 +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "modify new keys during forkless bgsave" { + + # Populate database with all data types + createComplexDatasetForVerification r 20 + set original_keys [r dbsize] + + # Start forkless save with very slow save (high delay per key) + r config set rdb-key-save-delay 10000 + r config set bgsave-default-method forkless + r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless bgsave did not start" + } + + # Create new keys of all data types while save is running + for {set i 0} {$i < 100} {incr i} { + r set after_$i "value_after_$i" + r set after_i_$i 42 + r lpush after_lst_$i "L2" "L1" + r rpush after_lst_$i "R1" "R2" + r sadd after_set_$i "B1" + r sadd after_iset_$i 12 34 + r zadd after_zset_$i 1 "Z1" + r hset after_hash_$i "H1" "a" + r pfadd after_hll_$i "PF1" + r set after_bits_$i "\x0f" + r bitfield after_bits_$i SET u4 0 0 INCRBY u4 0 1 + r geoadd after_geo_$i -122.345 47.775 "costco" + r geosearchstore after_geo_set_$i after_geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi + r geosearchstore after_geo_set_dist_$i after_geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi ASC COUNT 10 STOREDIST + r xadd after_stream_$i "*" "D1" "V2" + r xgroup create after_stream_$i after_group_$i 0 + r hsetex after_hashttl_$i EX 10000 FIELDS 1 HTTL1 a + } + + # Verify new keys were created (14 key types per iteration × 100 iterations) + set expected_keys [expr {$original_keys + 100 * 14}] + assert_equal [r dbsize] $expected_keys + assert_equal [s rdb_bgsave_in_progress] 1 + + # Speed up and complete save + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Verify save completed successfully + assert_equal [s rdb_last_bgsave_status] ok + + # Reload and verify ONLY original keys exist (new keys should NOT be in snapshot) + catch {r debug reload nosave} + assert_equal [r dbsize] $original_keys + + # Verify all original data types preserved + for {set i 0} {$i < 20} {incr i} { + assert_equal [r get before_$i] "value_before_$i" + assert_equal [r get int_$i] [expr {42 + $i}] + assert_equal [r llen lst_$i] 4 + assert_equal [r lrange lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [r scard set_$i] 2 + assert_equal [r zscore zset_$i "Z1"] 1 + assert_equal [r hget hash_$i "H1"] "a" + assert_equal [r pfcount hll_$i] 1 + assert_equal [r zcard geo_$i] 1 + assert_equal [r zcard geo_set_$i] 1 + } + + # Verify new keys do NOT exist in snapshot + for {set i 0} {$i < 100} {incr i} { + assert_equal [r exists after_$i] 0 + assert_equal [r exists after_lst_$i] 0 + assert_equal [r exists after_set_$i] 0 + assert_equal [r exists after_zset_$i] 0 + assert_equal [r exists after_hash_$i] 0 + assert_equal [r exists after_hll_$i] 0 + assert_equal [r exists after_geo_$i] 0 + assert_equal [r exists after_geo_set_$i] 0 + assert_equal [r exists after_geo_set_dist_$i] 0 + assert_equal [r exists after_stream_$i] 0 + assert_equal [r exists after_hashttl_$i] 0 + } + } {} {needs:debug} +} +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "SWAPDB during forkless bgsave" { + + # Populate 5 databases with all data types + for {set db 0} {$db < 5} {incr db} { + r select $db + createComplexDatasetForVerification r 20 "db${db}_" + } + r select 0 + + # Start slow forkless save + r config set rdb-key-save-delay 10000 + r config set bgsave-default-method forkless r bgsave wait_for_condition 50 100 { [s rdb_bgsave_in_progress] == 1 } else { - fail "bgsave did not start in time" + fail "forkless bgsave did not start" + } + + # Keep swapping databases while save is running + set perm [list 0 1 2 3 4] + set swaps 0 + while {[s rdb_bgsave_in_progress] == 1 && $swaps < 200} { + incr swaps + # Shuffle permutation + for {set i 4} {$i > 0} {incr i -1} { + set j [expr {int(rand() * ($i + 1))}] + set temp [lindex $perm $i] + lset perm $i [lindex $perm $j] + lset perm $j $temp + } + # Swap each database with its permuted target + for {set db 0} {$db < 5} {incr db} { + r swapdb $db [lindex $perm $db] + } + } + + # Speed up save and wait for completion + r config set rdb-key-save-delay 0 + waitForBgsave r + assert {$swaps > 100} + + # Verify save completed successfully + assert_equal [s rdb_last_bgsave_status] ok + + # Reload from RDB and verify keys are in ORIGINAL databases + # (SWAPDB is ignored for consistent snapshots) + r select 0 + catch {r debug reload nosave} + for {set db 0} {$db < 5} {incr db} { + r select $db + for {set i 0} {$i < 20} {incr i} { + assert_equal [r get db${db}_before_$i] "value_before_$i" + assert_equal [r get db${db}_int_$i] [expr {42 + $i}] + assert_equal [r lrange db${db}_lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers db${db}_set_$i]] [list "B1" "B2"] + assert_equal [r zscore db${db}_zset_$i "Z1"] 1 + assert_equal [r hget db${db}_hash_$i "H1"] "a" + assert_equal [r pfcount db${db}_hll_$i] 1 + assert_equal [r zcard db${db}_geo_$i] 1 + assert_equal [r zcard db${db}_geo_set_$i] 1 + } + } + } {} {needs:debug} +} + +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "delete all keys after SWAPDB during forkless bgsave" { + + # Populate 5 databases with all data types + for {set db 0} {$db < 5} {incr db} { + r select $db + createComplexDatasetForVerification r 20 "db${db}_" } - set fork_child_pid [get_child_pid 0] + r select 0 - assert {[r bgsave cancel] eq {Background saving cancelled}} - set temp_rdb [file join [lindex [r config get dir] 1] temp-${fork_child_pid}.rdb] - # Temp rdb must be deleted + # Start slow forkless save + r config set rdb-key-save-delay 10000 + r config set bgsave-default-method forkless + r bgsave wait_for_condition 50 100 { - ![file exists $temp_rdb] + [s rdb_bgsave_in_progress] == 1 } else { - fail "bgsave temp file was not deleted after cancel" + fail "forkless bgsave did not start" + } + + # Swap databases with fixed permutation [2, 3, 4, 0, 1] + set perm [list 2 3 4 0 1] + for {set db 0} {$db < 5} {incr db} { + r swapdb $db [lindex $perm $db] + } + + # Delete all keys in all databases + for {set db 4} {$db >= 0} {incr db -1} { + r select $db + set keys [r keys *] + foreach key $keys { + r del $key + } + assert_equal [r dbsize] 0 + } + + # Speed up and complete save + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Reload from RDB and verify ORIGINAL keys still exist + # Consistent snapshot should preserve state before SWAPDB and deletions + r select 0 + catch {r debug reload nosave} + for {set db 0} {$db < 5} {incr db} { + r select $db + for {set i 0} {$i < 20} {incr i} { + assert_equal [r get db${db}_before_$i] "value_before_$i" + assert_equal [r get db${db}_int_$i] [expr {42 + $i}] + assert_equal [r lrange db${db}_lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers db${db}_set_$i]] [list "B1" "B2"] + assert_equal [r zscore db${db}_zset_$i "Z1"] 1 + assert_equal [r hget db${db}_hash_$i "H1"] "a" + assert_equal [r pfcount db${db}_hll_$i] 1 + assert_equal [r zcard db${db}_geo_$i] 1 + assert_equal [r zcard db${db}_geo_set_$i] 1 + } } + } {} {needs:debug} +} - # Make sure no save is running and that bgsave return an error - wait_for_condition 50 100 { - [s rdb_bgsave_in_progress] == 0 +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "deleting keys during forkless bgsave" { + + # Populate database with all data types + createComplexDatasetForVerification r 20 + + # Start forkless save with very slow save (high delay per key) + r config set rdb-key-save-delay 10000 + r config set bgsave-default-method forkless + r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 } else { - fail "bgsave is currently running" + fail "forkless bgsave did not start" } - assert_error "ERR Background saving is currently not in progress or scheduled" {r bgsave cancel} - } + + # Delete all keys in the database + set keys [r keys *] + foreach key $keys { + r del $key + } + + # Verify all keys deleted and save still in progress + assert_equal [r dbsize] 0 + assert_equal [s rdb_bgsave_in_progress] 1 + + # Speed up and complete save + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Reload from RDB and verify ORIGINAL keys still exist + catch {r debug reload nosave} + for {set i 0} {$i < 20} {incr i} { + assert_equal [r get before_$i] "value_before_$i" + assert_equal [r get int_$i] [expr {42 + $i}] + assert_equal [r lrange lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers set_$i]] [list "B1" "B2"] + assert_equal [r zscore zset_$i "Z1"] 1 + assert_equal [r hget hash_$i "H1"] "a" + assert_equal [r pfcount hll_$i] 1 + assert_equal [r zcard geo_$i] 1 + assert_equal [r zcard geo_set_$i] 1 + } + } {} {needs:debug} +} + +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "blocking commands during forkless bgsave" { + + # Create initial dataset with 100 keys + createComplexDatasetForVerification r 100 + + # Start blocking commands on nonexistent keys BEFORE save starts + set rd1 [valkey_deferring_client] + set rd2 [valkey_deferring_client] + set rd3 [valkey_deferring_client] + set rd4 [valkey_deferring_client] + set rd5 [valkey_deferring_client] + set rd6 [valkey_deferring_client] + set rd7 [valkey_deferring_client] + + # Consume an item from a nonexistent key + $rd1 blpop new1 0 + + # Set up a cascade of brpoplpush's on nonexistent keys + $rd2 brpoplpush new2 new3 0 + $rd3 brpoplpush new3 new4 0 + + # Nonexistent keys + $rd4 brpoplpush new5 new6 0 + + # Cascade of brpoplpush's onto an existing key + $rd5 brpoplpush new88 new7 0 + $rd6 brpoplpush new7 lst_2 0 + + # Destination exists + $rd7 brpoplpush new8 lst_70 0 + + # Start save with slow speed + r config set rdb-key-save-delay 100000 + r config set bgsave-default-method forkless + r bgsave + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "bgsave didn't start" + } + + # Start more blocking commands during save + set rd8 [valkey_deferring_client] + set rd9 [valkey_deferring_client] + set rd10 [valkey_deferring_client] + set rd11 [valkey_deferring_client] + set rd12 [valkey_deferring_client] + + # Existing keys with new destinations, setting off some of the waiters + $rd8 brpoplpush lst_33 new1 0 + $rd9 brpoplpush lst_27 new2 0 + + # Duplicate another brpoplpush above + $rd10 brpoplpush new5 new6 0 + + # New key but existing destination + $rd11 brpoplpush new9 lst_3 0 + + # Consume an item from a nonexistent key + $rd12 brpop new100 0 + + # Set off more waiters + r rpush new5 foobar + r rpush new88 foobar + + assert_equal [s rdb_bgsave_in_progress] 1 + + # Resume save at normal speed + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Wait for blocking commands to complete and read responses + after 1000 + $rd8 read + $rd9 read + $rd1 read + $rd2 read + $rd3 read + $rd5 read + $rd6 read + + # Don't read from rd4, rd7, rd10, rd11, rd12 - they remain blocked or timeout + + # Verify the blocking commands executed correctly + assert_equal [r llen new1] 0 + assert_equal [r llen new2] 0 + assert_equal [r llen new3] 0 + assert_equal [lindex [r lrange new4 -1 -1] 0] "R2" + assert_equal [r llen new5] 0 + assert_equal [lindex [r lrange new6 -1 -1] 0] "foobar" + assert_equal [r llen new88] 0 + assert_equal [r llen new7] 0 + assert_equal [lindex [r lrange lst_2 0 0] 0] "foobar" + assert_equal [r llen new8] 0 + assert_equal [r llen lst_70] 4 + assert_equal [r llen lst_33] 3 + assert_equal [r llen lst_27] 3 + assert_equal [r llen lst_3] 4 + + # Close deferred clients (those that didn't complete will be force-closed) + $rd1 close + $rd2 close + $rd3 close + $rd4 close + $rd5 close + $rd6 close + $rd7 close + $rd8 close + $rd9 close + $rd10 close + $rd11 close + $rd12 close + + # Verify snapshot contains original keys (blocking commands should not affect snapshot) + catch {r debug reload nosave} + + # All original data types should be preserved in snapshot + for {set i 0} {$i < 100} {incr i} { + assert_equal [r get before_$i] "value_before_$i" + assert_equal [r get int_$i] [expr {42 + $i}] + assert_equal [r lrange lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers set_$i]] [list "B1" "B2"] + assert_equal [r zscore zset_$i "Z1"] 1 + assert_equal [r hget hash_$i "H1"] "a" + assert_equal [r pfcount hll_$i] 1 + assert_equal [r zcard geo_$i] 1 + assert_equal [r zcard geo_set_$i] 1 + } + + # New keys created during save should NOT be in snapshot + assert_equal [r exists new1] 0 + assert_equal [r exists new2] 0 + assert_equal [r exists new3] 0 + assert_equal [r exists new4] 0 + assert_equal [r exists new5] 0 + assert_equal [r exists new6] 0 + assert_equal [r exists new7] 0 + assert_equal [r exists new8] 0 + assert_equal [r exists new88] 0 + assert_equal [r exists new9] 0 + assert_equal [r exists new100] 0 + } {} {needs:debug} +} - test {bgsave cancel schedulled request} { - r config set save "" - # Generating RDB will take some 100 seconds - r config set rdb-key-save-delay 1000000 - populate 100 "" 16 +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "TTL expiration during forkless bgsave" { + + # Create initial dataset with 100 keys + set num_keys 100 + createComplexDatasetForVerification r $num_keys + + # Set TTLs on all keys - key i expires in (i/10 + 1) seconds + set start_time [clock milliseconds] + for {set i 0} {$i < $num_keys} {incr i} { + set ttl [expr {$i/10 + 1}] + foreach prefix {before int lst set zset hash hll bits geo geo_set stream iset} { + r expire ${prefix}_${i} $ttl + } + } + + # Start save and wait for completion + r config set bgsave-default-method forkless + r bgsave + waitForBgsave r + + # Reload from RDB + catch {r debug reload nosave} + + # Check keycount is reasonable + set keycount [r dbsize] + assert {$keycount <= $num_keys * 12} + + # Verify keys based on elapsed time + set verified [list] + for {set i 0} {$i < $num_keys} {incr i} { + lappend verified $i + } + + while {[llength $verified] > 0} { + set elapsed_time [expr {([clock milliseconds] - $start_time) / 1000.0}] + + foreach i $verified { + # If not yet expired, verify all data types exist + if {$elapsed_time < [expr {$i/10.0}]} { + assert_equal [r exists before_${i}] 1 + assert_equal [r exists int_${i}] 1 + assert_equal [r exists lst_${i}] 1 + assert_equal [r exists set_${i}] 1 + assert_equal [r exists zset_${i}] 1 + assert_equal [r exists hash_${i}] 1 + assert_equal [r exists hll_${i}] 1 + assert_equal [r exists bits_${i}] 1 + assert_equal [r exists geo_${i}] 1 + assert_equal [r exists geo_set_${i}] 1 + assert_equal [r exists stream_${i}] 1 + assert_equal [r exists iset_${i}] 1 + } + + # If expired for more than 2 seconds, verify all data types are gone + if {$elapsed_time > [expr {$i/10.0 + 2}]} { + assert_equal [r exists before_${i}] 0 + assert_equal [r exists int_${i}] 0 + assert_equal [r exists lst_${i}] 0 + assert_equal [r exists set_${i}] 0 + assert_equal [r exists zset_${i}] 0 + assert_equal [r exists hash_${i}] 0 + assert_equal [r exists hll_${i}] 0 + assert_equal [r exists bits_${i}] 0 + assert_equal [r exists geo_${i}] 0 + assert_equal [r exists geo_set_${i}] 0 + assert_equal [r exists stream_${i}] 0 + assert_equal [r exists iset_${i}] 0 + set verified [lsearch -all -inline -not -exact $verified $i] + } + } + + after 100 + } + } {} {needs:debug} +} - # start a long AOF child - r bgrewriteaof +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "evictions during forkless bgsave" { + + # Create initial dataset + createComplexDatasetForVerification r 1000 + + # Start save with stopped speed + r config set rdb-key-save-delay 10000 + r config set bgsave-default-method forkless + r bgsave + wait_for_condition 50 100 { - [s aof_rewrite_in_progress] == 1 + [s rdb_bgsave_in_progress] == 1 } else { - fail "aof not started" + fail "bgsave didn't start" } - # Make sure cancel return valid status - assert {[r bgsave schedule] eq {Background saving scheduled}} + # Trigger evictions by setting maxmemory below current usage + set current_memory [s used_memory] + set target_memory [expr {$current_memory * 3 / 4}] + r config set maxmemory $target_memory + r config set maxmemory-policy allkeys-lru + + # Generate evictions by adding new data + r set foo bar + + # Verify evictions occurred + set evicted_keys [s evicted_keys] + assert {$evicted_keys > 0} + assert_equal [s rdb_bgsave_in_progress] 1 + + # Resume save at normal speed + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Verify snapshot contains original keys + catch {r debug reload nosave} + for {set i 0} {$i < 1000} {incr i} { + assert_equal [r get before_$i] "value_before_$i" + assert_equal [r get int_$i] [expr {42 + $i}] + assert_equal [r lrange lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers set_$i]] [list "B1" "B2"] + assert_equal [r zscore zset_$i "Z1"] 1 + assert_equal [r hget hash_$i "H1"] "a" + assert_equal [r pfcount hll_$i] 1 + assert_equal [r zcard geo_$i] 1 + assert_equal [r zcard geo_set_$i] 1 + } + } {} {needs:debug} +} - # Cancel the scheduled save - assert {[r bgsave cancel] eq {Scheduled background saving cancelled}} +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "comprehensive modifications on all data types during forkless bgsave" { + + # Create initial dataset with 1000 keys + createComplexDatasetForVerification r 1000 + + # Start save with slow speed to keep it running during modifications + r config set rdb-key-save-delay 1000 + r config set bgsave-default-method forkless + r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "bgsave didn't start" + } + + # Overwrite keys during save - all data types. + set rd [valkey_deferring_client] + set outstanding 0 + set saw_save_in_progress 0 + for {set i 0} {$i < 1000} {incr i} { + $rd append before_$i "value_after_$i" + $rd incr int_$i + $rd set after_$i "VALUE_AFTER_$i" + $rd lpush lst_$i LL2 LL1 + $rd rpush lst_$i RR1 RR2 + $rd sadd set_$i BB1 BB2 + $rd zadd zset_$i 5 Z2 + $rd hset hash_$i H1 c + $rd pfadd hll_$i PF2 + $rd bitfield bits_$i SET u4 0 0 INCRBY u4 0 1 + $rd geoadd geo_$i -122.1592 47.5976 bellevue + $rd geosearchstore geo_set_$i geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi + $rd geosearchstore geo_set_dist_$i geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi ASC COUNT 10 STOREDIST + $rd xadd stream_$i "*" D1 V2 + $rd xreadgroup GROUP group_$i consumer_after_$i COUNT 1 STREAMS stream_$i > + $rd hsetex hashttl_$i EX 10000 FIELDS 1 HTTL1 a + # There is a chance that our client is blocked but we don't know it, because + # as a deferring client we never read replies. If we are blocked we would + # keep sending commands forever, which accumulate on the server side and can + # overflow the buffers. So stop periodically and consume replies - that is + # the mechanism that waits until we are unblocked. + incr outstanding 16 + if {$outstanding >= 320} { + for {set j 0} {$j < $outstanding} {incr j} { $rd read } + set outstanding 0 + if {[s rdb_bgsave_in_progress] == 1} { set saw_save_in_progress 1 } + } + } + + # Verify changes happened while the save was running + assert {[s rdb_changes_since_last_save] > 0} + assert_equal $saw_save_in_progress 1 + + # Speed up save and wait for completion + r config set rdb-key-save-delay 0 + waitForBgsave r + $rd close + + # Verify snapshot contains original keys + catch {r debug reload nosave} + for {set i 0} {$i < 1000} {incr i} { + assert_equal [r get before_$i] "value_before_$i" + assert_equal [r get int_$i] [expr {42 + $i}] + assert_equal [r lrange lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers set_$i]] [list "B1" "B2"] + assert_equal [r zscore zset_$i "Z1"] 1 + assert_equal [r hget hash_$i "H1"] "a" + assert_equal [r pfcount hll_$i] 1 + assert_equal [r zcard geo_$i] 1 + assert_equal [r zcard geo_set_$i] 1 + } + } {} {needs:debug} +} - # Make sure a second call to bgsave cancel return an error - assert_error "ERR Background saving is currently not in progress or scheduled" {r bgsave cancel} - } +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "store key deletion by georadius during forkless bgsave" { + + # Create initial dataset with geo data + createComplexDatasetForVerification r 1000 + + # Create additional zsets for georadius STORE operations + r zadd georad_zset_delete_test 1 Z1 2 Z2 + r zadd georadmem_zset_test 1 Z1 2 Z2 3 Z3 + + # Start save with stopped speed + r config set rdb-key-save-delay 10000 + r config set bgsave-default-method forkless + r bgsave + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "bgsave didn't start" + } + + # Use GEORADIUS with STORE - deletes georad_zset_delete_test key + r georadius geo_1 -122.191729 47.685821 5 mi STORE georad_zset_delete_test + + # Use GEORADIUSBYMEMBER with STORE - does not delete georadmem_zset_test as it returns 1 member + r georadiusbymember geo_1 seattle 5 mi STORE georadmem_zset_test + + # Verify changes were made + assert {[s rdb_changes_since_last_save] > 0} + assert_equal [s rdb_bgsave_in_progress] 1 + + # Resume save at normal speed + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Verify snapshot contains original keys + catch {r debug reload nosave} + + # Original geo_1 key should be preserved + assert_equal [r zcard geo_1] 1 + + # Original zsets should be preserved (not deleted by STORE operations) + assert_equal [r zcard georad_zset_delete_test] 2 + assert_equal [r zcard georadmem_zset_test] 3 + } {} {needs:debug} +} + +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test "transactions during forkless bgsave" { + + # Populate 5 databases + for {set db 0} {$db < 5} {incr db} { + r select $db + createComplexDatasetForVerification r 100 + } + r select 0 + + # Prepare transactions before save starts + set rd0 [valkey_deferring_client] + set rd1 [valkey_deferring_client] + + $rd0 select 0 + $rd0 multi + $rd0 set int_1 bad + $rd0 incrby int_2 2 + + $rd1 select 1 + $rd1 multi + $rd1 set int_1 bad + $rd1 set int_2 bad1 + $rd1 lpush lst_3 bad1 + $rd1 sadd set_3 bad1 + $rd1 set newkey bad1 + + # Start save with slow speed + r config set rdb-key-save-delay 10000 + r config set bgsave-default-method forkless + r bgsave + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "bgsave didn't start" + } + + # Start more transactions during save + set rd4 [valkey_deferring_client] + set rd3 [valkey_deferring_client] + + $rd4 select 4 + $rd4 multi + $rd4 hset hash_49 bad1 a + $rd4 hset hash_10 H1 b + $rd4 zadd zset_3 2 bad1 + $rd4 set newkey bad1 + $rd4 set another_newkey bad55 + $rd4 xadd newstream * D1 V2 + + $rd3 select 3 + $rd3 multi + $rd3 set aftersave bad1 + $rd3 xadd another_newstream * D1 V3 + + assert_equal [s rdb_bgsave_in_progress] 1 + + # Execute first 3 transactions + $rd0 exec + $rd1 exec + $rd4 exec + + # Read all responses: select, multi, queued commands, exec + # rd0: select(OK) multi(OK) set(QUEUED) incrby(QUEUED) exec(result) + for {set i 0} {$i < 5} {incr i} { $rd0 read } + # rd1: select(OK) multi(OK) set(QUEUED) set(QUEUED) lpush(QUEUED) sadd(QUEUED) set(QUEUED) exec(result) + for {set i 0} {$i < 8} {incr i} { $rd1 read } + # rd4: select(OK) multi(OK) hset(QUEUED) hset(QUEUED) zadd(QUEUED) set(QUEUED) set(QUEUED) xadd(QUEUED) exec(result) + for {set i 0} {$i < 9} {incr i} { $rd4 read } + + # Verify transactions executed + r select 0 + assert_equal [r get int_1] "bad" + r select 1 + assert_equal [r get int_2] "bad1" + r select 4 + assert_equal [r get newkey] "bad1" + r select 3 + assert_equal [r exists aftersave] 0 + r select 4 + assert_equal [r xlen newstream] 1 + r select 3 + assert_equal [r xlen another_newstream] 0 + + assert_equal [s rdb_bgsave_in_progress] 1 + + # Resume save at normal speed + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Execute last transaction after save completes + r select 3 + assert_equal [r exists aftersave] 0 + $rd3 exec + # consume rd3 replies: select(OK) multi(OK) set(QUEUED) xadd(QUEUED) exec(result) + for {set i 0} {$i < 5} {incr i} { $rd3 read } + assert_equal [r get aftersave] "bad1" + assert_equal [r xlen another_newstream] 1 + + # Close deferred clients + $rd0 close + $rd1 close + $rd3 close + $rd4 close + + # Verify snapshot contains original keys + catch {r debug reload nosave} + + # Original keys should be preserved in all databases + r select 0 + assert_equal [r get before_0] "value_before_0" + assert_equal [r get int_1] "43" + r select 1 + assert_equal [r get int_2] "44" + assert_equal [r llen lst_3] 4 + r select 4 + assert_equal [r exists newkey] 0 + assert_equal [r exists another_newkey] 0 + assert_equal [r exists newstream] 0 + r select 3 + assert_equal [r exists aftersave] 0 + assert_equal [r exists another_newstream] 0 + } {} {needs:debug} +} +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + foreach first_type {fork forkless} { + foreach second_type {fork forkless} { + test "$first_type bgsave blocks $second_type bgsave" { + r config set rdb-key-save-delay 1000000 + populate 100 "" 16 + + r config set bgsave-default-method $first_type + r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "$first_type bgsave did not start" + } + assert_equal [s rdb_current_bgsave_type] $first_type + + r config set bgsave-default-method $second_type + assert_error "ERR Background save already in progress" {r bgsave} + r bgsave cancel + r config set rdb-key-save-delay 0 + waitForBgsave r + } + } + } } test {client freed during loading} { @@ -512,52 +1603,77 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "scriptbackup } } -start_server {} { - test "failed bgsave prevents writes" { - # Make sure the server saves an RDB on shutdown - r config set save "900 1" - - r config set rdb-key-save-delay 10000000 - populate 1000 - r set x x - r bgsave - set pid1 [get_child_pid 0] - catch {exec kill -9 $pid1} - waitForBgsave r - - # make sure a read command succeeds - assert_equal [r get x] x - - # make sure a write command fails - assert_error {MISCONF *} {r set x y} +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + foreach bgsave_type {"fork" "forkless"} { + test "failed bgsave $bgsave_type prevents writes" { + # Make sure the server saves an RDB on shutdown + r config set save "900 1" + + r config set rdb-key-save-delay 10000000 + populate 1000 + r set x x + r config set bgsave-default-method $bgsave_type + if {$bgsave_type eq "forkless"} { + # Inject a failure to make the save fail: a directory whose name + # collides with the RDB file makes the final rename fail. We + # can't just kill -9 like fork-based bgsave since there is no + # child process. + set rdb_path [file join [lindex [r config get dir] 1] [lindex [r config get dbfilename] 1]] + file delete -force $rdb_path + file mkdir $rdb_path + } + r bgsave + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "$bgsave_type bgsave didn't start" + } + if {$bgsave_type ne "forkless"} { + set pid1 [get_child_pid 0] + catch {exec kill -9 $pid1} + } + r config set rdb-key-save-delay 0 + waitForBgsave r - # repeat with script - assert_error {MISCONF *} {r eval { - return redis.call('set','x',1) - } 1 x - } - assert_equal {x} [r eval { - return redis.call('get','x') - } 1 x - ] + # make sure a read command succeeds + assert_equal [r get x] x - # again with script using shebang - assert_error {MISCONF *} {r eval {#!lua - return redis.call('set','x',1) - } 1 x - } - assert_equal {x} [r eval {#!lua flags=no-writes - return redis.call('get','x') - } 1 x - ] + # make sure a write command fails + assert_error {MISCONF *} {r set x y} - r config set rdb-key-save-delay 0 - r bgsave - waitForBgsave r + # repeat with script + assert_error {MISCONF *} {r eval { + return redis.call('set','x',1) + } 1 x + } + assert_equal {x} [r eval { + return redis.call('get','x') + } 1 x + ] + + # again with script using shebang + assert_error {MISCONF *} {r eval {#!lua + return redis.call('set','x',1) + } 1 x + } + assert_equal {x} [r eval {#!lua flags=no-writes + return redis.call('get','x') + } 1 x + ] + + r config set rdb-key-save-delay 0 + if {$bgsave_type eq "forkless"} { + # Remove the blocking directory so the recovery save can succeed. + file delete [file join [lindex [r config get dir] 1] [lindex [r config get dbfilename] 1]] + } + r config set bgsave-default-method $bgsave_type + r bgsave + waitForBgsave r - # server is writable again - r set x y - } {OK} + # server is writable again + r set x y + } {OK} + } } start_server {} { @@ -588,4 +1704,54 @@ start_server {} { } } + +start_server {overrides {forkless-infrastructure-enabled yes save ""}} { + test {bgsave-default-method can be set to forkless with forkless-infrastructure-enabled} { + r config set bgsave-default-method forkless + assert_equal [lindex [r config get bgsave-default-method] 1] "forkless" + } +} + +start_server {overrides {forkless-infrastructure-enabled yes bgsave-default-method forkless}} { + test {BGSAVE uses forkless when bgsave-default-method is forkless} { + r set key value + set result [r bgsave] + assert_match "*Background saving started*" $result + waitForBgsave r + assert_equal [s rdb_last_bgsave_type] "forkless" + } +} + +start_server {overrides {bgsave-default-method fork}} { + test {BGSAVE uses fork when bgsave-default-method is fork} { + r set key value + set result [r bgsave] + assert_match "*Background saving started*" $result + waitForBgsave r + assert_equal [s rdb_last_bgsave_type] "fork" + } +} + +start_server {overrides {save ""}} { + test {bgsave-default-method forkless is rejected without forkless-infrastructure-enabled} { + # forkless-infrastructure-enabled defaults to no here. + assert_error "*forkless-infrastructure-enabled yes*" { + r config set bgsave-default-method forkless + } + # The value is unchanged and remains fork. + assert_equal [lindex [r config get bgsave-default-method] 1] "fork" + } +} + +test {Server refuses to start with bgsave-default-method forkless and no forkless-infrastructure-enabled} { + catch {exec $::VALKEY_SERVER_BIN --bgsave-default-method forkless} err + assert_match {*forkless-infrastructure-enabled yes*} $err +} + +test {Server starts with bgsave-default-method before forkless-infrastructure-enabled yes} { + start_server {overrides {bgsave-default-method forkless forkless-infrastructure-enabled yes save ""}} { + assert_equal [lindex [r config get bgsave-default-method] 1] "forkless" + } +} + } ;# tags diff --git a/tests/integration/repl-compression.tcl b/tests/integration/repl-compression.tcl new file mode 100644 index 000000000..170e0f7b4 --- /dev/null +++ b/tests/integration/repl-compression.tcl @@ -0,0 +1,1052 @@ +tags {"repl external:skip"} { + +# repl_uncompressed_bytes= from the replica line of the primary's INFO replication. +proc replica_line_uncompressed_bytes {primary} { + set info [$primary info replication] + assert {[regexp {repl_uncompressed_bytes=([0-9]+)} $info -> uncompressed_bytes]} + return $uncompressed_bytes +} + +# Start a fake primary that completes a size-framed full sync, then sends the +# supplied steady-state replication bytes and waits for the replica to close. +proc start_fake_primary_with_stream {rdb_payload stream_payload} { + set rdb_file [tmpfile fake-primary-rdb] + set stream_file [tmpfile fake-primary-stream] + write_binary_file $rdb_file $rdb_payload + write_binary_file $stream_file $stream_payload + set port [find_available_port $::baseport $::portcount] + set pid [exec [info nameofexecutable] tests/helpers/fake_primary.tcl \ + $port $rdb_file [string length $rdb_payload] $stream_file &] + wait_for_condition 50 50 { + [ping_server 127.0.0.1 $port] + } else { + fail "Failed to start fake primary" + } + return [list $pid $port] +} + +# Start a fake dual-channel primary that sends a valid RDB on the RDB channel +# and the supplied buffered command-stream bytes on the main channel. +proc start_fake_dual_channel_primary_with_stream {rdb_payload stream_payload} { + set rdb_file [tmpfile fake-dual-primary-rdb] + set stream_file [tmpfile fake-dual-primary-stream] + write_binary_file $rdb_file $rdb_payload + write_binary_file $stream_file $stream_payload + set port [find_available_port $::baseport $::portcount] + set pid [exec [info nameofexecutable] tests/helpers/fake_dual_channel_primary.tcl \ + $port $rdb_file $stream_file &] + wait_for_condition 50 50 { + [ping_server 127.0.0.1 $port] + } else { + fail "Failed to start fake dual-channel primary" + } + return [list $pid $port] +} + +# ============================================================ +# Config CRUD — single-server tests, no replication needed +# ============================================================ + +start_server {overrides {save "" repl-compression no}} { + + test {repl-compression config: default, set, and survives CONFIG REWRITE and restart} { + assert_equal "no" [lindex [r config get repl-compression] 1] + + r config set repl-compression yes + assert_equal "yes" [lindex [r config get repl-compression] 1] + r config set repl-compression lz4 + assert_equal "lz4" [lindex [r config get repl-compression] 1] + r config rewrite + + restart_server 0 true false + + assert_equal "lz4" [lindex [r config get repl-compression] 1] + + r config set repl-compression no + } +} + +# ============================================================ +# Replication handshake behavior — primary + replica tests +# ============================================================ + +start_server {tags {"repl"} overrides {save ""}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + # Negotiation and the compressed incremental stream are load-mode + # independent. Keep both explicit LZ4 load modes and prove that "yes" + # selects the current default algorithm (LZ4). + foreach {compression_mode diskless_load} { + lz4 swapdb + lz4 disabled + yes swapdb + } { + test "Replica negotiates $compression_mode compression (repl-diskless-load $diskless_load)" { + $primary config set repl-compression $compression_mode + set _code [catch { + start_server [list overrides [list save "" repl-compression $compression_mode repl-diskless-load $diskless_load]] { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + # The same negotiated capability covers diskless full sync + # and the post-sync incremental stream. + wait_for_condition 50 100 { + [regexp -all "repl_compression=lz4" [$primary info replication]] >= 1 + } else { + fail "Compression not negotiated" + } + + # Exercise the compressed incremental stream. + for {set i 0} {$i < 100} {incr i} { + $primary set "negotiated:$i" [string repeat "v" 50] + } + wait_for_condition 50 100 { + [$replica get "negotiated:99"] eq [string repeat "v" 50] + } else { + fail "Replica did not receive compressed incremental stream" + } + assert_equal [$primary debug digest] [$replica debug digest] + + $replica replicaof no one + } + } _res _opts] + $primary config set repl-compression no + return -options $_opts $_res + } + } + + test {Compressed replication handles compressible and incompressible values across batch boundaries} { + $primary config set repl-compression lz4 + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [regexp {state=online.*repl_compression=lz4,repl_compressed_bytes=[0-9]+,repl_uncompressed_bytes=[0-9]+} \ + [$primary info replication]] + } else { + fail "Compressed replication not established" + } + + # The compressible value spans several 1 MiB raw batches. + set bigval [string repeat "abcdefghij0123456789" 209715] + $primary set batch:compressible $bigval + wait_for_condition 50 200 { + [$replica get batch:compressible] eq $bigval + } else { + fail "Compressible value did not replicate intact" + } + + # A deterministic ratio≈1 payload exercises worst-case compressed + # output sizing across multiple raw batches. + expr {srand(424242)} + set payload "" + while {[string length $payload] < 1572864} { + set chunk "" + for {set i 0} {$i < 4096} {incr i} { + append chunk [format %c [expr {int(rand()*256)}]] + } + append payload $chunk + } + $primary set batch:incompressible $payload + wait_for_condition 50 200 { + [$replica get batch:incompressible] eq $payload + } else { + fail "Incompressible value did not replicate intact" + } + + $replica replicaof no one + } + $primary config set repl-compression no + } + + test {Backlog cursor stays pinned until the compressed batch fully drains} { + # The cursor advances only on full out_buf drain: pause the replica and + # uncompressed_bytes must freeze while master_repl_offset grows. + $primary config set repl-compression lz4 + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + set replica_pid [srv 0 pid] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + wait_for_condition 50 200 { + [string match {*state=online*repl_compression=lz4*} [$primary info replication]] + } else { + fail "Compression not active on replica" + } + + $primary set pin:baseline baseline_val + wait_for_ofs_sync $primary $replica + # The link must survive on the same connection (no resync). + set sync_full_before [status $primary sync_full] + set sync_partial_before [status $primary sync_partial_ok] + + pause_process $replica_pid + set pause_code [catch { + + # Pseudo-random 100KB block (past LZ4's 64KB window): ratio ~1 + # overfills the socket buffers and leaves out_buf mid-batch. + expr {srand(51555)} + set payload "" + while {[string length $payload] < 102400} { + set chunk "" + for {set i 0} {$i < 4096} {incr i} { + append chunk [format %c [expr {int(rand()*256)}]] + } + append payload $chunk + } + for {set i 0} {$i < 200} {incr i} { + $primary set "pin:burst:$i" $payload + } + + # Wait for the residual kernel-buffer drain to settle. + set previous_uncompressed_bytes [replica_line_uncompressed_bytes $primary] + set stable_samples 0 + for {set i 0} {$i < 100 && $stable_samples < 3} {incr i} { + after 100 + set current_uncompressed_bytes [replica_line_uncompressed_bytes $primary] + if {$current_uncompressed_bytes == $previous_uncompressed_bytes} { + incr stable_samples + } else { + set stable_samples 0 + } + set previous_uncompressed_bytes $current_uncompressed_bytes + } + assert_equal 3 $stable_samples + + # Frozen cursor: two samples with writes in between must be equal. + set uncompressed_bytes_before [replica_line_uncompressed_bytes $primary] + set repl_offset_before [status $primary master_repl_offset] + for {set i 0} {$i < 20} {incr i} { + $primary set "pin:tick:$i" tick_val + } + after 300 + set uncompressed_bytes_after [replica_line_uncompressed_bytes $primary] + set repl_offset_after [status $primary master_repl_offset] + + assert {$repl_offset_after > $repl_offset_before} + assert_equal $uncompressed_bytes_before $uncompressed_bytes_after + } pause_result pause_opts] + resume_process $replica_pid + if {$pause_code} { + return -options $pause_opts $pause_result + } + + # Pinned batches drain after the replica resumes. + wait_for_ofs_sync $primary $replica + assert {[$replica get pin:burst:199] eq $payload} + assert_equal tick_val [$replica get pin:tick:19] + assert {[replica_line_uncompressed_bytes $primary] > $uncompressed_bytes_after} + assert_equal $sync_full_before [status $primary sync_full] + assert_equal $sync_partial_before [status $primary sync_partial_ok] + + $replica replicaof no one + } + + $primary config set repl-compression no + } + + test {Compressed replica obeys the hard output buffer limit and resynchronizes} { + $primary config set repl-compression lz4 + $primary config set repl-backlog-size 1mb + $primary config set client-output-buffer-limit "replica 4mb 0 0" + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + set replica_pid [srv 0 pid] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [string match {*state=online*repl_compression=lz4*} [$primary info replication]] + } else { + fail "Compressed replication not established" + } + + set sync_full_before [status $primary sync_full] + set cob_disconnections_before [s -1 client_output_buffer_limit_disconnections] + expr {srand(99173)} + set payload [randstring [expr {256 * 1024}] [expr {256 * 1024}]] + set last_key "" + + pause_process $replica_pid + set pause_code [catch { + # Incompressible writes fill the compressed staging buffer and + # pin raw backlog blocks while the replica cannot read. + for {set i 0} {$i < 128} {incr i} { + set last_key "cob:$i" + $primary set $last_key $payload + if {[status $primary connected_slaves] == 0} break + } + + wait_for_condition 100 100 { + [status $primary connected_slaves] == 0 + } else { + fail "Primary did not disconnect compressed replica at the hard output buffer limit" + } + wait_for_condition 100 100 { + [s -1 client_output_buffer_limit_disconnections] > $cob_disconnections_before + } else { + fail "Primary did not record the output buffer limit disconnection" + } + + assert_equal "" [string trim [$primary client list type replica]] + wait_for_condition 100 100 { + [status $primary repl_backlog_histlen] <= 2 * 1024 * 1024 + } else { + fail "Compressed replica backlog reference was not released" + } + } pause_result pause_options] + resume_process $replica_pid + if {$pause_code} { + return -options $pause_options $pause_result + } + + # The burst is larger than the 1 MiB backlog, so reconnection needs + # a full sync. Incremental compression must be active afterward. + wait_for_condition 300 100 { + [status $primary sync_full] == $sync_full_before + 1 && + [s 0 master_link_status] eq {up} && + [string match {*state=online*repl_compression=lz4*} [$primary info replication]] + } else { + fail "Replica did not resynchronize with compression after the output buffer disconnect" + } + assert_equal $payload [$replica get $last_key] + + $primary set cob:after-resync delivered + wait_for_condition 50 100 { + [$replica get cob:after-resync] eq {delivered} + } else { + fail "Replication did not continue after compressed resynchronization" + } + + $replica replicaof no one + } + + $primary config set repl-compression no + $primary config set repl-backlog-size 10mb + $primary config set client-output-buffer-limit "replica 256mb 64mb 60" + } + + test {Compressed partial resync preserves data and decoded ACK offsets} { + $primary config set repl-compression lz4 + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [string match {*state=online*repl_compression=lz4*} [$primary info replication]] + } else { + fail "Compressed replication not established" + } + + $primary set key1 value1 + wait_for_condition 50 100 { + [$replica get key1] eq {value1} + } else { + fail "Initial replication failed" + } + + # Killing the primary-side connection preserves the replica's + # cached offset and exercises a compressed partial resync. + set full_before [status $primary sync_full] + set partial_before [status $primary sync_partial_ok] + $primary client kill type replica + + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [string match {*state=online*repl_compression=lz4*} [$primary info replication]] && + [status $primary sync_partial_ok] == $partial_before + 1 + } else { + fail "Compressed partial resync did not complete" + } + assert_equal $full_before [status $primary sync_full] + + # WAIT compares logical RESP offsets. A highly compressible write + # only reaches the target if the replica advances by decoded bytes, + # not by the much smaller number of wire bytes. + set ack_payload [string repeat x 262144] + $primary set key2 value2 + $primary set key3 $ack_payload + assert_equal 1 [$primary wait 1 5000] + assert_equal value1 [$replica get key1] + assert_equal value2 [$replica get key2] + assert_equal $ack_payload [$replica get key3] + + $replica replicaof no one + } + $primary config set repl-compression no + } + + test {Compressed replication survives an interrupted frame and reconverges} { + # A replication frame remains open for the life of its connection, so + # dropping the link interrupts the frame without a closing marker. + $primary config set repl-compression lz4 + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [string match {*state=online*repl_compression=lz4*} [$primary info replication]] + } else { + fail "Compressed replication not established" + } + + # Baseline sync so recovery is a clean reconverge from a known point. + $primary set trunc:baseline baseline_val + wait_for_ofs_sync $primary $replica + set full_before [status $primary sync_full] + set partial_before [status $primary sync_partial_ok] + + # Keep writing after the disconnect so either a partial or full + # resynchronization must converge to a newer offset. + set bigval [string repeat "abcdefghij0123456789" 52429] ;# ~1 MiB + for {set i 0} {$i < 60} {incr i} { + $primary set "trunc:load:$i" $bigval + if {$i == 30} { + # Kill from the primary side while the stream frame is open. + $primary client kill type replica + } + } + + # The truncated link forces the replica to reconnect and resync + # (partial or full, either is fine) and re-establish compression. + $primary set trunc:final final_val + wait_for_condition 100 200 { + [s 0 master_link_status] eq {up} && + [string match {*state=online*repl_compression=lz4*} [$primary info replication]] && + [status $primary sync_full] + [status $primary sync_partial_ok] > $full_before + $partial_before + } else { + fail "Replica did not reconnect with compression after mid-frame truncation" + } + + # (a) The replica survived the truncated frame: no assertion or + # crash was logged. + assert_equal 0 [count_log_message 0 "*=== ASSERTION FAILED ===*"] + assert_equal 0 [count_log_message 0 "*crashed by signal*"] + + # (b) The link is back up (asserted in the wait above) and (c) the + # datasets converge byte-identically once offsets align. + wait_for_ofs_sync $primary $replica + assert_equal [$primary debug digest] [$replica debug digest] + + $replica replicaof no one + } + $primary config set repl-compression no + } + + if {!$::tls} { + test {Corrupt compressed replication stream disconnects cleanly and recovers} { + $primary config set repl-compression lz4 + $primary flushall + $primary set corrupt:baseline baseline_val + $primary save + + set rdb_payload [read_binary_file [server_rdb_path $primary]] + # Valid VCS replication envelope followed by an invalid LZ4 frame. + set corrupt_stream [binary format H* 56435301010002] + append corrupt_stream [string repeat "\x00" 64] + + set fake_pid "" + with_cleanup { + lassign [start_fake_primary_with_stream $rdb_payload $corrupt_stream] fake_pid fake_port + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + set replica_loglines [count_log_lines 0] + $replica replicaof 127.0.0.1 $fake_port + + wait_for_log_messages 0 {"*replication stream decompression failure*"} \ + $replica_loglines 100 100 + wait_for_condition 50 100 { + [s 0 master_link_status] eq {down} + } else { + fail "Replica did not disconnect from the corrupt compressed stream" + } + assert_equal {PONG} [$replica ping] + assert_equal {baseline_val} [$replica get corrupt:baseline] + + # A fresh replication session must not retain any corrupt + # decoder state from the failed link. + $replica replicaof $primary_host $primary_port + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [$replica get corrupt:baseline] eq {baseline_val} && + [string match {*state=online*repl_compression=lz4*} [$primary info replication]] + } else { + fail "Replica did not recover after compressed stream corruption" + } + + $replica replicaof no one + } + } { + if {$fake_pid ne ""} {catch {exec kill $fake_pid}} + $primary config set repl-compression no + } + } + + test {Corrupt buffered dual-channel stream retries without promoting the replica} { + $primary config set repl-compression lz4 + $primary flushall + $primary set dual-corrupt:baseline baseline_val + $primary save + + set rdb_payload [read_binary_file [server_rdb_path $primary]] + set corrupt_stream [binary format H* 56435301010002] + append corrupt_stream [string repeat "\x00" 64] + + set fake_pid "" + with_cleanup { + lassign [start_fake_dual_channel_primary_with_stream $rdb_payload $corrupt_stream] fake_pid fake_port + + start_server {overrides {save "" repl-compression lz4 dual-channel-replication-enabled yes repl-diskless-load swapdb}} { + set replica [srv 0 client] + set replica_loglines [count_log_lines 0] + $replica replicaof 127.0.0.1 $fake_port + + wait_for_log_messages 0 {"*Dual-channel replication stream decompression failure*"} \ + $replica_loglines 100 100 + wait_for_condition 50 100 { + [lindex [$replica role] 0] eq {slave} && + [s 0 master_host] eq {127.0.0.1} && + [s 0 master_port] == $fake_port && + [s 0 master_link_status] eq {down} + } else { + fail "Replica did not retain its configured primary after buffered stream corruption" + } + assert_equal {baseline_val} [$replica get dual-corrupt:baseline] + + $replica replicaof $primary_host $primary_port + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [$replica get dual-corrupt:baseline] eq {baseline_val} + } else { + fail "Replica did not recover after dual-channel stream corruption" + } + + $replica replicaof no one + } + } { + if {$fake_pid ne ""} {catch {exec kill $fake_pid}} + $primary config set repl-compression no + } + } + } + + test {Replica with repl-compression lz4 handles a plaintext primary (passthrough)} { + # Primary has compression OFF, replica ON: the replica advertises the + # capability but the primary sends plaintext, so the replica must pass + # the stream through untouched rather than expecting a VCS envelope. + $primary config set repl-compression no + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started (primary plaintext, replica compression on)" + } + + # Incremental writes arrive as plaintext; passthrough must deliver them. + for {set i 0} {$i < 50} {incr i} { + $primary set "pt:$i" [string repeat "payload$i " 20] + } + wait_for_condition 50 100 { + [$replica get pt:49] eq [string repeat "payload49 " 20] + } else { + fail "Replica did not receive plaintext data via passthrough" + } + assert_equal [$primary dbsize] [$replica dbsize] + + # The link remained plaintext. + assert_equal 0 [string match {*repl_compression=lz4*} [$primary info replication]] + + # Disabling a link already classified as plaintext must not cause + # an unnecessary reconnect. + set full_before [status $primary sync_full] + set partial_before [status $primary sync_partial_ok] + $replica config set repl-compression no + after 1500 + assert_equal $full_before [status $primary sync_full] + assert_equal $partial_before [status $primary sync_partial_ok] + assert_equal up [s 0 master_link_status] + + $replica replicaof no one + } + } + + test {Replica config change waits for an in-progress full sync} { + $primary config set repl-compression lz4 + $primary config set rdb-key-save-delay 1000 + $primary flushall + $primary debug populate 5000 sync-config: 100 + + set full_before [status $primary sync_full] + set partial_before [status $primary sync_partial_ok] + set _code [catch { + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 100 100 { + [s 0 master_sync_in_progress] == 1 && + [status $primary sync_full] == $full_before + 1 + } else { + fail "Full sync did not start" + } + + $replica config set repl-compression no + after 1500 + assert_equal 1 [s 0 master_sync_in_progress] + assert_equal [expr {$full_before + 1}] [status $primary sync_full] + + wait_for_condition 200 100 { + [s 0 master_link_status] eq {up} && + [regexp -all {repl_compression=lz4} [$primary info replication]] == 0 && + [status $primary sync_partial_ok] == $partial_before + 1 + } else { + fail "Replica did not renegotiate after the full sync completed" + } + assert_equal [expr {$full_before + 1}] [status $primary sync_full] + + $replica replicaof no one + } + } _res _opts] + $primary config set rdb-key-save-delay 0 + $primary config set repl-compression no + return -options $_opts $_res + } + + test {Replica repl-compression flips renegotiate upstream in both directions} { + $primary config set repl-compression lz4 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [regexp -all {repl_compression=lz4} [$primary info replication]] == 1 + } else { + fail "Compressed replication not established" + } + + set full_before [status $primary sync_full] + set partial_before [status $primary sync_partial_ok] + + # The whole command rolls back, so the compressed link stays up. + assert_error {*argument 'maxmemory-policy'*} { + $replica config set repl-compression no maxmemory-policy not-a-policy + } + assert_equal lz4 [lindex [$replica config get repl-compression] 1] + after 1500 + assert_equal $partial_before [status $primary sync_partial_ok] + assert_equal 1 [regexp -all {repl_compression=lz4} [$primary info replication]] + + # Returning to the advertised state before cron runs does not + # require renegotiating the existing link. + $replica debug pause-cron 1 + set pause_code [catch { + $replica config set repl-compression no + $replica config set repl-compression lz4 + } pause_result pause_opts] + $replica debug pause-cron 0 + if {$pause_code} { + return -options $pause_opts $pause_result + } + after 1500 + assert_equal $partial_before [status $primary sync_partial_ok] + assert_equal 1 [regexp -all {repl_compression=lz4} [$primary info replication]] + + set transition 0 + foreach {mode expected_compressed} {no 0 lz4 1} { + incr transition + $replica config set repl-compression $mode + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [regexp -all {repl_compression=lz4} [$primary info replication]] == $expected_compressed && + [status $primary sync_partial_ok] == $partial_before + $transition + } else { + fail "Replica did not renegotiate after setting repl-compression $mode" + } + + $primary set "replica-flip:$mode" "value:$mode" + wait_for_condition 50 100 { + [$replica get "replica-flip:$mode"] eq "value:$mode" + } else { + fail "Data did not replicate after setting repl-compression $mode" + } + } + assert_equal $full_before [status $primary sync_full] + $replica replicaof no one + } + $primary config set repl-compression no + } + + test {Primary config flips preserve independent compressed and plaintext replica links} { + $primary config set repl-compression no + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set compressed_replica [srv 0 client] + $compressed_replica replicaof $primary_host $primary_port + wait_for_sync $compressed_replica + + start_server {overrides {save "" repl-compression no repl-diskless-load swapdb}} { + set plaintext_replica [srv 0 client] + $plaintext_replica replicaof $primary_host $primary_port + wait_for_sync $plaintext_replica + + assert_equal 0 [regexp -all {repl_compression=lz4} [$primary info replication]] + set full_before [status $primary sync_full] + set partial_before [status $primary sync_partial_ok] + + # Only the capable replica renegotiates when the primary enables + # compression; the opted-out replica remains on its link. + $primary config set repl-compression lz4 + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [status $compressed_replica master_link_status] eq {up} && + [regexp -all {repl_compression=lz4} [$primary info replication]] == 1 && + [status $primary sync_partial_ok] == $partial_before + 1 + } else { + fail "Mixed replica links did not converge after enabling compression" + } + $primary set mixed:compressed delivered + wait_for_condition 50 100 { + [$compressed_replica get mixed:compressed] eq {delivered} && + [$plaintext_replica get mixed:compressed] eq {delivered} + } else { + fail "Mixed replica links did not both receive compressed-phase data" + } + + # Disabling compression again reconnects only the compressed + # link and leaves both replicas receiving plaintext. + $primary config set repl-compression no + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [status $compressed_replica master_link_status] eq {up} && + [regexp -all {repl_compression=lz4} [$primary info replication]] == 0 && + [status $primary sync_partial_ok] == $partial_before + 2 + } else { + fail "Mixed replica links did not converge after disabling compression" + } + assert_equal $full_before [status $primary sync_full] + + $primary set mixed:plaintext delivered + wait_for_condition 50 100 { + [$compressed_replica get mixed:plaintext] eq {delivered} && + [$plaintext_replica get mixed:plaintext] eq {delivered} + } else { + fail "Mixed replica links did not both receive plaintext-phase data" + } + assert_equal [$primary debug digest] [$compressed_replica debug digest] + assert_equal [$primary debug digest] [$plaintext_replica debug digest] + + $plaintext_replica replicaof no one + } + $compressed_replica replicaof no one + } + } + + test {Dual-channel full sync with compression delivers writes made during load} { + $primary config set repl-compression lz4 + $primary config set dual-channel-replication-enabled yes + $primary config set rdb-key-save-delay 100 + $primary flushall + $primary debug populate 10000 dc: 100 + + start_server {overrides {save "" repl-compression lz4 dual-channel-replication-enabled yes}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + # rdb-key-save-delay stretches the RDB stage; catch the sync window. + wait_for_condition 500 10 { + [s 0 master_sync_in_progress] == 1 && + [string match {*state=bg_transfer*repl_compression=lz4*} [$primary info replication]] + } else { + fail "Dual-channel sync did not start" + } + + # Writes made during load reach the replica via the compressed main + # channel: +CONTINUE starts compression, put-online must not restart + # it (a second init would emit a new envelope mid-frame). + for {set i 0} {$i < 200} {incr i} { + $primary set "during_load:$i" "value_$i" + } + # This compresses below one socket read but expands past one decode + # budget, exercising streamReplDataBufToDb's resumable decode loop. + set during_load_payload [string repeat x [expr {2 * 1024 * 1024}]] + $primary set during_load:large $during_load_payload + assert_equal 1 [s 0 master_sync_in_progress] + assert_match {*state=bg_transfer*repl_compression=lz4*} [$primary info replication] + + wait_for_condition 100 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not up after dual-channel sync" + } + + wait_for_condition 50 100 { + [$replica get "during_load:199"] eq {value_199} + } else { + fail "Writes made during load did not reach the replica" + } + for {set i 0} {$i < 200} {incr i} { + assert_equal "value_$i" [$replica get "during_load:$i"] + } + assert_equal $during_load_payload [$replica get during_load:large] + assert_match "*repl_compression=lz4*" [$primary info replication] + wait_for_ofs_sync $primary $replica + + # A double init would emit a second envelope mid-frame on the first + # post-online write, corrupting the replica and forcing a resync. + # Stable sync counters prove the link survived that first write. + set sync_full_before [s -1 sync_full] + set sync_partial_before [s -1 sync_partial_ok] + $primary set post_online_probe delivered + wait_for_condition 50 100 { + [$replica get post_online_probe] eq {delivered} + } else { + fail "Post-online write did not reach the replica" + } + assert_equal $sync_full_before [s -1 sync_full] + assert_equal $sync_partial_before [s -1 sync_partial_ok] + + $replica replicaof no one + } + $primary config set rdb-key-save-delay 0 + $primary config set dual-channel-replication-enabled no + $primary config set repl-compression no + } + + foreach {initial final expected_compressed} {no lz4 1 lz4 no 0} { + test "Dual-channel load converges after primary repl-compression $initial->$final" { + $primary config set repl-compression $initial + $primary config set dual-channel-replication-enabled yes + $primary config set rdb-key-save-delay 100 + $primary flushall + $primary debug populate 10000 midload: 100 + + start_server {overrides {save "" repl-compression lz4 dual-channel-replication-enabled yes}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 500 10 { + [s 0 master_sync_in_progress] == 1 && + [string match {*state=bg_transfer*} [$primary info replication]] && + [regexp -all {repl_compression=lz4} [$primary info replication]] == [expr {$initial eq "lz4"}] + } else { + fail "Dual-channel $initial stream did not reach the load window" + } + + # The command-stream decision is fixed at +CONTINUE. Traffic + # buffered before the flip must survive cron reconciliation. + for {set i 0} {$i < 20} {incr i} { + $primary set "during_load:$i" "value_$i" + } + set full_before [status $primary sync_full] + set partial_before [status $primary sync_partial_ok] + $primary config set repl-compression $final + assert_equal 1 [s 0 master_sync_in_progress] + assert_match {*state=bg_transfer*} [$primary info replication] + assert_equal [expr {$initial eq "lz4"}] \ + [regexp -all {repl_compression=lz4} [$primary info replication]] + + wait_for_condition 100 100 { + [status $primary sync_partial_ok] == $partial_before + 1 && + [s 0 master_link_status] eq {up} && + [regexp -all {repl_compression=lz4} [$primary info replication]] == $expected_compressed + } else { + fail "Dual-channel link did not renegotiate to $final" + } + assert_equal $full_before [status $primary sync_full] + + for {set i 0} {$i < 20} {incr i} { + assert_equal "value_$i" [$replica get "during_load:$i"] + } + $primary set midload_probe delivered + wait_for_condition 50 100 { + [$replica get midload_probe] eq {delivered} + } else { + fail "Post-renegotiation write did not reach the replica" + } + assert_equal [expr {$partial_before + 1}] [status $primary sync_partial_ok] + assert_equal $full_before [status $primary sync_full] + + $replica replicaof no one + } + $primary config set rdb-key-save-delay 0 + $primary config set dual-channel-replication-enabled no + $primary config set repl-compression no + } + } + +} + +# ============================================================ +# Multi-replica compressed replication tests +# ============================================================ + +# Multiple replicas distribute across threads and stay in sync. +# io-threads-always-active starts off during the handshakes: an offloaded +# REPLCONF reply can leave pending output that makes the primary reject PSYNC, +# and the legacy-SYNC fallback suppresses the ACK that diskless sync waits for +# (upstream race). It is enabled once all replicas are streaming. +start_server {tags {"repl"} overrides {save "" io-threads 4 repl-compression lz4}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Multiple replicas all stay in sync under load} { + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica1 [srv 0 client] + $replica1 replicaof $primary_host $primary_port + wait_for_sync $replica1 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica2 [srv 0 client] + $replica2 replicaof $primary_host $primary_port + wait_for_sync $replica2 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica3 [srv 0 client] + $replica3 replicaof $primary_host $primary_port + wait_for_sync $replica3 + + wait_for_condition 50 200 { + [regexp -all "repl_compression=lz4" [$primary info replication]] >= 3 + } else { + fail "Not all replicas have compression active" + } + + # Handshakes are done; run the load phase on IO threads. + $primary config set io-threads-always-active yes + + for {set i 0} {$i < 500} {incr i} { + $primary set "multi_repl:$i" [string repeat "x" 100] + } + + wait_for_condition 100 200 { + [$replica1 dbsize] == [$primary dbsize] && + [$replica2 dbsize] == [$primary dbsize] && + [$replica3 dbsize] == [$primary dbsize] + } else { + fail "Not all replicas caught up: r1=[$replica1 dbsize] r2=[$replica2 dbsize] r3=[$replica3 dbsize] primary=[$primary dbsize]" + } + + set primary_digest [$primary debug digest] + assert_equal $primary_digest [$replica1 debug digest] + assert_equal $primary_digest [$replica2 debug digest] + assert_equal $primary_digest [$replica3 debug digest] + + $replica3 replicaof no one + } + $replica2 replicaof no one + } + $replica1 replicaof no one + } + } +} + +# Chained replication: each hop negotiates compression independently, and the +# middle node simultaneously decodes its primary link on the main thread while +# encoding for its own replica on IO threads. +start_server {tags {"repl"} overrides {save "" repl-compression lz4}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Chained replication compresses each hop independently} { + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb io-threads 4 io-threads-always-active yes}} { + set middle [srv 0 client] + set middle_host [srv 0 host] + set middle_port [srv 0 port] + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set leaf [srv 0 client] + + $middle replicaof $primary_host $primary_port + wait_for_sync $middle + $leaf replicaof $middle_host $middle_port + wait_for_sync $leaf + + # Both hops negotiated compression. + wait_for_condition 100 200 { + [regexp -all "repl_compression=lz4" [$primary info replication]] >= 1 && + [regexp -all "repl_compression=lz4" [$middle info replication]] >= 1 + } else { + fail "Compression not active on both hops" + } + + # Writes flow primary -> middle -> leaf across two compressed hops. + for {set i 0} {$i < 200} {incr i} { + $primary set "chain:$i" "chain_value_$i" + } + wait_for_condition 100 200 { + [$leaf dbsize] == [$primary dbsize] + } else { + fail "Leaf did not catch up: leaf=[$leaf dbsize] primary=[$primary dbsize]" + } + assert_equal "chain_value_0" [$leaf get chain:0] + assert_equal "chain_value_99" [$leaf get chain:99] + assert_equal "chain_value_199" [$leaf get chain:199] + + # The leaf flip automatically renegotiates only the second hop. + set full_before [status $middle sync_full] + set partial_before [status $middle sync_partial_ok] + $leaf config set repl-compression no + wait_for_condition 100 200 { + [status $leaf master_link_status] eq {up} && + [regexp -all {repl_compression=lz4} [$middle info replication]] == 0 && + [status $middle sync_partial_ok] == $partial_before + 1 + } else { + fail "Hop2 did not renegotiate to plaintext" + } + assert_equal $full_before [status $middle sync_full] + assert {[regexp -all {repl_compression=lz4} [$primary info replication]] >= 1} + + # Data still flows end-to-end over mixed hops. + $primary set chain:final final_val + wait_for_condition 100 200 { + [$leaf get chain:final] eq {final_val} + } else { + fail "Write did not reach leaf after hop2 renegotiated plaintext" + } + + $leaf replicaof no one + } + $middle replicaof no one + } + } +} + + +} diff --git a/tests/integration/repl-fullsync-compression.tcl b/tests/integration/repl-fullsync-compression.tcl new file mode 100644 index 000000000..40bbc3213 --- /dev/null +++ b/tests/integration/repl-fullsync-compression.tcl @@ -0,0 +1,771 @@ +# Full-sync streaming compression: end-to-end coverage across disk-based, +# diskless, and dual-channel full sync, both replica load paths, negative cases +# (truncation, corruption, premature EOF, non-capable cohorts), and byte +# accounting. Asymmetric policy: an all-capable group gets a compressed round; any +# non-capable member forces one plaintext round for all. + +# --- helpers -------------------------------------------------------------- + +proc rdb_is_compressed {client} { + set header [read_binary_file_prefix [server_rdb_path $client] 3] + return [expr {$header eq "VCS"}] +} + +# Compressible, multi-type dataset so an LZ4 frame is actually produced. +proc populate_compressible_dataset {client prefix {n 300}} { + $client flushall + for {set i 0} {$i < $n} {incr i} { + $client set "${prefix}:str:$i" [string repeat "${prefix}:payload:$i " 16] + } + $client rpush "${prefix}:list" a b c d e f g h + $client sadd "${prefix}:set" alpha beta gamma delta + $client zadd "${prefix}:zset" 1 one 2 two 3 three 4 four + $client hset "${prefix}:hash" f1 v1 f2 [string repeat "${prefix}:hashval " 8] + $client xadd "${prefix}:stream" * f1 s1 f2 [string repeat "${prefix}:streamval " 4] + $client xadd "${prefix}:stream" * f1 s2 f2 tail +} + +# Wait for the link up, then assert digests match. Generous 300x100 budget for +# slow/shared primaries and the macOS timing fix for the delay-based piggyback tests. +proc assert_replica_synced {primary replica {tag ""}} { + wait_for_condition 300 100 { + [status $replica master_link_status] eq "up" + } else { + fail "replica link not up after full sync $tag" + } + assert_equal [$primary debug digest] [$replica debug digest] "replica digest mismatch after full sync $tag" +} + +# Single-shot fake primary (tests/helpers/fake_primary.tcl): answers the handshake, +# announces $announce bytes, sends $payload, closes. Returns {pid port}. +proc start_fake_primary {payload announce} { + set payload_file [tmpfile fake-primary-payload] + write_binary_file $payload_file $payload + set port [find_available_port $::baseport $::portcount] + set pid [exec [info nameofexecutable] tests/helpers/fake_primary.tcl $port $payload_file $announce &] + wait_for_condition 50 50 { + [ping_server 127.0.0.1 $port] + } else { + fail "Failed to start fake primary" + } + return [list $pid $port] +} + +# Park both replicas in WAIT_BGSAVE_START together: a slow manual BGSAVE +# occupies the RDB child slot (syncCommand defers to replicationCron), so the +# following grouped round makes its group-AND capability decision over both +# waiters deterministically. Resetting the delay lets that round run fast. +proc park_replicas_for_grouped_bgsave {primary replica1 replica2 primary_host primary_port} { + $primary config set rdb-key-save-delay 5000 + assert_match {*Background saving started*} [$primary bgsave] + wait_for_condition 200 10 { + [status $primary rdb_bgsave_in_progress] eq 1 + } else { + $primary config set rdb-key-save-delay 0 + fail "manual BGSAVE did not start" + } + + $replica1 replicaof $primary_host $primary_port + $replica2 replicaof $primary_host $primary_port + wait_for_condition 200 10 { + [status $primary connected_slaves] == 2 && + [status $primary rdb_bgsave_in_progress] eq 1 + } else { + $primary config set rdb-key-save-delay 0 + fail "both replicas did not register before manual BGSAVE finished (grouping not achieved)" + } + + $primary config set rdb-key-save-delay 0 +} + +# ============================================================================ +# Disk-based full sync: ONE shared primary fixture. rdbcompression is MODIFIABLE, +# so each test sets its mode and repopulates its own prefix; the compression-off +# test restores the fixture default (lz4) at its end for later shared tests. +# Replicas are per-test nested start_servers because their repl-compression +# settings control whether they advertise the LZ4 capability. +# ============================================================================ + +start_server {tags {"repl rdb-compression external:skip needs:debug"} overrides {save "" enable-debug-command local}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + $primary config set repl-diskless-sync no + $primary config set rdbcompression lz4 + $primary config set rdb-del-sync-files no + + start_server {overrides {save "" enable-debug-command local repl-compression lz4}} { + set replica [srv 0 client] + + test {Disk full sync: all-capable group produces a compressed RDB that loads} { + $primary config set rdbcompression lz4 + populate_compressible_dataset $primary "cap" + set primary_loglines [count_log_lines -1] + + $replica replicaof $primary_host $primary_port + assert_replica_synced $primary $replica "(case1 capable)" + + assert {[file exists [server_rdb_path $primary]]} + assert_equal 1 [rdb_is_compressed $primary] + wait_for_log_messages -1 {"*Disk-based full sync with compression: lz4*"} $primary_loglines 50 100 + + $primary set cap:post "after-sync" + wait_for_value_to_propagate_to_replica $primary $replica cap:post + + $replica replicaof no one + } + } + + # Both scenarios fall back to plaintext: the primary must select LZ4 from + # rdbcompression and the replica must advertise it via repl-compression. + start_server {overrides {save "" enable-debug-command local repl-compression lz4}} { + set replica [srv 0 client] + + foreach {scenario primary_mode replica_mode prefix} { + primary-compression-off yes lz4 off + replica-not-capable lz4 no nocap + } { + test "Disk full sync: $scenario yields a plaintext RDB that loads" { + $primary config set rdbcompression $primary_mode + $replica config set repl-compression $replica_mode + populate_compressible_dataset $primary $prefix + + $replica replicaof $primary_host $primary_port + assert_replica_synced $primary $replica "($scenario)" + + assert_equal 0 [rdb_is_compressed $primary] + + $replica replicaof no one + } + } + # Restore the fixture default for subsequent shared-primary tests. + $primary config set rdbcompression lz4 + } + + # Regression: a size-framed ($, no EOF mark) compressed disk RDB, loaded + # directly from the socket via replicaLoadPrimaryRDBFromSocket. Decompression + # was once gated on the EOF mark (usemark), so the VCS frame was read as a raw + # RDB ("Wrong signature ... VCS"), causing an infinite resync loop. + start_server {overrides {save "" enable-debug-command local repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + + test {Disk-based compressed full sync is decompressed on a diskless-load (swapdb) replica} { + $primary config set rdbcompression lz4 + populate_compressible_dataset $primary "diskmaster" + set replica_loglines [count_log_lines 0] + + $replica replicaof $primary_host $primary_port + assert_replica_synced $primary $replica "(disk-master, diskless-load swapdb)" + + assert_equal 1 [rdb_is_compressed $primary] + # Socket decode path logs "from primary"; match it so a file load cannot satisfy this. + wait_for_log_messages 0 {"*Loading compressed RDB (algo=lz4) from primary*"} $replica_loglines 50 100 + + $primary set diskmaster:post "after-sync" + wait_for_value_to_propagate_to_replica $primary $replica diskmaster:post + + $replica replicaof no one + } + } +} + +# Mixed disk group: force both replicas to park in WAIT_BGSAVE_START together so +# the group-AND decision is exercised directly. A slow manual BGSAVE occupies the +# RDB child slot; a disk replica can't start its own BGSAVE (syncCommand defers to +# replicationCron), so both park. The cron then groups both waiters and the AND +# clears the LZ4 capability. Deterministic because the manual BGSAVE is still +# running when both register (asserted via connected_slaves==2); one round is +# asserted via the "Starting BGSAVE for SYNC" delta. + +start_server {tags {"repl rdb-compression external:skip needs:debug"} overrides {save "" enable-debug-command local}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + $primary config set repl-diskless-sync no + $primary config set rdbcompression lz4 + $primary config set rdb-del-sync-files no + # Enough keys so the manual BGSAVE stays alive for both replicas to register. + populate_compressible_dataset $primary "grp" 800 + + start_server {overrides {save "" enable-debug-command local repl-compression lz4}} { + set capable [srv 0 client] + + start_server {overrides {save "" enable-debug-command local repl-compression no}} { + set noncap [srv 0 client] + + test {Disk full sync: a single grouped BGSAVE with a non-capable member is plaintext for all} { + # One round == one "Starting BGSAVE for SYNC" in the primary log. + # srv -2 is the primary inside this doubly-nested scope. + set rounds_before [count_log_message -2 {Starting BGSAVE for SYNC}] + set primary_loglines [count_log_lines -2] + + park_replicas_for_grouped_bgsave $primary $capable $noncap $primary_host $primary_port + + assert_replica_synced $primary $capable "(case3b capable, grouped)" + assert_replica_synced $primary $noncap "(case3b non-capable, grouped)" + + # Exactly one BGSAVE served both -> they were grouped (AND precondition). + assert_equal 1 [expr {[count_log_message -2 {Starting BGSAVE for SYNC}] - $rounds_before}] + + # AND result: the grouped RDB is plaintext (capable replica downgraded too). + assert_equal 0 [rdb_is_compressed $primary] + verify_no_log_message -2 "*Disk-based full sync with compression: lz4*" $primary_loglines + + $noncap replicaof no one + $capable replicaof no one + } + } + } +} + +# ============================================================================ +# Piggyback (in-flight BGSAVE join): all four quadrants of (running save +# format) x (joiner capability). A joiner attaches whenever it can load the +# running save's format; a non-capable joiner must NOT attach to a compressed +# save and waits for the next, plaintext, save. Capability is advertised from +# repl-compression at handshake time, so one shared replica pair serves every +# row via CONFIG SET. rdb-key-save-delay 13000 (~2s over 155 keys x 13ms: above +# one PSYNC round trip, under wait_for_sync's ~5s budget on slow runners) keeps +# the save in flight while the joiner arrives. +# ============================================================================ + +start_server {tags {"repl rdb-compression external:skip needs:debug"} overrides {save "" enable-debug-command local}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + $primary config set repl-diskless-sync no + $primary config set rdb-del-sync-files no + + start_server {overrides {save "" enable-debug-command local repl-compression lz4}} { + set trigger [srv 0 client] + + start_server {overrides {save "" enable-debug-command local repl-compression lz4}} { + set joiner [srv 0 client] + + foreach {name primary_mode joiner_mode should_join expected_compressed} { + "plain save, non-capable joiner attaches" yes no 1 0 + "plain save, capable joiner attaches" yes lz4 1 0 + "compressed save, non-capable joiner waits for a new save" lz4 no 0 0 + "compressed save, capable joiner attaches" lz4 lz4 1 1 + } { + test "Piggyback: $name" { + with_cleanup { + $primary config set rdbcompression $primary_mode + $joiner config set repl-compression $joiner_mode + $primary config set rdb-key-save-delay 13000 + populate_compressible_dataset $primary piggy 150 + set primary_loglines [count_log_lines -2] + + $trigger replicaof $primary_host $primary_port + if {$primary_mode eq "lz4"} { + wait_for_log_messages -2 {"*Disk-based full sync with compression: lz4*"} \ + $primary_loglines 50 100 + } else { + wait_for_log_messages -2 {"*Starting BGSAVE for SYNC with target: disk*"} \ + $primary_loglines 50 100 + } + + $joiner replicaof $primary_host $primary_port + if {$should_join} { + wait_for_log_messages -2 {"*Waiting for end of BGSAVE for SYNC*"} \ + $primary_loglines 50 100 + # Prove the joiner piggybacked onto the running save rather than + # triggering a second BGSAVE round: the primary is -2 in this + # doubly-nested start_server scope, so neither the attach-refusal + # nor the next-save-wait message must appear. + verify_no_log_message -2 "*Can't attach the replica to the current BGSAVE*" $primary_loglines + verify_no_log_message -2 "*Waiting for next BGSAVE for SYNC*" $primary_loglines + } else { + wait_for_log_messages -2 {"*Can't attach the replica to the current BGSAVE*"} \ + $primary_loglines 50 100 + } + + $primary config set rdb-key-save-delay 0 + assert_replica_synced $primary $trigger "($name, trigger)" + assert_replica_synced $primary $joiner "($name, joiner)" + assert_equal $expected_compressed [rdb_is_compressed $primary] + } { + $primary config set rdb-key-save-delay 0 + $joiner replicaof no one + $trigger replicaof no one + } + } + } + } + } +} + +# ============================================================================ +# Cross-cutting transfer-path coverage: disk-receive (BIO) load, dual-channel, +# diskless payload compression, and byte-accounting. Diskless compression is +# deliberately enabled with repl-compression while rdbcompression is disabled. +# ============================================================================ + +tags {"repl external:skip"} { + +# --- dual-channel: ONE shared primary fixture ------------------------------ +# The byte-accounting test raises repl-diskless-sync-delay and +# repl-diskless-sync-max-replicas (both MODIFIABLE) at its start and restores them +# at its end so the shared primary stays neutral for the other tests. + +start_server {overrides {save "" rdbcompression no repl-compression lz4 repl-diskless-sync yes repl-diskless-sync-delay 0 dual-channel-replication-enabled yes}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + # Both load paths under dual-channel: disabled -> RDB to disk (BIO), rdbLoad() + # decompresses; swapdb -> load compressed payload from the socket (also exercises + # LZ4 capability advertising in the dual-channel handshake). + foreach load_mode {disabled swapdb} { + test "Dual-channel + compression full sync delivers identical data (repl-diskless-load $load_mode)" { + set primary_loglines [count_log_lines 0] + populate_compressible_dataset $primary "dc-$load_mode" + + start_server [list overrides [list save "" repl-compression lz4 repl-diskless-load $load_mode dual-channel-replication-enabled yes]] { + set replica [srv 0 client] + set replica_loglines [count_log_lines 0] + $replica replicaof $primary_host $primary_port + + assert_replica_synced $primary $replica "(dual-channel $load_mode)" + + wait_for_log_messages -1 {"*using: dual-channel*"} $primary_loglines 50 100 + wait_for_log_messages -1 {"*Diskless full sync with compression: lz4*"} $primary_loglines 50 100 + # swapdb loads inline from the socket ("from primary"); disabled loads from the received file. + if {$load_mode eq "swapdb"} { + wait_for_log_messages 0 {"*Loading compressed RDB (algo=lz4) from primary*"} $replica_loglines 50 100 + } else { + wait_for_log_messages 0 {"*Loading compressed RDB (algo=lz4) from *.rdb*"} $replica_loglines 50 100 + } + + $replica replicaof no one + } + } + } + + # Aggregate output accounting: one BGSAVE serving TWO dual-channel replicas + # writes the payload to both sockets from a single connset, so + # total_net_repl_output_bytes must cover both. Old single-stream accounting + # reported ~half and fails the >=75% check below. Needs the delay window + 2-replica cap (restored at the end). + test {Dual-channel diskless full sync counts output bytes across all sockets} { + $primary config set repl-diskless-sync-delay 1000 + $primary config set repl-diskless-sync-max-replicas 2 + populate_compressible_dataset $primary agg 2000 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb dual-channel-replication-enabled yes}} { + set replica1 [srv 0 client] + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb dual-channel-replication-enabled yes}} { + set replica2 [srv 0 client] + + # Attach both within the delay window so one BGSAVE groups both channels into one connset. + set rounds_before [count_log_message -2 {Starting BGSAVE for SYNC}] + $primary config resetstat + $replica1 replicaof $primary_host $primary_port + $replica2 replicaof $primary_host $primary_port + + assert_replica_synced $primary $replica1 "(agg r1)" + assert_replica_synced $primary $replica2 "(agg r2)" + + # One grouped BGSAVE served both: precondition for the aggregation to be exercised. + assert_equal 1 [expr {[count_log_message -2 {Starting BGSAVE for SYNC}] - $rounds_before}] + + # The child reports per-connset output bytes over the child-info pipe, + # which the parent drains asynchronously after link-up. Wait for that + # to land before sampling the counters. + wait_for_condition 50 100 { + [status $primary total_net_repl_output_bytes] > 0 + } else { + fail "primary did not aggregate dual-channel output bytes" + } + + set out [status $primary total_net_repl_output_bytes] + set in1 [status $replica1 total_net_repl_input_bytes] + set in2 [status $replica2 total_net_repl_input_bytes] + set total_in [expr {$in1 + $in2}] + + assert_morethan $in1 0 + assert_morethan $in2 0 + # Lower bound catches old single-stream (~half) accounting; upper bound guards double-counting. + assert {$out >= $total_in * 0.75} + assert {$out <= $total_in * 1.25} + + $replica2 replicaof no one + $replica1 replicaof no one + } + } + + # Restore the fixture defaults for the shared dual-channel primary. + $primary config set repl-diskless-sync-delay 0 + $primary config set repl-diskless-sync-max-replicas 0 + } +} + +# --- ordinary diskless: ONE shared primary fixture ------------------------- +# Shared by the receive-to-disk (disabled) and direct-socket-load (swapdb) tests; +# replicas differ only in repl-diskless-load so they stay per-test. + +start_server {overrides {save "" rdbcompression no repl-compression lz4 repl-diskless-sync yes repl-diskless-sync-delay 0}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + # disabled: replica copies the payload to a temp file, then rdbLoad() auto-decompresses. + test {Disk-receive (repl-diskless-load disabled) + diskless compressed save loads correctly} { + set primary_loglines [count_log_lines 0] + populate_compressible_dataset $primary "diskrecv" + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load disabled}} { + set replica [srv 0 client] + set replica_loglines [count_log_lines 0] + $replica replicaof $primary_host $primary_port + + assert_replica_synced $primary $replica "(disk-receive default load)" + + wait_for_log_messages -1 {"*target: replicas sockets*"} $primary_loglines 50 100 + wait_for_log_messages -1 {"*Diskless full sync with compression: lz4*"} $primary_loglines 50 100 + # Path B: disabled load reads from the received file; match "from .rdb", not "from primary". + wait_for_log_messages 0 {"*Loading compressed RDB (algo=lz4) from *.rdb*"} $replica_loglines 50 100 + + $replica replicaof no one + } + } + + # Diskless compresses the RDB payload as an LZ4 VCS frame over the socket; the + # $EOF: framing stays uncompressed so transfer boundaries are unchanged. + test {Diskless full sync compresses the RDB payload for a capable replica} { + set primary_loglines [count_log_lines 0] + populate_compressible_dataset $primary diskless-full-sync + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + set replica_loglines [count_log_lines 0] + $replica replicaof $primary_host $primary_port + + assert_replica_synced $primary $replica "(compressed diskless full sync)" + + wait_for_log_messages -1 {"*target: replicas sockets*"} $primary_loglines 50 100 + wait_for_log_messages -1 {"*Diskless full sync with compression: lz4*"} $primary_loglines 50 100 + wait_for_log_messages 0 {"*Loading compressed RDB (algo=lz4) from primary*"} $replica_loglines 50 100 + + $primary set diskless-full-sync:post after + wait_for_value_to_propagate_to_replica $primary $replica diskless-full-sync:post + + $replica replicaof no one + } + } + + test {Diskless full sync follows repl-compression instead of rdbcompression} { + set primary_loglines [count_log_lines 0] + populate_compressible_dataset $primary diskless-policy + + $primary config set rdbcompression lz4 + $primary config set repl-compression no + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + assert_replica_synced $primary $replica "(diskless policy)" + wait_for_log_messages -1 {"*target: replicas sockets*"} $primary_loglines 50 100 + verify_no_log_message -1 "*Diskless full sync with compression: lz4*" $primary_loglines + + $replica replicaof no one + } + $primary config set rdbcompression no + $primary config set repl-compression lz4 + } + +} + +# Checksum interaction: a compressed diskless full sync loads under rdbchecksum no. +# Isolated because rdbchecksum is an immutable startup-only override. + +start_server {overrides {save "" rdbcompression no repl-compression lz4 repl-diskless-sync yes repl-diskless-sync-delay 0 rdbchecksum no}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Compressed diskless full sync loads with rdbchecksum no on the primary} { + assert_equal "no" [lindex [$primary config get rdbchecksum] 1] + set primary_loglines [count_log_lines 0] + populate_compressible_dataset $primary "cksum-off" + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load disabled}} { + set replica [srv 0 client] + set replica_loglines [count_log_lines 0] + $replica replicaof $primary_host $primary_port + + assert_replica_synced $primary $replica "(checksum off)" + + wait_for_log_messages -1 {"*Diskless full sync with compression: lz4*"} $primary_loglines 50 100 + # Disabled load reads from the received file; match "from .rdb", not "from primary". + wait_for_log_messages 0 {"*Loading compressed RDB (algo=lz4) from *.rdb*"} $replica_loglines 50 100 + + $replica replicaof no one + } + } +} + +# Mid-transfer link drop on a compressed diskless full sync: swapdb's inline +# socket decompressor (replicaLoadPrimaryRDBFromSocket TRUNCATED path) sees a clean +# EOF before the LZ4 frame end. That is recoverable truncation, NOT corruption, so +# the replica must survive, retry, and resync. Determinism: compression finishes +# almost instantly, so stretch the child with a per-key save delay over a large +# dataset; watch the replica log (not INFO) since it is blocked in the synchronous load. + +start_server {overrides {save "" rdbcompression no repl-compression lz4 repl-diskless-sync yes repl-diskless-sync-delay 0}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Compressed diskless full sync recovers from a mid-transfer link drop (truncation, not corruption)} { + populate_compressible_dataset $primary "trunc" 2000 + set sync_full_before [status $primary sync_full] + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + set replica_loglines [count_log_lines 0] + + # 5ms/key over ~2000 keys keeps the compressed stream in flight long enough to interrupt. + $primary config set rdb-key-save-delay 5000 + + $replica replicaof $primary_host $primary_port + + # Catch the replica once it starts the inline socket decode (the TRUNCATED + # path under test); the log line is emitted at decode start. + wait_for_log_messages 0 {"*Loading compressed RDB*"} $replica_loglines 300 100 + + # Kill the in-flight transfer so the stream reader hits a clean EOF mid-frame. + # The ~5s remaining load window dwarfs the detect-and-kill latency. + $primary client kill type replica + + $primary config set rdb-key-save-delay 0 + + assert_equal {PONG} [$replica ping] + + # Retries and converges on a clean resync (reconnect driven by the cron, ~1s). + assert_replica_synced $primary $replica "(truncation recovery)" + + # Interrupted attempt + retry each count one full sync, proving the kill landed mid-transfer. + assert {[status $primary sync_full] >= $sync_full_before + 2} + + # The interrupted transfer was handled as a recoverable load failure, not a + # crash or corruption abort (no panic/assertion below). + assert {[count_log_message 0 "Loading compressed RDB"] >= 1} + assert {[count_log_message 0 "Failed trying to load the PRIMARY synchronization DB"] >= 1} + assert_equal 0 [count_log_message 0 "=== ASSERTION FAILED ==="] + assert {[count_log_message 0 "REPLICA sync: Finished with success"] >= 1} + + $primary set trunc:post "after-recovery" + wait_for_value_to_propagate_to_replica $primary $replica trunc:post + + $replica replicaof no one + } + } +} + +# Mixed diskless cohort (one capable, one not) falls back to plaintext for all: the +# group-AND clears the LZ4 capability. A slow manual BGSAVE occupies the child +# slot so both replicas park in WAIT_BGSAVE_START together, exercising the AND deterministically. +start_server {overrides {save "" rdbcompression no repl-compression lz4 repl-diskless-sync yes repl-diskless-sync-delay 0}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + # Enough keys so the manual BGSAVE stays alive for both replicas to register. + populate_compressible_dataset $primary mixed-diskless 800 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica_capable [srv 0 client] + + start_server {overrides {save "" repl-compression no repl-diskless-load swapdb}} { + set replica_plain [srv 0 client] + + test {Mixed diskless cohort falls back to a plaintext RDB payload} { + set rounds_before [count_log_message -2 {Starting BGSAVE for SYNC}] + set primary_loglines [count_log_lines -2] + + park_replicas_for_grouped_bgsave $primary $replica_capable $replica_plain $primary_host $primary_port + + assert_replica_synced $primary $replica_capable "(mixed diskless capable)" + assert_replica_synced $primary $replica_plain "(mixed diskless plain)" + + # A single grouped diskless round served both replicas. + assert_equal 1 [expr {[count_log_message -2 {Starting BGSAVE for SYNC}] - $rounds_before}] + + # AND result: plaintext round, so no compression NOTICE. + verify_no_log_message -2 "*Diskless full sync with compression: lz4*" $primary_loglines + + $replica_plain replicaof no one + $replica_capable replicaof no one + } + } + } +} + +# Fake-primary negative coverage for the size-framed compressed socket load. A +# complete VCS/LZ4 stream is captured from a throwaway server's dump.rdb (SAVE with +# rdbcompression lz4 writes a full VCS stream), then replayed by fake_primary.tcl +# with a controlled bulk header and payload. The fake primary is plain TCP, so skip under TLS. + +if {!$::tls} { + +# Capture a complete compressed stream once for all fake-primary tests. +set compressed_payload "" +start_server {overrides {save "" rdbcompression lz4}} { + set src [srv 0 client] + for {set i 0} {$i < 32} {incr i} { + $src set src:key:$i [string repeat "srcval:$i " 8] + } + $src save + set src_dump [server_rdb_path $src] + set compressed_payload [read_binary_file $src_dump] + assert_equal "VCS" [string range $compressed_payload 0 2] +} + +# Recoverable outcomes share one replica. Each test clears the dataset and +# counters, then always detaches and stops its fake primary. + +start_server {overrides {save "" rdbcompression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + + # Both size-framing mismatches decode a complete frame but fail the + # consumed-equals-announced check: either fewer bytes arrive than announced, + # or trailing bytes remain after the frame boundary. + set padded_payload "$compressed_payload[string repeat J 100]" + foreach {name payload announced_size} [list \ + "stream ends before the announced size" \ + $compressed_payload [expr {[string length $compressed_payload] + 100}] \ + "trailing bytes remain inside the announced size" \ + $padded_payload [string length $padded_payload] \ + ] { + test "Compressed socket load fails when $name" { + set fake_pid "" + with_cleanup { + $replica replicaof no one + $replica flushall + $replica config resetstat + set replica_loglines [count_log_lines 0] + lassign [start_fake_primary $payload $announced_size] fake_pid fake_port + + $replica replicaof 127.0.0.1 $fake_port + wait_for_log_messages 0 {"*ended before the announced transfer size*"} \ + $replica_loglines 100 100 + + assert_equal {PONG} [$replica ping] + assert_equal 0 [$replica dbsize] + } { + $replica replicaof no one + if {$fake_pid ne ""} {catch {exec kill $fake_pid}} + } + } + } + + # Complete frame with the exact announced size: the load succeeds and input + # accounting must count every encoded wire byte: total_net_repl_input_bytes == + # N + "$\r\n" bulk header. Tight enough to catch a lost footer. + test {Compressed socket load counts exactly the encoded wire bytes on a successful load} { + set fake_pid "" + with_cleanup { + $replica replicaof no one + $replica flushall + $replica config resetstat + set n [string length $compressed_payload] + lassign [start_fake_primary $compressed_payload $n] fake_pid fake_port + + $replica replicaof 127.0.0.1 $fake_port + wait_for_condition 100 100 { + [$replica dbsize] > 0 + } else { + fail "replica did not load the compressed payload" + } + # Stop before the dropped link (fake primary closed) can reconnect. + $replica replicaof no one + + # Bulk header "$\r\n" = 1 ('$') + digits(N) + 2 ("\r\n"). + set meta [expr {3 + [string length $n]}] + set expected [expr {$n + $meta}] + set got [status $replica total_net_repl_input_bytes] + assert_equal $expected $got \ + "net-input $got != N $n + bulk header $meta (expected $expected)" + } { + $replica replicaof no one + if {$fake_pid ne ""} {catch {exec kill $fake_pid}} + } + } + + # Announce N but send a truncated prefix and close: the load fails before the + # frame end, but residual wire bytes must be counted first. Truncation is + # recoverable, so the replica survives; input == bytes sent + "$\r\n" header. + test {Compressed socket load counts every wire byte received on a failing (truncated) transfer} { + set fake_pid "" + with_cleanup { + $replica replicaof no one + $replica flushall + $replica config resetstat + set full_len [string length $compressed_payload] + set bytes_sent [expr {$full_len / 2}] + set truncated [string range $compressed_payload 0 [expr {$bytes_sent - 1}]] + set announce $full_len + + set replica_loglines [count_log_lines 0] + lassign [start_fake_primary $truncated $announce] fake_pid fake_port + + $replica replicaof 127.0.0.1 $fake_port + wait_for_log_messages 0 {"*Failed trying to load the PRIMARY synchronization DB*"} \ + $replica_loglines 100 100 + + # Stop retrying before reading stats to keep the accounting window clean. + $replica replicaof no one + assert_equal {PONG} [$replica ping] + + # "$\r\n" bulk header = 1 ('$') + digits(N) + 2 ("\r\n"). + set meta [expr {3 + [string length $announce]}] + set expected [expr {$bytes_sent + $meta}] + set got [status $replica total_net_repl_input_bytes] + assert_equal $expected $got \ + "net-input $got != bytes_sent $bytes_sent + bulk header $meta (expected $expected)" + } { + $replica replicaof no one + if {$fake_pid ne ""} {catch {exec kill $fake_pid}} + } + } +} + +# Corrupt compressed payloads: a flipped byte mid-frame (caught during parse) and a +# flipped last byte (checksum mismatch at frame finish) converge on the same fatal +# path. A corrupt diskless load is terminal (rdbReportError exits), so each variant runs in its own fresh replica. +test {Corrupted compressed stream (mid-frame and footer) aborts the diskless load} { + foreach {label offset_expr} { + mid-frame {[string length $compressed_payload] / 2} + footer {[string length $compressed_payload] - 1} + } { + start_server {overrides {save "" rdbcompression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + set corrupt_at [expr $offset_expr] + assert {$corrupt_at > 8} ;# flip a byte well past the 7-byte VCS envelope + binary scan [string index $compressed_payload $corrupt_at] c byte_val + set corrupted [string replace $compressed_payload $corrupt_at $corrupt_at \ + [binary format c [expr {$byte_val ^ 0xff}]]] + lassign [start_fake_primary $corrupted [string length $corrupted]] fake_pid fake_port + + $replica replicaof 127.0.0.1 $fake_port + + wait_for_condition 100 100 { + ![is_alive [srv 0 pid]] + } else { + fail "replica did not exit on a corrupt compressed stream ($label)" + } + set stdout [srv 0 stdout] + assert_equal 1 [count_message_lines $stdout "Corrupt streaming-compressed RDB input"] + assert_equal 1 [count_message_lines $stdout "Terminating server after rdb file reading failure."] + catch {exec kill $fake_pid} + } + } +} + +} ;# end !tls + +} ;# end tags diff --git a/tests/integration/replication-aof-sync.tcl b/tests/integration/replication-aof-sync.tcl index 14113bd07..dca02bb65 100644 --- a/tests/integration/replication-aof-sync.tcl +++ b/tests/integration/replication-aof-sync.tcl @@ -164,6 +164,63 @@ tags {"repl external:skip"} { } } + # A streaming-compressed disk-based sync RDB (rdbcompression lz4 on the + # primary and repl-compression lz4 on the replica for capability advertisement) + # cannot be reused as an AOF base, so the replica falls back to BGREWRITEAOF. + # Inverse of the plaintext RDB-reuse tests above. + test "Disk-based full sync with rdbcompression lz4 falls back to BGREWRITEAOF for AOF base" { + start_server {overrides {repl-diskless-sync no rdbcompression lz4 save ""}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + for {set i 0} {$i < 40} {incr i} { + $primary set "rcomp-key:$i" "value:$i" + } + + start_server {overrides {appendonly yes aof-use-rdb-preamble yes repl-diskless-sync no rdbcompression lz4 repl-compression lz4 save ""}} { + set replica [srv 0 client] + set replica_log [srv 0 stdout] + + $replica replicaof $primary_host $primary_port + wait_for_sync $replica + + # Replica detects the compressed sync RDB and falls back to BGREWRITEAOF. + wait_for_condition 50 100 { + [log_file_matches $replica_log "*falling back to BGREWRITEAOF instead of reusing it as an AOF base*"] + } else { + fail "Expected streaming-compression AOF fallback log not found" + } + + # And it must NOT have reused the sync RDB as the AOF base. + assert {![log_file_matches $replica_log "*Reused RDB file from primary sync as AOF base file*"]} + + # AOF comes up via BGREWRITEAOF; a base file must exist. + waitForBgrewriteaof $replica + set manifest_path [get_aof_manifest_path $replica] + set base_name [get_cur_base_aof_name $manifest_path] + assert {$base_name ne ""} + + # Data correct at runtime (loaded from the compressed socket stream). + assert_equal 40 [$replica dbsize] + for {set i 0} {$i < 40} {incr i} { + assert_equal "value:$i" [$replica get "rcomp-key:$i"] + } + + # After restart: AOF loads from the rewritten base, not the compressed RDB. + $replica replicaof no one + restart_server 0 true false + set replica [srv 0 client] + wait_done_loading $replica + + assert_equal 40 [$replica dbsize] + for {set i 0} {$i < 40} {incr i} { + assert_equal "value:$i" [$replica get "rcomp-key:$i"] + } + } + } + } + # Test 4: aof-use-rdb-preamble no should fall back to bgrewriteaof test "Disk-based sync with aof-use-rdb-preamble no uses bgrewriteaof" { start_server {overrides {appendonly yes aof-use-rdb-preamble no repl-diskless-sync no save ""}} { diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 72fe60cab..80bd8781d 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -85,6 +85,71 @@ start_server {tags {"repl external:skip"}} { assert_equal [$A debug digest] [$B debug digest] } + test {INCREX replication, should not remove expire} { + r set test 1 EX 100 + r increx test byint 1 + wait_for_ofs_sync $A $B + assert_equal [$A debug digest] [$B debug digest] + } + + test {INCREX BYFLOAT replication, should not remove expire} { + r set test 1 EX 100 + r increx test byfloat 0.1 + wait_for_ofs_sync $A $B + assert_equal [$A debug digest] [$B debug digest] + } + + test {INCREX byint with EX propagates the correct TTL to replica} { + r del test + r increx test ex 100 byint 1 + wait_for_ofs_sync $A $B + assert_equal [$A get test] [$B get test] + assert_range [$B ttl test] 1 100 + } + + test {INCREX byfloat with EX propagates the correct TTL to replica} { + r del test + r increx test ex 100 byfloat 1.1 + wait_for_ofs_sync $A $B + assert_equal [$A get test] [$B get test] + assert_range [$B ttl test] 1 100 + } + + test {INCREX BYFLOAT without expire replicates deterministically} { + # Guards against float drift: BYFLOAT results must replicate as + # the resolved value, not as the literal INCREX/BYFLOAT command, + # the same way INCRBYFLOAT always rewrites to SET. + r del test + r increx test byfloat 0.1 + r increx test byfloat 0.2 + wait_for_ofs_sync $A $B + assert_equal [$A get test] [$B get test] + assert_equal [$A debug digest] [$B debug digest] + } + + test {INCREX BYFLOAT with NX without expire replicates deterministically} { + # Guards against float drift: BYFLOAT results must replicate as + # the resolved value, not as the literal INCREX/BYFLOAT command, + # the same way INCRBYFLOAT always rewrites to SET. + r del test + r increx test byfloat 0.1 NX + wait_for_ofs_sync $A $B + assert_equal [$A get test] [$B get test] + assert_equal [$A debug digest] [$B debug digest] + } + + test {INCREX BYFLOAT with XX without expire replicates deterministically} { + # Guards against float drift: BYFLOAT results must replicate as + # the resolved value, not as the literal INCREX/BYFLOAT command, + # the same way INCRBYFLOAT always rewrites to SET. + r del test + r increx test byfloat 0.1 + r increx test byfloat 0.1 XX + wait_for_ofs_sync $A $B + assert_equal [$A get test] [$B get test] + assert_equal [$A debug digest] [$B debug digest] + } + test {GETSET replication} { $A config resetstat $A config set loglevel debug @@ -313,6 +378,75 @@ start_server {tags {"repl external:skip"}} { close_replication_stream $repl } + test {INCREX with expire propagates as SET with PXAT} { + r -1 del foo + set repl [attach_to_replication_stream] + r -1 increx foo ex 100 byint 5 + assert_replication_stream $repl { + {set foo 5 PXAT *} + } + close_replication_stream $repl + } + + test {INCREX BYFLOAT propagates as SET, not literal command} { + r -1 del foo + set repl [attach_to_replication_stream] + r -1 increx foo byfloat 0.1 + assert_replication_stream $repl { + {set foo *} + } + close_replication_stream $repl + } + + test {INCREX BYINT without expire propagates verbatim} { + # BYINT-only increments are deterministic, so unlike BYFLOAT they + # don't need rewriting to SET for replication safety. + r -1 del foo + set repl [attach_to_replication_stream] + r -1 increx foo byint 5 + assert_replication_stream $repl { + {increx foo byint 5} + } + close_replication_stream $repl + } + + test {INCREX NX no-op does not propagate} { + r -1 set foo 1 + set repl [attach_to_replication_stream] + r -1 increx foo nx byint 1 + r -1 set marker 1 + assert_replication_stream $repl { + {set marker 1} + } + close_replication_stream $repl + } + + test {INCREX XX no-op does not propagate} { + r -1 del foo + set repl [attach_to_replication_stream] + r -1 increx foo xx byint 1 + r -1 set marker 1 + assert_replication_stream $repl { + {set marker 1} + } + close_replication_stream $repl + } + + test {INCREX BYFLOAT arithmetic overflow does not propagate} { + set big [ldbl_overflow_operand -1] + r -1 del foo + r -1 set foo $big + set repl [attach_to_replication_stream] + # Overflows to infinity; no change to DB, should not propagate + r -1 increx foo byfloat $big + assert_equal $big [r -1 get foo] + r -1 set marker 1 + assert_replication_stream $repl { + {set marker 1} + } + close_replication_stream $repl + } + test {ROLE in master reports master with a slave} { set res [r -1 role] lassign $res role offset slaves @@ -1114,6 +1248,69 @@ start_server {tags {"repl external:skip"} overrides {save ""}} { } } +# Compressed sibling of the drop-during-pipe family above. Compression finishes +# the transfer too quickly for the size-based throttling used there, so a +# per-key save delay keeps the compressed diskless transfer in flight while one +# replica is killed. The primary's RDB child must complete without crashing and +# the surviving replica must converge. +start_server {tags {"repl external:skip"} overrides {save "" rdbcompression lz4 repl-compression lz4}} { + set master [srv 0 client] + $master config set repl-diskless-sync yes + $master config set repl-diskless-sync-delay 5 + $master config set repl-diskless-sync-max-replicas 2 + $master config set dual-channel-replication-enabled "no"; # dual-channel-replication doesn't use pipe + set master_host [srv 0 host] + set master_port [srv 0 port] + $master debug populate 4000 test 1000 + # 1ms per key over 4k keys keeps the compressed transfer in flight for + # about 4 seconds; resetting the delay later does not speed up the + # already-forked child, so the kill below always lands mid-transfer. + $master config set rdb-key-save-delay 1000 + + test "diskless replica drops during compressed rdb pipe" { + start_server {overrides {save "" rdbcompression lz4 repl-compression lz4 repl-diskless-load swapdb}} { + set survivor [srv 0 client] + start_server {overrides {save "" rdbcompression lz4 repl-compression lz4}} { + set loglines [count_log_lines -2] + $survivor replicaof $master_host $master_port + [srv 0 client] replicaof $master_host $master_port + + # Wait for a compressed transfer to be in flight: the cohort + # negotiated compression and the survivor began the socket load. + wait_for_log_messages -2 {"*Diskless full sync with compression: lz4*"} $loglines 1500 10 + wait_for_log_messages -1 {"*Loading DB in memory*"} 0 1500 10 + + # Kill one replica mid-transfer. + exec kill [srv 0 pid] + + wait_for_condition 2400 100 { + [s -2 rdb_bgsave_in_progress] == 0 + } else { + fail "rdb child didn't terminate" + } + wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 1 replicas still up*"} $loglines 1000 10 + $master config set rdb-key-save-delay 0 + + # Verify the surviving replica converged on the compressed sync. + wait_for_condition 600 100 { + [lindex [$survivor role] 3] eq {connected} + } else { + fail "surviving replica still not connected after some time" + } + wait_for_condition 50 100 { + [$master dbsize] == [$survivor dbsize] + } else { + fail "Different number of keys between master and surviving replica after too long time." + } + set digest [$master debug digest] + set digest0 [$survivor debug digest] + assert {$digest ne 0000000000000000000000000000000000000000} + assert {$digest eq $digest0} + } + } + } +} + test "diskless replication child being killed is collected" { # when diskless master is waiting for the replica to become writable # it removes the read event from the rdb pipe so if the child gets killed @@ -1682,6 +1879,90 @@ start_server {tags {"repl external:skip"}} { } } +# Verify that after a diskless (socket) replication sync, save metrics +# are correctly reset and rdb_last_bgsave_time_sec is a plausible duration. +start_server {tags {"repl external:skip"}} { + start_server {} { + test {diskless sync: save metrics are plausible after socket transfer} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set replica [srv 0 client] + + $master config set repl-diskless-sync yes + $master config set repl-diskless-sync-delay 0 + $master config set save "" + $replica config set save "" + + $master debug populate 100 + + $replica replicaof $master_host $master_port + + wait_for_condition 100 100 { + [string match {*master_link_status:up*} [$replica info replication]] + } else { + fail "Replica didn't complete sync" + } + + # After diskless sync, master metrics should be sane + set time_sec [$master info persistence] + set bgsave_time [getInfoProperty $time_sec rdb_last_bgsave_time_sec] + assert {$bgsave_time >= 0 && $bgsave_time < 3600} + + # Save state should be cleared + assert_equal [getInfoProperty $time_sec rdb_bgsave_in_progress] "0" + assert_equal [getInfoProperty $time_sec current_save_keys_processed] "0" + assert_equal [getInfoProperty $time_sec current_save_keys_total] "0" + } + } +} + +start_server {tags {"repl external:skip"}} { + start_server {} { + test {diskless sync: save metrics are plausible after failed socket transfer} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set replica [srv 0 client] + + $master config set repl-diskless-sync yes + $master config set repl-diskless-sync-delay 0 + $master config set save "" + $replica config set save "" + + $master debug populate 1000 + $master config set rdb-key-save-delay 100000 + + $replica replicaof $master_host $master_port + + # Wait for bgsave to start on master + wait_for_condition 100 100 { + [getInfoProperty [$master info persistence] rdb_bgsave_in_progress] == 1 + } else { + fail "diskless bgsave didn't start" + } + + # Kill the replica connection to abort the transfer + $replica replicaof no one + + # Wait for bgsave to finish on master + wait_for_condition 100 100 { + [getInfoProperty [$master info persistence] rdb_bgsave_in_progress] == 0 + } else { + fail "diskless bgsave didn't stop after replica disconnect" + } + + # Metrics should still be sane after failure + set time_sec [$master info persistence] + set bgsave_time [getInfoProperty $time_sec rdb_last_bgsave_time_sec] + assert {$bgsave_time >= 0 && $bgsave_time < 3600} + assert_equal [getInfoProperty $time_sec current_save_keys_processed] "0" + assert_equal [getInfoProperty $time_sec current_save_keys_total] "0" + + $master config set rdb-key-save-delay 0 + } + } +} start_server {tags {"repl external:skip"}} { set replica [srv 0 client] $replica config set repl-diskless-load disabled diff --git a/tests/integration/throttle-repl.tcl b/tests/integration/throttle-repl.tcl new file mode 100644 index 000000000..609370a45 --- /dev/null +++ b/tests/integration/throttle-repl.tcl @@ -0,0 +1,399 @@ +# Integration tests for replication throttle (throttle_repl.c). +# +# We drive the throttling by SIGSTOP-ing the replica, +# so its output buffer on the primary grows and never drains. + +proc throttle_rate {r} { + getInfoProperty [{*}$r info throttling] repl_throttle_rate +} + +# Check whether the given client is currently throttled. +proc client_throttled {r wid} { + set flags "" + regexp {flags=(\S+)} [{*}$r CLIENT LIST ID $wid] -> flags + string match {*h*} $flags +} + +# Keep issuing writes until the throttler activates. +# +# A fixed-size burst followed by a plain wait_for_condition is not enough: the +# primary keeps writing the replication stream into the socket until the kernel +# buffers fill, so a burst that fits in the primary's send buffer plus the frozen +# replica's receive buffer never grows the primary-side COB past the activation +# threshold. Once the burst is over nothing can make the wait succeed, so the +# stimulus has to continue until activation is observed. +proc wait_throttle_activated {r writer} { + set payload [string repeat w 2000] + for {set i 0} {$i < 200} {incr i} { + for {set j 0} {$j < 200} {incr j} { + $writer set key:$j $payload + } + if {[throttle_rate $r] >= 0} { + return 1 + } + } + return 0 +} + +# Keep issuing writes until the writer client is observed being throttled. +proc wait_throttled_client {r writer wid} { + for {set k 0} {$k < 1000} {incr k} { + for {set j 0} {$j < 500} {incr j} { + $writer set nudge v + } + if {[client_throttled $r $wid] && + [getInfoProperty [{*}$r info debug] repl_throttle_current_clients] > 0} { + return 1 + } + } + return 0 +} + +# Set up primary/replica replication with throttling enabled and a COB limit configured. +proc setup_throttle_replication {primary replica primary_host primary_port} { + $primary replicaof no one + $primary flushall + $primary config set repl-throttling-enabled yes + $primary config set repl-backlog-size 1mb + $primary config set client-output-buffer-limit "replica 1024mb 1mb 3600" + $primary config set repl-timeout 1800 + $replica replicaof no one + $replica flushall + $replica replicaof $primary_host $primary_port + wait_for_sync $replica + wait_replica_online $primary + wait_for_condition 50 100 { + [throttle_rate $primary] == -1 + } else { + fail "repl throttler doesn't setup correctly" + } +} + +# Tear down after a test so the next test starts from a clean state. +proc teardown_throttle_replication {primary replica} { + + if {[catch {$primary ping} err]} { + fail "primary stopped responding during teardown: $err" + } + + catch {$primary config set repl-throttling-enabled no} + wait_for_condition 100 100 { + [throttle_rate $primary] == -1 && + [getInfoProperty [$primary info debug] repl_throttle_current_clients] == 0 + } else { + fail "repl throttler didn't tear down after the test" + } + + # The replica must be fully synced and hold the same dataset. + wait_for_sync $replica + wait_replica_online $primary + wait_for_ofs_sync $primary $replica + assert_equal [$primary dbsize] [$replica dbsize] + + # Detach replication + catch {$replica replicaof no one} +} + +start_server {tags {"throttle repl external:skip"}} { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + set replica_pid [srv 0 pid] + start_server {} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Steady-state throttle happy case} { + setup_throttle_replication $primary $replica $primary_host $primary_port + + # Freeze the replica so its output buffer on the primary grows monotonically. + pause_process $replica_pid + + set writer [valkey_deferring_client] + $writer CLIENT ID + set wid [$writer read] + + # Flood writes to grow the replica's COB and activate the throttler. + if {![wait_throttle_activated $primary $writer]} { + resume_process $replica_pid + fail "throttle did not activate while the replica's COB was growing" + } + # Keep writing until the client is actually throttled. + if {![wait_throttled_client $primary $writer $wid]} { + resume_process $replica_pid + fail "client was not throttled while the replica's COB was growing" + } + + set ti [$primary info throttling] + set td [$primary info debug] + # Throttling section + assert {[getInfoProperty $ti repl_throttle_rate] >= 0} + assert {[getInfoProperty $ti repl_throttle_activation_events] >= 1} + assert {[getInfoProperty $ti repl_throttle_below_guardrail_secs] >= 0} + assert {[getInfoProperty $ti repl_throttle_total_commands] > 0} + assert {[getInfoProperty $ti total_throttled_commands] > 0} + + # Debug section + assert {[getInfoProperty $td repl_throttle_more_events] >= 1} + assert {[getInfoProperty $td repl_throttle_less_events] >= 0} + $writer close + resume_process $replica_pid + + teardown_throttle_replication $primary $replica + } + + test {Throttling protects a replica above the soft COB limit} { + setup_throttle_replication $primary $replica $primary_host $primary_port + $primary config set client-output-buffer-limit "replica [expr {1024 * 1024 * 1024}] [expr {1 * 1024 * 1024}] 0" + + set writer [valkey_deferring_client] + $writer CLIENT ID + set wid [$writer read] + + pause_process $replica_pid + + if {![wait_throttle_activated $primary $writer]} { + resume_process $replica_pid + fail "throttler never began queueing clients" + } + + # Write 30MB total (30 x 1MB values). This is well above the 1mb + # soft limit and well below the 1024mb hard limit, so the replica's + # COB lands in between. + set value_size [expr {1 * 1024 * 1024}] + set num_writes 30 + for {set i 0} {$i < $num_writes} {incr i} { + $writer set key:$i [string repeat x $value_size] + } + + if {[status $primary connected_slaves] != 1} { + resume_process $replica_pid + fail "replica was disconnected while above soft but below hard COB limit" + } + wait_for_condition 50 100 { + [throttle_rate $primary] >= 0 + } else { + resume_process $replica_pid + fail "throttle did not activate while the replica's COB was growing" + } + if {![wait_throttled_client $primary $writer $wid]} { + resume_process $replica_pid + fail "client was not throttled while the replica's COB was growing" + } + + $writer close + resume_process $replica_pid + teardown_throttle_replication $primary $replica + } + + test {Throttling not protect a replica above the hard COB limit} { + setup_throttle_replication $primary $replica $primary_host $primary_port + $primary config set client-output-buffer-limit "replica 10mb 1mb 0" + + set writer [valkey_deferring_client] + $writer CLIENT ID + set wid [$writer read] + + pause_process $replica_pid + + if {![wait_throttle_activated $primary $writer]} { + resume_process $replica_pid + fail "throttler never began queueing clients" + } + + # Write 100MB total (100 x 1MB values). This is well above the 10mb + # hard limit, so the replica will be disconnected. + set value_size [expr {1 * 1024 * 1024}] + set num_writes 100 + for {set i 0} {$i < $num_writes} {incr i} { + $writer set key:$i [string repeat x $value_size] + } + + wait_for_condition 50 100 { + [throttle_rate $primary] == -1 && + ![client_throttled $primary $wid] && + [status $primary connected_slaves] == 0 + } else { + resume_process $replica_pid + fail "throttle did not tear down after the replica was disconnected" + } + + $writer close + resume_process $replica_pid + teardown_throttle_replication $primary $replica + } + + test {Throttling tears down when failover happened} { + setup_throttle_replication $primary $replica $primary_host $primary_port + + pause_process $replica_pid + set writer [valkey_deferring_client] + $writer CLIENT ID + set wid [$writer read] + + # Activate throttling. + if {![wait_throttle_activated $primary $writer]} { + resume_process $replica_pid + fail "throttle did not activate before failover" + } + + if {![wait_throttled_client $primary $writer $wid]} { + resume_process $replica_pid + fail "Client is not throttled." + } + + # Trigger the failover. The throttling must tear + # down even though the still-frozen replica's COB is high. + $primary replicaof $replica_host $replica_port + wait_for_condition 50 100 { + [throttle_rate $primary] == -1 && ![client_throttled $primary $wid] + } else { + resume_process $replica_pid + fail "throttle was not torn down after the primary was demoted" + } + + $writer close + resume_process $replica_pid + teardown_throttle_replication $replica $primary + } + + test {Throttling tears down when disabling config} { + setup_throttle_replication $primary $replica $primary_host $primary_port + + pause_process $replica_pid + set writer [valkey_deferring_client] + $writer CLIENT ID + set wid [$writer read] + + # Activate throttling. + if {![wait_throttle_activated $primary $writer]} { + resume_process $replica_pid + fail "Throttler did not activate." + } + if {![wait_throttled_client $primary $writer $wid]} { + resume_process $replica_pid + fail "Client is not throttled." + } + + # Disable the feature while COB is still high. The throttler must tear + # down AND release its throttled client. + $primary config set repl-throttling-enabled no + wait_for_condition 50 100 { + [throttle_rate $primary] == -1 && ![client_throttled $primary $wid] + } else { + resume_process $replica_pid + fail "Throttle not torn down / client not released after disabling steady state throttling." + } + + $writer close + resume_process $replica_pid + teardown_throttle_replication $primary $replica + } + + test {Client disconnect while throttling} { + setup_throttle_replication $primary $replica $primary_host $primary_port + + pause_process $replica_pid + set writer [valkey_deferring_client] + $writer CLIENT ID + set wid [$writer read] + + # Activate throttling. + if {![wait_throttle_activated $primary $writer]} { + resume_process $replica_pid + fail "Throttler did not activate." + } + if {![wait_throttled_client $primary $writer $wid]} { + resume_process $replica_pid + fail "Client is not throttled." + } + + # While throttled, queue 500 counter increments. They are buffered behind + # the throttle, not executed yet. + for {set i 0} {$i < 500} {incr i} { + $writer incr counter + } + + # The client drops its own TCP connection while throttled. + # It must then be removed from the throttler. + $writer close + wait_for_condition 50 100 { + [getInfoProperty [$primary info debug] repl_throttle_current_clients] == 0 && + ![client_throttled $primary $wid] + } else { + resume_process $replica_pid + fail "throttled client was not removed after its connection dropped" + } + + # The disconnect prevented the buffered increments from all running. + set executed [$primary get counter] + if {$executed eq ""} {set executed 0} + assert {$executed < 500} + + resume_process $replica_pid + teardown_throttle_replication $primary $replica + } + + test {Client blocked before throttling and unblocked after throttling} { + setup_throttle_replication $primary $replica $primary_host $primary_port + + # Block on a key BEFORE any repl throttler exists. + set blocker [valkey_deferring_client] + $blocker blpop mylist 0 + wait_for_blocked_client + pause_process $replica_pid + + # Drive the replica COB up with a deferring writer until the throttler + # queues this client. + set writer [valkey_deferring_client] + $writer CLIENT ID + set wid [$writer read] + set throttled 0 + set payload [string repeat w 2000] + for {set i 0} {$i < 200 && !$throttled} {incr i} { + for {set j 0} {$j < 200} {incr j} { + $writer set key:$j $payload + } + if {[client_throttled $primary $wid]} { + set throttled 1 + } + } + if {!$throttled} { + resume_process $replica_pid + fail "throttler never began queueing clients" + } + + # Deferring hosers that never read their replies, so the token bucket + # is empty and the throttler queue is non-empty when the LPUSH lands. + set writers {} + for {set i 0} {$i < 4} {incr i} { + lappend writers [valkey_deferring_client] + } + + # Nothing may be read from the primary between this burst and the + # LPUSH. Commands are processed in arrival order, so the LPUSH lands + # behind the burst. + foreach w $writers { + for {set j 0} {$j < 500} {incr j} { + $w set key:$j $payload + } + } + set pusher [valkey_deferring_client] + $pusher lpush mylist v + + resume_process $replica_pid + wait_for_sync $replica + wait_replica_online $primary + + assert_equal {mylist v} [$blocker read] + assert_equal 0 [$primary llen mylist] + + catch {$blocker close} + catch {$pusher close} + catch {$writer close} + foreach w $writers { catch {$w close} } + teardown_throttle_replication $primary $replica + } + } +} diff --git a/tests/integration/valkey-check-rdb.tcl b/tests/integration/valkey-check-rdb.tcl index d8d98a50c..c36c8f98a 100644 --- a/tests/integration/valkey-check-rdb.tcl +++ b/tests/integration/valkey-check-rdb.tcl @@ -124,6 +124,35 @@ tags {"check-rdb network external:skip logreqres:skip"} { } } + test "valkey-check-rdb rejects a compressed RDB truncated mid-frame" { + r config set rdbcompression lz4 + set dir [lindex [r config get dir] 1] + set midframe_rdb [file join $dir midframe-vcs.rdb] + with_cleanup { + r flushall + r set lz4:midframe [string repeat "payload " 200] + r save + + set dump_rdb [file join $dir dump.rdb] + set data [read_binary_file $dump_rdb] + # Cut mid-frame rather than dropping the trailer: a file has no + # more bytes coming, so this is corruption. + set keep [expr {[string length $data] * 6 / 10}] + write_binary_file $midframe_rdb [string range $data 0 [expr {$keep - 1}]] + + set failed [catch { + exec $::VALKEY_CHECK_RDB_BIN $midframe_rdb + } result] + + assert_equal 1 $failed + assert_match {*Corrupt compressed RDB stream*} $result + assert_no_match {*RDB looks OK*} $result + } { + file delete -force $midframe_rdb + catch {r config set rdbcompression yes} + } + } + test "valkey-check-rdb ignores trailing data after a compressed RDB" { r config set rdbcompression lz4 set dir [lindex [r config get dir] 1] diff --git a/tests/modules/blockedclient.c b/tests/modules/blockedclient.c index e29bf71ab..bda0f5f3d 100644 --- a/tests/modules/blockedclient.c +++ b/tests/modules/blockedclient.c @@ -952,6 +952,18 @@ int stop_slow_fg_command(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int ar return VALKEYMODULE_OK; } +int get_repl_read_offset(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { + VALKEYMODULE_NOT_USED(argv); + if (argc != 1) return ValkeyModule_WrongArity(ctx); + + ValkeyModuleServerInfoData *info = ValkeyModule_GetServerInfo(ctx, "replication"); + int err = VALKEYMODULE_OK; + long long offset = ValkeyModule_ServerInfoGetFieldSigned(info, "slave_read_repl_offset", &err); + ValkeyModule_FreeServerInfo(ctx, info); + if (err != VALKEYMODULE_OK) return ValkeyModule_ReplyWithError(ctx, "replication offset unavailable"); + return ValkeyModule_ReplyWithLongLong(ctx, offset); +} + /* used to enable or disable slow operation in do_bg_rm_call */ static int set_slow_bg_operation(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { if (argc != 2) { @@ -1105,6 +1117,9 @@ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int arg if (ValkeyModule_CreateCommand(ctx, "stop_slow_fg_command", stop_slow_fg_command,"allow-busy", 0, 0, 0) == VALKEYMODULE_ERR) return VALKEYMODULE_ERR; + if (ValkeyModule_CreateCommand(ctx, "get_repl_read_offset", get_repl_read_offset, "allow-busy", 0, 0, 0) == VALKEYMODULE_ERR) + return VALKEYMODULE_ERR; + if (ValkeyModule_CreateCommand(ctx, "set_slow_bg_operation", set_slow_bg_operation, "allow-busy", 0, 0, 0) == VALKEYMODULE_ERR) return VALKEYMODULE_ERR; diff --git a/tests/modules/defragtest.c b/tests/modules/defragtest.c index da176b705..b6aeca08e 100644 --- a/tests/modules/defragtest.c +++ b/tests/modules/defragtest.c @@ -14,6 +14,7 @@ struct FragObject { /* Make sure we get the expected cursor */ unsigned long int last_set_cursor = 0; +unsigned long int last_set_global_cursor = 0; unsigned long int datatype_attempts = 0; unsigned long int datatype_defragged = 0; @@ -21,9 +22,14 @@ unsigned long int datatype_resumes = 0; unsigned long int datatype_wrong_cursor = 0; unsigned long int global_attempts = 0; unsigned long int global_defragged = 0; +unsigned long int global_resumes = 0; +unsigned long int global_wrong_cursor = 0; int global_strings_len = 0; ValkeyModuleString **global_strings = NULL; +/* If non-zero, the global defrag callback stops after this many strings per + * invocation, forcing it to resume via the cursor on later calls. */ +int global_maxstep = 0; static void createGlobalStrings(ValkeyModuleCtx *ctx, int count) { @@ -37,14 +43,39 @@ static void createGlobalStrings(ValkeyModuleCtx *ctx, int count) static void defragGlobalStrings(ValkeyModuleDefragCtx *ctx) { - for (int i = 0; i < global_strings_len; i++) { + unsigned long i = 0; + int steps = 0; + + /* Resume from the saved cursor, validating it's what we set last time. */ + if (ValkeyModule_DefragCursorGet(ctx, &i) == VALKEYMODULE_OK) { + if (i > 0) global_resumes++; + if (i != last_set_global_cursor) global_wrong_cursor++; + } else { + if (last_set_global_cursor != 0) global_wrong_cursor++; + } + + for (; i < (unsigned long)global_strings_len; i++) { ValkeyModuleString *new = ValkeyModule_DefragValkeyModuleString(ctx, global_strings[i]); global_attempts++; if (new != NULL) { global_strings[i] = new; global_defragged++; } + + /* Stop after maxstep strings, or when out of time, saving progress in + * the cursor so the next invocation resumes here. */ + if ((global_maxstep && ++steps >= global_maxstep) || + ValkeyModule_DefragShouldStop(ctx)) + { + ValkeyModule_DefragCursorSet(ctx, i + 1); + last_set_global_cursor = i + 1; + return; + } } + + /* Finished: reset the cursor to 0 so core sees this module as done. */ + ValkeyModule_DefragCursorSet(ctx, 0); + last_set_global_cursor = 0; } static void FragInfo(ValkeyModuleInfoCtx *ctx, int for_crash_report) { @@ -57,6 +88,8 @@ static void FragInfo(ValkeyModuleInfoCtx *ctx, int for_crash_report) { ValkeyModule_InfoAddFieldLongLong(ctx, "datatype_wrong_cursor", datatype_wrong_cursor); ValkeyModule_InfoAddFieldLongLong(ctx, "global_attempts", global_attempts); ValkeyModule_InfoAddFieldLongLong(ctx, "global_defragged", global_defragged); + ValkeyModule_InfoAddFieldLongLong(ctx, "global_resumes", global_resumes); + ValkeyModule_InfoAddFieldLongLong(ctx, "global_wrong_cursor", global_wrong_cursor); } struct FragObject *createFragObject(unsigned long len, unsigned long size, int maxstep) { @@ -83,6 +116,8 @@ static int fragResetStatsCommand(ValkeyModuleCtx *ctx, ValkeyModuleString **argv datatype_wrong_cursor = 0; global_attempts = 0; global_defragged = 0; + global_resumes = 0; + global_wrong_cursor = 0; ValkeyModule_ReplyWithSimpleString(ctx, "OK"); return VALKEYMODULE_OK; @@ -204,10 +239,19 @@ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int arg } long long glen; - if (argc != 1 || ValkeyModule_StringToLongLong(argv[0], &glen) == VALKEYMODULE_ERR) { + if (argc < 1 || argc > 2 || ValkeyModule_StringToLongLong(argv[0], &glen) == VALKEYMODULE_ERR) { return VALKEYMODULE_ERR; } + /* Optional 2nd arg: global defrag step limit per callback invocation. */ + if (argc == 2) { + long long gmaxstep; + if (ValkeyModule_StringToLongLong(argv[1], &gmaxstep) == VALKEYMODULE_ERR) { + return VALKEYMODULE_ERR; + } + global_maxstep = gmaxstep; + } + createGlobalStrings(ctx, glen); ValkeyModuleTypeMethods tm = { diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c index 8f2111182..569ac6d6b 100644 --- a/tests/modules/infotest.c +++ b/tests/modules/infotest.c @@ -1,7 +1,10 @@ #include "valkeymodule.h" +#include #include +static size_t external_memory_used = 0; + void InfoFunc(ValkeyModuleInfoCtx *ctx, int for_crash_report) { ValkeyModule_InfoAddSection(ctx, ""); ValkeyModule_InfoAddFieldLongLong(ctx, "global", -2); @@ -96,6 +99,51 @@ int info_getd(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { return info_get(ctx, argv, argc, 'd'); } +int info_setexternal(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { + long long amount_ll; + size_t amount; + + if (argc != 2) { + ValkeyModule_WrongArity(ctx); + return VALKEYMODULE_OK; + } + + if (ValkeyModule_StringToLongLong(argv[1], &amount_ll) != VALKEYMODULE_OK || amount_ll < 0 || + (unsigned long long)amount_ll > SIZE_MAX) { + ValkeyModule_ReplyWithError(ctx, "ERR invalid external memory value"); + return VALKEYMODULE_OK; + } + + amount = amount_ll; + if (amount > external_memory_used) { + if (ValkeyModule_IncrExternalMemory(amount - external_memory_used) != VALKEYMODULE_OK) { + ValkeyModule_ReplyWithError(ctx, "ERR external memory increment failed"); + return VALKEYMODULE_OK; + } + } else if (amount < external_memory_used) { + if (ValkeyModule_DecrExternalMemory(external_memory_used - amount) != VALKEYMODULE_OK) { + ValkeyModule_ReplyWithError(ctx, "ERR external memory decrement failed"); + return VALKEYMODULE_OK; + } + } + + external_memory_used = amount; + ValkeyModule_ReplyWithLongLong(ctx, amount_ll); + return VALKEYMODULE_OK; +} + +int ValkeyModule_OnUnload(ValkeyModuleCtx *ctx) { + VALKEYMODULE_NOT_USED(ctx); + + if (external_memory_used != 0 && + ValkeyModule_DecrExternalMemory(external_memory_used) != VALKEYMODULE_OK) { + return VALKEYMODULE_ERR; + } + + external_memory_used = 0; + return VALKEYMODULE_OK; +} + int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { VALKEYMODULE_NOT_USED(argv); VALKEYMODULE_NOT_USED(argc); @@ -114,6 +162,8 @@ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int arg return VALKEYMODULE_ERR; if (ValkeyModule_CreateCommand(ctx,"info.getd", info_getd,"",0,0,0) == VALKEYMODULE_ERR) return VALKEYMODULE_ERR; + if (ValkeyModule_CreateCommand(ctx,"info.setexternal", info_setexternal,"",0,0,0) == VALKEYMODULE_ERR) + return VALKEYMODULE_ERR; return VALKEYMODULE_OK; } diff --git a/tests/modules/scan.c b/tests/modules/scan.c index dfdc30050..3e986a05e 100644 --- a/tests/modules/scan.c +++ b/tests/modules/scan.c @@ -103,6 +103,56 @@ int scan_key(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) return VALKEYMODULE_OK; } +typedef struct { + ValkeyModuleCtx *ctx; + size_t nreplies; +} scan_key_raw_pd; + +void scan_key_raw_callback(ValkeyModuleKey *key, const char* field, size_t field_len, const char* value, size_t value_len, void *privdata) { + VALKEYMODULE_NOT_USED(key); + scan_key_raw_pd* pd = privdata; + ValkeyModule_ReplyWithArray(pd->ctx, 2); + + // The callback delivers borrowed (const char*, size_t) byte ranges instead of + // a ValkeyModuleString, so we reply with the raw buffers directly. + ValkeyModule_ReplyWithStringBuffer(pd->ctx, field, field_len); + if(value){ + ValkeyModule_ReplyWithStringBuffer(pd->ctx, value, value_len); + } else { + ValkeyModule_ReplyWithNull(pd->ctx); + } + + pd->nreplies++; +} + +int scan_key_raw(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) +{ + if (argc != 2) { + ValkeyModule_WrongArity(ctx); + return VALKEYMODULE_OK; + } + scan_key_raw_pd pd = { + .ctx = ctx, + .nreplies = 0, + }; + + ValkeyModuleKey *key = ValkeyModule_OpenKey(ctx, argv[1], VALKEYMODULE_READ); + if (!key) { + ValkeyModule_ReplyWithError(ctx, "not found"); + return VALKEYMODULE_OK; + } + + ValkeyModule_ReplyWithArray(ctx, VALKEYMODULE_POSTPONED_ARRAY_LEN); + + ValkeyModuleScanCursor* cursor = ValkeyModule_ScanCursorCreate(); + while(ValkeyModule_ScanKeyRawBorrowed(key, cursor, scan_key_raw_callback, &pd)); + ValkeyModule_ScanCursorDestroy(cursor); + + ValkeyModule_ReplySetArrayLength(ctx, pd.nreplies); + ValkeyModule_CloseKey(key); + return VALKEYMODULE_OK; +} + int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { VALKEYMODULE_NOT_USED(argv); VALKEYMODULE_NOT_USED(argc); @@ -115,6 +165,9 @@ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int arg if (ValkeyModule_CreateCommand(ctx, "scan.scan_key", scan_key, "", 0, 0, 0) == VALKEYMODULE_ERR) return VALKEYMODULE_ERR; + if (ValkeyModule_CreateCommand(ctx, "scan.scan_key_raw", scan_key_raw, "", 0, 0, 0) == VALKEYMODULE_ERR) + return VALKEYMODULE_ERR; + return VALKEYMODULE_OK; } diff --git a/tests/rdma/rdma-test.c b/tests/rdma/rdma-test.c index 19288012c..19f745430 100644 --- a/tests/rdma/rdma-test.c +++ b/tests/rdma/rdma-test.c @@ -135,6 +135,12 @@ static int valkeySetFdBlocking(int fd, int blocking) { assert(0); \ } while (0) +#define rdmaFatalf(fmt, ...) \ + do { \ + fprintf(stderr, "%s:%d " fmt "\n", __func__, __LINE__, __VA_ARGS__); \ + assert(0); \ + } while (0) + static inline long valkeyNowMs(void) { struct timeval tv; @@ -148,7 +154,7 @@ static int rdmaPostRecv(RdmaContext *ctx, struct rdma_cm_id *cm_id, valkeyRdmaCm struct ibv_sge sge; size_t length = sizeof(valkeyRdmaCmd); struct ibv_recv_wr recv_wr, *bad_wr; - + int ret; sge.addr = (uint64_t)(uintptr_t)cmd; sge.length = length; @@ -159,7 +165,9 @@ static int rdmaPostRecv(RdmaContext *ctx, struct rdma_cm_id *cm_id, valkeyRdmaCm recv_wr.num_sge = 1; recv_wr.next = NULL; - if (ibv_post_recv(cm_id->qp, &recv_wr, &bad_wr)) { + ret = ibv_post_recv(cm_id->qp, &recv_wr, &bad_wr); + if (ret) { + rdmaFatalf("RDMA: post recv failed: %s (%d)", strerror(ret), ret); return -1; } @@ -202,7 +210,7 @@ static int rdmaSetupIoBuf(RdmaContext *ctx, struct rdma_cm_id *cm_id) { ctx->cmd_buf = calloc(length, 1); ctx->cmd_mr = ibv_reg_mr(ctx->pd, ctx->cmd_buf, length, access); if (!ctx->cmd_mr) { - rdmaFatal("RDMA: reg recv mr failed"); + rdmaFatalf("RDMA: reg command MR failed: %s (%d)", strerror(errno), errno); goto destroy_iobuf; } @@ -210,7 +218,6 @@ static int rdmaSetupIoBuf(RdmaContext *ctx, struct rdma_cm_id *cm_id) { cmd = ctx->cmd_buf + i; if (rdmaPostRecv(ctx, cm_id, cmd) == -1) { - rdmaFatal("RDMA: post recv failed"); goto destroy_iobuf; } } @@ -227,7 +234,7 @@ static int rdmaSetupIoBuf(RdmaContext *ctx, struct rdma_cm_id *cm_id) { ctx->recv_length = length; ctx->recv_mr = ibv_reg_mr(ctx->pd, ctx->recv_buf, length, access); if (!ctx->recv_mr) { - rdmaFatal("RDMA: reg send mr failed"); + rdmaFatalf("RDMA: reg receive buffer MR failed: %s (%d)", strerror(errno), errno); goto destroy_iobuf; } @@ -257,7 +264,7 @@ static int rdmaAdjustSendbuf(RdmaContext *ctx, unsigned int length) { ctx->send_length = length; ctx->send_mr = ibv_reg_mr(ctx->pd, ctx->send_buf, length, access); if (!ctx->send_mr) { - rdmaFatal("RDMA: reg send buf mr failed"); + rdmaFatalf("RDMA: reg send buffer MR failed: %s (%d)", strerror(errno), errno); free(ctx->send_buf); ctx->send_buf = NULL; ctx->send_length = 0; @@ -298,6 +305,7 @@ static int rdmaSendCommand(RdmaContext *ctx, struct rdma_cm_id *cm_id, valkeyRdm send_wr.next = NULL; ret = ibv_post_send(cm_id->qp, &send_wr, &bad_wr); if (ret) { + rdmaFatalf("RDMA: post send command failed: %s (%d)", strerror(ret), ret); return -1; } @@ -376,7 +384,7 @@ static int connRdmaHandleCq(RdmaContext *ctx) { if (ibv_get_cq_event(ctx->comp_channel, &ev_cq, &ev_ctx) < 0) { if (errno != EAGAIN) { - rdmaFatal("RDMA: get cq event failed"); + rdmaFatalf("RDMA: get cq event failed: %s (%d)", strerror(errno), errno); return -1; } @@ -384,22 +392,24 @@ static int connRdmaHandleCq(RdmaContext *ctx) { } ibv_ack_cq_events(ctx->cq, 1); - if (ibv_req_notify_cq(ev_cq, 0)) { - rdmaFatal("RDMA: notify cq failed"); + ret = ibv_req_notify_cq(ev_cq, 0); + if (ret) { + rdmaFatalf("RDMA: notify cq failed: %s (%d)", strerror(ret), ret); return -1; } pollcq: ret = ibv_poll_cq(ctx->cq, 1, &wc); if (ret < 0) { - rdmaFatal("RDMA: poll cq failed"); + rdmaFatalf("RDMA: poll cq failed: %s (%d)", strerror(-ret), ret); return -1; } else if (ret == 0) { return 0; } if (wc.status != IBV_WC_SUCCESS) { - rdmaFatal("RDMA: send/recv failed"); + rdmaFatalf("RDMA: send/recv failed: %s (status %d), opcode 0x%x", + ibv_wc_status_str(wc.status), wc.status, wc.opcode); return -1; } @@ -433,7 +443,7 @@ static int connRdmaHandleCq(RdmaContext *ctx) { break; default: - rdmaFatal("RDMA: unexpected opcode"); + rdmaFatalf("RDMA: unexpected opcode 0x%x", wc.opcode); return -1; } @@ -448,6 +458,7 @@ static ssize_t valkeyRdmaRead(RdmaContext *ctx, char *buf, size_t data_len) { long timed = 1000; long start = valkeyNowMs(); uint32_t toread, remained; + int ret; copy: if (ctx->recv_offset < ctx->rx_offset) { @@ -477,7 +488,13 @@ static ssize_t valkeyRdmaRead(RdmaContext *ctx, char *buf, size_t data_len) { pfd.fd = ctx->comp_channel->fd; pfd.events = POLLIN; pfd.revents = 0; - if (poll(&pfd, 1, 1000) < 0) { + ret = poll(&pfd, 1, 1000); + if (ret < 0) { + rdmaFatalf("RDMA: poll completion channel failed: %s (%d)", strerror(errno), errno); + return -1; + } + if (pfd.revents & (POLLERR | POLLHUP | POLLNVAL)) { + rdmaFatalf("RDMA: completion channel poll returned revents 0x%x", pfd.revents); return -1; } @@ -524,6 +541,7 @@ static size_t connRdmaSend(RdmaContext *ctx, struct rdma_cm_id *cm_id, const voi send_wr.next = NULL; ret = ibv_post_send(cm_id->qp, &send_wr, &bad_wr); if (ret) { + rdmaFatalf("RDMA: post RDMA write failed: %s (%d)", strerror(ret), ret); return -1; } @@ -539,6 +557,7 @@ static ssize_t valkeyRdmaWrite(RdmaContext *ctx, char *buf, size_t data_len) { long start = valkeyNowMs(); uint32_t towrite, wrote = 0; size_t ret; + int poll_ret; /* try to pollcq to */ goto pollcq; @@ -547,7 +566,13 @@ static ssize_t valkeyRdmaWrite(RdmaContext *ctx, char *buf, size_t data_len) { pfd.fd = ctx->comp_channel->fd; pfd.events = POLLIN; pfd.revents = 0; - if (poll(&pfd, 1, 1) < 0) { + poll_ret = poll(&pfd, 1, 1); + if (poll_ret < 0) { + rdmaFatalf("RDMA: poll completion channel failed: %s (%d)", strerror(errno), errno); + return -1; + } + if (pfd.revents & (POLLERR | POLLHUP | POLLNVAL)) { + rdmaFatalf("RDMA: completion channel poll returned revents 0x%x", pfd.revents); return -1; } @@ -610,32 +635,34 @@ static int valkeyRdmaConnect(RdmaContext *ctx, struct rdma_cm_id *cm_id) { struct ibv_pd *pd = NULL; struct ibv_qp_init_attr init_attr = {0}; struct rdma_conn_param conn_param = {0}; + int ret; pd = ibv_alloc_pd(cm_id->verbs); if (!pd) { - rdmaFatal("RDMA: alloc pd failed"); + rdmaFatalf("RDMA: alloc pd failed: %s (%d)", strerror(errno), errno); goto error; } comp_channel = ibv_create_comp_channel(cm_id->verbs); if (!comp_channel) { - rdmaFatal("RDMA: alloc pd failed"); + rdmaFatalf("RDMA: create comp channel failed: %s (%d)", strerror(errno), errno); goto error; } if (valkeySetFdBlocking(comp_channel->fd, 0) != 0) { - rdmaFatal("RDMA: set recv comp channel fd non-block failed"); + rdmaFatalf("RDMA: set recv comp channel fd non-block failed: %s (%d)", strerror(errno), errno); goto error; } cq = ibv_create_cq(cm_id->verbs, VALKEY_RDMA_MAX_WQE * 2, ctx, comp_channel, 0); if (!cq) { - rdmaFatal("RDMA: create send cq failed"); + rdmaFatalf("RDMA: create CQ failed: %s (%d)", strerror(errno), errno); goto error; } - if (ibv_req_notify_cq(cq, 0)) { - rdmaFatal("RDMA: notify send cq failed"); + ret = ibv_req_notify_cq(cq, 0); + if (ret) { + rdmaFatalf("RDMA: notify CQ failed: %s (%d)", strerror(ret), ret); goto error; } @@ -648,7 +675,7 @@ static int valkeyRdmaConnect(RdmaContext *ctx, struct rdma_cm_id *cm_id) { init_attr.send_cq = cq; init_attr.recv_cq = cq; if (rdma_create_qp(cm_id, pd, &init_attr)) { - rdmaFatal("RDMA: create qp failed"); + rdmaFatalf("RDMA: create qp failed: %s (%d)", strerror(errno), errno); goto error; } @@ -666,7 +693,7 @@ static int valkeyRdmaConnect(RdmaContext *ctx, struct rdma_cm_id *cm_id) { conn_param.retry_count = 7; conn_param.rnr_retry_count = 7; if (rdma_connect(cm_id, &conn_param)) { - rdmaFatal("RDMA: connect failed"); + rdmaFatalf("RDMA: connect failed: %s (%d)", strerror(errno), errno); goto destroy_iobuf; } @@ -708,7 +735,7 @@ static int valkeyRdmaCM(RdmaContext *ctx, int timeout) { timeout = 100; /* at most 100ms to resolve route */ ret = rdma_resolve_route(event->id, timeout); if (ret) { - rdmaFatal("RDMA: route resolve failed"); + rdmaFatalf("RDMA: route resolve failed: %s (%d)", strerror(errno), errno); } break; case RDMA_CM_EVENT_ROUTE_RESOLVED: @@ -729,20 +756,27 @@ static int valkeyRdmaCM(RdmaContext *ctx, int timeout) { case RDMA_CM_EVENT_DISCONNECTED: case RDMA_CM_EVENT_ADDR_CHANGE: default: - snprintf(errorstr, sizeof(errorstr), "RDMA: connect failed - %s", rdma_event_str(event->event)); + snprintf(errorstr, sizeof(errorstr), "RDMA: connect failed - %s (status %d)", + rdma_event_str(event->event), event->status); rdmaFatal(errorstr); ret = -1; break; } - rdma_ack_cm_event(event); + if (rdma_ack_cm_event(event)) { + rdmaFatalf("RDMA: ack CM event failed: %s (%d)", strerror(errno), errno); + } + } + + if (errno != EAGAIN) { + rdmaFatalf("RDMA: get CM event failed: %s (%d)", strerror(errno), errno); } return ret; } static int valkeyRdmaWaitConn(RdmaContext *ctx, long timeout) { - int timed; + int ret, timed; struct pollfd pfd; long now = valkeyNowMs(); long start = now; @@ -753,7 +787,17 @@ static int valkeyRdmaWaitConn(RdmaContext *ctx, long timeout) { pfd.fd = ctx->cm_channel->fd; pfd.events = POLLIN; pfd.revents = 0; - if (poll(&pfd, 1, timed) < 0) { + ret = poll(&pfd, 1, timed); + if (ret < 0) { + rdmaFatalf("RDMA: poll CM channel failed: %s (%d)", strerror(errno), errno); + return -1; + } + if (ret == 0) { + rdmaFatalf("RDMA: poll CM channel timed out after %d ms", timed); + return -1; + } + if (pfd.revents & (POLLERR | POLLHUP | POLLNVAL)) { + rdmaFatalf("RDMA: CM channel poll returned revents 0x%x", pfd.revents); return -1; } @@ -768,6 +812,7 @@ static int valkeyRdmaWaitConn(RdmaContext *ctx, long timeout) { now = valkeyNowMs(); } + rdmaFatalf("RDMA: CM connection timed out after %ld ms", timeout); return -1; } @@ -800,17 +845,17 @@ static RdmaContext *valkeyContextConnectRdma(const char *addr, int port, int tim ctx->cm_channel = rdma_create_event_channel(); if (!ctx->cm_channel) { - rdmaFatal("RDMA: create event channel failed"); + rdmaFatalf("RDMA: create event channel failed: %s (%d)", strerror(errno), errno); goto free_rdma; } if (rdma_create_id(ctx->cm_channel, &ctx->cm_id, (void *)ctx, RDMA_PS_TCP)) { - rdmaFatal("RDMA: create id failed"); + rdmaFatalf("RDMA: create id failed: %s (%d)", strerror(errno), errno); goto free_rdma; } if ((valkeySetFdBlocking(ctx->cm_channel->fd, 0) != 0)) { - rdmaFatal("RDMA: set cm channel fd non-block failed"); + rdmaFatalf("RDMA: set cm channel fd non-block failed: %s (%d)", strerror(errno), errno); goto free_rdma; } @@ -828,6 +873,8 @@ static RdmaContext *valkeyContextConnectRdma(const char *addr, int port, int tim /* resolve addr as most 100ms */ if (rdma_resolve_addr(ctx->cm_id, NULL, (struct sockaddr *)&saddr, 100)) { + fprintf(stderr, "%s:%d RDMA: address resolve failed: %s (%d)\n", + __func__, __LINE__, strerror(errno), errno); continue; } diff --git a/tests/rdma/rdma_env.py b/tests/rdma/rdma_env.py index 872fbfaec..6aa2faefd 100755 --- a/tests/rdma/rdma_env.py +++ b/tests/rdma/rdma_env.py @@ -13,8 +13,13 @@ import os import subprocess import netifaces -import time import argparse +import json + + +TEST_NETDEV = "valkeyrdma0" +TEST_IP = "192.0.2.1" +TEST_IP_CIDR = TEST_IP + "/24" def prepare_ib(): @@ -29,6 +34,66 @@ def prepare_ib(): print("Valkey Over RDMA probe modules of IB [OK]") +def is_dummy_netdev(interface): + p = subprocess.run(["ip", "-details", "-json", "link", "show", "dev", interface], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + try: + link_info = json.loads(p.stdout) + except json.JSONDecodeError: + return False + return (not p.returncode and link_info + and link_info[0].get("linkinfo", {}).get("info_kind") == "dummy") + + +def prepare_test_netdev(): + p = subprocess.run(["modprobe", "dummy"], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + if p.returncode: + print("Valkey Over RDMA load dummy netdev driver [FAILED]") + print("---------------\n" + p.stdout + "---------------\n") + os._exit(1) + + if os.path.exists("/sys/class/net/" + TEST_NETDEV): + if not is_dummy_netdev(TEST_NETDEV): + print("Valkey Over RDMA existing interface <%s> is not a dummy netdev [FAILED]" % TEST_NETDEV) + os._exit(1) + else: + p = subprocess.run(["ip", "link", "add", TEST_NETDEV, "type", "dummy"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if p.returncode: + print("Valkey Over RDMA create dummy netdev <%s> [FAILED]" % TEST_NETDEV) + print("---------------\n" + p.stdout + "---------------\n") + os._exit(1) + + for interface in netifaces.interfaces(): + if interface == TEST_NETDEV: + continue + addresses = netifaces.ifaddresses(interface).get(netifaces.AF_INET, []) + if any(address.get("addr") == TEST_IP for address in addresses): + print("Valkey Over RDMA test IP <%s> is already in use by <%s> [FAILED]" + % (TEST_IP, interface)) + os._exit(1) + + addresses = netifaces.ifaddresses(TEST_NETDEV).get(netifaces.AF_INET, []) + if not any(address.get("addr") == TEST_IP for address in addresses): + p = subprocess.run(["ip", "address", "add", TEST_IP_CIDR, "dev", TEST_NETDEV], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if p.returncode: + print("Valkey Over RDMA configure dummy netdev <%s> [FAILED]" % TEST_NETDEV) + print("---------------\n" + p.stdout + "---------------\n") + os._exit(1) + + p = subprocess.run(["ip", "link", "set", TEST_NETDEV, "up"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if p.returncode: + print("Valkey Over RDMA enable dummy netdev <%s> [FAILED]" % TEST_NETDEV) + print("---------------\n" + p.stdout + "---------------\n") + os._exit(1) + + print("Valkey Over RDMA prepare dummy netdev <%s %s> [OK]" % (TEST_NETDEV, TEST_IP)) + return TEST_NETDEV + + def prepare_rxe(interface): # is there any builtin rdma_rxe.ko? p = subprocess.Popen("modprobe rdma_rxe 2> /dev/null", shell=True, stdout=subprocess.PIPE) @@ -70,55 +135,40 @@ def prepare_rxe(interface): print("Valkey Over RDMA add RXE device <%s> [OK]" % softrdma) -# find any IPv4 available networking interface -def find_iface(): - interfaces = netifaces.interfaces() - for interface in interfaces: - if interface == "lo": - continue - - addrs = netifaces.ifaddresses(interface) - if netifaces.AF_INET not in addrs: - continue - - return interface - - def setup_rdma(driver, interface): - if interface == None: - interface = find_iface() - prepare_ib() if driver == "rxe": + if interface is None: + interface = prepare_test_netdev() prepare_rxe(interface) else: print("rxe is currently supported only") os._exit(1); -# iterate /sys/class/infiniband, find any all virtual RDMA device, and remove them -def cleanup_rdma(): - # Ex, /sys/class/infiniband/mlx5_0 - # Ex, /sys/class/infiniband/rxe_eth0 - # Ex, /sys/class/infiniband/siw_eth0 - ibclass = "/sys/class/infiniband/" - try: - for dev in os.listdir(ibclass): - # Ex, /sys/class/infiniband/rxe_eth0/ports/1/gid_attrs/ndevs/0 - origpath = os.readlink(ibclass + dev) - if "virtual" in origpath: - subprocess.Popen("rdma link del " + dev, shell=True).wait() - print("Remove virtual RDMA device : " + dev + " [OK]") - except os.error: - return None +def cleanup_rdma(interface): + if interface is None: + interface = TEST_NETDEV - # try to remove RXE driver from kernel, ignore error - subprocess.Popen("rmmod rdma_rxe 2> /dev/null", shell=True).wait() - - # try to remove SIW driver from kernel, ignore error - subprocess.Popen("rmmod rdma_siw 2> /dev/null", shell=True).wait() - - return None + softrdma = "rxe_" + interface + if os.path.exists("/sys/class/infiniband/" + softrdma): + p = subprocess.run(["rdma", "link", "del", softrdma], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if p.returncode: + print("Valkey Over RDMA remove RXE device <%s> [FAILED]" % softrdma) + print("---------------\n" + p.stdout + "---------------\n") + else: + print("Valkey Over RDMA remove RXE device <%s> [OK]" % softrdma) + + if (interface == TEST_NETDEV and os.path.exists("/sys/class/net/" + TEST_NETDEV) + and is_dummy_netdev(TEST_NETDEV)): + p = subprocess.run(["ip", "link", "del", TEST_NETDEV], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if p.returncode: + print("Valkey Over RDMA remove dummy netdev <%s> [FAILED]" % TEST_NETDEV) + print("---------------\n" + p.stdout + "---------------\n") + else: + print("Valkey Over RDMA remove dummy netdev <%s> [OK]" % TEST_NETDEV) if __name__ == "__main__": @@ -130,7 +180,7 @@ def cleanup_rdma(): parser.add_argument("-d", "--driver", type=str, default="rxe", help="[rxe|siw] specify soft RDMA driver, rxe by default") parser.add_argument("-i", "--interface", type=str, - help="[IFACE] network interface, auto-select any available interface by default") + help="[IFACE] network interface, use a dedicated dummy interface by default") args = parser.parse_args() # test UID. none-root user must stop on none RDMA platform, show some hints and exit. @@ -142,7 +192,7 @@ def cleanup_rdma(): os._exit(1); if args.operation == "cleanup": - cleanup_rdma() + cleanup_rdma(args.interface) elif args.operation == "setup": setup_rdma(args.driver, args.interface) diff --git a/tests/rdma/run.py b/tests/rdma/run.py index 5d41c7553..93a4c710c 100755 --- a/tests/rdma/run.py +++ b/tests/rdma/run.py @@ -16,10 +16,14 @@ import argparse import sys import signal +import rdma_env RDMA_PORT = 6379 IO_THREADS = 4 BENCH_TIMEOUT = 120 +RXE_TEST_NETDEV = rdma_env.TEST_NETDEV +RXE_TEST_DEVICE = "rxe_" + RXE_TEST_NETDEV +RXE_TEST_IP = rdma_env.TEST_IP def build_program(): @@ -45,26 +49,15 @@ def ipaddr_from_iface(iface): return None -def find_default_iface(): - for interface in netifaces.interfaces(): - if interface == "lo": - continue - addrs = netifaces.ifaddresses(interface) - if netifaces.AF_INET in addrs: - return interface - return None - - -def is_rxe_device(ibclass, dev): - # RXE driver sets node_desc to "rxe" (see kernel drivers/infiniband/sw/rxe/rxe_verbs.c). +def is_rdma_port_active(ibclass, dev): try: - with open(os.path.join(ibclass, dev, "node_desc")) as fp: - return fp.read().strip() == "rxe" + with open(os.path.join(ibclass, dev, "ports", "1", "state")) as fp: + return fp.read().strip().startswith("4:") except OSError: return False -def find_rdma_ip_from_sysfs(rxe_only=False): +def find_rdma_ip_from_sysfs(expected_dev=None): # Ex, /sys/class/infiniband/mlx5_0 # Ex, /sys/class/infiniband/rxe_eth0 # Ex, /sys/class/infiniband/siw_eth0 @@ -74,49 +67,57 @@ def find_rdma_ip_from_sysfs(rxe_only=False): except OSError: return None - candidates = sorted(devices) - if rxe_only: - candidates = [dev for dev in candidates if is_rxe_device(ibclass, dev)] + candidates = [expected_dev] if expected_dev else sorted(devices) for dev in candidates: - # Ex, /sys/class/infiniband/rxe_eth0/ports/1/gid_attrs/ndevs/0 - netdev = ibclass + dev + "/ports/1/gid_attrs/ndevs/0" + if dev not in devices: + continue + if not is_rdma_port_active(ibclass, dev): + continue + + # A RoCE device can expose several GID entries. Use one that has a + # non-zero GID and maps to a netdev with an IP address. + ndevs = os.path.join(ibclass, dev, "ports", "1", "gid_attrs", "ndevs") try: - with open(netdev) as fp: - iface = fp.readline().strip() - if not iface: + gid_indexes = sorted(os.listdir(ndevs), key=int) + except (OSError, ValueError): + continue + + for gid_index in gid_indexes: + try: + with open(os.path.join(ndevs, gid_index)) as fp: + iface = fp.readline().strip() + with open(os.path.join(ibclass, dev, "ports", "1", "gids", gid_index)) as fp: + gid = fp.readline().strip() + except OSError: + continue + + if not iface or not gid.replace(":", "").strip("0"): + continue + if expected_dev and iface != RXE_TEST_NETDEV: continue ipaddr = ipaddr_from_iface(iface) - if ipaddr is None: + if ipaddr is None or (expected_dev and ipaddr != RXE_TEST_IP): continue - print("Valkey Over RDMA test prepare " + dev + " <" + ipaddr + "> [OK]") + print("Valkey Over RDMA test prepare " + dev + " <" + iface + " " + ipaddr + "> [OK]") return ipaddr - except (OSError, ValueError): - continue return None def find_rdma_dev(install_rxe=False): - # After rdma link add, gid_attrs/ndevs can lag behind the device node. - # Retry briefly when we just installed RXE. Prefer RXE over host NICs - # (e.g. GitHub Actions mana_0) which share an IP but fail rdma_resolve_addr. - retries = 10 if install_rxe else 1 + # After rdma link add, the port and GID table can lag behind the device + # node. When RXE was installed for this test, require that exact device; + # an IP address alone cannot distinguish RXE from a hardware RDMA provider. + expected_dev = RXE_TEST_DEVICE if install_rxe else None + retries = 20 if install_rxe else 1 for attempt in range(retries): - ipaddr = find_rdma_ip_from_sysfs(rxe_only=install_rxe) + ipaddr = find_rdma_ip_from_sysfs(expected_dev) if ipaddr is not None: return ipaddr if attempt + 1 < retries: time.sleep(0.2) - if install_rxe: - iface = find_default_iface() - if iface is not None: - ipaddr = ipaddr_from_iface(iface) - if ipaddr is not None: - print("Valkey Over RDMA test prepare rxe_" + iface + " <" + ipaddr + "> [OK]") - return ipaddr - return None @@ -242,8 +243,8 @@ def test_exit(retval, install_rxe): if install_rxe and not os.geteuid(): rdma_env_py = os.path.dirname(os.path.abspath(__file__)) + "/rdma_env.py" - cmd = rdma_env_py + " -o cleanup" - subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).wait() + cmd = [rdma_env_py, "-o", "cleanup", "-i", RXE_TEST_NETDEV] + subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) sys.stdout.flush() sys.stderr.flush() @@ -277,8 +278,8 @@ def handler(signum, frame): test_exit(1, False) rdma_env_py = os.path.dirname(os.path.abspath(__file__)) + "/rdma_env.py" - cmd = rdma_env_py + " -o setup -d rxe" - if subprocess.call(cmd, shell=True): + cmd = [rdma_env_py, "-o", "setup", "-d", "rxe"] + if subprocess.call(cmd): print("Valkey Over RDMA setup RXE [FAILED]") test_exit(1, args.install_rxe) diff --git a/tests/support/cluster_util.tcl b/tests/support/cluster_util.tcl index ec1f91fdf..c00920f46 100644 --- a/tests/support/cluster_util.tcl +++ b/tests/support/cluster_util.tcl @@ -208,6 +208,12 @@ proc cluster_allocate_replicas {masters replicas} { } } +# Replica allocator that does not attach any replica to a primary. Pass it as +# the replica_allocator argument of start_cluster when a test needs the extra +# nodes to stay unassigned at setup time, e.g. to add them later as replicas +# with a particular replication configuration. +proc no_replica_allocation {primaries replicas} {} + # Setup method to be executed to configure the cluster before the # tests run. proc cluster_setup {masters replicas node_count slot_allocator replica_allocator options} { @@ -276,7 +282,7 @@ proc start_cluster {masters replicas options code {slot_allocator continuous_slo # Configure the starting of multiple servers. Set cluster node timeout # aggressively since many tests depend on ping/pong messages. - set cluster_options [list overrides [list cluster-enabled yes cluster-ping-interval 100 cluster-node-timeout 3000 cluster-databases 16 cluster-slot-stats-enabled yes]] + set cluster_options [list overrides [list cluster-enabled yes cluster-ping-interval 100 cluster-node-timeout 3000 cluster-databases 16 cluster-slot-stats-enabled yes latency-monitor-threshold 1]] set options [concat $cluster_options $options] # Cluster mode only supports a single database, so before executing the tests @@ -292,6 +298,20 @@ proc cluster_has_flag {node flag} { expr {[lsearch -exact [dict get $node flags] $flag] != -1} } +# Returns 1 only when every server instance in `srv_idxs` sees every +# node id in `node_ids` carrying `flag` in its CLUSTER NODES output. +proc cluster_all_see_flag {srv_idxs node_ids flag} { + foreach idx $srv_idxs { + foreach id $node_ids { + set node [cluster_get_node_by_id $idx $id] + if {![cluster_has_flag $node $flag]} { + return 0 + } + } + } + return 1 +} + # Returns the parsed "myself" node entry as a dictionary. proc cluster_get_myself id { set nodes [get_cluster_nodes $id] @@ -355,6 +375,13 @@ proc get_myself id { return {} } +# Returns 1 if the instance 'instance_id' agrees that the node 'replica_id' is a +# replica and is a replica of the node 'primary_id'. +proc cluster_node_is_replica_of {instance_id replica_id primary_id} { + set node [cluster_get_node_by_id $instance_id $replica_id] + expr {[cluster_has_flag $node slave] && [dict get $node slaveof] eq $primary_id} +} + # Returns 1 if no node knows node_id, 0 if any node knows it. proc node_is_forgotten {node_id} { for {set j 0} {$j < [llength $::servers]} {incr j} { diff --git a/tests/support/util.tcl b/tests/support/util.tcl index f9960f19e..af07dd4c4 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -47,6 +47,47 @@ proc write_binary_file {path data} { close $fd } +# Create keys of all data types with predictable/consistent names for verification +proc createComplexDatasetForVerification {r count {prefix ""}} { + for {set i 0} {$i < $count} {incr i} { + # String keys + {*}$r set ${prefix}before_$i "value_before_$i" + {*}$r set ${prefix}int_$i [expr {42 + $i}] + {*}$r set ${prefix}bits_$i "\x0f" + + # List keys + {*}$r lpush ${prefix}lst_$i "L2" "L1" + {*}$r rpush ${prefix}lst_$i "R1" "R2" + + # Set keys + {*}$r sadd ${prefix}set_$i "B1" "B2" + {*}$r sadd ${prefix}iset_$i 12 34 + + # Sorted set keys + {*}$r zadd ${prefix}zset_$i 1 "Z1" 2 "Z2" + + # Hash keys + {*}$r hset ${prefix}hash_$i "H1" "a" + {*}$r hset ${prefix}hash_$i "H2" 1 + + # HyperLogLog + {*}$r pfadd ${prefix}hll_$i "PF1" + + # Geo + {*}$r geoadd ${prefix}geo_$i -122.335167 47.608013 "seattle" + {*}$r geosearchstore ${prefix}geo_set_$i ${prefix}geo_$i FROMLONLAT -122.335167 47.608013 BYRADIUS 10 mi + + # Stream + {*}$r xadd ${prefix}stream_$i "*" "D1" "V1" + {*}$r xgroup create ${prefix}stream_$i ${prefix}group_$i 0 MKSTREAM + } +} + +# Path of the RDB file a server saves to (dir + dbfilename). +proc server_rdb_path {client} { + return [file join [lindex [$client config get dir] 1] [lindex [$client config get dbfilename] 1]] +} + # Useful for some test proc zlistAlikeSort {a b} { if {[lindex $a 0] > [lindex $b 0]} {return 1} @@ -738,8 +779,16 @@ proc process_is_paused pid { } # Wait until the process enters a paused state. -proc wait_process_paused pid { - wait_for_condition 50 100 { +# +# Callers that arm a self-stopping debug point (DEBUG PAUSE-AFTER-FORK, +# DEBUG PAUSE-BEFORE-PSYNC) also wait for the server to reach it, which under +# valgrind can take longer than 5 seconds. Scale the budget for them, but only +# under valgrind so normal runs keep failing fast. +proc wait_process_paused {pid {retries auto}} { + if {$retries eq "auto"} { + if {$::valgrind} {set retries 1000} else {set retries 50} + } + wait_for_condition $retries 100 { [process_is_paused $pid] } else { puts [exec ps j $pid] @@ -749,7 +798,8 @@ proc wait_process_paused pid { proc pause_process pid { exec kill -SIGSTOP $pid - wait_process_paused $pid + # We sent the signal, so the stop is near-immediate. Keep the short budget. + wait_process_paused $pid 50 } proc resume_process pid { @@ -1419,3 +1469,29 @@ proc cluster_nodes_conf_path {id} { set conf [lindex [R $id config get cluster-config-file] 1] return [file join $dir $conf] } + +# Return a finite operand `x` such that `x + x` overflows the server's `long double`. +# +# The width of `long double` is platform dependent, so no single constant works +# everywhere: +# +# x86-64 / aarch64 Linux 80-bit or 128-bit, LDBL_MAX ~1.19e4932 +# Apple Silicon long double == double, LDBL_MAX ~1.80e308 +# +# Rather than branch on the build, ask the server: a value it cannot represent is +# rejected when parsed as a long double, so the first candidate it accepts is the +# right magnitude for this build. +# +# `level` selects the server instance, matching the convention of `r` (0 is the +# current server, -1 the previous one, and so on). +proc ldbl_overflow_operand {{level 0}} { + foreach candidate {1e4932 1e308} { + r $level set __ldbl_probe $candidate + if {![catch {r $level increx __ldbl_probe byfloat 0}]} { + r $level del __ldbl_probe + return $candidate + } + } + r $level del __ldbl_probe + error "no long double operand large enough to overflow on this platform" +} diff --git a/tests/unit/acl-role.tcl b/tests/unit/acl-role.tcl new file mode 100644 index 000000000..4ae77a961 --- /dev/null +++ b/tests/unit/acl-role.tcl @@ -0,0 +1,784 @@ +# Return the `user ...` line ACL LIST reports for the given user, or an +# empty string if there is none. +proc acl_list_entry {level name} { + foreach entry [r $level ACL LIST] { + if {[lindex $entry 0] eq "user" && [lindex $entry 1] eq $name} { + return $entry + } + } + return {} +} + +start_server {tags {"acl external:skip"}} { + test {ACL ROLES - initially empty} { + r ACL ROLES + } {} + + # --- ACL SETROLE --- + + test {ACL SETROLE - create a role} { + r ACL SETROLE myrole ~keys:* +@all -@dangerous + } {OK} + + test {ACL ROLES - lists the role} { + r ACL ROLES + } {myrole} + + test {ACL SETROLE - update existing role} { + r ACL SETROLE myrole ~keys:* +@all -@dangerous -@scripting + } {OK} + + test {ACL SETROLE - rejects password operations} { + catch {r ACL SETROLE myrole >password} err + assert_match {*Error*} $err + } + + test {ACL SETROLE - rejects on/off flags} { + catch {r ACL SETROLE myrole on} err + assert_match {*Error*} $err + + catch {r ACL SETROLE myrole off} err + assert_match {*Error*} $err + } + + test {ACL SETROLE - rejects nested roles} { + r ACL SETROLE otherrole +@read + catch {r ACL SETROLE myrole role=otherrole} err + assert_match {*Error*} $err + + # resetroles is a user rule too, a role has no roles to reset. + catch {r ACL SETROLE myrole resetroles} err + assert_match {*Error*} $err + } + + test {ACL SETROLE - unmatched parenthesis} { + catch {r ACL SETROLE badrole (+get} err + assert_match {*Unmatched parenthesis*} $err + } + + test {ACL SETROLE - clearselectors removes non-root selectors} { + r ACL SETROLE multisel +get ~a:* (+set ~b:*) + r ACL SETROLE multisel clearselectors +get ~a:* + # After clearselectors, only root selector remains (no extra selectors) + set info [r ACL GETROLE multisel] + set idx [lsearch $info "selectors"] + set sels [lindex $info [expr {$idx + 1}]] + assert_equal [llength $sels] 0 + } + + test {ACL SETROLE - role names accept printable ASCII} { + foreach name {read-only read_only app.reader v2 role:admin a=b} { + r ACL SETROLE $name +get ~* + assert_not_equal -1 [lsearch -exact [r ACL ROLES] $name] + } + # The names round trip through a user's role= list unchanged. + r ACL SETUSER asciiuser on >p role=read-only,app.reader,a=b + set info [r ACL GETUSER asciiuser] + set idx [lsearch $info "roles"] + assert_equal {a=b app.reader read-only} [lsort [lindex $info [expr {$idx + 1}]]] + r ACL DELUSER asciiuser + r ACL DELROLE read-only read_only app.reader v2 role:admin a=b + } + + test {ACL SETROLE - role names reject what the parsers cannot read back} { + # Space and the other control characters end a token, a comma separates + # the names in a `role=` list, and quotes and backslashes are special to + # sdssplitargs(), which reads the ACL file and valkey.conf back. + foreach {name reason} { + {bad name} {*printable ASCII*} + "tab\there" {*printable ASCII*} + "test\xc3\xa9" {*printable ASCII*} + a,b {*can't contain commas*} + q"x {*quotes or backslashes*} + q'x {*quotes or backslashes*} + {q\x} {*quotes or backslashes*} + } { + catch {r ACL SETROLE $name +@all} err + assert_match $reason $err + assert_equal -1 [lsearch -exact [r ACL ROLES] $name] + } + } + + test {ACL SETROLE - rejects an empty role name} { + # An empty name survives ACL SETROLE but not a config round trip: + # sdssplitargs() collapses the whitespace, so the first rule would come + # back as the role name and the server would refuse to start. + catch {r ACL SETROLE "" +get ~*} err + assert_match {*Role names can't be empty*} $err + assert_equal {} [lsearch -all -inline [r ACL ROLES] {}] + } + + test {ACL SETROLE - a role name may collide with a command or category} { + # `role=` keeps role names in their own namespace, so there is nothing + # to disambiguate against commands and categories. + r ACL SETROLE get ~g:* +get + r ACL SETROLE read ~r:* +get + r ACL SETUSER collide on >p role=get,read + assert_equal [r ACL DRYRUN collide GET g:key] {OK} + assert_equal [r ACL DRYRUN collide GET r:key] {OK} + r ACL DELUSER collide + r ACL DELROLE get read + } + + # --- ACL GETROLE --- + + test {ACL GETROLE - returns role info} { + set info [r ACL GETROLE myrole] + assert_match {*commands*} $info + assert_match {*keys*} $info + } + + test {ACL GETROLE - non-existent role returns nil} { + r ACL GETROLE nonexistent + } {} + + test {ACL GETROLE - shows selectors and users} { + r ACL SETROLE inforole +get ~info:* (+set ~info:*) + r ACL SETUSER infouser on >infopass role=inforole + set info [r ACL GETROLE inforole] + # Check the list of users holding the role + set idx [lsearch $info "users"] + set users [lindex $info [expr {$idx + 1}]] + assert_equal $users {infouser} + # Check selectors (should have one extra selector beyond root) + set idx [lsearch $info "selectors"] + set sels [lindex $info [expr {$idx + 1}]] + assert_equal [llength $sels] 1 + } + + # --- ACL SETUSER --- + + test {ACL SETUSER - assign a role to a user} { + r ACL SETUSER alice on >pass123 role=myrole + } {OK} + + test {ACL SETUSER - resetroles removes every role} { + r ACL SETUSER alice resetroles + set info [r ACL GETUSER alice] + set idx [lsearch $info "roles"] + set roles [lindex $info [expr {$idx + 1}]] + assert_equal $roles {} + } + + test {ACL SETUSER - role= replaces the whole set rather than adding to it} { + r ACL SETROLE repA +get ~a:* + r ACL SETROLE repB +get ~b:* + r ACL SETUSER repuser on >p role=repA,repB + set info [r ACL GETUSER repuser] + set idx [lsearch $info "roles"] + assert_equal {repA repB} [lsort [lindex $info [expr {$idx + 1}]]] + + r ACL SETUSER repuser role=repB + set info [r ACL GETUSER repuser] + set idx [lsearch $info "roles"] + assert_equal {repB} [lindex $info [expr {$idx + 1}]] + + # repA no longer lists the user, so it can be deleted. + assert_equal 1 [r ACL DELROLE repA] + r ACL SETUSER repuser resetroles + r ACL DELUSER repuser + r ACL DELROLE repB + } + + test {ACL SETUSER - the same role named twice is kept once} { + r ACL SETROLE dupe +get ~* + r ACL SETUSER dupeuser on >p role=dupe,dupe + set info [r ACL GETUSER dupeuser] + set idx [lsearch $info "roles"] + assert_equal {dupe} [lindex $info [expr {$idx + 1}]] + r ACL DELUSER dupeuser + r ACL DELROLE dupe + } + + test {ACL SETUSER - referencing non-existent role fails} { + catch {r ACL SETUSER dave on >pass role=nosuchrole} err + assert_match {*role does not exist*} $err + } + + test {ACL SETUSER - a failing role= leaves the user's roles untouched} { + r ACL SETROLE keptrole +get ~kept:* + r ACL SETUSER keeper on >p role=keptrole + catch {r ACL SETUSER keeper role=keptrole,nosuchrole} err + assert_match {*role does not exist*} $err + set info [r ACL GETUSER keeper] + set idx [lsearch $info "roles"] + assert_equal {keptrole} [lindex $info [expr {$idx + 1}]] + r ACL DELUSER keeper + r ACL DELROLE keptrole + } + + test {ACL SETUSER - empty and malformed role= lists are rejected} { + r ACL SETROLE listrole +get ~* + # role= has to name at least one role. resetroles is the way to leave a + # user with none. + foreach spec {role= role=, role=,listrole role=listrole, role=listrole,,listrole} { + catch {r ACL SETUSER alice $spec} err + assert_match {*Syntax error*} $err + } + r ACL DELROLE listrole + } + + # --- ACL GETUSER --- + + test {ACL GETUSER - shows role membership} { + r ACL SETUSER alice role=myrole + set info [r ACL GETUSER alice] + set idx [lsearch $info "roles"] + set roles [lindex $info [expr {$idx + 1}]] + assert_equal $roles {myrole} + } + + # --- ACL DELROLE --- + + test {ACL DELROLE - fails if the role is assigned to a user} { + r ACL SETUSER bob on >pass456 role=otherrole + catch {r ACL DELROLE otherrole} err + assert_match {*is assigned to one or more users*} $err + } + + test {ACL DELROLE - succeeds when no user holds the role} { + r ACL SETUSER bob resetroles + r ACL DELROLE otherrole + } {1} + + test {ACL DELROLE - non-existent role is not counted} { + r ACL DELROLE nonexistent + } {0} + + test {ACL DELROLE - delete multiple roles at once} { + r ACL SETROLE delA +get + r ACL SETROLE delB +set + r ACL SETROLE delC +del + assert_equal [r ACL DELROLE delA delB delC] 3 + # Verify they're all gone + assert_equal [r ACL GETROLE delA] {} + assert_equal [r ACL GETROLE delB] {} + assert_equal [r ACL GETROLE delC] {} + } + + # --- Permission checks --- + + test {Role permissions are effective for user} { + r AUTH alice pass123 + + r SET keys:hello world + assert_equal [r GET keys:hello] world + + catch {r SET other:key value} err + assert_match {*NOPERM*} $err + } {} {needs:reset} + + test {ACL DRYRUN respects role permissions} { + r AUTH default "" + + assert_equal [r ACL DRYRUN alice SET keys:test value] {OK} + + set result [r ACL DRYRUN alice SET other:test value] + assert_match {*no permissions*} $result + } + + test {After removing from role, permissions are revoked} { + r ACL SETUSER alice resetroles + set result [r ACL DRYRUN alice SET keys:test value] + assert_match {*no permissions*} $result + } + + test {Role changes are immediately visible to the users holding it} { + r ACL SETROLE liverole +@all ~* + r ACL SETUSER carol on >carolpass role=liverole + # Carol can do anything now + assert_equal [r ACL DRYRUN carol SET anykey value] {OK} + # Update role to restrict keys + r ACL SETROLE liverole resetkeys +@all ~restricted:* + # Carol should now only access restricted:* keys + catch {r ACL DRYRUN carol SET anykey value} err + assert_match {*no permissions*} $err + assert_equal [r ACL DRYRUN carol SET restricted:key value] {OK} + } + + test {Multiple roles - each role is a separate selector with OR logic} { + r ACL SETROLE roleA +get ~a:* + r ACL SETROLE roleB +set ~b:* + r ACL SETUSER multi on >multipass role=roleA,roleB + + # roleA allows GET on a:* keys + assert_equal [r ACL DRYRUN multi GET a:key] {OK} + # roleB allows SET on b:* keys + assert_equal [r ACL DRYRUN multi SET b:key value] {OK} + + # GET b:key is denied + set result [r ACL DRYRUN multi GET b:key] + assert_match {*no permissions*} $result + # Keys outside both roles are denied + set result [r ACL DRYRUN multi GET c:key] + assert_match {*no permissions*} $result + } + + test {Role with multiple selectors} { + # Create a role with two selectors: one for reads on r:*, one for writes on w:* + r ACL SETROLE multiselector +get ~r:* (+set ~w:*) + r ACL SETUSER msuser on >mspass role=multiselector + + # First selector allows GET on r:* + assert_equal [r ACL DRYRUN msuser GET r:key] {OK} + # Second selector allows SET on w:* + assert_equal [r ACL DRYRUN msuser SET w:key value] {OK} + + # Cross-selector: GET on w:* is denied (no single selector allows it) + set result [r ACL DRYRUN msuser GET w:key] + assert_match {*no permissions*} $result + # SET on r:* is also denied + set result [r ACL DRYRUN msuser SET r:key value] + assert_match {*no permissions*} $result + } + + test {User own permissions add on top of role (OR logic)} { + r ACL SETROLE onlyset +set ~data:* + r ACL SETUSER userplus on >pluspass role=onlyset +get ~data:* + + # Role allows SET on data:*, user's own selector allows GET on data:* + assert_equal [r ACL DRYRUN userplus SET data:key value] {OK} + assert_equal [r ACL DRYRUN userplus GET data:key] {OK} + + # Neither allows DEL + set result [r ACL DRYRUN userplus DEL data:key] + assert_match {*no permissions*} $result + } + + test {User cannot restrict role permissions} { + r ACL SETROLE permissive +@all ~* + r ACL SETUSER restricted on >rpass role=permissive -@admin + + # Even though user has no admin permissions, the role grants it + assert_equal [r ACL DRYRUN restricted FLUSHALL] {OK} + } + + test {Role with channel patterns} { + r ACL SETROLE channelrole +subscribe &news:* ~* + r ACL SETUSER chanuser on >chanpass role=channelrole + assert_equal [r ACL DRYRUN chanuser SUBSCRIBE news:sports] {OK} + set result [r ACL DRYRUN chanuser SUBSCRIBE private:msg] + assert_match {*no permissions*} $result + } + + test {SORT BY/GET honours full key access granted by a role} { + r RPUSH sortlist 1 2 3 + r ACL SETROLE allkeysrole ~* +@all + r ACL SETROLE onekeyrole ~sortlist +@all + r ACL SETUSER sortok on >p role=allkeysrole + r ACL SETUSER sortlimited on >p role=onekeyrole + + r AUTH sortok p + assert_equal {1 2 3} [r SORT sortlist BY weight_* GET #] + + r AUTH sortlimited p + assert_error {*BY option of SORT denied*} {r SORT sortlist BY weight_*} + + r AUTH default "" + } {OK} {needs:reset} + + # --- ACL LIST --- + + test {ACL LIST includes roles} { + set list [r ACL LIST] + assert_match "role *" [lindex $list 0] + } + + # --- Pubsub client disconnection --- + + test {SETROLE restricting channels kills pubsub clients} { + r ACL SETROLE pubrole +subscribe &news:* ~* + r ACL SETUSER pubuser on >pubpass role=pubrole + set rd [valkey_deferring_client] + $rd AUTH pubuser pubpass + $rd read + $rd SUBSCRIBE news:sports + assert_match {subscribe news:sports 1} [$rd read] + + # Restrict the role's channels + r ACL SETROLE pubrole resetchannels +subscribe &alerts:* ~* + + # Client should be disconnected + catch {$rd read} err + catch {$rd close} + assert_match {*I/O error*} $err + } + + test {SETROLE restricting channels kills shard pubsub clients} { + r ACL SETROLE shardrole +ssubscribe &shard:* ~* + r ACL SETUSER sharduser on >shardpass role=shardrole + set rd [valkey_deferring_client] + $rd AUTH sharduser shardpass + $rd read + $rd SSUBSCRIBE shard:one + assert_match {ssubscribe shard:one 1} [$rd read] + + r ACL SETROLE shardrole resetchannels +ssubscribe &other:* ~* + + catch {$rd read} err + catch {$rd close} + assert_match {*I/O error*} $err + } + + test {SETUSER removing role kills pubsub clients using role channels} { + r ACL SETROLE subrole +subscribe &events:* ~* + r ACL SETUSER subuser on >subpass role=subrole + set rd [valkey_deferring_client] + $rd AUTH subuser subpass + $rd read + $rd SUBSCRIBE events:live + assert_match {subscribe events:live 1} [$rd read] + + # Remove user from the role + r ACL SETUSER subuser resetroles + + # Client should be disconnected + catch {$rd read} err + catch {$rd close} + assert_match {*I/O error*} $err + } + + # --- User reset --- + + test {ACL DELUSER removes the user from the role user list} { + r ACL SETROLE delrole ~* +get + r ACL SETUSER deluser1 on >p role=delrole + r ACL SETUSER deluser2 on >p role=delrole + + set info [r ACL GETROLE delrole] + set idx [lsearch $info "users"] + assert_equal {deluser1 deluser2} [lsort [lindex $info [expr {$idx + 1}]]] + + r ACL DELUSER deluser1 + set info [r ACL GETROLE delrole] + set idx [lsearch $info "users"] + assert_equal {deluser2} [lindex $info [expr {$idx + 1}]] + + # With the last user gone the role becomes deletable. + r ACL DELUSER deluser2 + assert_equal 1 [r ACL DELROLE delrole] + } + + test {User reset clears role memberships} { + r ACL SETUSER carol reset + set info [r ACL GETUSER carol] + set idx [lsearch $info "roles"] + set roles [lindex $info [expr {$idx + 1}]] + assert_equal $roles {} + } + + # --- Roles are not users --- + + test {A role cannot be authenticated against or read as a user} { + r ACL SETROLE notauser ~* +@all + catch {r AUTH notauser anything} err + assert_match {*WRONGPASS*} $err + + # Roles live in their own table, so the user commands must not see them + # and the role commands must not see users. + assert_equal {} [r ACL GETUSER notauser] + assert_equal {} [r ACL GETROLE default] + assert_equal -1 [lsearch -exact [r ACL USERS] notauser] + assert_equal -1 [lsearch -exact [r ACL ROLES] default] + r ACL DELROLE notauser + } + + test {Role subcommands require admin permissions} { + r ACL SETROLE probed ~* +get + r ACL SETUSER plain on >p ~* +@all -@admin -@dangerous + + assert_match {*no permissions*} [r ACL DRYRUN plain ACL SETROLE x +get] + assert_match {*no permissions*} [r ACL DRYRUN plain ACL DELROLE probed] + assert_match {*no permissions*} [r ACL DRYRUN plain ACL GETROLE probed] + assert_match {*no permissions*} [r ACL DRYRUN plain ACL ROLES] + r ACL DELROLE probed + } + + test {ACL LOG records a denial for a user whose access comes from a role} { + r ACL LOG RESET + r ACL SETROLE logrole ~allowed:* +get + r ACL SETUSER loguser on >logpass role=logrole + + set rd [valkey_client] + $rd AUTH loguser logpass + catch {$rd GET denied:key} err + assert_match {*NOPERM*} $err + $rd close + + set entry [lindex [r ACL LOG] 0] + assert_equal [dict get $entry username] {loguser} + assert_equal [dict get $entry context] {toplevel} + assert_equal [dict get $entry reason] {key} + assert_equal [dict get $entry object] {denied:key} + } + + # --- Case sensitivity of role and user names --- + + test {Role names are case-sensitive} { + r ACL SETROLE Cache ~c:* +get + r ACL SETROLE cache ~d:* +set + assert_equal {Cache cache} [lsort [lsearch -all -inline [r ACL ROLES] {*ache}]] + + r ACL SETUSER caseuser on >p role=Cache,cache + set info [r ACL GETUSER caseuser] + set idx [lsearch $info "roles"] + assert_equal {Cache cache} [lsort [lindex $info [expr {$idx + 1}]]] + } + + test {User names are case-sensitive on the role user list} { + r ACL SETROLE rr ~* +get + r ACL SETUSER alice on >p role=rr + r ACL SETUSER ALICE on >p role=rr + + set info [r ACL GETROLE rr] + set idx [lsearch $info "users"] + assert_equal {ALICE alice} [lsort [lindex $info [expr {$idx + 1}]]] + } + + # Cleanup + test {Cleanup test users and roles} { + # Remove all non-default users (which also drops their role memberships) + foreach entry [r ACL LIST] { + if {[string match "user *" $entry]} { + set uname [lindex $entry 1] + if {$uname ne "default"} { + catch {r ACL DELUSER $uname} + } + } + } + # Now delete all roles (no users hold them any more) + foreach role [r ACL ROLES] { + catch {r ACL DELROLE $role} + } + } +} + +# Two servers fed the same ACL commands must end up with the same ACL state. +# Nine roles on purpose: up to ENTRIES_PER_BUCKET entries share a single +# hashtable bucket and happen to come back in insertion order, so a smaller +# role count would not notice an unordered role list. +start_server {tags {"acl external:skip"}} { + start_server {} { + test {Two servers given identical ACLs agree on ACL DIGEST} { + # Same commands, same order, on two freshly started servers. + for {set i 1} {$i <= 9} {incr i} { + r -1 ACL SETROLE r$i ~r$i:* +get + r ACL SETROLE r$i ~r$i:* +get + } + r -1 ACL SETUSER alice on >p role=r1,r2,r3,r4,r5,r6,r7,r8,r9 + r ACL SETUSER alice on >p role=r1,r2,r3,r4,r5,r6,r7,r8,r9 + + # Sanity: the two nodes really do grant the same access. + assert_equal [r -1 ACL DRYRUN alice GET r5:k] [r ACL DRYRUN alice GET r5:k] + assert_equal [lsort [r -1 ACL ROLES]] [lsort [r ACL ROLES]] + + set list_a [acl_list_entry -1 alice] + set list_b [acl_list_entry 0 alice] + set dig_a [r -1 ACL DIGEST] + set dig_b [r ACL DIGEST] + + assert_equal $list_a $list_b + assert_equal $dig_a $dig_b + } + + test {A user's role list is reported in the order it was set} { + r ACL SETUSER bob on >p role=r9,r8,r7,r6,r5,r4,r3,r2,r1 + set info [r ACL GETUSER bob] + assert_equal {r9 r8 r7 r6 r5 r4 r3 r2 r1} [lindex $info [expr {[lsearch $info "roles"] + 1}]] + assert_match {*role=r9,r8,r7,r6,r5,r4,r3,r2,r1*} [acl_list_entry 0 bob] + } + } +} + +# Test loading roles from ACL file +set server_path [tmpdir "server.role.acl"] +exec cp -f tests/assets/role.acl $server_path +start_server [list overrides [list "dir" $server_path "aclfile" "role.acl"] tags [list "external:skip"]] { + + test {Roles loaded from ACL file} { + lsort [r ACL ROLES] + } {customer viewer} + + test {Users loaded with role assignments from ACL file} { + set info [r ACL GETUSER alice] + set idx [lsearch $info "roles"] + set roles [lindex $info [expr {$idx + 1}]] + assert_equal $roles {customer} + } + + test {Role permissions work after loading from ACL file} { + # alice has customer role: all commands except admin/dangerous/scripting + assert_equal [r ACL DRYRUN alice SET anykey value] {OK} + + set result [r ACL DRYRUN alice FLUSHALL] + assert_match {*no permissions*} $result + } + + test {User-level permissions add on top of role from ACL file} { + assert_equal [r ACL DRYRUN carol EVAL "return 1" 0] {OK} + set result [r ACL DRYRUN alice EVAL "return 1" 0] + assert_match {*no permissions*} $result + } + + test {ACL SAVE and reload preserves roles} { + r ACL SAVE + r ACL LOAD + lsort [r ACL ROLES] + } {customer viewer} + + test {ACL SAVE and reload preserves a punctuated role name} { + r ACL SETROLE app.read-only ~ro:* +get + r ACL SETUSER punctuser on >p role=app.read-only,customer + r ACL SAVE + r ACL LOAD + + assert_equal {app.read-only customer viewer} [lsort [r ACL ROLES]] + set info [r ACL GETUSER punctuser] + set idx [lsearch $info "roles"] + assert_equal {app.read-only customer} [lsort [lindex $info [expr {$idx + 1}]]] + assert_equal [r ACL DRYRUN punctuser GET ro:key] {OK} + + r ACL DELUSER punctuser + r ACL DELROLE app.read-only + r ACL SAVE + } + + test {Default user keeps its role membership across ACL LOAD} { + for {set i 0} {$i < 3} {incr i} { + r ACL LOAD + + set info [r ACL GETUSER default] + set idx [lsearch $info "roles"] + assert_equal {viewer} [lindex $info [expr {$idx + 1}]] + + set info [r ACL GETROLE viewer] + set idx [lsearch $info "users"] + assert_equal {bob default} [lsort [lindex $info [expr {$idx + 1}]]] + } + } + + test {Role held by the default user cannot be deleted} { + r ACL SETUSER bob resetroles + catch {r ACL DELROLE viewer} err + assert_match {*is assigned to one or more users*} $err + + # Reading the user back must not dereference a stale role entry. + assert_match {*role=viewer*} [r ACL LIST] + assert_equal {PONG} [r PING] + } +} + +# Test ACL file error paths for roles +set server_path [tmpdir "server.role.errors.acl"] +exec cp -f tests/assets/role.acl $server_path +start_server [list overrides [list "dir" $server_path "aclfile" "role.acl"] tags [list "external:skip"]] { + + test {ACL LOAD - role with invalid rules fails} { + set fd [open "$server_path/role.acl" w] + puts $fd "role badrole >password" + close $fd + catch {r ACL LOAD} err + assert_match {*Error*} $err + } + + test {ACL LOAD - role line without name fails} { + set fd [open "$server_path/role.acl" w] + puts $fd "role" + close $fd + catch {r ACL LOAD} err + assert_match {*requires a role name*} $err + } + + test {ACL LOAD - duplicate role fails} { + set fd [open "$server_path/role.acl" w] + puts $fd "role dup ~* +@all" + puts $fd "role dup ~* +@read" + close $fd + catch {r ACL LOAD} err + assert_match {*Duplicate role*} $err + } + + test {Restore valid ACL file} { + exec cp -f tests/assets/role.acl $server_path + r ACL LOAD + } +} + +# Test loading roles from valkey.conf inline directives +set conf_lines [list "role" "inlinerole ~* +@read" "user" "inlineuser on >ipass role=inlinerole"] +start_server [list config_lines $conf_lines tags [list "external:skip"]] { + + test {Roles loaded from valkey.conf inline directives} { + r ACL ROLES + } {inlinerole} + + test {User with role from valkey.conf works} { + assert_equal [r ACL DRYRUN inlineuser GET anykey] {OK} + + set result [r ACL DRYRUN inlineuser SET anykey value] + assert_match {*no permissions*} $result + } + + test {CONFIG REWRITE persists runtime role changes} { + r ACL SETROLE runtimerole ~rt:* +get + r CONFIG REWRITE + assert_match {*role runtimerole*} [exec cat [srv 0 config_file]] + } + + test {CONFIG REWRITE drops roles deleted at runtime} { + r ACL DELROLE runtimerole + r CONFIG REWRITE + assert_equal 0 [string match {*role runtimerole*} [exec cat [srv 0 config_file]]] + } + + test {A user's role= survives CONFIG REWRITE and a restart} { + # Punctuated names are the interesting case: they have to come back + # from sdssplitargs() as one token and split on the comma the same way. + r ACL SETROLE app.read-only ~rw:* +get + r ACL SETROLE second ~sc:* +set + r ACL SETUSER rewriteuser on >p role=app.read-only,second + r CONFIG REWRITE + restart_server 0 true false + + assert_equal {app.read-only inlinerole second} [lsort [r ACL ROLES]] + set info [r ACL GETUSER rewriteuser] + set idx [lsearch $info "roles"] + assert_equal {app.read-only second} [lsort [lindex $info [expr {$idx + 1}]]] + assert_equal [r ACL DRYRUN rewriteuser GET rw:key] {OK} + assert_equal [r ACL DRYRUN rewriteuser SET sc:key v] {OK} + + # The roles are still held, so they cannot be deleted yet. + assert_error {*is assigned to one or more users*} {r ACL DELROLE app.read-only} + r ACL DELUSER rewriteuser + r ACL DELROLE app.read-only second + } +} + +# Test duplicate role in config on startup +test {Duplicate role in config on startup fails} { + catch {exec $::VALKEY_SERVER_BIN --role dup --role dup} err + assert_match {*Duplicate role*} $err +} {} {external:skip} + +# Test invalid role name in config on startup +test {Invalid role name in config on startup fails} { + catch {exec $::VALKEY_SERVER_BIN --role "" +get} err + assert_match {*Role names can't be empty*} $err + + set aclfile [tmpfile "role-invalid-name.acl"] + set fd [open $aclfile w] + puts $fd "role bad,name +get" + close $fd + catch {exec $::VALKEY_SERVER_BIN --aclfile $aclfile} err + assert_match {*invalid role name*commas*} $err +} {} {external:skip} + +# Test invalid role rule in config on startup +test {Invalid role rule in config on startup fails} { + set conffile [tmpfile "role-invalid-rule.conf"] + set fd [open $conffile w] + puts $fd "role badrole >password" + close $fd + catch {exec $::VALKEY_SERVER_BIN $conffile} err + assert_match {*Error in role declaration*} $err +} {} {external:skip} diff --git a/tests/unit/acl-v2.tcl b/tests/unit/acl-v2.tcl index 767818aa6..b25b66876 100644 --- a/tests/unit/acl-v2.tcl +++ b/tests/unit/acl-v2.tcl @@ -106,6 +106,51 @@ start_server {tags {"acl external:skip"}} { assert_match "*NOPERM*key*" $err } + test {EXEC conditions require read permission} { + r ACL SETUSER exec-condition-write-only on nopass %W~write* +@all + r set writecondition value + r del writelist + $r2 auth exec-condition-write-only password + + $r2 multi + $r2 lpush writelist value + catch {$r2 exec ifeq writecondition value} err + assert_match "*NOPERM*key*" $err + + $r2 multi + $r2 lpush writelist value + catch {$r2 exec ifne writecondition other} err + assert_match "*NOPERM*key*" $err + + $r2 multi + $r2 lpush writelist value + catch {$r2 exec nx writecondition} err + assert_match "*NOPERM*key*" $err + + $r2 multi + $r2 lpush writelist value + catch {$r2 exec xx writecondition} err + assert_match "*NOPERM*key*" $err + + $r2 multi + $r2 lpush writelist value + assert_error "EXECABORT*invalid check condition syntax*" {$r2 exec ifeq writecondition other invalid} + assert_equal 0 [r llen writelist] + } + + test {EXEC condition keys check permissions on the active database} { + r ACL SETUSER exec-db-selector-user on nopass (db=1 +@all ~*) (db=0 +@all ~public*) + r select 0 + r set secret secret-value + $r2 auth exec-db-selector-user password + $r2 select 0 + $r2 multi + $r2 select 1 + catch {$r2 exec ifeq secret guessed-value} err + assert_match "*NOPERM*key*" $err + r del secret + } + test {Test separate read and write permissions} { r ACL SETUSER key-permission-RW on nopass %R~read* %W~write* +@all $r2 auth key-permission-RW password @@ -628,6 +673,37 @@ start_server {tags {"acl external:skip"}} { r del v1 mylist } + test {Test SORT STORE destination that is named like an option} { + # Every decoy destination below is permitted, so the only reason to + # reject a command is the real destination the server writes to. + r ACL setuser test-sort-store on nopass ~allowed:* ~by ~get ~limit ~alpha +@all + r rpush allowed:src c b a + + # A dedicated client, because deleting the user below kills it. + set r3 [valkey_client] + $r3 auth test-sort-store nopass + + # A destination spelling an option that takes arguments hides the later + # STORE clause that SORT actually uses. + foreach keyword {by get limit} { + assert_equal "User test-sort-store has no permissions to access the 'forbidden:dst' key" \ + [r ACL DRYRUN test-sort-store SORT allowed:src ALPHA STORE $keyword STORE forbidden:dst] + assert_error "*NOPERM*key*" {$r3 sort allowed:src ALPHA STORE $keyword STORE forbidden:dst} + assert_equal 0 [r exists forbidden:dst] + } + + # A destination spelling STORE reports whatever follows it instead. + assert_equal "User test-sort-store has no permissions to access the 'store' key" \ + [r ACL DRYRUN test-sort-store SORT allowed:src STORE store alpha] + assert_error "*NOPERM*key*" {$r3 sort allowed:src STORE store alpha} + assert_equal 0 [r exists store] + + # cleanup + $r3 close + r ACL deluser test-sort-store + r del allowed:src + } + test {Test DRYRUN with wrong number of arguments} { r ACL setuser test-dry-run +@all ~v* diff --git a/tests/unit/cluster/cluster-migrateslots.tcl b/tests/unit/cluster/cluster-migrateslots.tcl index 7fad3d7d0..0238a8a96 100644 --- a/tests/unit/cluster/cluster-migrateslots.tcl +++ b/tests/unit/cluster/cluster-migrateslots.tcl @@ -233,6 +233,45 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster network} overrides assert_error "*No migrations ongoing*" {R 0 CLUSTER CANCELSLOTMIGRATIONS} } + test "CLUSTER MIGRATESLOTS AUTH syntax errors" { + # AUTH with no username or password + assert_error "*syntax error*" {R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 0 0 NODE $node1_id AUTH} + + # AUTH with only a username and no password + assert_error "*syntax error*" {R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 0 0 NODE $node1_id AUTH onlyuser} + + # Duplicate AUTH in the same group + assert_error "*syntax error*" {R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 0 0 NODE $node1_id AUTH u p AUTH u p} + + # None of the above started a migration + assert_equal {} [R 0 CLUSTER GETSLOTMIGRATIONS] + } + + test "CLUSTER MIGRATESLOTS AUTH credentials are redacted on synchronous failure" { + set old_threshold [lindex [R 0 CONFIG GET commandlog-execution-slower-than] 1] + R 0 CONFIG SET commandlog-execution-slower-than 0 + R 0 COMMANDLOG RESET slow + + # An invalid target is rejected before the parser reaches AUTH. + assert_error "*Invalid node name*" { + R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 0 0 NODE invalid AUTH aclusr authpwd + } + + R 0 CONFIG SET commandlog-execution-slower-than $old_threshold + set slowlog_resp [R 0 COMMANDLOG GET -1 slow] + + # Flatten all logged command args into one searchable string + set log_text {} + foreach entry $slowlog_resp { + append log_text " " [join [lindex $entry 3] " "] + } + + assert_no_match {*aclusr*} $log_text + assert_no_match {*authpwd*} $log_text + assert_match {*SLOTSRANGE 0 0 NODE invalid AUTH (redacted) (redacted)*} $log_text + assert_equal {} [R 0 CLUSTER GETSLOTMIGRATIONS] + } + test "CLUSTER MIGRATESLOTS already migrating" { set_debug_prevent_pause 1 assert_match "OK" [R 2 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node0_id] @@ -1487,6 +1526,7 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster network} overrides assert_error "*ERR CLUSTER SYNCSLOTS PAUSED should only be used by slot migration clients*" {R 0 CLUSTER SYNCSLOTS PAUSED} assert_error "*ERR CLUSTER SYNCSLOTS FAILOVER-GRANTED should only be used by slot migration clients*" {R 0 CLUSTER SYNCSLOTS FAILOVER-GRANTED} assert_error "*ERR CLUSTER SYNCSLOTS ACK should only be used by slot migration clients*" {R 0 CLUSTER SYNCSLOTS ACK} + assert_error "*ERR CLUSTER SYNCSLOTS FINISH should only be used by slot migration clients*" {R 0 CLUSTER SYNCSLOTS FINISH STATE failed NAME $fake_jobname} assert_error "*syntax error*" {R 0 CLUSTER SYNCSLOTS UNKNOWN} assert_causes_conn_drop 0 { @@ -1505,6 +1545,15 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster network} overrides } } + test "CLUSTER SYNCSLOTS ESTABLISH rejects a repeated establish on the same connection" { + assert_does_not_resync { + assert_causes_conn_drop 0 { + $client CLUSTER SYNCSLOTS ESTABLISH SOURCE $node2_id NAME $fake_jobname SLOTSRANGE 16383 16383 + $client CLUSTER SYNCSLOTS ESTABLISH SOURCE $node2_id NAME $fake_jobname SLOTSRANGE 10923 10923 + } + } + } + test "CLUSTER SYNCSLOTS ESTABLISH command interface" { assert_does_not_resync { # No arguments @@ -1765,6 +1814,205 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster network} overrides } } + test "CLUSTER MIGRATESLOTS with AUTH succeeds when target requires password" { + assert_does_not_resync { + R 0 CONFIG SET requirepass "targetpass" + + # Populate data before migration + populate 1000 "$16383_slot_tag:" 1000 -2 + + assert_match "OK" [R 2 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node0_id AUTH default targetpass] + set jobname [get_job_name 2 16383] + wait_for_migration 0 16383 + + # Keys successfully migrated + assert_match "1000" [R 0 CLUSTER COUNTKEYSINSLOT 16383] + assert_match "0" [R 2 CLUSTER COUNTKEYSINSLOT 16383] + + # Also eventually reflected in replicas + wait_for_countkeysinslot 3 16383 1000 + wait_for_countkeysinslot 5 16383 0 + + # Migration log shows success on both ends + assert {[dict get [get_migration_by_name 0 $jobname] state] eq "success"} + assert {[dict get [get_migration_by_name 2 $jobname] state] eq "success"} + + # Cleanup for next test + assert_match "OK" [R 0 FLUSHDB SYNC] + assert_match "OK" [R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node2_id] + wait_for_migration 2 16383 + R 0 CONFIG SET requirepass "" + } + } + + test "CLUSTER MIGRATESLOTS AUTH with WRONGPASS fails cleanly" { + assert_does_not_resync { + R 0 CONFIG SET requirepass "correctpass" + + # Perform one-shot import with wrong password in AUTH option + assert_match "OK" [R 2 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node0_id AUTH default wrongpass] + set jobname [get_job_name 2 16383] + + # Should be denied with clear error message + wait_for_migration_field 2 $jobname state failed + assert_match {*Failed to AUTH to target node*} [dict get [get_migration_by_name 2 $jobname] message] + + # Cleanup for next test + R 0 CONFIG SET requirepass "" + } + } + + test "CLUSTER MIGRATESLOTS with AUTH overrides primaryuser and primaryauth" { + assert_does_not_resync { + R 0 CONFIG SET requirepass "targetpass" + R 2 CONFIG SET primaryauth "wrongpass" + R 2 CONFIG SET primaryuser "wronguser" + + # Populate data before migration + populate 1000 "$16383_slot_tag:" 1000 -2 + + # AUTH overrides both configured credentials on the source + assert_match "OK" [R 2 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node0_id AUTH default targetpass] + set jobname [get_job_name 2 16383] + wait_for_migration 0 16383 + + # Keys successfully migrated + assert_match "1000" [R 0 CLUSTER COUNTKEYSINSLOT 16383] + assert_match "0" [R 2 CLUSTER COUNTKEYSINSLOT 16383] + + # Also eventually reflected in replicas + wait_for_countkeysinslot 3 16383 1000 + wait_for_countkeysinslot 5 16383 0 + + # Migration log shows success on both ends + assert {[dict get [get_migration_by_name 0 $jobname] state] eq "success"} + assert {[dict get [get_migration_by_name 2 $jobname] state] eq "success"} + + # Cleanup for next test + assert_match "OK" [R 0 FLUSHDB SYNC] + assert_match "OK" [R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node2_id] + wait_for_migration 2 16383 + R 0 CONFIG SET requirepass "" + R 2 CONFIG SET primaryauth "" + R 2 CONFIG SET primaryuser "" + } + } + + test "CLUSTER MIGRATESLOTS with AUTH succeeds for ACL user" { + assert_does_not_resync { + R 0 CONFIG SET requirepass "mustauth" + R 0 ACL SETUSER alice on >s3cret ~* &* +@all + + # Populate data before migration + populate 1000 "$16383_slot_tag:" 1000 -2 + + # Authenticate as the named ACL user + assert_match "OK" [R 2 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node0_id AUTH alice s3cret] + set jobname [get_job_name 2 16383] + wait_for_migration 0 16383 + + # Keys successfully migrated + assert_match "1000" [R 0 CLUSTER COUNTKEYSINSLOT 16383] + assert_match "0" [R 2 CLUSTER COUNTKEYSINSLOT 16383] + + # Also eventually reflected in replicas + wait_for_countkeysinslot 3 16383 1000 + wait_for_countkeysinslot 5 16383 0 + + # Migration log shows success on both ends + assert {[dict get [get_migration_by_name 0 $jobname] state] eq "success"} + assert {[dict get [get_migration_by_name 2 $jobname] state] eq "success"} + + # Cleanup for next test + assert_match "OK" [R 0 FLUSHDB SYNC] + assert_match "OK" [R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node2_id] + wait_for_migration 2 16383 + R 0 ACL DELUSER alice + R 0 CONFIG SET requirepass "" + } + } + + test "CLUSTER MIGRATESLOTS per-target AUTH differs in single command" { + # Explicit AUTH default pw0 for node0, primaryauth fallback pw1 for node1. + ensure_slot_on_node 2 16383 + ensure_slot_on_node 2 16382 + assert_does_not_resync { + R 0 CONFIG SET requirepass "pw0" + R 1 CONFIG SET requirepass "pw1" + R 2 CONFIG SET primaryauth "pw1" + + populate 500 "$16383_slot_tag:" 1000 -2 + populate 500 "$16382_slot_tag:" 1000 -2 + + # One command: explicit AUTH for node0, primaryauth fallback for node1 + assert_match "OK" [R 2 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node0_id AUTH default pw0 SLOTSRANGE 16382 16382 NODE $node1_id] + set jobname0 [get_job_name 2 16383] + set jobname1 [get_job_name 2 16382] + wait_for_migration 0 16383 + wait_for_migration 1 16382 + + # Keys migrated to correct targets + assert_match "500" [R 0 CLUSTER COUNTKEYSINSLOT 16383] + assert_match "0" [R 2 CLUSTER COUNTKEYSINSLOT 16383] + assert_match "500" [R 1 CLUSTER COUNTKEYSINSLOT 16382] + assert_match "0" [R 2 CLUSTER COUNTKEYSINSLOT 16382] + + # Replicas reflect the migration + wait_for_countkeysinslot 3 16383 500 + wait_for_countkeysinslot 4 16382 500 + wait_for_countkeysinslot 5 16383 0 + wait_for_countkeysinslot 5 16382 0 + + # Both migrations succeeded + assert {[dict get [get_migration_by_name 0 $jobname0] state] eq "success"} + assert {[dict get [get_migration_by_name 2 $jobname0] state] eq "success"} + assert {[dict get [get_migration_by_name 1 $jobname1] state] eq "success"} + assert {[dict get [get_migration_by_name 2 $jobname1] state] eq "success"} + + # Cleanup for next test + assert_match "OK" [R 0 FLUSHDB SYNC] + assert_match "OK" [R 1 FLUSHDB SYNC] + assert_match "OK" [R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node2_id] + wait_for_migration 2 16383 + assert_match "OK" [R 1 CLUSTER MIGRATESLOTS SLOTSRANGE 16382 16382 NODE $node2_id] + wait_for_migration 2 16382 + R 0 CONFIG SET requirepass "" + R 1 CONFIG SET requirepass "" + R 2 CONFIG SET primaryauth "" + } + } + + test "CLUSTER MIGRATESLOTS AUTH credentials are redacted in command log" { + ensure_slot_on_node 2 16383 + + # The commandlog entry is written synchronously when CLUSTER MIGRATESLOTS returns + # OK, before any async auth handshake with the target. No requirepass on node 0 + # means the async auth attempt will fail, which is fine — we only need the entry. + set old_threshold [lindex [R 2 CONFIG GET commandlog-execution-slower-than] 1] + R 2 CONFIG SET commandlog-execution-slower-than 0 + R 2 COMMANDLOG RESET slow + R 2 CLUSTER MIGRATESLOTS SLOTSRANGE 16383 16383 NODE $node0_id AUTH aclusr authpwd + set jobname [get_job_name 2 16383] + R 2 CONFIG SET commandlog-execution-slower-than $old_threshold + set slowlog_resp [R 2 COMMANDLOG GET -1 slow] + + # Flatten all logged command args into one searchable string + set log_text {} + foreach entry $slowlog_resp { + append log_text " " [join [lindex $entry 3] " "] + } + + # Neither credential may appear verbatim + assert_no_match {*authpwd*} $log_text + assert_no_match {*aclusr*} $log_text + + # Username and password are both replaced with (redacted) + assert_match {*SLOTSRANGE 16383 16383 NODE * AUTH (redacted) (redacted)*} $log_text + + # Migration fails as intended; wait for terminal state + wait_for_migration_field 2 $jobname state failed + } + test "Connection drop during import causes failure" { assert_does_not_resync { # Start an import @@ -1960,15 +2208,17 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster network} overrides test "Migration not cancelled when snapshot takes more time than repl-timeout" { assert_does_not_resync { # The target must not kill the import link while the source's - # snapshot is quiet for longer than repl-timeout (the source - # cannot send ACKs while its child is snapshotting). + # snapshot takes longer than repl-timeout (the source cannot send + # ACKs while its child is snapshotting). R 2 CONFIG SET repl-timeout 2 # Load keys before the snapshot to target a snapshot time > 2sec. - # 50 * 100ms = 5 sec. The delay must be on node 0: it is the - # source of this migration and runs the snapshot. + # 50 * 100ms = 5 sec. Values larger than PROTO_IOBUF_LEN are + # written directly instead of accumulating in rio's buffer, so + # the target continues receiving data during the slow snapshot. + # The delay must be on node 0: it is the source of this migration. R 0 CONFIG SET rdb-key-save-delay 100000 - populate 50 "$0_slot_tag:1:" 1000 -0 + populate 50 "$0_slot_tag:1:" 32768 -0 set errcode [catch { assert_match "OK" [R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 0 0 NODE $node2_id] @@ -2319,7 +2569,8 @@ start_cluster 3 0 {tags {logreqres:skip external:skip cluster network}} { test "Migration cannot connect to target" { # Shutdown to prevent connection success catch {R 2 shutdown nosave} - assert_match "OK" [R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 0 0 NODE $node2_id] + # Exercise credential cleanup when the connection fails before AUTH is sent. + assert_match "OK" [R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 0 0 NODE $node2_id AUTH default authpwd] set jobname [get_job_name 0 0] # Connecting will fail diff --git a/tests/unit/cluster/failover-if-empty.tcl b/tests/unit/cluster/failover-if-empty.tcl new file mode 100644 index 000000000..94cff35fd --- /dev/null +++ b/tests/unit/cluster/failover-if-empty.tcl @@ -0,0 +1,104 @@ +# Check that 'cluster-replica-no-failover if-empty' prevents a replica whose +# replication offset is still 0 (it never completed a sync with its primary) +# from starting an automatic failover, which would promote an empty node and +# lose all the data of the shard. + +# Run the zero-offset-replica failover test for the given sync type. sync_type +# is either "diskless" (the primary never starts the snapshot) or "disk-based" +# (the primary stalls while generating the RDB file), both leaving the new +# replica stuck at replication offset 0. +proc test_zero_offset_replica {sync_type} { + test "Zero-offset replica cannot fail over ($sync_type)" { + set R0_nodeid [R 0 CLUSTER MYID] + set R3_nodeid [R 3 CLUSTER MYID] + + # Fill primary 0 with some data so that later, after the failover promotes + # an empty replica, we can detect that the shard data was lost. + for {set i 0} {$i < 1000} {incr i} { + R 0 set "{key_991803}:$i" x + } + + # Make the new replica stall at replication offset 0, depending on the sync + # type under test. + if {$sync_type eq "diskless"} { + # Diskless replication with a large sync delay: the primary won't even + # start transferring the snapshot, so the new replica stays at offset 0. + R 0 CONFIG SET repl-diskless-sync yes + R 0 CONFIG SET repl-diskless-sync-delay 1000 + } else { + # Disk-based replication with a huge per-key save delay: rdb-key-save-delay + # is applied per key by the RDB child process, so the parent can still run + # its event loop (handling the replica's CLUSTER REPLICATE and propagating + # the new role) while the snapshot is never finished. The replica therefore + # stays at offset 0. + R 0 CONFIG SET repl-diskless-sync no + R 0 CONFIG SET rdb-key-save-delay 100000 + } + + # Turn on the guard on the replica: a zero-offset replica must not start an + # automatic failover. Validity factor 0 lets it become a candidate immediately + # once the primary is down. + R 3 CONFIG SET cluster-replica-no-failover if-empty + R 3 CONFIG SET cluster-replica-validity-factor 0 + + # Add the empty node as a replica of R0 and wait until the role change is + # agreed across the cluster (the replica and each primary agree that R3 is a + # replica of R0). + R 3 cluster replicate [R 0 CLUSTER MYID] + wait_for_condition 1000 50 { + [cluster_node_is_replica_of 0 $R3_nodeid $R0_nodeid] && + [cluster_node_is_replica_of 1 $R3_nodeid $R0_nodeid] && + [cluster_node_is_replica_of 2 $R3_nodeid $R0_nodeid] && + [cluster_node_is_replica_of 3 $R3_nodeid $R0_nodeid] + } else { + for {set j 0} {$j < [llength $::servers]} {incr j} { + puts "R $j cluster nodes output: [R $j cluster nodes]" + } + fail "R3 was not consistently recognized as a replica of R0 across the cluster" + } + + # Take the primary down so the replica should attempt a failover. + pause_process [srv 0 pid] + + # The replica must refuse to fail over and log the reason. Waiting for the + # log is a stable signal that it already tried and was blocked. + wait_for_log_messages -3 {"*Currently unable to failover*Replication offset is 0*"} 0 1000 50 + + # Disabling the guard lets the zero-offset replica win the election. + R 3 CONFIG SET cluster-replica-no-failover no + wait_for_condition 1000 50 { + [s -3 role] eq {master} + } else { + fail "Replica did not fail over after disabling the guard" + } + + # The new primary has an empty dataset: the shard data is lost. + assert_equal 0 [R 3 dbsize] + + # Bring the original primary back. After the failover R3 is the new primary, + # so R0 should rejoin the cluster as its replica. Both now have an empty + # dataset, because the failover promoted an empty node. + if {$sync_type eq "diskless"} { + R 3 CONFIG SET repl-diskless-sync yes + R 3 CONFIG SET repl-diskless-sync-delay 0 + resume_process [srv 0 pid] + } else { + resume_process [srv 0 pid] + R 0 CONFIG SET rdb-key-save-delay 0 + } + wait_for_condition 1000 50 { + [s 0 role] eq {slave} && + [R 0 dbsize] eq 0 + } else { + fail "Original primary did not rejoin the cluster as a replica of the new primary" + } + } +} + +start_cluster 3 1 {tags {external:skip cluster}} { + test_zero_offset_replica diskless +} continuous_slot_allocation no_replica_allocation + +start_cluster 3 1 {tags {external:skip cluster}} { + test_zero_offset_replica disk-based +} continuous_slot_allocation no_replica_allocation diff --git a/tests/unit/cluster/failover2.tcl b/tests/unit/cluster/failover2.tcl index 905299ac1..72e5aa05b 100644 --- a/tests/unit/cluster/failover2.tcl +++ b/tests/unit/cluster/failover2.tcl @@ -64,16 +64,51 @@ start_cluster 3 4 {tags {external:skip cluster} overrides {cluster-ping-interval } } ;# start_cluster -start_cluster 7 3 {tags {external:skip cluster} overrides {cluster-ping-interval 1000 cluster-node-timeout 15000}} { - test "Primaries will not time out then they are elected in the same epoch" { +# Needs to run in the body of +# start_cluster 7 3 {tags {external:skip cluster} overrides {cluster-ping-interval 1000 cluster-node-timeout 15000}} +proc test_same_epoch {delay} { + test "Primaries will not time out then they are elected in the same epoch - delay $delay" { # Since we have the delay time, so these node may not initiate the # election at the same time (same epoch). But if they do, we make # sure there is no failover timeout. + R 7 DEBUG CLUSTER-FAILOVER-DELAY $delay + R 8 DEBUG CLUSTER-FAILOVER-DELAY $delay + R 9 DEBUG CLUSTER-FAILOVER-DELAY $delay # Killing there primary nodes. - pause_process [srv 0 pid] - pause_process [srv -1 pid] - pause_process [srv -2 pid] + set primary_ids [list [R 0 cluster myid] [R 1 cluster myid] [R 2 cluster myid]] + exec kill -SIGSTOP [srv 0 pid] [srv -1 pid] [srv -2 pid] + + # Wait until every voter (idx 3..6) sees all three paused primaries + # as fail, so the upcoming election is granted on the first round. + # Otherwise voter might reply with NACK primary-up. + wait_for_condition 1000 50 { + [cluster_all_see_flag {3 4 5 6} $primary_ids fail] + } else { + fail "Voters did not mark all paused primaries as fail" + } + + # Force the three replicas to start their election in the very same + # epoch, so the same-epoch split vote is reproduced deterministically + # rather than depending on the timing of the (possibly zero) delay. + if {$delay == 0} { + set epoch [expr [CI 3 cluster_current_epoch] + 1] + R 7 DEBUG CLUSTER-FAILOVER-EPOCH $epoch + R 8 DEBUG CLUSTER-FAILOVER-EPOCH $epoch + R 9 DEBUG CLUSTER-FAILOVER-EPOCH $epoch + } + + # Now let the replicas proceed with the election. + R 7 CONFIG SET cluster-replica-no-failover no + R 8 CONFIG SET cluster-replica-no-failover no + R 9 CONFIG SET cluster-replica-no-failover no + + # All three must have contended in the very same (forced) epoch. + if {$delay == 0} { + wait_for_log_messages -7 [list "*Starting a failover election for epoch $epoch*"] 0 1000 50 + wait_for_log_messages -8 [list "*Starting a failover election for epoch $epoch*"] 0 1000 50 + wait_for_log_messages -9 [list "*Starting a failover election for epoch $epoch*"] 0 1000 50 + } # Wait for the failover wait_for_condition 1000 50 { @@ -99,6 +134,14 @@ start_cluster 7 3 {tags {external:skip cluster} overrides {cluster-ping-interval resume_process [srv -1 pid] resume_process [srv -2 pid] } +} + +start_cluster 7 3 {tags {external:skip cluster} overrides {cluster-ping-interval 1000 cluster-replica-no-failover yes}} { + test_same_epoch 500 +} ;# start_cluster + +start_cluster 7 3 {tags {external:skip cluster} overrides {cluster-ping-interval 1000 cluster-replica-no-failover yes}} { + test_same_epoch 0 } ;# start_cluster run_solo {cluster} { @@ -134,12 +177,26 @@ run_solo {cluster} { } ;# start_cluster } ;# run_solo +# Setup: R3 is the only replica of R0. While R3 is network-isolated, +# we bump R0's configEpoch and let R1/R2 learn the new value through +# gossip. R3's view is therefore stale. We then pause R0 and let R3 +# attempt failover: the voters reject it with STALE_CONFIG, send back +# both an UPDATE (with R0's fresh slot config) and a NACK, and R3 must +# eventually win a second election with the refreshed configEpoch. +# +# `type` : "automatic" or "manual" failover. +# `drop_nack` : 1 -> drop FAILOVER_AUTH_NACK on R3 to exercise the +# legacy timeout-based retry. +# 0 -> let NACKs through and exercise the fast-fail +# path that retries without waiting for the timeout. +# # Needs to run in the body of # start_cluster 3 1 {tags {external:skip cluster} overrides {cluster-replica-validity-factor 0}} -proc test_replica_config_epoch_failover {type} { - test "Replica can update the config epoch when trigger the failover - $type" { +proc test_replica_config_epoch_failover {type drop_nack} { + test "Replica can update the config epoch when trigger the failover - $type - drop_nack $drop_nack" { set CLUSTER_PACKET_TYPE_NONE -1 set CLUSTER_PACKET_TYPE_ALL -2 + set CLUSTER_PACKET_TYPE_FAILOVER_AUTH_NACK 11 if {$type == "automatic"} { R 3 CONFIG SET cluster-replica-no-failover no @@ -167,24 +224,61 @@ proc test_replica_config_epoch_failover {type} { # Make sure that replica do not update config epoch. assert_not_equal $R0_config_epoch [dict get [cluster_get_node_by_id 3 $R0_nodeid] config_epoch] - # Pause the R 0 and wait for the cluster to be down. + # Pause R0 and resume R3's debug. + # drop_nack=1 keeps NACKs filtered so the legacy timeout path is exercised. + # drop_nack=0 lets NACKs through for the fast-fail path. pause_process [srv 0 pid] - R 3 DEBUG DROP-CLUSTER-PACKET-FILTER $CLUSTER_PACKET_TYPE_NONE + if {$drop_nack} { + R 3 DEBUG DROP-CLUSTER-PACKET-FILTER $CLUSTER_PACKET_TYPE_FAILOVER_AUTH_NACK + } else { + R 3 DEBUG DROP-CLUSTER-PACKET-FILTER $CLUSTER_PACKET_TYPE_NONE + } R 3 DEBUG CLOSE-CLUSTER-LINK-ON-PACKET-DROP 0 + + # Wait for R3 to reconnect to both voters before triggering anything + # that depends on bidirectional traffic, otherwise an immediate failover + # request can race the link rebuild and the NACK reply may be lost. + set R1_nodeid [R 1 cluster myid] + set R2_nodeid [R 2 cluster myid] + set R3_nodeid [R 3 cluster myid] wait_for_condition 1000 50 { - [CI 1 cluster_state] == "fail" && - [CI 2 cluster_state] == "fail" && - [CI 3 cluster_state] == "fail" + [dict get [cluster_get_node_by_id 1 $R3_nodeid] linkstate] eq "connected" && + [dict get [cluster_get_node_by_id 3 $R1_nodeid] linkstate] eq "connected" && + [dict get [cluster_get_node_by_id 2 $R3_nodeid] linkstate] eq "connected" && + [dict get [cluster_get_node_by_id 3 $R2_nodeid] linkstate] eq "connected" } else { - fail "Cluster does not fail" + fail "R3 did not reconnect its bus links to the voters" } - # Make sure both the automatic and the manual failover will fail in the first time. - if {$type == "automatic"} { - wait_for_log_messages -3 {"*Failover attempt expired*"} 0 1200 50 - } elseif {$type == "manual"} { + if {$drop_nack} { + wait_for_condition 1000 50 { + [CI 1 cluster_state] == "fail" && + [CI 2 cluster_state] == "fail" && + [CI 3 cluster_state] == "fail" + } else { + fail "Cluster does not fail" + } + } + + if {$type == "manual"} { R 3 cluster failover force - wait_for_log_messages -3 {"*Manual failover timed out*"} 0 1200 50 + } + + if {$drop_nack} { + # Make sure both the automatic and the manual failover will fail in the first time. + if {$type == "automatic"} { + wait_for_log_messages -3 {"*Failover attempt expired*"} 0 1200 50 + } elseif {$type == "manual"} { + wait_for_log_messages -3 {"*Manual failover timed out*"} 0 1200 50 + } + } else { + # Fast-fail path: NACK accounting trips the quorum check + # and "attempt expired" / "timed out" must never appear. + wait_for_log_messages -3 {"*cannot reach quorum*"} 0 1200 50 + verify_no_log_message -3 "*Failover attempt expired*" 0 + if {$type == "manual"} { + verify_no_log_message -3 "*Manual failover timed out*" 0 + } } # Make sure the primaries prints the relevant logs. @@ -200,7 +294,7 @@ proc test_replica_config_epoch_failover {type} { fail "The replica does not update the config epoch" } - if {$type == "manual"} { + if {$drop_nack && $type == "manual"} { # The second manual failure will succeed because the config epoch # has already propagated. R 3 cluster failover force @@ -228,9 +322,17 @@ proc test_replica_config_epoch_failover {type} { } start_cluster 3 1 {tags {external:skip cluster} overrides {cluster-replica-validity-factor 0}} { - test_replica_config_epoch_failover "automatic" + test_replica_config_epoch_failover "automatic" 1 +} + +start_cluster 3 1 {tags {external:skip cluster} overrides {cluster-replica-validity-factor 0}} { + test_replica_config_epoch_failover "manual" 1 +} + +start_cluster 3 1 {tags {external:skip cluster} overrides {cluster-replica-validity-factor 0}} { + test_replica_config_epoch_failover "automatic" 0 } start_cluster 3 1 {tags {external:skip cluster} overrides {cluster-replica-validity-factor 0}} { - test_replica_config_epoch_failover "manual" + test_replica_config_epoch_failover "manual" 0 } diff --git a/tests/unit/cluster/failure-marking.tcl b/tests/unit/cluster/failure-marking.tcl index 8cf610800..189808775 100644 --- a/tests/unit/cluster/failure-marking.tcl +++ b/tests/unit/cluster/failure-marking.tcl @@ -1,5 +1,5 @@ # Test a single primary can mark replica as `fail` -start_cluster 1 1 {tags {external:skip cluster}} { +start_cluster 1 1 {tags {external:skip cluster network}} { test "Verify that single primary marks replica as failed" { set primary [srv -0 client] @@ -22,7 +22,7 @@ start_cluster 1 1 {tags {external:skip cluster}} { } # Test multiple primaries wait for a quorum and then mark a replica as `fail` -start_cluster 2 1 {tags {external:skip cluster}} { +start_cluster 2 1 {tags {external:skip cluster network}} { test "Verify that multiple primaries mark replica as failed" { set primary1 [srv -0 client] @@ -56,7 +56,7 @@ start_cluster 2 1 {tags {external:skip cluster}} { } } -tags {external:skip tls:skip cluster singledb} { +tags {external:skip tls:skip cluster singledb network} { set base_conf [list cluster-enabled yes cluster-ping-interval 100 cluster-node-timeout 3000 save ""] start_multiple_servers 5 [list overrides $base_conf] { test "Only primary with slots has the right to mark a node as failed" { @@ -116,7 +116,7 @@ tags {external:skip tls:skip cluster singledb} { } # Test that no new failure-report is added once the node is already marked as FAIL -start_cluster 3 1 {tags {external:skip cluster}} { +start_cluster 3 1 {tags {external:skip cluster network}} { test "Primaries do not add failure-report after replica is already marked FAIL" { # Primary nodes set primary0 [srv 0 client]; diff --git a/tests/unit/cluster/faster-failover.tcl b/tests/unit/cluster/faster-failover.tcl index 2120ae56b..0020c1191 100644 --- a/tests/unit/cluster/faster-failover.tcl +++ b/tests/unit/cluster/faster-failover.tcl @@ -315,3 +315,123 @@ start_cluster 5 7 {tags {external:skip cluster tls:skip} overrides {cluster-ping assert_morethan $best_ranked_triggered 0 } } two_primaries_slot_allocation cluster_allocate_replicas ;# start_cluster + +# Regression test for a stale CLUSTER_NODE_MY_PRIMARY_FAIL flag. +# +# CLUSTER_NODE_MY_PRIMARY_FAIL means "I am a replica and my primary is FAIL in +# my view". It is set on myself and gossiped so that other replicas in the shard +# can tell whether everybody has already seen the failure (and therefore whether +# the exchanged replication offsets are fresh). clusterAllReplicasThinkPrimaryIsFail() +# uses it to let the best ranked replica skip the election delay entirely. +# +# If the flag is not cleared when the replica is reconfigured under a new primary, +# it keeps advertising "my primary is dead" forever. A later, unrelated failover +# then sees a bogus unanimous vote and takes the no-delay fast path even though +# the offsets were never re-exchanged. +# +# The deployment is 3 primaries + 2 replicas, both replicas under R0: +# Primary: R0 R1 R2 +# Replica: R3 (replica of R0) <- will win the first failover +# R4 (replica of R0) <- cluster-replica-no-failover, the flag carrier +# +# R0 owns slots 0-5461, key key_977613 belongs to slot 0. +proc stale_flag_replica_allocation {masters replicas} { + set master0_id [R 0 CLUSTER MYID] + R 3 CLUSTER REPLICATE $master0_id + R 4 CLUSTER REPLICATE $master0_id +} + +start_cluster 3 2 {tags {external:skip cluster} overrides {cluster-ping-interval 1000 cluster-node-timeout 5000}} { + test "MY_PRIMARY_FAIL is cleared when a replica is reconfigured under a new primary" { + set R0_id [R 0 CLUSTER MYID] + set R3_id [R 3 CLUSTER MYID] + set R4_id [R 4 CLUSTER MYID] + + # R4 must never run an election. It only exists to carry (or not carry) + # the MY_PRIMARY_FAIL flag, which makes the first failover deterministic: + # R3 is the only node that can win it. + R 4 config set cluster-replica-no-failover yes + + # Give the replicas a non-zero replication offset. + for {set i 0} {$i < 10} {incr i} { + R 0 incr key_977613 + } + wait_for_ofs_sync [srv 0 client] [srv -3 client] + wait_for_ofs_sync [srv 0 client] [srv -4 client] + + ########################################################## + # Phase 1: make R4 set MY_PRIMARY_FAIL, then move it to a + # brand new primary while the flag is still set. + ########################################################## + pause_process [srv 0 pid] + + # R4 marks R0 as FAIL, so it sets MY_PRIMARY_FAIL on itself. + wait_node_marked_fail 4 $R0_id + + # Only R3 is allowed to fail over, so this is not a race. + wait_for_condition 1000 50 { + [s -3 role] == "master" + } else { + fail "R3 did not take over" + } + + # R4 is reconfigured under R3 via clusterSetPrimary(). This is where the + # stale flag is (or is not) cleared. + wait_for_condition 1000 50 { + [dict get [cluster_get_node_by_id 4 $R4_id] slaveof] eq $R3_id + } else { + fail "R4 did not become a replica of R3" + } + + # Bring R0 back as the second replica of R3. + resume_process [srv 0 pid] + wait_for_condition 1000 50 { + [s 0 role] == "slave" && + [dict get [cluster_get_node_by_id 0 $R0_id] slaveof] eq $R3_id + } else { + fail "R0 did not come back as a replica of R3" + } + + # R0 must know that R4 is a sibling replica, otherwise R4 would not be + # part of the clusterAllReplicasThinkPrimaryIsFail() scan at all. + wait_for_condition 1000 50 { + [dict get [cluster_get_node_by_id 0 $R4_id] slaveof] eq $R3_id + } else { + fail "R0 does not see R4 as a replica of R3" + } + wait_for_cluster_propagation + wait_for_cluster_state "ok" + + ########################################################## + # Phase 2: freeze R4 and fail R3. R0 is the only replica + # that can actually observe the failure. + ########################################################## + + # Freezing R4 pins whatever flags R0 last learned about it, and stops its + # replication offset, so R0 is unambiguously the better ranked replica. + pause_process [srv -4 pid] + + for {set i 0} {$i < 10} {incr i} { + R 3 incr key_977613 + } + wait_for_ofs_sync [srv -3 client] [srv 0 client] + + set loglines [count_log_lines 0] + pause_process [srv -3 pid] + + # R0 sees R3 fail and schedules its election. + wait_for_log_messages 0 {"*Start of election delayed for*"} $loglines 1000 50 + + # R4 is frozen and never saw R3 die, so not every replica thinks the + # primary is failing: R0 must take the regular ranked delay. + # + # With the stale flag bug, R4 still advertises MY_PRIMARY_FAIL from the + # first failover, the vote looks unanimous and R0 elects immediately. + verify_no_log_message 0 "*This is the best ranked replica and can initiate the election immediately*" $loglines + verify_no_log_message 0 "*Myself become the best ranked replica, initiate the election immediately*" $loglines + verify_no_log_message 0 "*Start of election delayed for 0 milliseconds*" $loglines + + resume_process [srv -4 pid] + resume_process [srv -3 pid] + } +} continuous_slot_allocation stale_flag_replica_allocation ;# start_cluster diff --git a/tests/unit/cluster/info.tcl b/tests/unit/cluster/info.tcl index 6a1b6545a..1fd1e2c4d 100644 --- a/tests/unit/cluster/info.tcl +++ b/tests/unit/cluster/info.tcl @@ -52,6 +52,22 @@ start_cluster 3 0 {tags {external:skip cluster} overrides {cluster-node-timeout } } + test "Cluster info reports established links counters" { + # After the cluster is up, both peers must have (re)established + # their links with each other: each node accepts one inbound link + # and initiates one outbound link. + wait_for_condition 1000 50 { + [CI 0 total_cluster_links_established_inbound] >= 2 && + [CI 0 total_cluster_links_established_outbound] >= 2 && + [CI 1 total_cluster_links_established_inbound] >= 2 && + [CI 1 total_cluster_links_established_outbound] >= 2 && + [CI 2 total_cluster_links_established_inbound] >= 2 && + [CI 2 total_cluster_links_established_outbound] >= 2 + } else { + fail "Cluster established links counters are not as expected" + } + } + test "fail reason changed" { # Kill one primary, so the cluster fail with not-full-coverage. pause_process [srv 0 pid] @@ -80,7 +96,9 @@ start_cluster 3 0 {tags {external:skip cluster} overrides {cluster-node-timeout [CI 0 cluster_stats_messages_received] >= 1 && [CI 0 cluster_stats_bytes_sent] >= 1 && [CI 0 cluster_stats_bytes_received] >= 1 && - [CI 0 total_cluster_links_buffer_limit_exceeded] >= 1 + [CI 0 total_cluster_links_buffer_limit_exceeded] >= 1 && + [CI 0 total_cluster_links_established_inbound] >= 1 && + [CI 0 total_cluster_links_established_outbound] >= 1 } else { fail "R 0 related info fields are not as expected" } @@ -99,6 +117,8 @@ start_cluster 3 0 {tags {external:skip cluster} overrides {cluster-node-timeout assert_equal [getInfoProperty $info cluster_stats_module_bytes_sent] 0 assert_equal [getInfoProperty $info cluster_stats_module_bytes_received] 0 assert_equal [getInfoProperty $info total_cluster_links_buffer_limit_exceeded] 0 + assert_equal [getInfoProperty $info total_cluster_links_established_inbound] 0 + assert_equal [getInfoProperty $info total_cluster_links_established_outbound] 0 R 0 config set cluster-link-sendbuf-limit 0 } diff --git a/tests/unit/cluster/links.tcl b/tests/unit/cluster/links.tcl index 68b07048c..9cf61d9f7 100644 --- a/tests/unit/cluster/links.tcl +++ b/tests/unit/cluster/links.tcl @@ -67,7 +67,7 @@ proc publish_messages {server num_msgs msg_size} { } } -start_cluster 1 2 {tags {external:skip cluster}} { +start_cluster 1 2 {tags {external:skip cluster network}} { set primary_id 0 set replica1_id 1 @@ -122,7 +122,7 @@ start_cluster 1 2 {tags {external:skip cluster}} { } {} {needs:debug} } -start_cluster 3 0 {tags {external:skip cluster}} { +start_cluster 3 0 {tags {external:skip cluster network}} { test "Each node has two links with each peer" { for {set id 0} {$id < [llength $::servers]} {incr id} { # Assert that from point of view of each node, there are two links for @@ -159,6 +159,29 @@ start_cluster 3 0 {tags {external:skip cluster}} { } } + test "Cluster bus I/O is offloaded only when I/O threads are enabled" { + if {[lindex [R 0 config get io-threads] 1] > 1} { + # Gossip is continuous, so completed threaded reads and writes must + # show up. These are counted on completion, not on dispatch. + wait_for_condition 50 100 { + [CI 0 cluster_io_threaded_reads_processed] > 0 && + [CI 0 cluster_io_threaded_writes_processed] > 0 + } else { + fail "cluster bus I/O was never offloaded to the I/O threads" + } + } else { + # With the pool disabled, every dispatch has to take the + # main-thread fallback and nothing may be counted as threaded. + wait_for_condition 50 100 { + [CI 0 cluster_io_main_thread_fallbacks] > 0 + } else { + fail "cluster bus I/O did not fall back to the main thread" + } + assert_equal 0 [CI 0 cluster_io_threaded_reads_processed] + assert_equal 0 [CI 0 cluster_io_threaded_writes_processed] + } + } + test {Validate cluster links format} { set lines [R 0 cluster links] foreach l $lines { diff --git a/tests/unit/cluster/misc.tcl b/tests/unit/cluster/misc.tcl index 7a6dfc3e2..42c286cb7 100644 --- a/tests/unit/cluster/misc.tcl +++ b/tests/unit/cluster/misc.tcl @@ -40,6 +40,36 @@ start_cluster 1 1 {tags {external:skip cluster}} { assert_error {CROSSSLOT *} {r exec} } + test {Conditional EXEC rejects cross-slot condition keys} { + set condition1 "{condition1}key" + set condition2 "{condition2}key" + set destination "{condition1}destination" + assert {[R 0 cluster keyslot $condition1] != [R 0 cluster keyslot $condition2]} + + R 0 del $condition1 + R 0 del $condition2 + R 0 del $destination + R 0 set $condition1 value + R 0 multi + R 0 set $destination should-not-execute + assert_error {CROSSSLOT Keys*} {R 0 exec ifeq $condition1 value nx $condition2} + assert_equal 0 [R 0 exists $destination] + } + + test {Conditional EXEC rejects a queued key in a different slot} { + set condition "{condition}key" + set destination "{destination}key" + assert {[R 0 cluster keyslot $condition] != [R 0 cluster keyslot $destination]} + + R 0 del $condition + R 0 del $destination + R 0 set $condition value + R 0 multi + R 0 set $destination should-not-execute + assert_error {CROSSSLOT Keys*} {R 0 exec ifeq $condition value} + assert_equal 0 [R 0 exists $destination] + } + # Regression tests for WATCHed keys that hash to a different slot than the transaction's commands. EXEC # committed such a transaction even when the WATCHed key had expired or had been removed. test {WATCHed key in another slot that expired aborts EXEC} { @@ -51,15 +81,23 @@ start_cluster 1 1 {tags {external:skip cluster}} { # the key having expired can abort the transaction. R 0 debug set-active-expire 0 R 0 del $transaction_key - R 0 set $watched_key alive px 50 - R 0 watch $watched_key - set keys_before [R 0 dbsize] - after 100 - assert_equal $keys_before [R 0 dbsize] - R 0 multi - R 0 set $transaction_key committed - set reply [R 0 exec] + # Retry if timing issues occur on slow CI runners + for {set j 0} {$j < 10} {incr j} { + R 0 del $transaction_key + R 0 set $watched_key alive px 100 + R 0 watch $watched_key + set keys_before [R 0 dbsize] + after 101 + assert_equal $keys_before [R 0 dbsize] + + R 0 multi + R 0 set $transaction_key committed + set reply [R 0 exec] + if {$reply eq {}} break + } + if {$::verbose} { puts "WATCHed key in another slot that expired aborts EXEC attempts: $j" } + R 0 debug set-active-expire 1 assert_equal {} $reply @@ -103,16 +141,66 @@ start_cluster 1 1 {tags {external:skip cluster}} { } } -# Create a folder called "nodes.conf" to trigger temp nodes.conf rename -# failure and it will cause cluster config file save to fail at the rename. -proc create_nodes_conf_folder {srv_idx} { +start_cluster 2 0 {tags {external:skip cluster}} { + test {EXEC with conditions without MULTI returns EXEC without MULTI, not MOVED or CROSSSLOT} { + set remote_key key_for_other_node + while {![catch {R 0 get $remote_key} err] || ![string match {MOVED *} $err]} { + append remote_key x + } + + set slotA "{slotA}key" + set slotB "{slotB}key" + assert {[R 0 cluster keyslot $slotA] != [R 0 cluster keyslot $slotB]} + assert_error {*EXEC without MULTI*} {R 0 exec ifeq $remote_key val} + assert_error {*EXEC without MULTI*} {R 0 exec ifeq $slotA val1 ifeq $slotB val2} + } + + test {Queue-time MOVED followed by EXEC with remote condition returns EXECABORT} { + set other_key key_for_other_node + while {![catch {R 0 get $other_key} err] || ![string match {MOVED *} $err]} { + append other_key x + } + + R 0 multi + assert_error {MOVED *} {R 0 set $other_key val} + assert_error {EXECABORT*previous errors*} {R 0 exec ifeq $other_key val} + } + + test {Keyless queued commands with remote condition key on EXEC returns MOVED} { + set other_key key_for_other_node + while {![catch {R 0 get $other_key} err] || ![string match {MOVED *} $err]} { + append other_key x + } + + R 0 multi + assert_equal {QUEUED} [R 0 dbsize] + assert_error {MOVED *} {R 0 exec ifeq $other_key val} + assert_equal {PONG} [R 0 ping] + } +} + +# Get the path to the cluster config file. +proc get_nodes_conf_path {srv_idx} { set dir [lindex [R $srv_idx config get dir] 1] set cluster_conf [lindex [R $srv_idx config get cluster-config-file] 1] set cluster_conf_path [file join $dir $cluster_conf] - if {[file exists $cluster_conf_path]} { exec rm -f $cluster_conf_path } + return $cluster_conf_path +} + +# Create a folder called "nodes.conf" to trigger temp nodes.conf rename +# failure and it will cause cluster config file save to fail at the rename. +proc create_nodes_conf_folder {srv_idx} { + set cluster_conf_path [get_nodes_conf_path $srv_idx] + if {[file exists $cluster_conf_path]} { exec rm -rf $cluster_conf_path } exec mkdir -p $cluster_conf_path } +# Remove the folder (or nodes.conf) that we created from create_nodes_conf_folder. +proc remove_nodes_conf_folder {srv_idx} { + set cluster_conf_path [get_nodes_conf_path $srv_idx] + exec rm -rf $cluster_conf_path +} + start_cluster 1 1 {tags {external:skip cluster} overrides {cluster-config-save-behavior sync}} { test {cluster-config-save-behavior sync mode - node exits when config save fails} { # Create folder that can cause the rename fail. @@ -136,6 +224,15 @@ start_cluster 1 1 {tags {external:skip cluster} overrides {cluster-config-save-b start_cluster 1 1 {tags {external:skip cluster} overrides {cluster-config-save-behavior best-effort}} { test {cluster-config-save-behavior best-effort mode - node continues running when config save fails} { + assert_equal "ok" [getInfoProperty [R 0 cluster info] cluster_config_save_status] + assert_equal "ok" [getInfoProperty [R 1 cluster info] cluster_config_save_status] + + # cluster_config_last_save_time should be set to a non-zero unix time on startup. + set last_save_time_0 [getInfoProperty [R 0 cluster info] cluster_config_last_save_time] + set last_save_time_1 [getInfoProperty [R 1 cluster info] cluster_config_last_save_time] + assert_morethan $last_save_time_0 0 + assert_morethan $last_save_time_1 0 + # Create folder that can cause the rename fail. create_nodes_conf_folder 0 create_nodes_conf_folder 1 @@ -156,6 +253,8 @@ start_cluster 1 1 {tags {external:skip cluster} overrides {cluster-config-save-b assert_equal 1 [process_is_alive [srv -1 pid]] # Make sure relevant logs are printed. + R 0 debug bio-drain BIO_CLUSTER_SAVE + R 1 debug bio-drain BIO_CLUSTER_SAVE verify_log_message 0 "*Could not rename tmp cluster config file*" 0 verify_log_message -1 "*Could not rename tmp cluster config file*" 0 verify_log_message 0 "*Cluster config updated even though writing the cluster config file to disk failed*" 0 @@ -170,10 +269,61 @@ start_cluster 1 1 {tags {external:skip cluster} overrides {cluster-config-save-b } else { fail "The failover does not happen" } + R 0 debug bio-drain BIO_CLUSTER_SAVE + R 1 debug bio-drain BIO_CLUSTER_SAVE assert_morethan_equal [count_log_message 0 "Could not rename tmp cluster config file"] 2 assert_equal [count_log_message 0 "Cluster config updated even though writing the cluster config file to disk failed"] 1 assert_morethan_equal [count_log_message -1 "Could not rename tmp cluster config file"] 2 assert_equal [count_log_message -1 "Cluster config updated even though writing the cluster config file to disk failed"] 1 + + # Check the info field is err. + assert_equal "err" [getInfoProperty [R 0 cluster info] cluster_config_save_status] + assert_equal "err" [getInfoProperty [R 1 cluster info] cluster_config_save_status] + + # Remove the test folder to trigger a config save again. + remove_nodes_conf_folder 0 + remove_nodes_conf_folder 1 + + # Trigger a takeover so that cluster will need to update the config file. + R 1 cluster failover takeover + wait_for_condition 1000 50 { + [s 0 role] eq {slave} && + [s -1 role] eq {master} + } else { + fail "The failover does not happen" + } + + # Check the info field is ok. + wait_for_condition 1000 50 { + [getInfoProperty [R 0 cluster info] cluster_config_save_status] eq "ok" && + [getInfoProperty [R 1 cluster info] cluster_config_save_status] eq "ok" + } else { + fail "The config save status is not ok" + } + + # Create folder that can cause the rename fail. + create_nodes_conf_folder 0 + create_nodes_conf_folder 1 + + # saveconfig will fail and info field is err. + assert_error {ERR *} {R 0 CLUSTER saveconfig} + assert_error {ERR *} {R 1 CLUSTER saveconfig} + assert_equal "err" [getInfoProperty [R 0 cluster info] cluster_config_save_status] + assert_equal "err" [getInfoProperty [R 1 cluster info] cluster_config_save_status] + + # Remove the test folder to trigger a config save again. + remove_nodes_conf_folder 0 + remove_nodes_conf_folder 1 + + # saveconfig will success and info field is ok. + assert_equal {OK} [R 0 CLUSTER saveconfig] + assert_equal {OK} [R 1 CLUSTER saveconfig] + assert_equal "ok" [getInfoProperty [R 0 cluster info] cluster_config_save_status] + assert_equal "ok" [getInfoProperty [R 1 cluster info] cluster_config_save_status] + + # cluster_config_last_save_time must have advanced after a successful save. + assert_morethan_equal [getInfoProperty [R 0 cluster info] cluster_config_last_save_time] $last_save_time_0 + assert_morethan_equal [getInfoProperty [R 1 cluster info] cluster_config_last_save_time] $last_save_time_1 } } @@ -205,3 +355,61 @@ start_cluster 3 0 {tags {external:skip cluster} overrides {cluster-require-full- wait_for_cluster_state ok } } + +start_cluster 2 0 {tags {cluster external:skip needs:debug}} { + test "A lost PING does not leave a node stuck in PFAIL when the peer keeps sending PINGs" { + set CLUSTER_PACKET_TYPE_MEET 2 + set CLUSTER_PACKET_TYPE_NONE -1 + set CLUSTER_PACKET_TYPE_ALL -2 + set R1_nodeid [R 1 cluster myid] + + # Configure different timeout values to better reproduce the issue. + R 0 config set cluster-node-timeout 15000 + R 1 config set cluster-node-timeout 1500 + + # Drop all packets on R0 and wait for pfail. + R 0 debug drop-cluster-packet-filter $CLUSTER_PACKET_TYPE_ALL + wait_for_condition 1000 50 { + [cluster_has_flag [cluster_get_node_by_id 0 $R1_nodeid] "fail?"] + } else { + puts "R 0 cluster nodes:" + puts [R 0 cluster nodes] + fail "R0 did not mark R1 as PFAIL" + } + + # Remember some information after the PFAIL. + set R0_ping_sent [dict get [cluster_get_node_by_id 0 $R1_nodeid] ping_sent] + set R0_ping_received [CI 0 cluster_stats_messages_ping_received] + + # Restore the DEBUG setting on R0, but exclude MEET first to avoid + # multiple MEET reconnections. We will restore it after R0 receives + # the PING and refreshes data_received. + R 0 debug drop-cluster-packet-filter $CLUSTER_PACKET_TYPE_MEET + wait_for_condition 1000 50 { + [CI 0 cluster_stats_messages_ping_received] > $R0_ping_received + } else { + fail "R0 did not receive the PING" + } + R 0 debug drop-cluster-packet-filter $CLUSTER_PACKET_TYPE_NONE + + # Ensure ping_sent does not get stuck and R0 can send PINGs. + wait_for_condition 1000 50 { + [dict get [cluster_get_node_by_id 0 $R1_nodeid] ping_sent] != $R0_ping_sent + } else { + puts "R 0 cluster nodes:" + puts [R 0 cluster nodes] + fail "R0 did not send the PING" + } + + # All packets are being sent and received normally, and R0 should be + # able to remove the PFAIL flag. + wait_for_condition 1000 50 { + ![cluster_has_flag [cluster_get_node_by_id 0 $R1_nodeid] "fail?"] + } else { + puts "R 0 cluster nodes:" + puts [R 0 cluster nodes] + fail "R0 did not remove the PFAIL flag" + } + wait_for_cluster_state ok + } +} diff --git a/tests/unit/cluster/socket-prioritization.tcl b/tests/unit/cluster/socket-prioritization.tcl new file mode 100644 index 000000000..9fb6b5bba --- /dev/null +++ b/tests/unit/cluster/socket-prioritization.tcl @@ -0,0 +1,122 @@ +# Verify socket prioritization in Cluster mode (with and without TLS) +start_cluster 2 2 {tags {socket-prioritization external:skip cluster}} { + test "Cluster is up and running" { + wait_for_cluster_state ok + } + + test "Verify CLIENT LIST qos filters for replica links" { + set high_clients [R 0 client list flags H] + assert_match "*flags=*H*" $high_clients + + set normal_cl [valkey 127.0.0.1 [srv 0 port] 0 $::tls] + $normal_cl client setname clusternorm + + assert_match "*name=clusternorm*flags=N*" [R 0 client list not-flags H name clusternorm] + assert_equal "" [R 0 client list flags H name clusternorm] + $normal_cl close + } + + test "Verify cluster slot synchronization under pipeline load" { + wait_for_cluster_state ok + + set load_clients {} + foreach idx {0 1} { + set port [srv [expr -1*$idx] port] + for {set c 0} {$c < 3} {incr c} { + lappend load_clients [valkey 127.0.0.1 $port 0 $::tls] + } + } + + set val [string repeat "y" 128] + set total_ops 0 + for {set iter 0} {$iter < 5} {incr iter} { + foreach cl $load_clients { + catch { + $cl write "*3\r\n\$3\r\nSET\r\n\$7\r\npipekey\r\n\$128\r\n$val\r\n" + $cl flush + $cl read + incr total_ops 1 + } + } + } + + assert {$total_ops > 0} + + wait_for_condition 100 50 { + [R 0 dbsize] + [R 1 dbsize] == 1 && + [R 2 dbsize] == [R 0 dbsize] && + [R 3 dbsize] == [R 1 dbsize] + } else { + fail "Replicas failed to complete sync during pipelined load" + } + + foreach cl $load_clients { catch { $cl close } } + } + + proc local_slot_ranges_contains_slot {slot_ranges slot} { + set ranges [split $slot_ranges " "] + foreach slot_range $ranges { + lassign [split $slot_range -] start end + if {$end == {}} {set end $start} + if {$slot >= $start && $slot <= $end} { + return 1 + } + } + return 0 + } + + proc local_is_slot_migrated {node_idx slot} { + set target_id [R $node_idx CLUSTER MYID] + set nodes [get_cluster_nodes $node_idx] + foreach n $nodes { + set node_id [dict get $n id] + if {$node_id eq $target_id} { + set slot_ranges [dict get $n slots] + if {[local_slot_ranges_contains_slot $slot_ranges $slot]} { + return 1 + } + } + } + return 0 + } + + proc check_prioritized_client_count {node_idx expected_min} { + set prioritized [string trim [R $node_idx client list flags H]] + if {$prioritized eq ""} { + set count 0 + } else { + set count [llength [split $prioritized "\n"]] + } + return [expr {$count > $expected_min}] + } + + test "Verify CLUSTER MIGRATESLOTS connection is prioritized" { + set target_id [R 1 CLUSTER MYID] + set prioritized_before [string trim [R 1 client list flags H]] + if {$prioritized_before eq ""} { + set expected_min 0 + } else { + set expected_min [llength [split $prioritized_before "\n"]] + } + + R 1 DEBUG slotmigration prevent-failover 1 + + assert_equal "OK" [R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 0 0 NODE $target_id] + + wait_for_condition 100 50 { + [check_prioritized_client_count 1 $expected_min] + } else { + puts "Before: $prioritized_before" + puts "After: [R 1 client list flags H]" + fail "Migration connection did not appear on target as prioritized" + } + + R 1 DEBUG slotmigration prevent-failover 0 + + wait_for_condition 100 50 { + [local_is_slot_migrated 1 0] + } else { + fail "Slot 0 was not migrated to R1" + } + } +} diff --git a/tests/unit/hashexpire.tcl b/tests/unit/hashexpire.tcl index 5d9d6f1da..79e7636cc 100644 --- a/tests/unit/hashexpire.tcl +++ b/tests/unit/hashexpire.tcl @@ -1638,11 +1638,19 @@ start_server {tags {"hashexpire"}} { if {$cmd eq "RESTORE"} { assert_equal 2 [get_keys r] assert_equal 2 [get_keys_with_volatile_items r] + # RESTORE rebuilds the object; the listpack is byte-identical + # but with libc malloc the allocation's usable size (what + # MEMORY USAGE reports) can differ by an allocator chunk. + # Assert what matters: the encoding is preserved, and memory + # stays in the same ballpark. + assert_encoding listpack $newhash + assert_range $memory_after [expr {$mem_before - 16}] [expr {$mem_before + 16}] } else { assert_equal 1 [get_keys r] assert_equal 1 [get_keys_with_volatile_items r] + # RENAME does not touch the object: memory must be identical. + assert_equal $mem_before $memory_after } - assert_equal $mem_before $memory_after } {} {needs:debug} } @@ -1728,6 +1736,24 @@ start_server {tags {"hashexpire"}} { r DEBUG SET-ACTIVE-EXPIRE 1 } {OK} {needs:debug} + set original_max [lindex [r config get hash-max-listpack-entries] 1] + r config set hash-max-listpack-entries 0 + test {HMGET batch lookup skips expired hash fields} { + r DEBUG SET-ACTIVE-EXPIRE 0 + + r del hmgetbatchhfetest + r hset hmgetbatchhfetest alive value expired stale + assert_encoding hashtable hmgetbatchhfetest + assert_equal {1} [r hpexpire hmgetbatchhfetest 1 fields 1 expired] + after 2 + + assert_equal {value {} {}} [r hmget hmgetbatchhfetest alive expired missing] + assert_equal {} [r hget hmgetbatchhfetest expired] + + r DEBUG SET-ACTIVE-EXPIRE 1 + } {OK} {needs:debug} + r config set hash-max-listpack-entries $original_max + test {HGETALL skips expired fields} { r FLUSHALL r DEBUG SET-ACTIVE-EXPIRE 0 @@ -4831,7 +4857,7 @@ start_server {tags {"hash"}} { r config set import-mode yes assert_equal [r hsetex myhash exat 0 fields 2 f2 v2 f3 v3] 1 assert_equal [r hlen myhash] 3 - assert_equal [r OBJECT ENCODING myhash] "hashtable" + assert_equal [r OBJECT ENCODING myhash] "listpack" r config set import-mode no wait_for_condition 30 100 { [r hlen myhash] == 1 @@ -4981,6 +5007,107 @@ start_server {tags {"hashexpire"}} { } {OK} {needs:debug} } +start_server {tags {"hash expire listpack"}} { + r config set hash-max-listpack-entries 128 + set original_max_value [lindex [r config get hash-max-listpack-value] 1] + + test "Volatile-count header tracks listpack expiry transitions" { + r del myhash + r hset myhash f1 v1 f2 v2 f3 v3 + assert_encoding listpack myhash + assert_equal 0 [get_keys_with_volatile_items r] + + # 0 -> 1: first expiry creates the aggregate header + assert_equal {1} [r hexpire myhash 1000 FIELDS 1 f1] + assert_equal 1 [get_keys_with_volatile_items r] + + # 1 -> 2 -> 1: add another, then persist one + assert_equal {1} [r hexpire myhash 1000 FIELDS 1 f2] + assert_equal {1} [r hpersist myhash FIELDS 1 f1] + assert_equal 1 [get_keys_with_volatile_items r] + + # 1 -> 0: last volatile field persisted, header removed + assert_equal {1} [r hpersist myhash FIELDS 1 f2] + assert_equal 0 [get_keys_with_volatile_items r] + assert_equal 3 [r hlen myhash] + } + + test "Volatile-count header follows HDEL of a volatile field" { + r del myhash + r hset myhash f1 v1 f2 v2 + r hexpire myhash 1000 FIELDS 1 f1 + assert_equal 1 [get_keys_with_volatile_items r] + r hdel myhash f1 + assert_equal 0 [get_keys_with_volatile_items r] + assert_equal {v2} [r hget myhash f2] + } + + test "Volatile-count header survives RDB reload and DUMP/RESTORE" { + r del myhash + r hset myhash f1 v1 f2 v2 + r hsetex myhash EX 1000 FIELDS 1 t1 x1 + assert_encoding listpack myhash + r debug reload + assert_encoding listpack myhash + assert_equal 1 [get_keys_with_volatile_items r] + assert_range [lindex [r httl myhash FIELDS 1 t1] 0] 1 1000 + + set d [r dump myhash] + r del myhash + r restore myhash 0 $d + assert_equal 1 [get_keys_with_volatile_items r] + assert_range [lindex [r httl myhash FIELDS 1 t1] 0] 1 1000 + } {} {needs:debug} + + test "Volatile-count header cleared when active expiry reaps last field" { + r del myhash + r hset myhash f1 v1 + r hpexpire myhash 50 FIELDS 1 f1 + assert_equal 1 [get_keys_with_volatile_items r] + wait_for_condition 50 100 { + [get_keys_with_volatile_items r] == 0 + } else { + fail "volatile tracking not cleared after reap" + } + } + + # A HASH_2 payload that makes the loader convert to a hashtable only after + # a volatile field has landed in the listpack: 'a' and 'b' are one byte and + # stay under the lowered value threshold, field 'cc' does not. The loader + # installs the aggregate volatile-count header after its listpack loop, so + # at conversion time the listpack does not have one yet. + r config set hash-max-listpack-value $original_max_value + r del myhash + r hset myhash a b cc dd + r hexpire myhash 1000 FIELDS 1 a + assert_encoding listpack myhash + set mid_load_payload [r dump myhash] + r config set hash-max-listpack-value 1 + + test "RESTORE tracks field TTLs when the load converts mid-listpack" { + r del myhash + r restore myhash 0 $mid_load_payload + assert_encoding hashtable myhash + + assert_equal 1 [get_keys_with_volatile_items r] + assert_range [lindex [r httl myhash FIELDS 1 a] 0] 1 1000 + assert_equal 1 [r hdel myhash a] + assert_equal 0 [get_keys_with_volatile_items r] + assert_equal {dd} [r hget myhash cc] + } + + test "Field TTLs survive a save after a mid-listpack conversion" { + # An untracked expiry also makes the save pick RDB_TYPE_HASH over + # RDB_TYPE_HASH_2, dropping the TTL instead of crashing. + r del myhash + r restore myhash 0 $mid_load_payload + r debug reload + assert_range [lindex [r httl myhash FIELDS 1 a] 0] 1 1000 + } {} {needs:debug} + + r config set hash-max-listpack-value $original_max_value +} + start_server {tags {"hashexpire"}} { # Regression: HPEXPIREAT with timestamps at/near the top of the int64 range # used to crash the server via the vset bucket-timestamp math. Two flows: @@ -5034,3 +5161,56 @@ start_server {tags {"hashexpire"}} { r DEL myhash } {1} } + +start_server {tags {"hashexpire external:skip"}} { + # HGETEX changes field TTLs (and can delete a field via a past EXAT/PXAT), + # so its key spec requires both read and write permission on the key. + set r2 [valkey_client] + + test {HGETEX under a read-only (%R~) ACL grant is denied} { + r DEL myhash + r HSET myhash f1 v1 f2 v2 + + r ACL SETUSER hgetex-ro on nopass %R~myhash* +@all + $r2 auth hgetex-ro password + assert_equal PONG [$r2 PING] + + assert_equal "User hgetex-ro has no permissions to access the 'myhash' key" \ + [r ACL DRYRUN hgetex-ro HGETEX myhash FIELDS 1 f1] + + assert_error {*NOPERM*key*} {$r2 HGETEX myhash FIELDS 1 f1} + assert_error {*NOPERM*key*} {$r2 HGETEX myhash PERSIST FIELDS 1 f1} + assert_error {*NOPERM*key*} {$r2 HGETEX myhash EX 100 FIELDS 1 f1} + assert_error {*NOPERM*key*} {$r2 HGETEX myhash EXAT 1 FIELDS 1 f1} + + assert_equal 2 [r HLEN myhash] + assert_equal v1 [r HGET myhash f1] + assert_equal -1 [r HTTL myhash FIELDS 1 f1] + } + + test {HGETEX under a write-only (%W~) ACL grant is denied} { + r DEL myhash + r HSET myhash f1 v1 + + r ACL SETUSER hgetex-wo on nopass %W~myhash* +@all + $r2 auth hgetex-wo password + assert_equal PONG [$r2 PING] + + assert_error {*NOPERM*key*} {$r2 HGETEX myhash EX 100 FIELDS 1 f1} + } + + test {HGETEX with read+write (%RW~) ACL grant is permitted} { + r DEL myhash + r HSET myhash f1 v1 f2 v2 + + r ACL SETUSER hgetex-rw on nopass %RW~myhash* +@all + $r2 auth hgetex-rw password + assert_equal PONG [$r2 PING] + + assert_equal v1 [$r2 HGETEX myhash FIELDS 1 f1] + assert_equal v1 [$r2 HGETEX myhash EX 1000 FIELDS 1 f1] + assert_morethan [r HTTL myhash FIELDS 1 f1] 0 + } + + $r2 close +} diff --git a/tests/unit/hotkeys.tcl b/tests/unit/hotkeys.tcl new file mode 100644 index 000000000..3f4b27862 --- /dev/null +++ b/tests/unit/hotkeys.tcl @@ -0,0 +1,411 @@ +start_server {tags {"hotkey external:skip"}} { + # QPS accounting uses a fixed window; HOTKEYS GET reports the last + # *completed* window. Rather than sleep a fixed time (racy: the accessed + # window can split, or a too-long wait empties the snapshot), poll GET until + # the accessed window has been frozen, capturing the first non-empty result + # so we never overshoot into the emptied next window. + proc hk_wait_hotkeys {} { + global _hk + wait_for_condition 50 100 { + [llength [set _hk [r hotkeys get]]] > 0 + } else { + fail "no hot keys reported within the timeout" + } + return $_hk + } + + # Key names reported by the last completed window. + proc hk_names {hotkeys} { + set names {} + foreach e $hotkeys { lappend names [dict get $e key] } + return $names + } + + # The entry for a given key, or an empty string if it was not reported. + proc hk_entry {hotkeys key} { + foreach e $hotkeys { + if {[dict get $e key] eq $key} { return $e } + } + return "" + } + + proc hk_enable {} { + r config set hotkeys-top-k 16 + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-window-seconds 1 + } + + test "Enable hotkey functionality" { + r config set hotkeys-top-k 16 + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-window-seconds 1 + set hotkey_status [r config get hotkeys-top-k] + assert_equal [lindex $hotkey_status 1] "16" + } + + test "HOTKEYS GET returns empty when no hot keys" { + r hotkeys reset + set all_hotkeys [r hotkeys get] + assert_equal [llength $all_hotkeys] 0 + } + + test "Generate hot keys through repeated access" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 16 + + r set "hot_read_key" "value" + for {set j 1} {$j <= 300} {incr j} { + r get "hot_read_key" + r set "hot_write_key" "value_$j" + } + + set all_hotkeys [hk_wait_hotkeys] + assert {[llength $all_hotkeys] > 0} + + # Each entry is a map: {key db qps }. + set first [lindex $all_hotkeys 0] + assert_equal [lsort [dict keys $first]] {db key qps} + assert {[string length [dict get $first key]] > 0} + } + + test "Reads and writes share one combined summary" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 16 + + r set "combined_key" "val" + for {set i 0} {$i < 300} {incr i} { + r get "combined_key" + r set "combined_key" "val_$i" + } + + set hotkeys [hk_wait_hotkeys] + assert {[llength $hotkeys] > 0} + # The key appears once (a single summary), not split by access type. + set names {} + foreach e $hotkeys { lappend names [dict get $e key] } + assert_equal [llength [lsearch -all $names "combined_key"]] 1 + } + + test "HOTKEYS RESET clears all statistics" { + set reset_result [r hotkeys reset] + assert_equal $reset_result "OK" + set all_hotkeys [r hotkeys get] + assert_equal [llength $all_hotkeys] 0 + } + + test "Hotkey detection with different data types" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 16 + + r set "hot_string" "value" + r hset "hot_hash" "field_1" "value" + r rpush "hot_list" "item" + r sadd "hot_set" "member" + r zadd "hot_zset" 1.0 "member" + + for {set i 1} {$i <= 300} {incr i} { + r get "hot_string" + r hget "hot_hash" "field_1" + r lrange "hot_list" 0 -1 + r smembers "hot_set" + r zrange "hot_zset" 0 -1 + } + + set all_hotkeys [hk_wait_hotkeys] + assert {[llength $all_hotkeys] > 0} + } + + test "Invalid HOTKEYS command syntax" { + catch {r hotkeys invalid} err + assert_match "*unknown*subcommand*" $err + catch {r hotkeys} err + assert_match "*wrong number of arguments*" $err + } + + test "Disable hotkey functionality" { + r config set hotkeys-top-k 0 + assert_equal [lindex [r config get hotkeys-top-k] 1] "0" + # Disabled reports an empty result rather than an error, so a polling + # client has a single reply shape to handle (as SLOWLOG GET does). + assert_equal [r hotkeys get] {} + assert_equal [r hotkeys reset] "OK" + } + + test "HOTKEYS HELP lists the subcommands" { + set help [r hotkeys help] + assert_match "*GET*" $help + assert_match "*RESET*" $help + } + + test "Re-enable hotkey functionality" { + r config set hotkeys-top-k 16 + assert_equal [lindex [r config get hotkeys-top-k] 1] "16" + assert_equal [r hotkeys reset] "OK" + } + + test "Hotkey detection with high frequency access" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 16 + + for {set i 1} {$i <= 300} {incr i} { + r set "super_hot_write" "value_$i" + r get "super_hot_read" + } + + set all_hotkeys [hk_wait_hotkeys] + assert {[llength $all_hotkeys] > 0} + } + + test "HOTKEYS GET returns sorted by QPS descending" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 16 + + r set "low_freq" "val" + r set "mid_freq" "val" + r set "high_freq" "val" + + # Run all reads inside one MULTI/EXEC so they land in a single detection + # window: EXEC executes the whole batch in one event-loop tick, so the + # window clock cannot advance mid-batch and split the keys. + r multi + for {set i 0} {$i < 100} {incr i} { r get "low_freq" } + for {set i 0} {$i < 500} {incr i} { r get "mid_freq" } + for {set i 0} {$i < 1000} {incr i} { r get "high_freq" } + r exec + + set hotkeys [hk_wait_hotkeys] + assert {[llength $hotkeys] >= 2} + + # QPS is at field index 5 + set prev_qps [dict get [lindex $hotkeys 0] qps] + for {set i 1} {$i < [llength $hotkeys]} {incr i} { + set cur_qps [dict get [lindex $hotkeys $i] qps] + assert {$prev_qps >= $cur_qps} + set prev_qps $cur_qps + } + } + + test "HOTKEYS GET limits results to top-k" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 3 + + r multi + foreach key {k1 k2 k3 k4 k5 k6 k7 k8} { + for {set i 0} {$i < 200} {incr i} { r get $key } + } + r exec + + set hotkeys [hk_wait_hotkeys] + assert {[llength $hotkeys] <= 3} + + r config set hotkeys-top-k 16 + } + + test "Hotkey entries report the db the key was accessed in" { + hk_enable + # Access the key in a specific, non-default db and assert that db is the + # one reported back. + r select 5 + r hotkeys reset + r set "db_field_key" "val" + r multi + for {set i 0} {$i < 300} {incr i} { r get "db_field_key" } + r exec + + set entry [hk_entry [hk_wait_hotkeys] "db_field_key"] + assert {$entry ne ""} + assert_equal [dict get $entry db] 5 + r select 9 + } + + test "FLUSHALL purges all hotkey state" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 16 + + r set "flush_key" "val" + for {set i 0} {$i < 300} {incr i} { r get "flush_key" } + + set hotkeys_before [hk_wait_hotkeys] + assert {[llength $hotkeys_before] > 0} + + r flushall + + set hotkeys_after [r hotkeys get] + assert_equal [llength $hotkeys_after] 0 + } + + test "Test memory cleanup on manager recreation" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 16 + + for {set i 1} {$i <= 10} {incr i} { + for {set j 1} {$j <= 100} {incr j} { + r get "memory_test_key_$i" + } + } + + set hotkeys_before [hk_wait_hotkeys] + assert {[llength $hotkeys_before] > 0} + + r config set hotkeys-top-k 0 + r config set hotkeys-top-k 16 + set hotkeys_after [r hotkeys get] + assert_equal [llength $hotkeys_after] 0 + assert_equal [r ping] "PONG" + } + + test "QPS reported as a positive rate over the window" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 16 + r config set hotkeys-window-seconds 1 + + r set "qps_key" "val" + for {set i 0} {$i < 400} {incr i} { r get "qps_key" } + + set hotkeys [hk_wait_hotkeys] + assert {[llength $hotkeys] > 0} + set first [lindex $hotkeys 0] + assert_equal [dict get $first key] "qps_key" + # QPS is count over the last completed window; assert a sane positive + # value rather than an exact figure (sampling-boundary noise applies). + set qps [dict get $first qps] + assert {$qps > 0} + } + + test "Cold keys drop after a completed idle window" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 16 + r config set hotkeys-window-seconds 1 + + r set "fading_key" "val" + for {set i 0} {$i < 200} {incr i} { r get "fading_key" } + + # The key is reportable once its window completes. + set before [hk_wait_hotkeys] + assert {[llength $before] > 0} + + # No further access: once a full window elapses with no traffic the live + # window freezes empty and replaces the previous (hot) snapshot. Poll for + # that rather than sleeping a fixed time, which is racy on a loaded box. + wait_for_condition 50 100 { + [llength [r hotkeys get]] == 0 + } else { + fail "cold key was still reported after an idle window" + } + } + + test "Max top-k tracks the hottest keys" { + r hotkeys reset + r config set hotkeys-sampling-percentage 100 + r config set hotkeys-top-k 3 + + r set "cold_key" "val" + r set "warm_key" "val" + r set "hot_key" "val" + + r multi + for {set i 0} {$i < 100} {incr i} { r get "cold_key" } + for {set i 0} {$i < 500} {incr i} { r get "warm_key" } + for {set i 0} {$i < 1000} {incr i} { r get "hot_key" } + r exec + + set hotkeys [hk_wait_hotkeys] + assert {[llength $hotkeys] <= 3} + assert {[llength $hotkeys] > 0} + + r config set hotkeys-top-k 16 + } + + test "CLIENT NO-TOUCH does not suppress hot-key accounting" { + # A no-touch client only skips the LRU/LFU update. Accounting must be + # identical either way, and in particular a hit must not be dropped + # while a miss on an equally hot key is still counted. + hk_enable + r set "nt_hit" "val" + r del "nt_miss" + r hotkeys reset + + r client no-touch on + # One window, equal access counts: an existing key and a missing one. + r multi + for {set i 0} {$i < 300} {incr i} { + r get "nt_hit" + r get "nt_miss" + } + r exec + r client no-touch off + + set hotkeys [hk_wait_hotkeys] + set names [hk_names $hotkeys] + if {[lsearch $names "nt_hit"] < 0} { + fail "a hit on an existing key was not counted for a no-touch client" + } + if {[lsearch $names "nt_miss"] < 0} { + fail "a miss was not counted for a no-touch client" + } + + # Same number of accesses in the same window, so the same rate. + set qps_hit -1 + set qps_miss -2 + foreach e $hotkeys { + if {[dict get $e key] eq "nt_hit"} { set qps_hit [dict get $e qps] } + if {[dict get $e key] eq "nt_miss"} { set qps_miss [dict get $e qps] } + } + assert_equal $qps_hit $qps_miss + } + + test "EXISTS, TYPE and TTL count as key accesses" { + # These read the key without touching it (LOOKUP_NOTOUCH). They are + # still genuine client accesses and must be reported. + hk_enable + r set "meta_key" "val" + r expire "meta_key" 1000 + + foreach cmd {exists type ttl} { + # Reset first so only this command's accesses are in the window. + r hotkeys reset + r multi + for {set i 0} {$i < 200} {incr i} { r $cmd "meta_key" } + r exec + + set names [hk_names [hk_wait_hotkeys]] + if {[lsearch $names "meta_key"] < 0} { + fail "$cmd on a hot key was not reported" + } + } + } + + test "OBJECT ENCODING is introspection and is not counted" { + # OBJECT looks keys up with LOOKUP_NOHOTKEY. Drive it in a tight loop + # alongside a real GET on another key in the SAME window: the real + # access must be reported and the introspection must not, so an empty + # or not-yet-frozen window cannot make this pass vacuously. + hk_enable + r set "obj_probe" "val" + r set "real_probe" "val" + r hotkeys reset + + r multi + for {set i 0} {$i < 300} {incr i} { + r object encoding "obj_probe" + r get "real_probe" + } + r exec + + set names [hk_names [hk_wait_hotkeys]] + if {[lsearch $names "real_probe"] < 0} { + fail "a real access in the same window was not reported" + } + assert {[lsearch $names "obj_probe"] < 0} + } +} diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl index 5c876b08e..0343a97ab 100644 --- a/tests/unit/info.tcl +++ b/tests/unit/info.tcl @@ -356,8 +356,12 @@ start_server {tags {"info" "external:skip" "debug_defrag:skip"}} { # make sure debug info is hidden set info [r info] assert_equal [getInfoProperty $info eventloop_duration_aof_sum] {} + assert_equal [getInfoProperty $info eventloop_priority_duration_max] {} + assert_equal [getInfoProperty $info eventloop_priority_cmd_per_cycle_max] {} set info_all [r info all] assert_equal [getInfoProperty $info_all eventloop_duration_aof_sum] {} + assert_equal [getInfoProperty $info_all eventloop_priority_duration_max] {} + assert_equal [getInfoProperty $info_all eventloop_priority_cmd_per_cycle_max] {} set info1 [r info debug] @@ -369,6 +373,10 @@ start_server {tags {"info" "external:skip" "debug_defrag:skip"}} { assert {$cycle_max1 > 0} set duration_max1 [getInfoProperty $info1 eventloop_duration_max] assert {$duration_max1 > 0} + set priority_duration_max1 [getInfoProperty $info1 eventloop_priority_duration_max] + assert {$priority_duration_max1 >= 0} + set priority_cycle_max1 [getInfoProperty $info1 eventloop_priority_cmd_per_cycle_max] + assert {$priority_cycle_max1 >= 0} after 110 ;# hz is 10, wait for a cron tick. set info2 [r info debug] @@ -570,3 +578,221 @@ start_server {tags {"info" "external:skip"}} { assert_equal [dict get $mem_stats db.dict.rehashing.count] {1} } } + +start_server {tags {"info" "external:skip"} overrides {save "" forkless-infrastructure-enabled yes}} { + test {INFO forkless save metrics show default values when no save is running} { + r config set save "" + r flushall + + # When no forkless save is running, time metrics should be -1 + set dbg [r info debug] + assert_match "*forkless_current_item_ms:-1*" $dbg + assert_match "*forkless_estimated_seconds_remaining:-1*" [r info persistence] + + # Debug metrics should be 0 + assert_match "*forkless_current_queue_length:0*" $dbg + assert_match "*forkless_queue_length_target:0*" $dbg + assert_match "*forkless_dbentries_queued:0*" $dbg + assert_match "*forkless_dbentries_processed:0*" $dbg + } + + test {INFO forkless save metrics are present during active save} { + r config set save "" + r flushall + r debug populate 1000 + + # Start slow forkless save + r config set rdb-key-save-delay 100000 + r config set bgsave-default-method forkless + r bgsave + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + set dbg [r info debug] + + # Verify time metrics are present in debug section + assert_match "*forkless_current_item_ms:*" $dbg + assert_match "*forkless_estimated_seconds_remaining:*" [r info persistence] + + # Verify queue metrics are present in debug section + assert_match "*forkless_current_queue_length:*" $dbg + assert_match "*forkless_queue_length_target:*" $dbg + assert_match "*forkless_dbentries_queued:*" $dbg + assert_match "*forkless_dbentries_processed:*" $dbg + + # Verify queue_length_target has a reasonable value + set target [getInfoProperty $dbg forkless_queue_length_target] + assert {$target > 0} + + r bgsave cancel + r config set rdb-key-save-delay 0 + waitForBgsave r + } + + test {INFO forkless save cumulative metrics increase during save} { + r config set save "" + r flushall + r debug populate 100 + + # Start slow forkless save + r config set rdb-key-save-delay 50000 + r config set bgsave-default-method forkless + r bgsave + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + # Wait a bit for some processing + after 200 + + set dbg [r info debug] + set queued1 [getInfoProperty $dbg forkless_dbentries_queued] + set processed1 [getInfoProperty $dbg forkless_dbentries_processed] + + # Wait more + after 200 + + set dbg [r info debug] + set queued2 [getInfoProperty $dbg forkless_dbentries_queued] + set processed2 [getInfoProperty $dbg forkless_dbentries_processed] + + # Cumulative metrics should increase or stay same (never decrease) + assert {$queued2 >= $queued1} + assert {$processed2 >= $processed1} + + # At least one should have increased + assert {$queued2 > $queued1 || $processed2 > $processed1} + + r bgsave cancel + r config set rdb-key-save-delay 0 + waitForBgsave r + } + + test {INFO forkless save current_item_ms is counted} { + r config set save "" + r flushall + r debug populate 10 + + # Start very slow forkless save - 2 seconds per key + r config set rdb-key-save-delay 2000000 + r config set bgsave-default-method forkless + r bgsave + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + # Wait until the item has been processing for at least 1 second + wait_for_condition 50 100 { + [getInfoProperty [r info debug] forkless_current_item_ms] > 1000 + } else { + fail "forkless_current_item_ms never exceeded 1000" + } + + set item_time [getInfoProperty [r info debug] forkless_current_item_ms] + + # Should be processing an item for ~1 second (1000+ ms) + assert {$item_time > 1000} + + r bgsave cancel + r config set rdb-key-save-delay 0 + waitForBgsave r + } + + test {INFO forkless save estimated_seconds_remaining is reasonable} { + r config set save "" + r flushall + r debug populate 100 + + # Set 1 second delay per key + r config set rdb-key-save-delay 1000000 + waitForBgsave r + r config set bgsave-default-method forkless + r bgsave + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + # Wait for ~1 key to be processed (1.2 seconds to be safe) + after 1200 + + set dbg [r info debug] + set processed [getInfoProperty $dbg forkless_dbentries_processed] + set estimated [getInfoProperty [r info persistence] forkless_estimated_seconds_remaining] + + # Should have processed at least 1 key + assert {$processed >= 1} + + # With 100 keys and ~1 processed at 1sec/key, the estimate should be + # in the tens of seconds range. Use a wide band to avoid timing flakes. + assert {$estimated > 0 && $estimated < 300} + + r bgsave cancel + r config set rdb-key-save-delay 0 + waitForBgsave r + } + + test {INFO forkless save metrics show default values after save completes} { + r config set save "" + r flushall + r debug populate 100 + + # Start and complete a fast forkless save + r config set rdb-key-save-delay 0 + r config set bgsave-default-method forkless + r bgsave + waitForBgsave r + + # After save completes, time metrics should be -1 + set dbg [r info debug] + assert_match "*forkless_current_item_ms:-1*" $dbg + assert_match "*forkless_estimated_seconds_remaining:-1*" [r info persistence] + + # Debug metrics should be 0 + assert_match "*forkless_current_queue_length:0*" $dbg + assert_match "*forkless_queue_length_target:0*" $dbg + assert_match "*forkless_dbentries_queued:0*" $dbg + assert_match "*forkless_dbentries_processed:0*" $dbg + } + + test {INFO rdb_current_bgsave_time_sec increases during forkless save} { + r config set save "" + r flushall + r debug populate 100 + + # Start slow forkless save + r config set rdb-key-save-delay 100000 + r config set bgsave-default-method forkless + r bgsave + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + set time1 [s rdb_current_bgsave_time_sec] + + wait_for_condition 50 100 { + [s rdb_current_bgsave_time_sec] >= $time1 + 2 + } else { + fail "rdb_current_bgsave_time_sec did not advance during forkless save" + } + + r bgsave cancel + r config set rdb-key-save-delay 0 + waitForBgsave r + } +} diff --git a/tests/unit/introspection-2.tcl b/tests/unit/introspection-2.tcl index 157ebb37d..682c04825 100644 --- a/tests/unit/introspection-2.tcl +++ b/tests/unit/introspection-2.tcl @@ -139,6 +139,11 @@ start_server {tags {"introspection"}} { assert_equal {key} [r command getkeys get key] } + test {COMMAND GETKEYS EXEC} { + assert_equal {} [r command getkeys exec] + assert_equal {key} [r command getkeys exec ifeq key value] + } + test {COMMAND GETKEYSANDFLAGS} { assert_equal {{k1 {OW update}}} [r command getkeysandflags set k1 v1] assert_equal {{k1 {OW update}} {k2 {OW update}}} [r command getkeysandflags mset k1 v1 k2 v2] @@ -146,6 +151,11 @@ start_server {tags {"introspection"}} { assert_equal {{k1 {RO access}} {k2 {OW update}}} [r command getkeysandflags sort k1 store k2] } + test {COMMAND GETKEYSANDFLAGS EXEC} { + assert_equal {} [r command getkeysandflags exec] + assert_equal {{key {RO access}}} [r command getkeysandflags exec ifeq key value] + } + test {COMMAND GETKEYS MEMORY USAGE} { assert_equal {key} [r command getkeys memory usage key] } @@ -316,7 +326,7 @@ start_server {tags {"introspection"}} { } } - foreach cmd {ZUNIONSTORE XREAD EVAL SORT SORT_RO MIGRATE GEORADIUS} { + foreach cmd {ZUNIONSTORE XREAD EVAL EXEC SORT SORT_RO MIGRATE GEORADIUS} { test "$cmd command is marked with movablekeys" { set info [lindex [r command info $cmd] 0] assert_match {*movablekeys*} [lindex $info 2] diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 324100f6e..e4c6d7a11 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -990,38 +990,7 @@ start_server {tags {"introspection"}} { assert_error "ERR timeout is negative" {r client pause -1} } - test "CLIENT KILL close the client connection during bgsave" { - # Start a slow bgsave, trigger an active fork. - r flushall - r set k v - r config set rdb-key-save-delay 10000000 - r bgsave - wait_for_condition 1000 10 { - [s rdb_bgsave_in_progress] eq 1 - } else { - fail "bgsave did not start in time" - } - - # Kill (close) the connection - r client kill skipme no - - # In the past, client connections needed to wait for bgsave - # to end before actually closing, now they are closed immediately. - assert_error "*I/O error*" {r ping} ;# get the error very quickly - assert_equal "PONG" [r ping] - # Make sure the bgsave is still in progress - assert_equal [s rdb_bgsave_in_progress] 1 - - # Stop the child before we proceed to the next test - r config set rdb-key-save-delay 0 - r flushall - wait_for_condition 1000 10 { - [s rdb_bgsave_in_progress] eq 0 - } else { - fail "bgsave did not stop in time" - } - } {} {needs:save} test "CLIENT REPLY OFF/ON: disable all commands reply" { set rd [valkey_deferring_client] @@ -1393,6 +1362,7 @@ start_server {tags {"introspection"}} { rdma-rx-size rdma-bind rdma-port + forkless-infrastructure-enabled } if {!$::tls} { @@ -1694,7 +1664,7 @@ start_server {tags {"introspection"}} { # Get the tot-net-out of the replica before sending the command. set info_list [$primary client list] foreach info [split $info_list "\r\n"] { - if {[string match "* flags=S *" $info]} { + if {[string match "* flags=*S* *" $info]} { set out_before [get_field_in_client_info $info "tot-net-out"] break } @@ -1707,7 +1677,7 @@ start_server {tags {"introspection"}} { # Get the tot-net-out of the replica after sending the command. set info_list [$primary client list] foreach info [split $info_list "\r\n"] { - if {[string match "* flags=S *" $info]} { + if {[string match "* flags=*S* *" $info]} { set out_after [get_field_in_client_info $info "tot-net-out"] break } @@ -2115,3 +2085,38 @@ test {CONFIG hash-seed is immutable and settable at startup} { } } } {} {external:skip} + +start_server {overrides {forkless-infrastructure-enabled yes} tags {"introspection" "external:skip"}} { + foreach bgsave_type {"fork" "forkless"} { + test "CLIENT KILL close the client connection during bgsave - $bgsave_type" { + r flushall + r set k v + r config set rdb-key-save-delay 10000000 + r config set bgsave-default-method $bgsave_type + r bgsave + wait_for_condition 1000 10 { + [s rdb_bgsave_in_progress] eq 1 + } else { + fail "bgsave did not start in time" + } + + set expected_type [expr {$bgsave_type eq "forkless" ? "forkless" : "fork"}] + assert_equal [s rdb_current_bgsave_type] $expected_type + + r client kill skipme no + + assert_error "*I/O error*" {r ping} + assert_equal "PONG" [r ping] + + assert_equal [s rdb_bgsave_in_progress] 1 + + r config set rdb-key-save-delay 0 + r flushall + wait_for_condition 1000 10 { + [s rdb_bgsave_in_progress] eq 0 + } else { + fail "bgsave did not stop in time" + } + } {} {needs:save} + } +} diff --git a/tests/unit/moduleapi/aclcheck.tcl b/tests/unit/moduleapi/aclcheck.tcl index 3de59431e..eaf592b80 100644 --- a/tests/unit/moduleapi/aclcheck.tcl +++ b/tests/unit/moduleapi/aclcheck.tcl @@ -216,6 +216,13 @@ start_server {tags {"modules acl"}} { r ACL DELUSER selcmduser } + test {Module unload blocked by ACL role rule} { + r ACL SETROLE modrole +subcommands.parent_get_fullname + catch {r module unload subcommands} e + assert_match {*one or more ACL users reference commands from this module*} $e + r ACL DELROLE modrole + } + test {Unload the module - subcommands} { r ACL DELUSER subcmduser basecmduser denycmduser selcmduser assert_equal {OK} [r module unload subcommands] @@ -277,6 +284,16 @@ start_server {tags {"modules acl"}} { } } +start_server {tags {"modules acl"}} { + test {test existing roles to have access to module commands loaded on runtime} { + r acl SETROLE writerole -@all +@WRITE + r acl SETUSER j7 on >password -@all role=writerole + assert_equal [r module load $testmodule] OK + assert_equal [r acl DRYRUN j7 aclcheck.module.command.aclcategories.write] OK + assert_equal {OK} [r module unload aclcheck] + } +} + start_server {tags {"modules acl"}} { test {test existing users without permissions, do not have access to module commands loaded on runtime.} { r acl SETUSER j4 on >password -@all +@READ @@ -319,3 +336,34 @@ start_server {tags {"modules acl"}} { assert_error {ERR Error loading module: module initialization failed} {r module load $testmodule 1} } } + +start_server {tags {"modules acl"}} { + r module load $testmodule + + test {test module check acl for key and channel perm granted by a role} { + r acl SETROLE modperms ~x resetchannels &ch1 + r acl setuser default on nopass +@all resetkeys resetchannels role=modperms + + assert_equal [r aclcheck.set.check.key "~" x 5] OK + assert_error "*DENIED KEY*" {r aclcheck.set.check.key "~" v 5} + assert_equal [r aclcheck.publish.check.channel ch1 msg] 0 + assert_error "*DENIED CHANNEL*" {r aclcheck.publish.check.channel ch2 msg} + + # Restore the default user so the module can be unloaded. + r acl setuser default resetroles on nopass ~* &* +@all alldbs + r acl DELROLE modperms + } + + test {Unload the module - aclcheck role perms} { + assert_equal {OK} [r module unload aclcheck] + } +} + +set modrole "modconfrole -@all +aclcheck.module.command.aclcategories.write" +start_server [list tags {"modules acl"} overrides [list loadmodule $testmodule role $modrole]] { + test {role in config can reference a module command} { + assert_equal [r ACL ROLES] {modconfrole} + r acl SETUSER j10 on >password -@all role=modconfrole + assert_equal [r acl DRYRUN j10 aclcheck.module.command.aclcategories.write] OK + } +} diff --git a/tests/unit/moduleapi/defrag.tcl b/tests/unit/moduleapi/defrag.tcl index 6d8f55bd0..949047efc 100644 --- a/tests/unit/moduleapi/defrag.tcl +++ b/tests/unit/moduleapi/defrag.tcl @@ -1,7 +1,9 @@ set testmodule [file normalize tests/modules/defragtest.so] start_server {tags {"modules"} overrides {{save ""}}} { - r module load $testmodule 10000 + # Load with 10000 global strings and a global defrag step limit of 100, so + # the global callback must resume across many invocations via its cursor. + r module load $testmodule 10000 100 r config set active-defrag-ignore-bytes 1 r config set active-defrag-threshold-lower 0 r config set active-defrag-cycle-min 99 @@ -41,5 +43,33 @@ start_server {tags {"modules"} overrides {{save ""}}} { set info [r info defragtest_stats] assert {[getInfoProperty $info defragtest_global_attempts] > 0} } + + test {Module defrag: global defrag resumes via cursor} { + r flushdb + r frag.resetstats + + # With the module's global step limit, the 10000 global strings + # can't be defragged in one invocation, so the callback must be + # re-invoked and resume from its saved cursor. This exercises the + # endtime + per-module cursor forwarded to the global callback. + after 2000 + set info [r info defragtest_stats] + assert {[getInfoProperty $info defragtest_global_resumes] > 0} + assert_equal 0 [getInfoProperty $info defragtest_global_wrong_cursor] + } + + test {Module defrag: global defrag is revisited on later cycles} { + r flushdb + r frag.resetstats + + # Once the callback finishes a pass it resets its cursor to 0 (done). + # A finished module is skipped for the rest of that cycle but must be + # revisited on later cycles, so over time the callback runs many more + # times than the single pass needed to walk all global strings. + after 3000 + set info [r info defragtest_stats] + assert {[getInfoProperty $info defragtest_global_attempts] > 10000} + assert_equal 0 [getInfoProperty $info defragtest_global_wrong_cursor] + } } } diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl index 2659d43d2..5f044493e 100644 --- a/tests/unit/moduleapi/infotest.tcl +++ b/tests/unit/moduleapi/infotest.tcl @@ -123,6 +123,18 @@ start_server {tags {"modules"}} { field $info infotest_dos } {2} + test {module external memory is reported only in info debug} { + assert_equal 321 [r info.setexternal 321] + + set debug_info [r info debug] + set memory_info [r info memory] + + assert_equal 321 [getInfoProperty $debug_info used_memory_module_external] + assert { ![string match "*used_memory_module_external*" $memory_info] } + + assert_equal 0 [r info.setexternal 0] + } + test "Unload the module - infotest" { assert_equal {OK} [r module unload infotest] } diff --git a/tests/unit/moduleapi/repl-compression.tcl b/tests/unit/moduleapi/repl-compression.tcl new file mode 100644 index 000000000..aa850069e --- /dev/null +++ b/tests/unit/moduleapi/repl-compression.tcl @@ -0,0 +1,105 @@ +set testmodule [file normalize tests/modules/blockedclient.so] + +tags {"modules repl external:skip"} { + +# repl_uncompressed_bytes= from the replica line of the primary's INFO replication. +proc replica_line_uncompressed_bytes {primary} { + set info [$primary info replication] + assert {[regexp {repl_uncompressed_bytes=([0-9]+)} $info -> uncompressed_bytes]} + return $uncompressed_bytes +} + +start_server {overrides {save "" repl-compression lz4}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Compressed replication keeps decoding while a module command yields} { + $primary flushall + + start_server [list overrides [list save "" repl-compression lz4 repl-diskless-load swapdb \ + busy-reply-threshold 10 loadmodule $testmodule]] { + set replica [srv 0 client] + set replica_pid [srv 0 pid] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [string match {*state=online*repl_compression=lz4*} [$primary info replication]] + } else { + fail "Compressed replication not established" + } + + set sync_full_before [status $primary sync_full] + set sync_partial_before [status $primary sync_partial_ok] + set busy [valkey_deferring_client] + set busy_started 0 + set replica_paused 0 + set test_code [catch { + $busy slow_fg_command 0 + $busy flush + set busy_started 1 + wait_for_condition 50 20 { + [catch {$replica ping} busy_error] && [string match {*BUSY*} $busy_error] + } else { + fail "Module command did not enter its yielding loop" + } + + set read_offset_before [$replica get_repl_read_offset] + set uncompressed_before [replica_line_uncompressed_bytes $primary] + set payload [string repeat x [expr {2 * 1024 * 1024}]] + + # Queue the complete compressed write in the socket before the + # replica resumes, so no later read event is needed to finish it. + pause_process $replica_pid + set replica_paused 1 + $primary set blocked:large $payload + wait_for_condition 50 20 { + [replica_line_uncompressed_bytes $primary] >= + $uncompressed_before + [string length $payload] + } else { + fail "Primary did not finish writing the compressed batches" + } + resume_process $replica_pid + set replica_paused 0 + + # Replicated commands are not applied while the module command + # is busy, but decoding must continue past the first 1 MiB + # scheduling slice while the module yields to the event loop. + wait_for_condition 50 20 { + [$replica get_repl_read_offset] > $read_offset_before + 1024 * 1024 + } else { + fail "Compressed replication stopped decoding during a yielding command" + } + } test_result test_options] + + if {$replica_paused} { + resume_process $replica_pid + } + if {$busy_started} { + $replica stop_slow_fg_command + $busy read + } + $busy close + if {$test_code} { + return -options $test_options $test_result + } + + # A fresh read event lets the primary client apply the data that + # was intentionally left unparsed while the module was busy. + $primary set blocked:probe delivered + wait_for_condition 50 100 { + [$replica get blocked:large] eq $payload && + [$replica get blocked:probe] eq {delivered} + } else { + fail "Write decoded during the yielding command was not applied" + } + assert_equal $sync_full_before [status $primary sync_full] + assert_equal $sync_partial_before [status $primary sync_partial_ok] + + $replica replicaof no one + } + } +} + +} diff --git a/tests/unit/moduleapi/scan.tcl b/tests/unit/moduleapi/scan.tcl index b0f7d6f35..89c954437 100644 --- a/tests/unit/moduleapi/scan.tcl +++ b/tests/unit/moduleapi/scan.tcl @@ -67,6 +67,84 @@ start_server {tags {"modules"}} { lsort [r scan.scan_key ss1] } {{a {}} {b {}} {c {}}} + # ---- VM_ScanKeyRawBorrowed: borrowed (ptr,len) scan, must match native scan ---- + + test {Module scan_key_raw hash listpack} { + r del rh + r hmset rh f1 v1 f2 v2 + assert_encoding listpack rh + lsort [r scan.scan_key_raw rh] + } {{f1 v1} {f2 v2}} + + test {Module scan_key_raw hash listpack with int value} { + r del rh1 + r hmset rh1 f1 1 + assert_encoding listpack rh1 + lsort [r scan.scan_key_raw rh1] + } {{f1 1}} + + test {Module scan_key_raw hash dict} { + r del rh3 + r hmset rh3 f1 v1 f2 v2 f3 v3 + assert_encoding hashtable rh3 + lsort [r scan.scan_key_raw rh3] + } {{f1 v1} {f2 v2} {f3 v3}} + + test {Module scan_key_raw zset listpack} { + r del rz + r zadd rz 1 f1 2 f2 + assert_encoding listpack rz + lsort [r scan.scan_key_raw rz] + } {{f1 1} {f2 2}} + + test {Module scan_key_raw zset btree} { + r del rz1 + r zadd rz1 1 f1 2 f2 3 f3 + assert_encoding btree rz1 + lsort [r scan.scan_key_raw rz1] + } {{f1 1} {f2 2} {f3 3}} + + test {Module scan_key_raw zset fractional score (d2string form)} { + r del rz2 + r zadd rz2 1.5 f1 2 f2 3 f3 + assert_encoding btree rz2 + assert_equal [lsort [r zrange rz2 0 -1 withscores]] [lsort {f1 1.5 f2 2 f3 3}] + lsort [r scan.scan_key_raw rz2] + } {{f1 1.5} {f2 2} {f3 3}} + + test {Module scan_key_raw set intset} { + r del rs + r sadd rs 1 2 + assert_encoding intset rs + lsort [r scan.scan_key_raw rs] + } {{1 {}} {2 {}}} + + test {Module scan_key_raw set dict} { + r del rsa + r sadd rsa 1 2 ; # Created as intset + r sadd rsa 3 ; # Converted to hashtable + assert_encoding hashtable rsa + lsort [r scan.scan_key_raw rsa] + } {{1 {}} {2 {}} {3 {}}} + + test {Module scan_key_raw set listpack} { + r del rs1 + r sadd rs1 a b c + assert_encoding listpack rs1 + lsort [r scan.scan_key_raw rs1] + } {{a {}} {b {}} {c {}}} + + test {Module scan_key_raw unsupported key type returns empty} { + # VM_ScanKeyRawBorrowed returns 0 with errno=EINVAL for a wrong-type key, + # so the command's scan loop terminates immediately and replies with an + # empty array rather than looping forever. + r del rstr rlist + r set rstr hello + r rpush rlist a b c + assert_equal {} [r scan.scan_key_raw rstr] + assert_equal {} [r scan.scan_key_raw rlist] + } + test "Unload the module - scan" { assert_equal {OK} [r module unload scan] } diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl index d80617008..01b84d7a5 100644 --- a/tests/unit/moduleapi/testrdb.tcl +++ b/tests/unit/moduleapi/testrdb.tcl @@ -304,3 +304,31 @@ tags "modules" { } } } + +start_server {tags {"modules"} overrides {forkless-infrastructure-enabled yes save "" enable-debug-command yes enable-module-command yes}} { + test {MODULE LOAD is blocked during forkless save} { + r debug populate 100 + + # Start slow forkless save + r config set rdb-key-save-delay 200000 + r config set bgsave-default-method forkless + r bgsave + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + # Try to load a module - should fail during forkless save + catch {r module load $testmodule} err + assert_match "*Error*" $err + + r bgsave cancel + r config set rdb-key-save-delay 0 + waitForBgsave r + + # After forkless save completes, module load should succeed + assert_equal {OK} [r module load $testmodule] + } +} diff --git a/tests/unit/moduleapi/usercall.tcl b/tests/unit/moduleapi/usercall.tcl index e8029aedd..e4f6985d3 100644 --- a/tests/unit/moduleapi/usercall.tcl +++ b/tests/unit/moduleapi/usercall.tcl @@ -180,3 +180,46 @@ start_server {tags {"modules usercall network"}} { } } } + +# Module users live outside the Users radix tree, so ACL LOAD does not replace +# them. A role they hold must be re-pointed at the reloaded role rather than left +# dangling at the freed one. +set server_path [tmpdir "server.usercall.role.acl"] +exec cp -f tests/assets/user.acl $server_path +start_server [list overrides [list "dir" $server_path "aclfile" "user.acl"] tags [list "modules usercall external:skip"]] { + r module load $testmodule + + test {module user keeps a working role across ACL LOAD} { + r ACL SETROLE myrole +ping ~* + r usercall.reset_user + r usercall.add_to_acl "on role=myrole ~* &*" + assert_match {*role=myrole*} [r usercall.get_acl] + + set info [r ACL GETROLE myrole] + set idx [lsearch $info "users"] + assert_equal {module_user} [lindex $info [expr {$idx + 1}]] + + r ACL SAVE + r ACL LOAD + + # The membership survives, and both directions still agree. + assert_match {*role=myrole*} [r usercall.get_acl] + set info [r ACL GETROLE myrole] + set idx [lsearch $info "users"] + assert_equal {module_user} [lindex $info [expr {$idx + 1}]] + + # The role is still referenced, so it cannot be deleted. + assert_error {*is assigned to one or more users*} {r ACL DELROLE myrole} + assert_equal {PONG} [r PING] + } + + test {module user loses the membership when ACL LOAD drops the role} { + set fd [open "$server_path/user.acl" w] + close $fd + r ACL LOAD + + assert_equal {} [r ACL ROLES] + assert_match {*-@all*} [r usercall.get_acl] + assert_equal {PONG} [r PING] + } +} diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 0b3f38e63..4048501fa 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -21,6 +21,115 @@ start_server {tags {"multi"}} { list $v1 $v2 $v3 } {QUEUED QUEUED {{a b c} PONG}} + test {EXEC conditions} { + r del condition{t} destination{t} + r set condition{t} value + + r multi + r set destination{t} committed + set committed [r exec ifeq condition{t} value ifne condition{t} other xx condition{t} nx missing{t}] + + r multi + r set destination{t} not-committed + set ifeq_failed [r exec ifeq condition{t} other] + + r multi + r set destination{t} not-committed + set ifne_failed [r exec ifne condition{t} value] + + r multi + r set destination{t} not-committed + set nx_failed [r exec nx condition{t}] + + r multi + r set destination{t} not-committed + set xx_failed [r exec xx missing{t}] + + list $committed $ifeq_failed $ifne_failed $nx_failed $xx_failed [r get destination{t}] + } {OK {} {} {} {} committed} + + test {EXEC IFNE matches a missing key} { + r del condition{t} destination{t} + r multi + r set destination{t} committed + list [r exec ifne condition{t} value] [r get destination{t}] + } {OK committed} + + test {EXEC NX lazy-deletes expired condition key on primary} { + r flushdb + r debug set-active-expire 0 + r psetex expired_cond{t} 10 old_val + after 50 + # The expired condition key remains in the dictionary while active expiration is off. + assert_equal 1 [r dbsize] + r multi + r set destination{t} committed + assert_equal {OK} [r exec nx expired_cond{t}] + assert_equal {committed} [r get destination{t}] + # The condition key was lazy-deleted; only destination{t} remains. + assert_equal 1 [r dbsize] + r debug set-active-expire 1 + } {OK} {needs:debug} + + test {EXEC string comparisons return WRONGTYPE for non-string keys} { + r del condition{t} destination{t} + r lpush condition{t} value + r multi + r set destination{t} not-committed + assert_error {EXECABORT*WRONGTYPE*} {r exec ifeq condition{t} value} + r multi + r set destination{t} not-committed + assert_error {EXECABORT*WRONGTYPE*} {r exec ifne condition{t} value} + assert_equal {} [r get destination{t}] + } + + test {EXEC condition syntax errors abort the transaction} { + r del condition{t} destination{t} + r set condition{t} value + r multi + r set destination{t} committed + assert_error {EXECABORT*invalid check condition syntax*} {r exec ifeq condition{t}} + list [r ping] [r get destination{t}] + } {PONG {}} + + test {EXEC condition syntax error logs to MONITOR only once} { + set rd [valkey_deferring_client] + $rd monitor + assert_match {*OK*} [$rd read] + r multi + r set destination{t} committed + assert_error {EXECABORT*invalid check condition syntax*} {r exec ifeq condition{t}} + r ping + set m1 [$rd read] + set m2 [$rd read] + set m3 [$rd read] + $rd close + assert_match {*"multi"*} $m1 + assert_match {*"exec" "ifeq"*} $m2 + assert_match {*"ping"*} $m3 + } + + test {EXEC skips conditions when WATCH already aborted the transaction} { + r del watched{t} condition{t} destination{t} + r set watched{t} value + r lpush condition{t} value + r watch watched{t} + r set watched{t} changed + r multi + r set destination{t} should-not-execute + list [r exec ifeq condition{t} value] [r get destination{t}] + } {{} {}} + + test {EXEC skips conditions when queueing already aborted the transaction} { + r del condition{t} destination{t} + r lpush condition{t} value + r multi + catch {r non-existing-command} + r set destination{t} should-not-execute + assert_error {EXECABORT*} {r exec ifeq condition{t} value} + assert_equal {} [r get destination{t}] + } + test {DISCARD} { r del mylist r rpush mylist a diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl index 41d5c2354..780660b19 100644 --- a/tests/unit/other.tcl +++ b/tests/unit/other.tcl @@ -240,17 +240,6 @@ start_server {tags {"other"}} { } } - test {BGSAVE} { - # Use FLUSHALL instead of FLUSHDB, FLUSHALL do a foreground save - # and reset the dirty counter to 0, so we won't trigger an unexpected bgsave. - r flushall - r save - r set x 10 - r bgsave - waitForBgsave r - r debug reload - r get x - } {10} {needs:debug needs:save} test {SELECT an out of range DB} { catch {r select 1000000} err @@ -311,21 +300,6 @@ start_server {tags {"other"}} { } {1} {needs:debug} } - test {EXPIRES after a reload (snapshot + append only file rewrite)} { - r flushdb - r set x 10 - r expire x 1000 - r save - r debug reload - set ttl [r ttl x] - set e1 [expr {$ttl > 900 && $ttl <= 1000}] - r bgrewriteaof - waitForBgrewriteaof r - r debug loadaof - set ttl [r ttl x] - set e2 [expr {$ttl > 900 && $ttl <= 1000}] - list $e1 $e2 - } {1 1} {needs:debug needs:save} test {EXPIRES after AOF reload (without rewrite)} { r flushdb @@ -778,3 +752,40 @@ if {$::verbose} { } close $tempFileId file delete $tempFileName + +start_server {overrides {forkless-infrastructure-enabled yes} tags {"other" "external:skip"}} { + foreach bgsave_type {"fork" "forkless"} { + test "BGSAVE $bgsave_type" { + r flushall + r save + r set x 10 + r config set bgsave-default-method $bgsave_type + r bgsave + waitForBgsave r + + set expected_type [expr {$bgsave_type eq "forkless" ? "forkless" : "fork"}] + assert_equal [s rdb_last_bgsave_type] $expected_type + + r debug reload + r get x + } {10} {needs:debug needs:save} + + test "EXPIRES after a reload ($bgsave_type snapshot + append only file rewrite)" { + r flushdb + r set x 10 + r expire x 1000 + r config set bgsave-default-method $bgsave_type + r bgsave + waitForBgsave r + r debug reload + set ttl [r ttl x] + set e1 [expr {$ttl > 900 && $ttl <= 1000}] + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set ttl [r ttl x] + set e2 [expr {$ttl > 900 && $ttl <= 1000}] + list $e1 $e2 + } {1 1} {needs:debug needs:save} + } +} diff --git a/tests/unit/qos.tcl b/tests/unit/qos.tcl new file mode 100644 index 000000000..ccef428d6 --- /dev/null +++ b/tests/unit/qos.tcl @@ -0,0 +1,759 @@ +start_server {tags {"qos"}} { + # Helper to get the IP address of the current test client as seen by the server. + proc get_current_client_ip {} { + set my_id [r client id] + set client_list [r client list] + foreach line [split $client_list "\n"] { + if {[regexp "id=$my_id " $line]} { + if {[regexp {addr=([^ ]+)} $line -> my_addr]} { + # my_addr is ip:port or [ip]:port + if {[string match "*\[*" $my_addr]} { + regexp {\[([^\]]+)\]} $my_addr -> my_ip + } else { + set my_ip [lindex [split $my_addr ":"] 0] + } + return $my_ip + } + } + } + error "Could not find current client IP" + } + + proc get_current_client_ip_with_mask {} { + set ip [get_current_client_ip] + if {[string match "*:*" $ip]} { + return "$ip/128" + } else { + return "$ip/32" + } + } + + # Helper to match connection rejection error across plain, cluster, and TLS modes. + # In cluster mode, the error string is "-ERR max number of clients + cluster connections reached". + # In standalone mode, the error string is "-ERR max number of clients reached". + # In TLS mode, connection drops before handshake completing, resulting in I/O error. + proc get_maxclients_error_pattern {} { + if {$::tls} { + return "*I/O error*" + } else { + return "*max number of clients*reached*" + } + } + + proc can_bind_loopback_ip {ip} { + if {[catch { + set s [socket -myaddr $ip [srv 0 "host"] [srv 0 "port"]] + close $s + }] == 0} { + return 1 + } + return 0 + } + + proc valkey_from_ip {myaddr server port {defer 0}} { + if {$::tls} { + package require tls + ::tls::init \ + -cafile "$::tlsdir/ca.crt" \ + -certfile "$::tlsdir/client.crt" \ + -keyfile "$::tlsdir/client.key" + set fd [::tls::socket -myaddr $myaddr $server $port] + } else { + set fd [socket -myaddr $myaddr $server $port] + } + fconfigure $fd -translation binary + set id [incr ::valkey::id] + set ::valkey::fd($id) $fd + set ::valkey::addr($id) [list $server $port] + set ::valkey::blocking($id) 1 + set ::valkey::deferred($id) $defer + set ::valkey::readraw($id) 0 + set ::valkey::reconnect($id) 0 + set ::valkey::curr_argv($id) 0 + set ::valkey::testing_resp3($id) 0 + set ::valkey::tls($id) $::tls + ::valkey::valkey_reset_state $id + interp alias {} ::valkey::valkeyHandle$id {} ::valkey::__dispatch__ $id + } + + proc valkey_deferring_client_from_ip {myaddr} { + set client [valkey_from_ip $myaddr [srv 0 "host"] [srv 0 "port"] 1] + if {!$::singledb} { + $client select 9 + $client read + } else { + $client echo goodday + $client read + } + return $client + } + + # Save original configs for global restoration + set global_old_maxclients [lindex [r config get maxclients] 1] + set global_old_maxclients_reserved [lindex [r config get maxclients-reserved] 1] + set global_old_priority_subnets [lindex [r config get priority-subnets] 1] + + set qos_test_script_err "" + set qos_test_script_status [catch { + + test {CONFIG SET / GET priority-subnets} { + r config set priority-subnets "127.0.0.1/32 10.0.0.0/8" + assert_equal {127.0.0.1/32 10.0.0.0/8} [lindex [r config get priority-subnets] 1] + + r config set priority-subnets "::1/128,2001:db8::/32" + assert_equal {::1/128,2001:db8::/32} [lindex [r config get priority-subnets] 1] + + r config set priority-subnets "127.0.0.1 ::1" + assert_equal {127.0.0.1 ::1} [lindex [r config get priority-subnets] 1] + + r config set priority-subnets "" + assert_equal {} [lindex [r config get priority-subnets] 1] + } + + test {CONFIG SET priority-subnets invalid inputs} { + catch {r config set priority-subnets "127.0.0.1/99"} err + assert_match "*Invalid IP address or CIDR subnet*" $err + + catch {r config set priority-subnets "invalid/24"} err + assert_match "*Invalid IP address or CIDR subnet*" $err + } + + test {CONFIG SET / GET maxclients-reserved} { + r config set maxclients-reserved 100 + assert_equal 100 [lindex [r config get maxclients-reserved] 1] + + r config set maxclients-reserved 200 + assert_equal 200 [lindex [r config get maxclients-reserved] 1] + + catch {r config set maxclients-reserved -1} err + assert_match "*argument must be*" $err + + # Values >= maxclients can be set without order dependency + set cur_maxclients [lindex [r config get maxclients] 1] + r config set maxclients-reserved [expr {$cur_maxclients + 100}] + assert_equal [expr {$cur_maxclients + 100}] [lindex [r config get maxclients-reserved] 1] + + # maxclients can also be changed freely regardless of maxclients-reserved + r config set maxclients 10 + assert_equal 10 [lindex [r config get maxclients] 1] + + r config set maxclients $cur_maxclients + r config set maxclients-reserved 0 + } + + test {Admission control with maxclients-reserved and priority-subnets} { + # Current active clients = 1 (r). + # Set maxclients to 4 (allows 4 total connections). + # Set maxclients-reserved to 2. + # Normal client threshold = 4 - 2 = 2 (r + 1 normal client). + r config set maxclients 4 + r config set maxclients-reserved 2 + set my_ip_mask [get_current_client_ip_with_mask] + # Without priority-subnets, reservation is inactive. + r config set priority-subnets "" + + # Connect 1 normal client (total normal = 2: r + c1) + set c1 [valkey_deferring_client] + $c1 client id + set c1_id [$c1 read] + + # Enable priority by specifying priority-subnets for loopback. + # Dynamic re-classification immediately promotes all existing clients matching + # the subnet (r and c1) to prioritized status. + r config set priority-subnets $my_ip_mask + + # With reservation active, total clients = 2 (r + c1). + # Normal client ceiling is maxclients - reserved = 4 - 2 = 2. + # Since loopback is now prioritized, connections from loopback will be prioritized. + # Connect prioritized client p1 (total = 3 < 4 maxclients) - succeeds! + set p1 [valkey_deferring_client] + $p1 client id + set p1_id [$p1 read] + $p1 ping + assert_equal {PONG} [$p1 read] + + # Connect prioritized client p2 (total = 4 = maxclients) - succeeds! + set p2 [valkey_deferring_client] + $p2 client id + set p2_id [$p2 read] + $p2 ping + assert_equal {PONG} [$p2 read] + + # Verify INFO clients metrics: all 4 clients (r, c1, p1, p2) are prioritized + set info_clients [r info clients] + assert_match "*connected_priority_clients:4*" $info_clients + + # Close p1 and verify active prioritized count decrements to 3 + $p1 close + wait_for_condition 50 100 { + [string match "*connected_priority_clients:3*" [r info clients]] + } else { + fail "connected_priority_clients did not decrement to 3 after closing p1" + } + + # Close p2 and verify active prioritized count decrements to 2 + $p2 close + wait_for_condition 50 100 { + [string match "*connected_priority_clients:2*" [r info clients]] + } else { + fail "connected_priority_clients did not decrement to 2 after closing p2" + } + + # Close c1 and verify active prioritized count decrements to 1 (only r remains) + catch {$c1 close} + wait_for_condition 50 100 { + [string match "*connected_priority_clients:1*" [r info clients]] + } else { + fail "connected_priority_clients did not decrement to 1 after closing c1" + } + + # Clearing priority-subnets dynamically demotes r to normal + r config set priority-subnets "" + set info_clients [r info clients] + assert_match "*connected_priority_clients:0*" $info_clients + + r config set maxclients-reserved 0 + } + + test {Admission control when maxclients-reserved >= maxclients} { + r config set maxclients 3 + r config set maxclients-reserved 10 + # Set subnet to an unrelated IP so loopback is normal (non-prioritized) + r config set priority-subnets "192.0.2.1/32" + + # Active clients: r (1). Since reserved (10) >= maxclients (3), + # normal limit is clamped to 0. Any new normal connection must be rejected. + set expected_code [get_maxclients_error_pattern] + catch { + set c_normal [valkey_deferring_client] + $c_normal ping + $c_normal read + } err_normal + assert_match $expected_code $err_normal + + # Now set priority-subnets to current client IP (making loopback prioritized) + r config set priority-subnets [get_current_client_ip_with_mask] + + # Prioritized client should connect successfully (active: r + p1 = 2 <= 3) + set p1 [valkey_deferring_client] + $p1 ping + assert_equal {PONG} [$p1 read] + + # Second prioritized client connects successfully (active: r + p1 + p2 = 3 <= 3) + set p2 [valkey_deferring_client] + $p2 ping + assert_equal {PONG} [$p2 read] + + # Third prioritized client rejected at maxclients ceiling (3) + set expected_p_code [get_maxclients_error_pattern] + catch { + set p3 [valkey_deferring_client] + $p3 ping + $p3 read + } err_p3 + assert_match $expected_p_code $err_p3 + + catch {$p1 close} + catch {$p2 close} + r config set maxclients-reserved 0 + r config set priority-subnets "" + } + + test {Maxclients ceiling rejection with rejected_priority_connections stat} { + r config set maxclients 3 + r config set maxclients-reserved 1 + r config set priority-subnets [get_current_client_ip_with_mask] + + # Active clients: r (1) + p1 (1) + p2 (1) = 3 (reaches maxclients) + set p1 [valkey_deferring_client] + $p1 ping + assert_equal {PONG} [$p1 read] + + set p2 [valkey_deferring_client] + $p2 ping + assert_equal {PONG} [$p2 read] + + r config resetstat + + # 3rd prioritized client should fail because total reached maxclients (3) + set expected_code [get_maxclients_error_pattern] + catch { + set p3 [valkey_deferring_client] + $p3 ping + $p3 read + } err_p3 + assert_match $expected_code $err_p3 + + # Verify INFO stats contains rejected_connections:1 and rejected_priority_connections:1 + set info_stats [r info stats] + assert_match "*rejected_connections:1*" $info_stats + assert_match "*rejected_priority_connections:1*" $info_stats + + catch {$p1 close} + catch {$p2 close} + r config set maxclients-reserved 0 + r config set priority-subnets "" + } + + test {Admission control with comma-separated priority-subnets} { + r config set maxclients 10 + r config set maxclients-reserved 2 + + # Test comma-separated list (with and without space) + set client_ip_mask [get_current_client_ip_with_mask] + r config set priority-subnets "$client_ip_mask,1.1.1.1/32" + + set p1 [valkey_deferring_client] + $p1 client id + set p1_id [$p1 read] + $p1 ping + assert_equal {PONG} [$p1 read] + + # r (1) + p1 (1) = 2 prioritized clients due to dynamic re-classification + assert_match "*connected_priority_clients:2*" [r info clients] + + catch {$p1 close} + + # Test mixed comma and space list + r config set priority-subnets "$client_ip_mask, 1.1.1.1/32" + + set p2 [valkey_deferring_client] + $p2 client id + set p2_id [$p2 read] + $p2 ping + assert_equal {PONG} [$p2 read] + + assert_match "*connected_priority_clients:2*" [r info clients] + + catch {$p2 close} + r config set maxclients-reserved 0 + r config set priority-subnets "" + } + + test {Admission control with raw IP priority-subnets} { + r config set maxclients 10 + r config set maxclients-reserved 2 + r config set priority-subnets "[get_current_client_ip] 1.1.1.1" + + set p1 [valkey_deferring_client] + $p1 client id + set p1_id [$p1 read] + $p1 ping + assert_equal {PONG} [$p1 read] + + # r (1) + p1 (1) = 2 prioritized clients + assert_match "*connected_priority_clients:2*" [r info clients] + + catch {$p1 close} + r config set maxclients-reserved 0 + r config set priority-subnets "" + } + + test {Priority clients exceeding reservation do not starve normal clients} { + r config set maxclients 10 + r config set maxclients-reserved 5 + + # If system supports binding to 127.0.0.2, test multi-subnet priority separation + if {[can_bind_loopback_ip "127.0.0.2"]} { + r config set priority-subnets "127.0.0.2/32" + + set prio_clients {} + for {set i 0} {$i < 6} {incr i} { + set p [valkey_deferring_client_from_ip "127.0.0.2"] + $p ping + assert_equal {PONG} [$p read] + lappend prio_clients $p + } + + # Active: r (normal from 127.0.0.1, 1) + 6 prioritized (from 127.0.0.2) = 7 total. + assert_match "*connected_priority_clients:6*" [r info clients] + + # Normal limit is 10 - 5 = 5. + # Current normal clients = 1 (r). Normal quota has 4 slots available. + # Normal clients from 127.0.0.1 can connect despite priority clients exceeding reserved. + set c1 [valkey_deferring_client] + $c1 ping + assert_equal {PONG} [$c1 read] + + set c2 [valkey_deferring_client] + $c2 ping + assert_equal {PONG} [$c2 read] + + # Normal clients = 3 (r + c1 + c2), prioritized = 6, total = 9 <= 10 + assert_match "*connected_priority_clients:6*" [r info clients] + + catch {$c1 close} + catch {$c2 close} + foreach p $prio_clients { + catch {$p close} + } + } + r config set maxclients-reserved 0 + r config set priority-subnets "" + } + + test {Dynamic Re-Classification of connected clients on priority-subnets change} { + if {[can_bind_loopback_ip "127.0.0.2"]} { + r config set maxclients 10 + r config set maxclients-reserved 3 + r config set priority-subnets "" + + # Step 1: Initial state without priority subnets + set c_normal [valkey_deferring_client] + $c_normal ping + assert_equal {PONG} [$c_normal read] + + set c_alt [valkey_deferring_client_from_ip "127.0.0.2"] + $c_alt ping + assert_equal {PONG} [$c_alt read] + + # r, c_normal (127.0.0.1) and c_alt (127.0.0.2) are all normal clients + assert_match "*connected_priority_clients:0*" [r info clients] + + # Step 2: Configure priority-subnets to 127.0.0.2/32 + # Dynamic re-classification immediately promotes c_alt to prioritized; + # r and c_normal remain normal. + r config set priority-subnets "127.0.0.2/32" + assert_match "*connected_priority_clients:1*" [r info clients] + + # Step 3: Switch priority-subnets to 127.0.0.1/32 + # Dynamic re-classification immediately promotes r and c_normal to prioritized, + # and demotes c_alt back to normal. + r config set priority-subnets "127.0.0.1/32" + assert_match "*connected_priority_clients:2*" [r info clients] + + # Step 4: Include both subnets in priority-subnets + # Dynamic re-classification promotes all 3 clients to prioritized. + r config set priority-subnets "127.0.0.1/32 127.0.0.2/32" + assert_match "*connected_priority_clients:3*" [r info clients] + + # Step 5: Disconnect c_alt; prioritized count decrements to 2 + $c_alt close + wait_for_condition 50 100 { + [string match "*connected_priority_clients:2*" [r info clients]] + } else { + fail "connected_priority_clients did not decrement to 2 after closing c_alt" + } + + # Step 6: Clear priority-subnets; remaining clients demoted to normal + r config set priority-subnets "" + assert_match "*connected_priority_clients:0*" [r info clients] + + # Step 7: Disconnect c_normal; ensures no underflow desync + $c_normal close + wait_for_condition 50 100 { + [string match "*connected_priority_clients:0*" [r info clients]] + } else { + fail "connected_priority_clients did not remain 0 after closing c_normal" + } + + r config set maxclients-reserved 0 + } + } + + test {Normal clients rejected when normal quota exhausted, while priority clients still connect} { + r config set maxclients 6 + r config set maxclients-reserved 3 + + # Loopback is normal + r config set priority-subnets "192.0.2.1/32" + + # Normal limit is 6 - 3 = 3. + # Currently 1 normal client (r). + set c1 [valkey_deferring_client] + $c1 ping + assert_equal {PONG} [$c1 read] + + set c2 [valkey_deferring_client] + $c2 ping + assert_equal {PONG} [$c2 read] + + # Now normal clients = 3 (r + c1 + c2) == normal_limit (3). + # Total clients = 3 < maxclients (6). + # A new normal client must be rejected because normal quota is full. + set expected_code [get_maxclients_error_pattern] + catch { + set c3 [valkey_deferring_client] + $c3 ping + $c3 read + } err_c3 + assert_match $expected_code $err_c3 + + # Switch loopback to priority: prioritized clients can still connect into reserved slots + r config set priority-subnets [get_current_client_ip_with_mask] + + set p1 [valkey_deferring_client] + $p1 ping + assert_equal {PONG} [$p1 read] + + set p2 [valkey_deferring_client] + $p2 ping + assert_equal {PONG} [$p2 read] + + set p3 [valkey_deferring_client] + $p3 ping + assert_equal {PONG} [$p3 read] + + # Total is now 6 == maxclients. A 4th priority client should fail at global ceiling. + catch { + set p4 [valkey_deferring_client] + $p4 ping + $p4 read + } err_p4 + assert_match $expected_code $err_p4 + + catch {$c1 close} + catch {$c2 close} + catch {$p1 close} + catch {$p2 close} + catch {$p3 close} + r config set maxclients-reserved 0 + r config set priority-subnets "" + } + + test {Normal client rejected when total reaches maxclients even if normal quota has headroom} { + r config set maxclients 5 + r config set maxclients-reserved 3 + + # Normal limit = 5 - 3 = 2. + # Active normal: r (1 < 2). Normal quota has 1 headroom slot. + # Fill total capacity to maxclients (5) using priority clients. + r config set priority-subnets [get_current_client_ip_with_mask] + + set prio_clients {} + for {set i 0} {$i < 4} {incr i} { + set p [valkey_deferring_client] + $p ping + assert_equal {PONG} [$p read] + lappend prio_clients $p + } + + # Total clients = 1 (r) + 4 (prioritized) = 5 == maxclients. + # Now switch loopback to normal. + r config set priority-subnets "192.0.2.1/32" + + set expected_code [get_maxclients_error_pattern] + catch { + set c1 [valkey_deferring_client] + $c1 ping + $c1 read + } err_c1 + assert_match $expected_code $err_c1 + + # Priority client should also be rejected because total == maxclients + r config set priority-subnets [get_current_client_ip_with_mask] + catch { + set p5 [valkey_deferring_client] + $p5 ping + $p5 read + } err_p5 + assert_match $expected_code $err_p5 + + foreach p $prio_clients { + catch {$p close} + } + r config set maxclients-reserved 0 + r config set priority-subnets "" + } + + test {Dynamic reconfiguration of maxclients-reserved and priority-subnets} { + r config set maxclients 5 + r config set maxclients-reserved 0 + r config set priority-subnets "" + + # Connect 3 normal clients without QoS (active: r + 3 = 4 < 5) + set c1 [valkey_deferring_client] + $c1 ping + assert_equal {PONG} [$c1 read] + + set c2 [valkey_deferring_client] + $c2 ping + assert_equal {PONG} [$c2 read] + + set c3 [valkey_deferring_client] + $c3 ping + assert_equal {PONG} [$c3 read] + + # Dynamically set maxclients-reserved 3 (normal_limit = 5 - 3 = 2) + # Existing 4 normal clients (r + c1..c3) exceed normal_limit (2). + # Existing clients must continue working without disruption. + r config set priority-subnets "192.0.2.1/32" + r config set maxclients-reserved 3 + + $c1 ping + assert_equal {PONG} [$c1 read] + $c2 ping + assert_equal {PONG} [$c2 read] + $c3 ping + assert_equal {PONG} [$c3 read] + + # New normal connection must be rejected because normal_clients (4) >= normal_limit (2) + set expected_code [get_maxclients_error_pattern] + catch { + set c4 [valkey_deferring_client] + $c4 ping + $c4 read + } err_c4 + assert_match $expected_code $err_c4 + + # Dynamically change maxclients-reserved to 0: reservation disabled + r config set maxclients-reserved 0 + + # Now normal client can connect since total (4) < maxclients (5) + set c5 [valkey_deferring_client] + $c5 ping + assert_equal {PONG} [$c5 read] + + catch {$c1 close} + catch {$c2 close} + catch {$c3 close} + catch {$c5 close} + r config set maxclients-reserved 0 + r config set priority-subnets "" + } + + test {Zero maxclients-reserved or empty priority-subnets disables reservation} { + r config set maxclients 4 + + # Case 1: priority-subnets configured, but maxclients-reserved is 0 + r config set maxclients-reserved 0 + r config set priority-subnets "192.0.2.1/32" + + # Active: r (1). Connect 3 normal clients up to maxclients (4) + set c1 [valkey_deferring_client] + $c1 ping + assert_equal {PONG} [$c1 read] + set c2 [valkey_deferring_client] + $c2 ping + assert_equal {PONG} [$c2 read] + set c3 [valkey_deferring_client] + $c3 ping + assert_equal {PONG} [$c3 read] + + # 4th normal connection fails at maxclients + set expected_code [get_maxclients_error_pattern] + catch { + set c4 [valkey_deferring_client] + $c4 ping + $c4 read + } err_c4 + assert_match $expected_code $err_c4 + + catch {$c1 close} + catch {$c2 close} + catch {$c3 close} + + # Case 2: maxclients-reserved > 0, but priority-subnets is empty + r config set maxclients-reserved 2 + r config set priority-subnets "" + + set c1 [valkey_deferring_client] + $c1 ping + assert_equal {PONG} [$c1 read] + set c2 [valkey_deferring_client] + $c2 ping + assert_equal {PONG} [$c2 read] + set c3 [valkey_deferring_client] + $c3 ping + assert_equal {PONG} [$c3 read] + + catch { + set c4 [valkey_deferring_client] + $c4 ping + $c4 read + } err_c4 + assert_match $expected_code $err_c4 + + catch {$c1 close} + catch {$c2 close} + catch {$c3 close} + r config set maxclients-reserved 0 + r config set priority-subnets "" + } + + } qos_test_script_err] + + # Restore global configs + r config set maxclients $global_old_maxclients + r config set maxclients-reserved $global_old_maxclients_reserved + r config set priority-subnets $global_old_priority_subnets + + if {$qos_test_script_status != 0} { + error $qos_test_script_err $::errorInfo + } +} + +start_server {tags {"qos external:skip"} overrides {priority-subnets {"127.0.0.0/8,::1/128"} maxclients 5 maxclients-reserved 2}} { + test {Priority subnets configured on startup enable priority admission} { + assert_match "*connected_priority_clients:1*" [r info clients] + set c1 [valkey_client] + assert_match "*connected_priority_clients:2*" [r info clients] + $c1 close + wait_for_condition 50 100 { + [string match "*connected_priority_clients:1*" [r info clients]] + } else { + fail "connected_priority_clients did not decrement to 1 after closing c1" + } + } +} + +start_server {tags {"qos external:skip"}} { + test {CONFIG REWRITE and reload persists priority-subnets and maxclients-reserved} { + # Configure non-default settings + r config set priority-subnets "127.0.0.0/8,::1/128" + r config set maxclients-reserved 10 + r config set maxclients 20 + + # Verify initial values + assert_equal {127.0.0.0/8,::1/128} [lindex [r config get priority-subnets] 1] + assert_equal 10 [lindex [r config get maxclients-reserved] 1] + assert_equal 20 [lindex [r config get maxclients] 1] + + # Trigger CONFIG REWRITE to write changes to valkey.conf on disk + assert_equal "OK" [r config rewrite] + set config_file [srv 0 config_file] + assert_equal 1 [count_message_lines $config_file "priority-subnets"] + assert_equal 1 [count_message_lines $config_file "maxclients-reserved"] + + # Restart server to reload configuration from disk + restart_server 0 true false + + # Verify configuration is reloaded accurately from disk + assert_equal {127.0.0.0/8,::1/128} [lindex [r config get priority-subnets] 1] + assert_equal 10 [lindex [r config get maxclients-reserved] 1] + assert_equal 20 [lindex [r config get maxclients] 1] + + # Verify priority admission control functions properly after config reload + assert_match "*connected_priority_clients:1*" [r info clients] + set c1 [valkey_client] + assert_match "*connected_priority_clients:2*" [r info clients] + $c1 close + + # Reset to default (empty priority-subnets and 0 maxclients-reserved) and rewrite again + r config set priority-subnets "" + r config set maxclients-reserved 0 + assert_equal "OK" [r config rewrite] + + # Restart server to verify clearing config persists after reload + restart_server 0 true false + assert_equal {} [lindex [r config get priority-subnets] 1] + assert_equal 0 [lindex [r config get maxclients-reserved] 1] + assert_match "*connected_priority_clients:0*" [r info clients] + + # Verify debug config-rewrite-force-all rewrite and reload + r config set priority-subnets "10.0.0.0/8" + r config set maxclients-reserved 5 + assert_equal [r debug config-rewrite-force-all] "OK" + restart_server 0 true false + assert_equal {10.0.0.0/8} [lindex [r config get priority-subnets] 1] + assert_equal 5 [lindex [r config get maxclients-reserved] 1] + + # Clean up + r config set priority-subnets "" + r config set maxclients-reserved 0 + r config rewrite + } +} + + + diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 4399209c5..b39e3ba83 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -1885,7 +1885,6 @@ start_server {tags {"scripting external:skip"}} { test {Lua scripts promoted from eval to script load} { r script flush - r config resetstat r eval "return 'hello world'" 0 set sha [r script load "return 'hello world'"] @@ -1895,6 +1894,168 @@ start_server {tags {"scripting external:skip"}} { } assert_equal {hello world} [r evalsha $sha 0] } + + test {Lua scripts memory for LRU script SHA copies} { + r script flush + + # Perform 500 EVAL cycles, then use script to load the same data to + # discard all LRU list nodes. + for {set j 1} {$j <= 500} {incr j} { + r eval "return $j" 0 + } + set mem_before [s used_memory_scripts_eval] + for {set j 1} {$j <= 500} {incr j} { + r script load "return $j" + } + set mem_after [s used_memory_scripts_eval] + + # Each script differs by at least 40 bytes SHA + 24 (or 12) bytes listNode. + set arch_bits [s arch_bits] + set diff [expr $mem_before - $mem_after] + if {$arch_bits == 64} { + assert_morethan $diff [expr 64 * 500] + } elseif {$arch_bits == 32} { + assert_morethan $diff [expr 52 * 500] + } + } + + test {maxmemory-scripts does not change the EVAL script count limit} { + r script flush sync + r config resetstat + r config set maxmemory-scripts 100MB + + for {set j 1} {$j <= 501} {incr j} { + assert_equal $j [r eval "return $j" 0] + } + assert_equal 500 [s number_of_cached_scripts] + assert_equal 1 [s evicted_scripts] + + r config set maxmemory-scripts 0 + } + + test {maxmemory-scripts will trigger eviction during large EVAL scripts} { + r script flush sync + r config resetstat + r config set maxmemory-scripts 1MB + + set padding [string repeat x 100000] + for {set j 1} {$j <= 500} {incr j} { + assert_equal $j [r eval "--$padding\nreturn $j" 0] + } + assert_morethan [s evicted_scripts] 0 + assert_lessthan [s number_of_cached_scripts] 500 + + r config set maxmemory-scripts 0 + } + + test {maxmemory-scripts set in runtime will trigger eviction} { + r script flush sync + r config resetstat + + set padding [string repeat x 100000] + for {set j 1} {$j <= 500} {incr j} { + assert_equal $j [r eval "--$padding\nreturn $j" 0] + } + assert_equal 500 [s number_of_cached_scripts] + + r config set maxmemory-scripts 1MB + wait_for_condition 1000 10 { + [s evicted_scripts] > 0 && + [s number_of_cached_scripts] < 500 + } else { + fail "scripts eviction did not start in time" + } + + r config set maxmemory-scripts 0 + } + + test {maxmemory-scripts only evicts EVAL scripts} { + r script flush sync + r config resetstat + r config set maxmemory-scripts 1MB + + set padding [string repeat x 100000] + set shas {} + for {set j 1} {$j <= 500} {incr j} { + set sha [r script load "--$padding\nreturn $j"] + lappend shas $sha + assert_equal $j [r evalsha $sha 0] + } + assert_equal 500 [s number_of_cached_scripts] + assert_equal 0 [s evicted_scripts] + + for {set j 1001} {$j <= 1500} {incr j} { + assert_equal $j [r eval "--$padding\nreturn $j" 0] + } + assert_morethan [s evicted_scripts] 0 + + for {set j 1} {$j <= 500} {incr j} { + set sha [lindex $shas [expr {$j - 1}]] + assert_equal $j [r evalsha $sha 0] + } + + r config set maxmemory-scripts 0 + } + + test {The combination of maxmemory-scripts and maxmemory} { + r script flush sync + r config resetstat + + # maxmemory-scripts percentage value is not working if maxmemory is 0 + r config set maxmemory 0 + r config set maxmemory-scripts 1% + set padding [string repeat x 100000] + for {set j 1} {$j <= 500} {incr j} { + assert_equal $j [r eval "--$padding\nreturn $j" 0] + } + assert_equal 500 [s number_of_cached_scripts] + assert_equal 0 [s evicted_scripts] + + r config set maxmemory 100MB + wait_for_condition 1000 10 { + [s evicted_scripts] > 0 && + [s number_of_cached_scripts] < 500 + } else { + fail "scripts eviction did not start in time" + } + + r config set maxmemory 0 + r config set maxmemory-scripts 0 + } + + test {SCRIPT LOAD returns OOM after evicting keys} { + r flushall sync + r script flush sync + r config resetstat + r config set maxmemory 0 + r config set maxmemory-policy allkeys-random + + set value [string repeat x [expr 1024 * 1024]] + for {set j 0} {$j < 50} {incr j} { + r set "script-load-key:$j" $value + } + + set used [expr {[s used_memory] - [s mem_not_counted_for_evict]}] + set limit [expr {$used + 10*1024}] + r config set maxmemory $limit + + set padding [string repeat x 100000] + for {set j 1} {$j <= 5000} {incr j} { + catch {r script load "--$padding\nreturn $j"} e + if {[string match "OOM *" $e]} { + break + } + } + r config set maxmemory 1 + assert_error {OOM command not allowed*} {r script load "--$padding\nreturn 0"} + assert_morethan [s evicted_keys] 0 + assert_equal 0 [s evicted_scripts] + assert_morethan [s number_of_cached_scripts] 0 + assert_lessthan [s number_of_cached_scripts] 5000 + + r config set maxmemory 0 + r config set maxmemory-policy noeviction + } } } ;# is_eval diff --git a/tests/unit/socket-prioritization.tcl b/tests/unit/socket-prioritization.tcl new file mode 100644 index 000000000..700156d30 --- /dev/null +++ b/tests/unit/socket-prioritization.tcl @@ -0,0 +1,184 @@ +# Copyright (c) Valkey Contributors +# All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +start_server {tags {"socket-prioritization"}} { + test {Socket Prioritization: CLIENT LIST and CLIENT KILL QOS filter} { + set c1 [valkey_client] + $c1 client setname qostestclient + + # By default client is normal priority + set res [r client list not-flags H name qostestclient] + assert_match "*name=qostestclient*flags=N*" $res + + set res_high [r client list flags H name qostestclient] + assert_equal "" $res_high + + # Test invalid flags argument + assert_error "*Unknown flags*" {r client list flags invalid_flag} + + # Kill client by flags + set killed [r client kill not-flags H name qostestclient] + assert_equal 1 $killed + assert_error "*I/O error*" {$c1 ping} + catch {$c1 close} + } + + test {Socket Prioritization: Replication QoS classification and CLIENT KILL} { + # Start a replica server and verify that replication links are upgraded to QoS priority + start_server {} { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + set primary [srv -1 client] + set primary_host [srv -1 host] + set primary_port [srv -1 port] + + # Connect replica to primary + $replica replicaof $primary_host $primary_port + wait_for_condition 50 100 { + [string match "*role:slave*master_link_status:up*" [$replica info replication]] + } else { + fail "Can't turn the instance into a replica" + } + + # On primary server, verify that a client connection has flags=H for the replica + set rep_list [$primary client list flags H] + assert_match "*flags=*H*" $rep_list + + set val [string repeat "a" 1024] + for {set i 0} {$i < 50} {incr i} { + $primary set "key:$i" $val + } + $primary ping + + # Wait for replica to sync the keys + wait_for_condition 50 100 { + [$replica dbsize] == 50 + } else { + fail "Replica failed to sync 50 keys" + } + + # Verify CLIENT KILL flags H kills the replica connection + set killed [$primary client kill flags H] + assert {$killed >= 1} + } + } {} {external:skip} + + foreach io_threads {1 4} { + test "Verify replication connection upgrade (io-threads=$io_threads)" { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + $primary CONFIG SET io-threads $io_threads + + set replica_mock [valkey $primary_host $primary_port 0 $::tls] + $replica_mock client setname replica_mock_$io_threads + + set res [$primary client list not-flags H name replica_mock_$io_threads] + assert_match "*name=replica_mock_$io_threads*flags=N*" $res + assert_equal "" [$primary client list flags H name replica_mock_$io_threads] + + $replica_mock write "PSYNC ? -1\r\n" + $replica_mock flush + + wait_for_condition 50 100 { + [string match "*name=replica_mock_${io_threads}*flags=*H*" [$primary client list flags H name replica_mock_$io_threads]] + } else { + fail "Replica connection was not upgraded to QoS priority (io-threads=$io_threads)" + } + + $replica_mock close + $primary CONFIG SET io-threads 1 + } + } +} + +start_server {tags {"socket-prioritization external:skip"}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test "Populate standalone primary with 1000 keys" { + for {set k 0} {$k < 1000} {incr k} { + $primary set "benchkey:$k" [string repeat "x" 128] + } + } + + start_server {} { + set replica1 [srv 0 client] + start_server {} { + set replica2 [srv 0 client] + + test "Benchmark standalone replica sync under pipeline load with QoS event loop" { + set load_clients {} + for {set c 0} {$c < 5} {incr c} { + lappend load_clients [valkey $primary_host $primary_port 0 $::tls] + } + + set val [string repeat "x" 256] + set pipeline "" + for {set p 0} {$p < 20} {incr p} { + append pipeline "*3\r\n\$3\r\nSET\r\n\$7\r\npipekey\r\n\$256\r\n$val\r\n" + } + + $replica1 replicaof $primary_host $primary_port + $replica2 replicaof $primary_host $primary_port + + set total_ops 0 + for {set iter 0} {$iter < 10} {incr iter} { + foreach cl $load_clients { + catch { + $cl write $pipeline + $cl flush + incr total_ops 20 + for {set r 0} {$r < 20} {incr r} { $cl read } + } + } + } + + wait_for_condition 100 50 { + [status $primary connected_slaves] == 2 && + [$replica1 dbsize] == 1000 && + [$replica2 dbsize] == 1000 + } else { + fail "Replicas failed to complete sync during pipelined load" + } + + foreach cl $load_clients { $cl close } + } + } + + test {Dynamic configuration of priority-preemptive-poll-interval-us} { + assert_equal [lindex [r config get priority-preemptive-poll-interval-us] 1] 2000 + r config set priority-preemptive-poll-interval-us 500 + assert_equal [lindex [r config get priority-preemptive-poll-interval-us] 1] 500 + r config set priority-preemptive-poll-interval-us 0 + assert_equal [lindex [r config get priority-preemptive-poll-interval-us] 1] 0 + assert_error "*argument must be between*" {r config set priority-preemptive-poll-interval-us -1} + assert_error "*argument couldn't be parsed into an integer*" {r config set priority-preemptive-poll-interval-us invalid} + r config set priority-preemptive-poll-interval-us 2000 + assert_equal [lindex [r config get priority-preemptive-poll-interval-us] 1] 2000 + } + + test {Socket Prioritization: INFO stats and debug QoS metrics} { + set info_stats [$primary info stats] + set info_debug [$primary info debug] + + assert_morethan [getInfoProperty $info_stats eventloop_priority_cycles] 0 + assert_morethan [getInfoProperty $info_stats eventloop_priority_duration_sum] 0 + assert_morethan [getInfoProperty $info_stats eventloop_priority_duration_cmd_sum] 0 + assert {[getInfoProperty $info_debug eventloop_priority_duration_max] >= 0} + assert {[getInfoProperty $info_debug eventloop_priority_cmd_per_cycle_max] >= 0} + + # Reset stats and verify + $primary config resetstat + set info_stats_reset [$primary info stats] + assert_equal [getInfoProperty $info_stats_reset eventloop_priority_cycles] 0 + assert_equal [getInfoProperty $info_stats_reset eventloop_priority_duration_sum] 0 + assert_equal [getInfoProperty $info_stats_reset eventloop_priority_duration_cmd_sum] 0 + assert_equal [getInfoProperty [$primary info debug] eventloop_priority_cmd_per_cycle_max] 0 + } + } +} diff --git a/tests/unit/sort.tcl b/tests/unit/sort.tcl index 0626aaa02..9241fcb7c 100644 --- a/tests/unit/sort.tcl +++ b/tests/unit/sort.tcl @@ -120,6 +120,22 @@ foreach command {SORT SORT_RO} { r command getkeys sort abc store invalid store stillbad store def } {abc def} + test "SORT extracts STORE correctly when the destination is named like an option" { + # A destination is a key name, never an option keyword, even when it + # spells one. Parsed as an option it would consume the arguments that + # follow it and hide the later STORE that SORT really writes to. + foreach keyword {by get limit} { + assert_equal {abc def} [r command getkeys sort abc store $keyword store def] + assert_equal [list abc $keyword] [r command getkeys sort abc store $keyword] + } + + # A destination spelling STORE would instead be taken for another STORE + # clause, reporting whatever follows it. + assert_equal {abc store} [r command getkeys sort abc store store alpha] + assert_equal {abc STORE} [r command getkeys sort abc store STORE alpha] + assert_equal {abc store} [r command getkeys sort abc store store by w_*] + } + test "SORT DESC" { assert_equal [lsort -decreasing -integer $result] [r sort tosort DESC] } diff --git a/tests/unit/tls.tcl b/tests/unit/tls.tcl index f002a4ab5..b83fc7b95 100644 --- a/tests/unit/tls.tcl +++ b/tests/unit/tls.tcl @@ -127,6 +127,129 @@ start_server {tags {"tls"}} { } } + test {TLS: basic dual certificates support} { + # backup current certificates + set orig_server_crt [lindex [r config get tls-cert-file] 1] + set orig_server_key [lindex [r config get tls-key-file] 1] + set orig_server_alt_crt [lindex [r config get tls-alt-cert-file] 1] + set orig_server_alt_key [lindex [r config get tls-alt-key-file] 1] + set ca_file [lindex [r config get tls-ca-cert-file] 1] + + set valkey_crt [format "%s/tests/tls/valkey.crt" [pwd]] + set valkey_key [format "%s/tests/tls/valkey.key" [pwd]] + set valkey_ec_crt [format "%s/tests/tls/valkey-ec.crt" [pwd]] + set valkey_ec_key [format "%s/tests/tls/valkey-ec.key" [pwd]] + try { + r CONFIG SET tls-cert-file $valkey_ec_crt tls-key-file $valkey_ec_key tls-alt-cert-file $valkey_crt tls-alt-key-file $valkey_key + set s [valkey_client] + assert_equal "PONG" [$s PING] + $s close + # also test connecting with openssl without ec ciphers support + set port [lindex [r config get tls-port] 1] + catch {exec openssl s_client -connect localhost:$port -CAfile $valkey_crt -sigalgs "rsa_pss_pss_sha256:rsa_pss_rsae_sha256" < /dev/null} out + assert_match {*Peer signature type: [rR][sS][aA][-_][pP][sS][sS]*} $out + } finally { + #cleanup + r CONFIG SET tls-cert-file $orig_server_crt tls-key-file $orig_server_key tls-alt-cert-file $orig_server_alt_crt tls-alt-key-file $orig_server_alt_key + } + } + + test {TLS: alt cert and key files must be provided together} { + # backup current certificates + set orig_server_crt [lindex [r config get tls-cert-file] 1] + set orig_server_key [lindex [r config get tls-key-file] 1] + set orig_server_alt_crt [lindex [r config get tls-alt-cert-file] 1] + set orig_server_alt_key [lindex [r config get tls-alt-key-file] 1] + + try { + r CONFIG SET tls-cert-file $orig_server_crt tls-key-file $orig_server_key tls-alt-cert-file "" tls-alt-key-file "" + + catch {r CONFIG SET tls-alt-cert-file $orig_server_crt} e + assert_match {*related to argument 'tls-alt-cert-file'*} $e + catch {r CONFIG SET tls-alt-key-file $orig_server_key} e + assert_match {*related to argument 'tls-alt-key-file'*} $e + } finally { + #cleanup + r CONFIG SET tls-cert-file $orig_server_crt tls-key-file $orig_server_key tls-alt-cert-file $orig_server_alt_crt tls-alt-key-file $orig_server_alt_key + } + } + + test {TLS: the same certificate twice not allowed} { + # backup current certificates + set orig_server_crt [lindex [r config get tls-cert-file] 1] + set orig_server_key [lindex [r config get tls-key-file] 1] + set orig_server_alt_crt [lindex [r config get tls-alt-cert-file] 1] + set orig_server_alt_key [lindex [r config get tls-alt-key-file] 1] + + try { + catch {r CONFIG SET tls-alt-cert-file $orig_server_crt tls-alt-key-file $orig_server_key} e + assert_match {*Unable to update TLS configuration*} $e + } finally { + #cleanup + r CONFIG SET tls-cert-file $orig_server_crt tls-key-file $orig_server_key tls-alt-cert-file $orig_server_alt_crt tls-alt-key-file $orig_server_alt_key + } + } + + test {TLS: Two certificates of the same type not allowed} { + # backup current certificates + set orig_server_crt [lindex [r config get tls-cert-file] 1] + set orig_server_key [lindex [r config get tls-key-file] 1] + set orig_server_alt_crt [lindex [r config get tls-alt-cert-file] 1] + set orig_server_alt_key [lindex [r config get tls-alt-key-file] 1] + + set valkey_crt [format "%s/tests/tls/valkey.crt" [pwd]] + set valkey_key [format "%s/tests/tls/valkey.key" [pwd]] + set valkey_ec_crt [format "%s/tests/tls/valkey-ec.crt" [pwd]] + set valkey_ec_key [format "%s/tests/tls/valkey-ec.key" [pwd]] + set valkey_pw_crt [format "%s/tests/tls/valkey-pw.crt" [pwd]] + set valkey_pw_key [format "%s/tests/tls/valkey-pw.key" [pwd]] + set valkey_ec_pw_crt [format "%s/tests/tls/valkey-ec-pw.crt" [pwd]] + set valkey_ec_pw_key [format "%s/tests/tls/valkey-ec-pw.key" [pwd]] + + try { + r CONFIG SET tls-cert-file $valkey_ec_crt tls-key-file $valkey_ec_key tls-alt-cert-file $valkey_crt tls-alt-key-file $valkey_key + set s [valkey_client] + assert_equal "PONG" [$s PING] + $s close + catch {r CONFIG SET tls-alt-cert-file $valkey_ec_pw_crt tls-alt-key-file $valkey_ec_pw_key tls-key-file-pass 1234} e + assert_match {*Unable to update TLS configuration*} $e + catch {r CONFIG SET tls-cert-file $valkey_pw_crt tls-key-file $valkey_pw_key tls-key-file-pass 1234} e + assert_match {*Unable to update TLS configuration*} $e + } finally { + #cleanup + r CONFIG SET tls-cert-file $orig_server_crt tls-key-file $orig_server_key tls-alt-cert-file $orig_server_alt_crt tls-alt-key-file $orig_server_alt_key tls-key-file-pass "" + } + } + + test {TLS: Dual certificates with passphrases} { + # backup current certificates + set orig_server_crt [lindex [r config get tls-cert-file] 1] + set orig_server_key [lindex [r config get tls-key-file] 1] + set orig_server_alt_crt [lindex [r config get tls-alt-cert-file] 1] + set orig_server_alt_key [lindex [r config get tls-alt-key-file] 1] + + set valkey_crt [format "%s/tests/tls/valkey.crt" [pwd]] + set valkey_key [format "%s/tests/tls/valkey.key" [pwd]] + set valkey_ec_crt [format "%s/tests/tls/valkey-ec.crt" [pwd]] + set valkey_ec_key [format "%s/tests/tls/valkey-ec.key" [pwd]] + set valkey_pw_crt [format "%s/tests/tls/valkey-pw.crt" [pwd]] + set valkey_pw_key [format "%s/tests/tls/valkey-pw.key" [pwd]] + set valkey_ec_pw_crt [format "%s/tests/tls/valkey-ec-pw.crt" [pwd]] + set valkey_ec_pw_key [format "%s/tests/tls/valkey-ec-pw.key" [pwd]] + + try { + r CONFIG SET tls-cert-file $valkey_ec_pw_crt tls-key-file $valkey_ec_pw_key tls-alt-cert-file $valkey_crt tls-alt-key-file $valkey_key tls-key-file-pass asdf tls-alt-key-file-pass 1234 + set s [valkey_client] + assert_equal "PONG" [$s PING] + $s close + r CONFIG SET tls-cert-file $valkey_ec_pw_crt tls-key-file $valkey_ec_pw_key tls-alt-cert-file $valkey_pw_crt tls-alt-key-file $valkey_pw_key + r CONFIG SET tls-cert-file $valkey_ec_crt tls-key-file $valkey_ec_key tls-alt-cert-file $valkey_pw_crt tls-alt-key-file $valkey_pw_key + } finally { + #cleanup + r CONFIG SET tls-cert-file $orig_server_crt tls-key-file $orig_server_key tls-alt-cert-file $orig_server_alt_crt tls-alt-key-file $orig_server_alt_key tls-key-file-pass "" tls-alt-key-file-pass "" + } + } + test {TLS: switch between tcp and tls ports} { set srv_port [srv 0 port] @@ -186,6 +309,25 @@ start_server {tags {"tls"}} { $s close } + test {TLS: Certificate CN with an embedded NUL does not authenticate as the truncated user} { + r ACL SETUSER {Client-only} on allcommands allkeys + r CONFIG SET tls-auth-clients-user CN + r CONFIG RESETSTAT + + # The CN is "Client-only\0attacker". Read as a C string it is "Client-only". + set s [valkey [srv 0 host] [srv 0 port]] + ::tls::import [$s channel] -cafile $::tlsdir/ca.crt \ + -certfile $::tlsdir/client-nul-cn.crt -keyfile $::tlsdir/client-nul-cn.key + assert_equal "default" [$s ACL WHOAMI] + $s close + + # The rejected identity reaches the ACL log. + assert_equal 1 [s acl_access_denied_tls_cert] + + r ACL DELUSER {Client-only} + r CONFIG SET tls-auth-clients-user off + } + test {TLS: Auto-authenticate using tls-auth-clients-user (URI)} { # Enable the feature to auto-authenticate based on URI r CONFIG SET tls-auth-clients-user URI @@ -258,17 +400,26 @@ start_server {tags {"tls"}} { # Get current certificate files set orig_server_crt [lindex [r config get tls-cert-file] 1] set orig_server_key [lindex [r config get tls-key-file] 1] + set orig_server_alt_crt [lindex [r config get tls-alt-cert-file] 1] + set orig_server_alt_key [lindex [r config get tls-alt-key-file] 1] + set valkey_alt_crt [format "%s/tests/tls/valkey-ec.crt" [pwd]] + set valkey_alt_key [format "%s/tests/tls/valkey-ec.key" [pwd]] + set orig_server_key_pass [lindex [r config get tls-alt-key-file-pass] 1] # Create temporary certificate files (copies of current ones) set temp_crt "$orig_server_crt.temp" set temp_key "$orig_server_key.temp" file copy -force $orig_server_crt $temp_crt file copy -force $orig_server_key $temp_key + set temp_alt_crt "$valkey_alt_crt.temp" + set temp_alt_key "$valkey_alt_key.temp" + file copy -force $valkey_alt_crt $temp_alt_crt + file copy -force $valkey_alt_key $temp_alt_key # Ensure cleanup happens even if test fails try { # Update server to use temporary certificate files - r CONFIG SET tls-cert-file $temp_crt tls-key-file $temp_key + r CONFIG SET tls-cert-file $temp_crt tls-key-file $temp_key tls-alt-cert-file $temp_alt_crt tls-alt-key-file $temp_alt_key tls-alt-key-file-pass "asdf" # Enable auto-reload with 1 second interval for faster testing r CONFIG SET tls-auto-reload-interval 1 @@ -281,7 +432,11 @@ start_server {tags {"tls"}} { if {![regexp {tls_server_cert_serial:([^\r\n]+)} $info1 -> serial1]} { fail "INFO tls missing tls_server_cert_serial" } + if {![regexp {tls_server_alt_cert_serial:([^\r\n]+)} $info1 -> alt_serial1]} { + fail "INFO tls missing tls_server_alt_cert_serial" + } assert {$serial1 ne "none"} + assert {$alt_serial1 ne "none"} # Wait for at least one auto-reload cycle to complete after 1100 @@ -307,6 +462,29 @@ start_server {tags {"tls"}} { assert {$serial2 ne "none"} assert {$serial1 ne $serial2} + set valkey_alt_crt [format "%s/tests/tls/valkey-ec-pw.crt" [pwd]] + set valkey_alt_key [format "%s/tests/tls/valkey-ec-pw.key" [pwd]] + file copy -force $valkey_alt_crt $temp_alt_crt + file copy -force $valkey_alt_key $temp_alt_key + + # Wait for another auto-reload cycle to complete + after 2100 + + # Wait for reload to actually complete by checking server logs + # Use generous timeout for slow/busy CI systems + wait_for_log_messages 0 {"*TLS materials reloaded successfully*"} 0 150 100 + + # Verify connection still works after reload + set s [valkey_client] + assert_equal "PONG" [$s PING] + $s close + set info3 [r info tls] + if {![regexp {tls_server_alt_cert_serial:([^\r\n]+)} $info3 -> alt_serial2]} { + fail "INFO tls missing tls_server_alt_cert_serial" + } + assert {$alt_serial2 ne "none"} + assert {$alt_serial1 ne $alt_serial2} + # Wait again to ensure filesystem timestamp will be different # for the second modification and next reload cycle can detect it after 1100 @@ -314,6 +492,8 @@ start_server {tags {"tls"}} { # Restore original certificate content to temporary files file copy -force $orig_server_crt $temp_crt file copy -force $orig_server_key $temp_key + file copy -force $valkey_alt_crt $temp_alt_crt + file copy -force $valkey_alt_key $temp_alt_key # Wait for second reload to complete # Use generous timeout for slow/busy CI systems @@ -325,13 +505,13 @@ start_server {tags {"tls"}} { $s close } finally { # Restore original configuration - r CONFIG SET tls-cert-file $orig_server_crt tls-key-file $orig_server_key + r CONFIG SET tls-cert-file $orig_server_crt tls-key-file $orig_server_key tls-alt-cert-file $orig_server_alt_crt tls-alt-key-file $orig_server_alt_key tls-alt-key-file-pass $orig_server_key_pass # Disable auto-reload r CONFIG SET tls-auto-reload-interval 0 # Clean up temporary files - file delete -force $temp_crt $temp_key + file delete -force $temp_crt $temp_key $temp_alt_crt $temp_alt_key } } @@ -475,6 +655,9 @@ start_server {tags {"tls"}} { # Not-yet-valid CA certificate directory test_tls_cert_rejection ca-dir $tlsdir/ca-notyet {*One or more loaded CA certificates are invalid*} + + # Empty CA certificate directory + test_tls_cert_rejection ca-dir $tlsdir/ca-empty {*No CA certificates loaded from directory*} } proc test_tls_cert_rejection_runtime {r cert_type cert_path} { @@ -521,6 +704,9 @@ start_server {tags {"tls"}} { # Not-yet-valid CA certificate directory test_tls_cert_rejection_runtime r ca-dir $tlsdir/ca-notyet + + # Empty CA certificate directory + test_tls_cert_rejection_runtime r ca-dir $tlsdir/ca-empty } } } diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index f8bdfb0e7..5fcd639ef 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -402,6 +402,28 @@ start_server {tags {"hash"}} { set _ $err } {} + set original_max [lindex [r config get hash-max-listpack-entries] 1] + r config set hash-max-listpack-entries 0 + test {HMGET uses hashtable batch lookup} { + r del hmgetbatchtest + for {set i 1} {$i <= 128} {incr i} { + r hset hmgetbatchtest [format "f%02d" $i] [format "v%02d" $i] + } + + assert_encoding hashtable hmgetbatchtest + assert_equal {v01} [r hmget hmgetbatchtest f01] + assert_equal {v01 {} v01 v04} [r hmget hmgetbatchtest f01 missing f01 f04] + + set fields {missing} + set expected [list {}] + for {set i 1} {$i <= 19} {incr i} { + lappend fields [format "f%02d" $i] + lappend expected [format "v%02d" $i] + } + assert_equal $expected [r hmget hmgetbatchtest {*}$fields] + } + r config set hash-max-listpack-entries $original_max + test {HKEYS - small hash} { lsort [r hkeys smallhash] } [lsort [array names smallhash *]] @@ -939,3 +961,94 @@ start_server {tags {"hash"}} { assert_equal 0 [r exists hfoo] } {} {valgrind:skip} } + +start_server {config "minimal.conf" tags {"hash" "external:skip"} overrides {io-threads 4 io-threads-always-active yes hash-max-listpack-entries 0}} { + test "Hash nested prefetch - HGET correctness with pipelined commands" { + for {set i 0} {$i < 200} {incr i} { + r hset myhash "field:$i" "value:$i" + } + assert_encoding hashtable myhash + + set rd [valkey_deferring_client] + for {set i 0} {$i < 50} {incr i} { + $rd hget myhash "field:$i" + } + $rd flush + for {set i 0} {$i < 50} {incr i} { + assert_equal "value:$i" [$rd read] + } + $rd close + } + + test "Hash nested prefetch - HMGET multi-field correctness" { + set rd [valkey_deferring_client] + $rd hmget myhash field:0 field:1 field:2 field:3 field:4 + $rd flush + set result [$rd read] + assert_equal [list value:0 value:1 value:2 value:3 value:4] $result + $rd close + } + + test "Hash nested prefetch - HDEL correctness with pipelined commands" { + set rd [valkey_deferring_client] + $rd hget myhash field:10 + $rd hdel myhash field:199 + $rd hget myhash field:199 + $rd flush + assert_equal "value:10" [$rd read] + assert_equal 1 [$rd read] + assert_equal {} [$rd read] + $rd close + } + + test "Hash nested prefetch - correctness with a forced multi-key batch" { + set server_pid [s process_id] + + # Suspend the server while the commands are sent so they are all read + # together, which is the case where the prefetch batch holds several keys + # and the nested walk is interleaved across them. + set clients {} + for {set c 0} {$c < 16} {incr c} { + lappend clients [valkey_deferring_client] + } + pause_process $server_pid + set idx 0 + foreach rd $clients { + $rd hget myhash "field:$idx" + $rd flush + incr idx + } + resume_process $server_pid + + set idx 0 + foreach rd $clients { + assert_equal "value:$idx" [$rd read] + incr idx + } + foreach rd $clients { $rd close } + } + + test "Hash nested prefetch - large non-embedded values exercise value phase" { + set big [string repeat "x" 512] + for {set i 0} {$i < 200} {incr i} { + r hset bighash "field:$i" "$big:$i" + } + assert_encoding hashtable bighash + + set clients {} + for {set c 0} {$c < 8} {incr c} { + set rd [valkey_deferring_client] + lappend clients $rd + for {set i 0} {$i < 100} {incr i} { + $rd hget bighash "field:[expr {$i % 200}]" + } + $rd flush + } + foreach rd $clients { + for {set i 0} {$i < 100} {incr i} { + assert_equal "$big:$i" [$rd read] + } + $rd close + } + } +} diff --git a/tests/unit/type/incr.tcl b/tests/unit/type/incr.tcl index 7a6bfbf72..8c28458ee 100644 --- a/tests/unit/type/incr.tcl +++ b/tests/unit/type/incr.tcl @@ -170,6 +170,370 @@ start_server {tags {"incr"}} { r get foo } {0} + test {INCREX keyspace notifications} { + set db [expr {$::singledb ? 0 : 9}] + r config set notify-keyspace-events KEA + set rd [valkey_deferring_client] + assert_equal {1} [psubscribe $rd *] + r del foo + assert_equal "pmessage * __keyspace@${db}__:foo del" [$rd read] + assert_equal "pmessage * __keyevent@${db}__:del foo" [$rd read] + # Integer mode -> incrby + r increx foo byint 5 + assert_equal "pmessage * __keyspace@${db}__:foo incrby" [$rd read] + assert_equal "pmessage * __keyevent@${db}__:incrby foo" [$rd read] + # Float mode -> incrbyfloat + r increx foo byfloat 1 + assert_equal "pmessage * __keyspace@${db}__:foo incrbyfloat" [$rd read] + assert_equal "pmessage * __keyevent@${db}__:incrbyfloat foo" [$rd read] + # An expiry emits a second, separate event. + r increx foo byint 1 ex 100 + assert_equal "pmessage * __keyspace@${db}__:foo incrby" [$rd read] + assert_equal "pmessage * __keyevent@${db}__:incrby foo" [$rd read] + assert_equal "pmessage * __keyspace@${db}__:foo expire" [$rd read] + assert_equal "pmessage * __keyevent@${db}__:expire foo" [$rd read] + $rd close + r config set notify-keyspace-events "" + } + + test {INCREX no negative zero} { + r del foo + r increx foo byfloat [expr double(1)/41] + r increx foo byfloat [expr double(-1)/41] + r get foo + } {0} + + test {INCREX default increment is 1} { + r del foo + r increx foo + } {1 1} + + test {INCREX BYINT increments by given amount} { + r del foo + r increx foo byint 5 + r increx foo byint 5 + } {10 5} + + test {INCREX BYFLOAT increments by the given amount} { + r del foo + assert_match {0.1* 0.1*} [r increx foo byfloat 0.1] + assert_match {0.3* 0.2*} [r increx foo byfloat 0.2] + assert_match {0.3*} [r get foo] + } + + test {INCREX NX only sets when key does not exist} { + r del foo + assert_equal {1 1} [r increx foo nx] + assert_equal {1 0} [r increx foo nx] + assert_equal {1} [r get foo] + } + + test {INCREX BYFLOAT NX only sets when key does not exist} { + r del foo + assert_match {*0.1* *0.1*} [r increx foo nx byfloat 0.1] + assert_match {*0.1* 0} [r increx foo nx byfloat 0.1] + assert_match {*0.1*} [r get foo] + } + + test {INCREX XX only sets when key already exists} { + r del foo + assert_equal {0 0} [r increx foo xx] + assert_equal {0} [r exists foo] + r set foo 10 + assert_equal {11 1} [r increx foo xx] + } + + test {INCREX BYFLOAT XX only sets when key already exist} { + r del foo + assert_match {0 0} [r increx foo xx byfloat 0.1] + r set foo 0.1 + assert_match {*0.2* *0.1*} [r increx foo xx byfloat 0.1] + assert_match {*0.2*} [r get foo] + } + + test {INCREX NX and XX are mutually exclusive} { + r del foo + catch {r increx foo nx xx} err + format $err + } {ERR*} + + test {INCREX with EX sets a TTL} { + r del foo + r increx foo ex 100 + assert_range [r ttl foo] 1 100 + } + + test {INCREX with PX sets a TTL in milliseconds} { + r del foo + r increx foo px 100000 + assert_range [r pttl foo] 1 100000 + } + + test {INCREX with EXAT in the past deletes/skips the key} { + r set foo 5 + r increx foo exat 1 + assert_equal {0} [r exists foo] + } + + test {INCREX combines EX and BYINT correctly} { + r del foo + r increx foo ex 100 byint 7 + assert_equal {7} [r get foo] + assert_range [r ttl foo] 1 100 + } + + test {INCREX combines EX and BYFLOAT correctly} { + r del foo + r increx foo ex 100 byfloat 2.5 + assert_equal {2.5} [r get foo] + assert_range [r ttl foo] 1 100 + } + + test {INCREX combines NX, EX, and BYINT correctly} { + r del foo + r increx foo nx ex 100 byint 3 + assert_equal {3} [r get foo] + assert_range [r ttl foo] 1 100 + # second call should no-op since key now exists + assert_equal {3 0} [r increx foo nx ex 100 byint 3] + } + + test {INCREX BYINT and BYFLOAT are mutually exclusive} { + r del foo + catch {r increx foo byint 1 byfloat 1.0} err + format $err + } {ERR*} + + test {INCREX overflow protection} { + r set foo 9223372036854775807 + assert_equal [r increx foo byint 1] {9223372036854775807 0} + } + + test {INCREX BYFLOAT does not allow Infinity} { + r set foo 0 + catch {r increx foo byfloat +inf} err + format $err + } {ERR *BYFLOAT increment cannot be Infinity*} {valgrind:skip} + + test {INCREX BYFLOAT does not allow nan} { + r set foo 0 + catch {r increx foo byfloat nan} err + format $err + } {ERR *Increment is not a valid float*} {valgrind:skip} + + test {INCREX BYFLOAT does not allow exponentials} { + r set foo 0 + catch {r increx foo byfloat 1e99999} err + format $err + } {ERR *Increment is not a valid float*} {valgrind:skip} + + test {INCREX BYFLOAT does not allow inf values} { + r set foo inf + catch {r increx foo byfloat 1} err + format $err + } {ERR *value cannot be Infinity*} {valgrind:skip} + + test {INCREX BYINT does not allow inf values} { + r set foo inf + catch {r increx foo byint 1} err + format $err + } {ERR *value is not an integer or out of range*} {valgrind:skip} + + test {INCREX distinguishes a bad increment from a bad stored value} { + # The two have opposite causes - the caller's argument is wrong, or the + # data is - so they must not report the same thing. + r set foo 10 + assert_error "ERR Increment is not an integer or out of range" {r increx foo byint abc} + assert_error "ERR Increment is not a valid float" {r increx foo byfloat abc} + r set foo abc + assert_error "ERR value is not an integer or out of range" {r increx foo byint 1} + assert_error "ERR value is not a valid float" {r increx foo byfloat 1} + } + + test {INCREX BYFLOAT positive arithmetic overflow returns [curr_val, 0]} { + set big [ldbl_overflow_operand] + r del foo + r set foo $big + # big + big overflows to infinity: should not error, and should leave the + # value alone while reporting a zero delta. + set res [r increx foo byfloat $big] + assert_equal 0 [lindex $res 1] + assert_equal $big [r get foo] + } + test {INCREX BYFLOAT negative arithmetic overflow returns [curr_val, 0]} { + set big [ldbl_overflow_operand] + r del foo + r set foo -$big + # -big + -big overflows to -infinity + set res [r increx foo byfloat -$big] + assert_equal 0 [lindex $res 1] + assert_equal -$big [r get foo] + } + test {INCREX BYFLOAT overflow preserves existing TTL} { + set big [ldbl_overflow_operand] + r del foo + r set foo $big ex 100 + r increx foo byfloat $big + assert_range [r ttl foo] 1 100 + assert_equal $big [r get foo] + } + test {INCREX BYFLOAT overflow does not apply the command's expiration} { + set big [ldbl_overflow_operand] + # A rejected operation should not set a TTL on a key that has none... + r del foo + r set foo $big + r increx foo byfloat $big ex 60 + assert_equal -1 [r ttl foo] + # ...nor overwrite one that already exists. + r del foo + r set foo $big ex 100 + r increx foo byfloat $big ex 60 + assert_range [r ttl foo] 61 100 + } + + test {INCREX reports the increment that was actually applied} { + # A long double cannot represent every integer at these magnitudes, so + # the addition rounds and the delta that lands can differ from the one + # that was asked for. By how much depends on how wide a long double is, + # which varies by platform, so assert the invariant rather than any + # particular value: the reply describes what happened, which means + # `new == old + applied` has to hold everywhere. + foreach {seed incr} { + 100000000000000000000 1 + 100000000000000000000 3 + 100000000000000000000 7 + 100000000000000000000 9 + 100000000000000000000 -1 + 1000000000000000000000000000000 1 + 1000000000000000000000000000000 7 + 1000000000000000000000000000000 100000000000 + } { + r del foo + r set foo $seed + # SET stores the literal string, but INCREX round-trips the value + # through a long double, so a magnitude this large can be rewritten + # just by being read. Normalize first - and check on the way past + # that a zero increment reports a zero delta. + assert_equal 0 [lindex [r increx foo byfloat 0] 1] + set old [r get foo] + set res [r increx foo byfloat $incr] + set new [lindex $res 0] + set applied [lindex $res 1] + assert_equal $new [r get foo] + assert_equal $new [expr {$old + $applied}] + } + } + + test {INCREX reports the requested increment when nothing is rounded} { + r del foo + r set foo 10 + assert_equal {15 5} [r increx foo byint 5] + assert_equal {12.5 -2.5} [r increx foo byfloat -2.5] + r del foo + assert_equal {5 5} [r increx foo byint 5] + } + + test {INCREX against key holding a list} { + r del mylist + r rpush mylist 1 + catch {r increx mylist} err + r del mylist + format $err + } {WRONGTYPE*} + + test {INCREX preserves existing TTL when expire option omitted} { + r del foo + r set foo 1 ex 100 + r increx foo byint 1 + assert_range [r ttl foo] 1 100 + } + + test {INCREX wrong number of arguments} { + assert_error "*ERR*" {r increx} + } + + test {INCREX against key holding a list, with already-expired EXAT} { + r del list_key + r rpush list_key a + assert_error {WRONGTYPE*} {r increx list_key exat 1} + r del list_key + } + + test {INCREX against non-numeric string value, with already-expired EXAT} { + r set str abc + assert_error {ERR*} {r increx str exat 1} + r del str + } + + test {INCREX against nonexistent key with already-expired EXAT does not store key} { + r del non_existing + assert_equal {1 1} [r increx non_existing exat 1] + assert_equal 0 [r exists non_existing] + } + + test {INCREX BYINT with missing value is a syntax error} { + r del key + assert_error {ERR*} {r increx key byint} + } + + test {INCREX reply types on the wire - RESP3} { + r hello 3 + r readraw 1 + r del foo + assert_equal {*2} [r increx foo byint 5] + assert_equal {:5} [r read] + assert_equal {:5} [r read] + # Default increment is integer mode. + assert_equal {*2} [r increx foo] + assert_equal {:6} [r read] + assert_equal {:1} [r read] + # Both elements are RESP3 doubles in float mode, not bulk strings. + r del foo + assert_equal {*2} [r increx foo byfloat 2.5] + assert_equal {,2.5} [r read] + assert_equal {,2.5} [r read] + # XX on nonexistent key returns [0, 0] + r del foo + assert_equal {*2} [r increx foo xx] + assert_equal {:0} [r read] + assert_equal {:0} [r read] + # XX on nonexistent key with BYFLOAT returns [0.0, 0.0] + assert_equal {*2} [r increx foo xx byfloat 1.5] + assert_equal {,0} [r read] + assert_equal {,0} [r read] + r readraw 0 + r hello 2 + } + + test {INCREX reply types on the wire - RESP2} { + if {!$::force_resp3} { + r readraw 1 + r del foo + assert_equal {*2} [r increx foo byint 5] + assert_equal {:5} [r read] + assert_equal {:5} [r read] + # BYFLOAT degrades to bulk strings under RESP2. + r del foo + assert_equal {*2} [r increx foo byfloat 2.5] + assert_equal {$3} [r read] + assert_equal {2.5} [r read] + assert_equal {$3} [r read] + assert_equal {2.5} [r read] + # XX on nonexistent key returns [0, 0] + r del foo + assert_equal {*2} [r increx foo xx] + assert_equal {:0} [r read] + assert_equal {:0} [r read] + # XX on nonexistent key with BYFLOAT returns ["0", "0"] + assert_equal {*2} [r increx foo xx byfloat 1.5] + assert_equal {$1} [r read] + assert_equal {0} [r read] + assert_equal {$1} [r read] + assert_equal {0} [r read] + r readraw 0 + } + } + test {INCRBY INCRBYFLOAT DECRBY against unhappy path} { r del mykeyincr assert_error "*ERR wrong number of arguments*" {r incr mykeyincr v} @@ -181,7 +545,7 @@ start_server {tags {"incr"}} { assert_error "*value is not a valid float*" {r incrbyfloat mykeyincr v} } - foreach cmd {"incr" "decr" "incrby" "decrby"} { + foreach cmd {"incr" "decr" "incrby" "decrby" "increx"} { test "$cmd operation should update encoding from raw to int" { set res {} set expected {1 12} diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index da58618cb..0b48ac73f 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -55,6 +55,26 @@ start_server { r memory usage myset } + test {SISMEMBER with XX parameter} { + r sadd myset foo bar baz + + assert_equal 1 [r sismember myset foo] + assert_equal 1 [r sismember myset foo XX] + assert_equal 0 [r sismember myset nonexist] + assert_equal 0 [r sismember myset nonexist XX] + + r del nonexistkey + assert_equal 0 [r sismember nonexistkey foo] + assert_equal -1 [r sismember nonexistkey foo XX] + + r set wrongtype "not a set" + assert_error WRONGTYPE* {r sismember wrongtype foo} + assert_error WRONGTYPE* {r sismember wrongtype foo XX} + + assert_error "ERR*syntax error*" {r sismember myset foo invalidparam} + assert_error "ERR*syntax error*" {r sismember myset foo XX invalidparam} + } + test {SMISMEMBER SMEMBERS SCARD against non set} { r lpush mylist foo assert_error WRONGTYPE* {r smismember mylist bar} @@ -78,6 +98,28 @@ start_server { assert_match {*ERR*wrong*number*arg*} $e } + set original_max [lindex [r config get set-max-listpack-entries] 1] + r config set set-max-listpack-entries 0 + test {SMISMEMBER uses hashtable batch lookup} { + r del smismembertest + for {set i 1} {$i <= 128} {incr i} { + r sadd smismembertest [format "m%02d" $i] + } + + assert_encoding hashtable smismembertest + assert_equal {1} [r smismember smismembertest m01] + assert_equal {1 0 1 1} [r smismember smismembertest m01 missing m01 m04] + + set members {missing} + set expected {0} + for {set i 1} {$i <= 19} {incr i} { + lappend members [format "m%02d" $i] + lappend expected 1 + } + assert_equal $expected [r smismember smismembertest {*}$members] + } + r config set set-max-listpack-entries $original_max + test {SADD against non set} { r lpush mylist foo assert_error WRONGTYPE* {r sadd mylist bar} @@ -1211,3 +1253,24 @@ if {[lindex [r config get proto-max-bulk-len] 1] == 10000000000} { } ;# skip 32bit builds } } ;# run_solo + +start_server {config "minimal.conf" tags {"set" "external:skip"} overrides {io-threads 4 io-threads-always-active yes set-max-listpack-entries 0}} { + test "Set nested prefetch - SISMEMBER correctness with pipelined commands" { + for {set i 0} {$i < 200} {incr i} { + r sadd myset "member:$i" + } + assert_encoding hashtable myset + + set rd [valkey_deferring_client] + for {set i 0} {$i < 50} {incr i} { + $rd sismember myset "member:$i" + } + $rd sismember myset "absent" + $rd flush + for {set i 0} {$i < 50} {incr i} { + assert_equal 1 [$rd read] + } + assert_equal 0 [$rd read] + $rd close + } +} diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index 047defecf..f8d9e7756 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -159,6 +159,48 @@ start_server { assert {[r xack s g $id1] eq 1} } + test {Stream consumer group commands should dirty WATCHed keys} { + r DEL mystream + r XADD mystream 1-0 f v + r XGROUP CREATE mystream mygroup 0 + + r WATCH mystream + r XGROUP CREATE mystream othergroup 0 + r MULTI + r PING + assert_equal {} [r EXEC] + + r WATCH mystream + r XGROUP SETID mystream mygroup 1-0 + r MULTI + r PING + assert_equal {} [r EXEC] + + r WATCH mystream + r XGROUP CREATECONSUMER mystream mygroup alice + r MULTI + r PING + assert_equal {} [r EXEC] + + r WATCH mystream + r XGROUP DELCONSUMER mystream mygroup alice + r MULTI + r PING + assert_equal {} [r EXEC] + + r WATCH mystream + r XSETID mystream 2-0 + r MULTI + r PING + assert_equal {} [r EXEC] + + r WATCH mystream + r XGROUP DESTROY mystream othergroup + r MULTI + r PING + assert_equal {} [r EXEC] + } + test {PEL NACK reassignment after XGROUP SETID event} { r del events r xadd events * f1 v1 @@ -213,6 +255,59 @@ start_server { assert {[lindex $res 0 1 1] == {2-0 {field1 B}}} } + test {XREADGROUP, XACK, XCLAIM and XAUTOCLAIM should dirty WATCHed keys} { + r DEL mystream + set id1 [r XADD mystream 1-0 a 1] + set id2 [r XADD mystream 2-0 b 2] + r XGROUP CREATE mystream mygroup 0 + + r WATCH mystream + r XREADGROUP GROUP mygroup consumer1 COUNT 1 STREAMS mystream > + r MULTI + r PING + assert_equal {} [r EXEC] + + r WATCH mystream + r XACK mystream mygroup $id1 + r MULTI + r PING + assert_equal {} [r EXEC] + + r XREADGROUP GROUP mygroup consumer1 COUNT 1 STREAMS mystream > + after 20 + + r WATCH mystream + r XCLAIM mystream mygroup consumer2 0 $id2 + r MULTI + r PING + assert_equal {} [r EXEC] + + after 20 + r WATCH mystream + r XAUTOCLAIM mystream mygroup consumer1 0 0-0 COUNT 1 + r MULTI + r PING + assert_equal {} [r EXEC] + } + + test {Blocking XREADGROUP consumer creation should dirty WATCHed keys} { + r DEL mystream + r XGROUP CREATE mystream mygroup $ MKSTREAM + + r WATCH mystream + set rd [valkey_deferring_client] + $rd XREADGROUP GROUP mygroup consumer1 BLOCK 0 NOACK STREAMS mystream ">" + wait_for_blocked_clients_count 1 + + r MULTI + r PING + assert_equal {} [r EXEC] + + r XADD mystream * f v + $rd read + $rd close + } + test {Blocking XREADGROUP will not reply with an empty array} { r del mystream r XGROUP CREATE mystream mygroup $ MKSTREAM @@ -1479,6 +1574,357 @@ start_server { } } + start_server {tags {"external:skip"}} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set replica [srv 0 client] + + test {XACKDEL replication: ack-only (no deletion) propagates PEL removal to replica} { + $replica replicaof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + + # Two groups both read the message so grp2 blocks deletion in ACKED mode + $master DEL stream + $master XADD stream 1-0 f v + $master XGROUP CREATE stream grp1 0 + $master XGROUP CREATE stream grp2 0 + $master XREADGROUP GROUP grp1 alice COUNT 1 STREAMS stream > + $master XREADGROUP GROUP grp2 bob COUNT 1 STREAMS stream > + + wait_for_ofs_sync $master $replica + + # Replica should have the pending entry in grp1 before ack + assert_equal [llength [$replica XPENDING stream grp1 - + 10]] 1 + + # ACKED mode: grp2 still has entry in PEL so deletion is suppressed + $master XACKDEL stream grp1 ACKED IDS 1 1-0 + + wait_for_ofs_sync $master $replica + + # grp1's PEL entry should be gone, stream entry still present on replica + assert_equal [llength [$replica XPENDING stream grp1 - + 10]] 0 + assert_equal [llength [$replica XRANGE stream - +]] 1 + } + + test {XACKDEL replication: ACKED mode deletion propagates stream removal to replica} { + $replica replicaof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + + $master DEL stream + $master XADD stream 1-0 f v + $master XGROUP CREATE stream grp 0 + $master XREADGROUP GROUP grp alice COUNT 1 STREAMS stream > + + wait_for_ofs_sync $master $replica + + # Replica should have entry in stream and PEL before ack + assert_equal [llength [$replica XRANGE stream - +]] 1 + assert_equal [llength [$replica XPENDING stream grp - + 10]] 1 + + # ACKED mode with only one group: triggers deletion + $master XACKDEL stream grp ACKED IDS 1 1-0 + + wait_for_ofs_sync $master $replica + + # Both PEL entry and stream entry should be gone on replica + assert_equal [llength [$replica XPENDING stream grp - + 10]] 0 + assert_equal [llength [$replica XRANGE stream - +]] 0 + } + + test {XACKDEL replication: DELREF clears other groups' PELs on replica} { + $replica replicaof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + + $master DEL stream + $master XADD stream 1-0 f v + $master XGROUP CREATE stream grp1 0 + $master XGROUP CREATE stream grp2 0 + $master XREADGROUP GROUP grp1 alice COUNT 1 STREAMS stream > + $master XREADGROUP GROUP grp2 bob COUNT 1 STREAMS stream > + + wait_for_ofs_sync $master $replica + + # Both groups should have the entry in their PEL on replica + assert_equal [llength [$replica XPENDING stream grp1 - + 10]] 1 + assert_equal [llength [$replica XPENDING stream grp2 - + 10]] 1 + + # DELREF: ack for grp1, force-delete from stream and clear all groups' PELs + $master XACKDEL stream grp1 DELREF IDS 1 1-0 + + wait_for_ofs_sync $master $replica + + # Stream entry and grp2's PEL entry should both be gone on replica + assert_equal [llength [$replica XRANGE stream - +]] 0 + assert_equal [llength [$replica XPENDING stream grp2 - + 10]] 0 + } + } + + start_server {tags {"external:skip"}} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set replica [srv 0 client] + + test {XDELEX replication: KEEPREF deletes stream entry but keeps dangling PEL ref on replica} { + $replica replicaof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + + $master DEL stream + $master XADD stream 1-0 f v + $master XADD stream 2-0 f v + $master XGROUP CREATE stream grp 0 + $master XREADGROUP GROUP grp alice COUNT 1 STREAMS stream > + + wait_for_ofs_sync $master $replica + + # Replica has both entries and grp's PEL contains 1-0 + assert_equal 2 [$replica XLEN stream] + assert_equal 1 [llength [$replica XPENDING stream grp - + 10]] + + # KEEPREF: deletes entry from stream but leaves PEL reference intact + $master XDELEX stream KEEPREF IDS 1 1-0 + + wait_for_ofs_sync $master $replica + + # Stream entry gone on replica, PEL reference still present + assert_equal 1 [$replica XLEN stream] + assert_equal 1 [llength [$replica XPENDING stream grp - + 10]] + } + + test {XDELEX replication: DELREF deletes stream entry and clears PEL on replica} { + $replica replicaof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + + $master DEL stream + $master XADD stream 1-0 f v + $master XADD stream 2-0 f v + $master XGROUP CREATE stream grp 0 + $master XREADGROUP GROUP grp alice COUNT 1 STREAMS stream > + + wait_for_ofs_sync $master $replica + + # Replica has both entries and grp's PEL contains 1-0 + assert_equal 2 [$replica XLEN stream] + assert_equal 1 [llength [$replica XPENDING stream grp - + 10]] + + # DELREF: deletes entry from stream AND removes it from all PELs + $master XDELEX stream DELREF IDS 1 1-0 + + wait_for_ofs_sync $master $replica + + # Stream entry gone and PEL cleared on replica + assert_equal 1 [$replica XLEN stream] + assert_equal 0 [llength [$replica XPENDING stream grp - + 10]] + } + + test {XDELEX replication: ACKED skips pending entries, deletes only after all groups ack} { + $replica replicaof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + + $master DEL stream + $master XADD stream 1-0 f v + $master XGROUP CREATE stream grp 0 + $master XREADGROUP GROUP grp alice COUNT 1 STREAMS stream > + + wait_for_ofs_sync $master $replica + + # Entry is pending; ACKED mode should not delete it + $master XDELEX stream ACKED IDS 1 1-0 + + wait_for_ofs_sync $master $replica + + # Entry still present on replica since it was not acked + assert_equal 1 [$replica XLEN stream] + assert_equal 1 [llength [$replica XPENDING stream grp - + 10]] + + # Now ack the entry and retry XDELEX ACKED + $master XACK stream grp 1-0 + $master XDELEX stream ACKED IDS 1 1-0 + + wait_for_ofs_sync $master $replica + + # Entry deleted on replica after all groups have acked + assert_equal 0 [$replica XLEN stream] + assert_equal 0 [llength [$replica XPENDING stream grp - + 10]] + } + } + + start_server {tags {"external:skip"}} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set replica [srv 0 client] + + # Number of times 'cmd' was executed on the replica, or 0 if never + # called (INFO omits zero counters). + proc get_replica_calls {client cmd} { + set info [$client INFO commandstats] + foreach line [split $info "\n"] { + if {[string match "cmdstat_$cmd:*" $line]} { + regexp {calls=(\d+)} $line -> count + return $count + } + } + return 0 + } + + $replica replicaof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + + test {XACKDEL ack-only propagates XACK but never XACKDEL or XDEL} { + $master DEL stream + $master XADD stream 1-0 f v + $master XGROUP CREATE stream grp1 0 + $master XGROUP CREATE stream grp2 0 + $master XREADGROUP GROUP grp1 alice COUNT 1 STREAMS stream > + $master XREADGROUP GROUP grp2 bob COUNT 1 STREAMS stream > + wait_for_ofs_sync $master $replica + + set xack_before [get_replica_calls $replica xack] + set xdel_before [get_replica_calls $replica xdel] + + # grp2 still holds the message pending, so nothing is deleted + $master XACKDEL stream grp1 ACKED IDS 1 1-0 + wait_for_ofs_sync $master $replica + + assert_equal 1 [expr {[get_replica_calls $replica xack] - $xack_before}] + assert_equal 0 [expr {[get_replica_calls $replica xdel] - $xdel_before}] + assert_equal 0 [get_replica_calls $replica xackdel] + assert_equal 1 [$replica XLEN stream] + assert_equal 0 [llength [$replica XPENDING stream grp1 - + 10]] + } + + test {XACKDEL KEEPREF propagates XACK + XDEL but never XACKDEL} { + $master DEL stream + $master XADD stream 1-0 f v + $master XGROUP CREATE stream grp 0 + $master XREADGROUP GROUP grp alice COUNT 1 STREAMS stream > + wait_for_ofs_sync $master $replica + + set xack_before [get_replica_calls $replica xack] + set xdel_before [get_replica_calls $replica xdel] + + $master XACKDEL stream grp KEEPREF IDS 1 1-0 + wait_for_ofs_sync $master $replica + + assert_equal 1 [expr {[get_replica_calls $replica xack] - $xack_before}] + assert_equal 1 [expr {[get_replica_calls $replica xdel] - $xdel_before}] + assert_equal 0 [get_replica_calls $replica xackdel] + assert_equal 0 [$replica XLEN stream] + assert_equal 0 [llength [$replica XPENDING stream grp - + 10]] + } + + test {XACKDEL DELREF propagates per-group XACK + XDEL but never XACKDEL} { + $master DEL stream + $master XADD stream 1-0 f v + $master XGROUP CREATE stream grp1 0 + $master XGROUP CREATE stream grp2 0 + $master XREADGROUP GROUP grp1 alice COUNT 1 STREAMS stream > + $master XREADGROUP GROUP grp2 bob COUNT 1 STREAMS stream > + wait_for_ofs_sync $master $replica + + set xack_before [get_replica_calls $replica xack] + set xdel_before [get_replica_calls $replica xdel] + + $master XACKDEL stream grp1 DELREF IDS 1 1-0 + wait_for_ofs_sync $master $replica + + # One XACK for the target group + one for grp2's cleared PEL ref + assert_equal 2 [expr {[get_replica_calls $replica xack] - $xack_before}] + assert_equal 1 [expr {[get_replica_calls $replica xdel] - $xdel_before}] + assert_equal 0 [get_replica_calls $replica xackdel] + assert_equal 0 [$replica XLEN stream] + assert_equal 0 [llength [$replica XPENDING stream grp2 - + 10]] + } + + test {XDELEX KEEPREF propagates XDEL only but never XDELEX} { + $master DEL stream + $master XADD stream 1-0 f v + wait_for_ofs_sync $master $replica + + set xack_before [get_replica_calls $replica xack] + set xdel_before [get_replica_calls $replica xdel] + + $master XDELEX stream KEEPREF IDS 1 1-0 + wait_for_ofs_sync $master $replica + + assert_equal 1 [expr {[get_replica_calls $replica xdel] - $xdel_before}] + assert_equal 0 [expr {[get_replica_calls $replica xack] - $xack_before}] + assert_equal 0 [get_replica_calls $replica xdelex] + assert_equal 0 [$replica XLEN stream] + } + + test {XDELEX DELREF propagates XDEL + XACK but never XDELEX} { + $master DEL stream + $master XADD stream 1-0 f v + $master XGROUP CREATE stream grp 0 + $master XREADGROUP GROUP grp alice COUNT 1 STREAMS stream > + wait_for_ofs_sync $master $replica + + set xack_before [get_replica_calls $replica xack] + set xdel_before [get_replica_calls $replica xdel] + + $master XDELEX stream DELREF IDS 1 1-0 + wait_for_ofs_sync $master $replica + + assert_equal 1 [expr {[get_replica_calls $replica xdel] - $xdel_before}] + assert_equal 1 [expr {[get_replica_calls $replica xack] - $xack_before}] + assert_equal 0 [get_replica_calls $replica xdelex] + assert_equal 0 [$replica XLEN stream] + assert_equal 0 [llength [$replica XPENDING stream grp - + 10]] + } + + test {XDELEX ACKED with nothing deleted propagates nothing} { + $master DEL stream + $master XADD stream 1-0 f v + $master XGROUP CREATE stream grp 0 + $master XREADGROUP GROUP grp alice COUNT 1 STREAMS stream > + wait_for_ofs_sync $master $replica + + set xack_before [get_replica_calls $replica xack] + set xdel_before [get_replica_calls $replica xdel] + + # Entry still pending in grp, so ACKED neither deletes nor clears + $master XDELEX stream ACKED IDS 1 1-0 + wait_for_ofs_sync $master $replica + + assert_equal 0 [expr {[get_replica_calls $replica xdel] - $xdel_before}] + assert_equal 0 [expr {[get_replica_calls $replica xack] - $xack_before}] + assert_equal 0 [get_replica_calls $replica xdelex] + assert_equal 1 [$replica XLEN stream] + assert_equal 1 [llength [$replica XPENDING stream grp - + 10]] + } + } + start_server {tags {"stream needs:debug"} overrides {appendonly yes aof-use-rdb-preamble no}} { test {Empty stream with no lastid can be rewrite into AOF correctly} { r XGROUP CREATE mystream group-name $ MKSTREAM diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 73a6f6432..99d65bcc4 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -634,6 +634,894 @@ start_server { } } + test {XACKDEL returns syntax error when IDS token is missing after mode} { + r DEL teststream + r XADD teststream 1 msg hello + r XGROUP CREATE teststream testgrp 0 + assert_error "*syntax error*" {r XACKDEL teststream testgrp KEEPREF NOIDS 1 1} + assert_error "*syntax error*" {r XACKDEL teststream testgrp DELREF NOIDS 1 1} + assert_error "*syntax error*" {r XACKDEL teststream testgrp ACKED NOIDS 1 1} + } + + test {XACKDEL wrong number of args} { + assert_error {*wrong number of arguments*} {r XACKDEL s} + assert_error {*wrong number of arguments*} {r XACKDEL s grp} + } + + test {XACKDEL w/ KEEPREF keeps refs in other consumer groups' PEL} { + r DEL testxadstream + r XADD testxadstream 1 msg hello + r XGROUP CREATE testxadstream testxadgrp1 1 + r XGROUP CREATE testxadstream testxadgrp2 1 + r XADD testxadstream 2 msg2 hello2 + + # Setup consumer groups w/ message in both groups PEL + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + r XREADGROUP GROUP testxadgrp2 testxadcnsmr COUNT 1 STREAMS testxadstream > + + # Group 1 + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2-0 [lindex $pend 1] + set ids [r XACKDEL testxadstream testxadgrp1 KEEPREF IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal {} [r xrange testxadstream 2 2] + + # Group 2 still has ref in PEL + set pend [r XPENDING testxadstream testxadgrp2] + assert_equal 2-0 [lindex $pend 1] + } + + test {XACKDEL uses KEEPREF by default} { + r DEL testxadstream + r XADD testxadstream 1 msg hello + r XGROUP CREATE testxadstream testxadgrp1 1 + r XGROUP CREATE testxadstream testxadgrp2 1 + r XADD testxadstream 2 msg2 hello2 + + # Setup consumer groups w/ message in both groups PEL + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + r XREADGROUP GROUP testxadgrp2 testxadcnsmr COUNT 1 STREAMS testxadstream > + + # Group 1 + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2-0 [lindex $pend 1] + set ids [r XACKDEL testxadstream testxadgrp1 IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal {} [r xrange testxadstream 2 2] + + # Group 2 still has ref in PEL + set pend [r XPENDING testxadstream testxadgrp2] + assert_equal 2-0 [lindex $pend 1] + } + + test {XACKDEL w/ ACKED doesn't delete when 2nd consumer group has message in PEL} { + r DEL testxadstream + r XADD testxadstream 1 msg hello + r XGROUP CREATE testxadstream testxadgrp1 1 + r XGROUP CREATE testxadstream testxadgrp2 1 + r XADD testxadstream 2 msg2 hello2 + + # Setup consumer groups w/ message in both groups PEL + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + r XREADGROUP GROUP testxadgrp2 testxadcnsmr COUNT 1 STREAMS testxadstream > + + # Group 1 w/ ACKED only ack's, doesn't delete b/c group2 still hasn't gotten there + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2-0 [lindex $pend 1] + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal 2 [lindex $ids 0] + assert_equal {{2-0 {msg2 hello2}}} [r xrange testxadstream 2 2] + + # Group 2 w/ ACKED both ack's and deletes now that all groups have ACK'd + set pend [r XPENDING testxadstream testxadgrp2] + assert_equal 2-0 [lindex $pend 1] + set ids [r XACKDEL testxadstream testxadgrp2 ACKED IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal {} [r xrange testxadstream 2 2] + } + + # The claim checking logic uses `last_id`. So using XCLAIM to FORCE setting the LAST_ID + # would naturally affect this. + test {XACKDEL w/ ACKED doesn't delete when 2nd consumer group hasn't claimed message yet} { + r DEL testxadstream + r XADD testxadstream 1 msg hello + r XGROUP CREATE testxadstream testxadgrp1 1 + r XGROUP CREATE testxadstream testxadgrp2 1 + r XADD testxadstream 2 msg2 hello2 + + # Group 1 w/ ACKED only ack's, doesn't delete b/c group2 still hasn't gotten there + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2-0 [lindex $pend 1] + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal 2 [lindex $ids 0] + assert_equal {{2-0 {msg2 hello2}}} [r xrange testxadstream 2 2] + + # Group 2 w/ ACKED both ack's and deletes now that all groups have ACK'd + r XREADGROUP GROUP testxadgrp2 testxadcnsmr COUNT 1 STREAMS testxadstream > + set ids [r XACKDEL testxadstream testxadgrp2 ACKED IDS 1 2-0] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal {} [r xrange testxadstream 2 2] + } + + test {XACKDEL w/ ACKED acks dangling PEL reference after plain XDEL} { + r DEL testxadstream + r XADD testxadstream 1-0 msg hello + r XGROUP CREATE testxadstream testxadgrp1 0 + r XGROUP CREATE testxadstream testxadgrp2 0 + + # Only group 1 delivers the message; group 2's last_id stays at 0-0. + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 1-0 [lindex $pend 1] + + # Plain XDEL removes the entry but leaves group 1's PEL reference dangling. + assert_equal 1 [r XDEL testxadstream 1-0] + + # Acking the dangling reference replies 1 (acked, nothing left to + # delete), not 2 (blocked by group 2): the entry no longer exists and + # can never be delivered to group 2. + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 1-0] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + + # The dangling reference is gone from group 1's PEL and the stream is + # still empty. + assert_equal 0 [r XLEN testxadstream] + assert_equal {} [lindex [r XPENDING testxadstream testxadgrp1] 1] + assert_equal {} [lindex [r XPENDING testxadstream testxadgrp2] 1] + } + + test {XACKDEL w/ ACKED doesn't delete when 2nd group has message pending after plain XDEL} { + r DEL testxadstream + r XADD testxadstream 1-0 msg hello + r XGROUP CREATE testxadstream testxadgrp1 0 + r XGROUP CREATE testxadstream testxadgrp2 0 + + # Both groups deliver the message, then plain XDEL removes the entry, + # leaving both PEL references dangling. + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + r XREADGROUP GROUP testxadgrp2 testxadcnsmr COUNT 1 STREAMS testxadstream > + assert_equal 1 [r XDEL testxadstream 1-0] + + # Group 2 still has the message pending, so deletion stays blocked (2). + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 1-0] + assert_equal 1 [llength $ids] + assert_equal 2 [lindex $ids 0] + + # Group 1 was acked, group 2's reference is untouched. + assert_equal {} [lindex [r XPENDING testxadstream testxadgrp1] 1] + assert_equal 1-0 [lindex [r XPENDING testxadstream testxadgrp2] 1] + } + + test {XACKDEL w/ DELREF deletes from stream and 2nd consumer group's PEL even if not ACK'd} { + r DEL testxadstream + r XADD testxadstream 1 msg hello + r XGROUP CREATE testxadstream testxadgrp1 1 + r XGROUP CREATE testxadstream testxadgrp2 1 + r XADD testxadstream 2 msg2 hello2 + + # Setup consumer groups w/ message in both groups PEL + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + r XREADGROUP GROUP testxadgrp2 testxadcnsmr COUNT 1 STREAMS testxadstream > + + # Group 1 w/ DELREF does ACK in group 1, removes from group 1's PEL, and deletes from stream + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2-0 [lindex $pend 1] + set ids [r XACKDEL testxadstream testxadgrp1 DELREF IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal {} [r xrange testxadstream 2 2] + + # And Group 2 still has the message in it's PEL + set pend [r XPENDING testxadstream testxadgrp2] + assert_equal {} [lindex $pend 1] + } + + test {XACKDEL w/ DELREF skips deleting refs when target group never received message} { + r DEL testxadstream + r XADD testxadstream 1-0 msg hello + # grp1 created first and reads the message; grp2 (target) never reads it. + # "grp1" < "grp2" so grp1 is iterated first. + r XGROUP CREATE testxadstream grp1 0 + r XGROUP CREATE testxadstream grp2 0 + r XREADGROUP GROUP grp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + + # Target grp2 never had 1-0 pending -> must reply -1 and not change any state. + set ids [r XACKDEL testxadstream grp2 DELREF IDS 1 1-0] + assert_equal 1 [llength $ids] + assert_equal -1 [lindex $ids 0] + + # Stream entry is still be present. + assert_equal {{1-0 {msg hello}}} [r XRANGE testxadstream 1-0 1-0] + + # grp1's PEL entry is untouched. + set pend [r XPENDING testxadstream grp1] + assert_equal 1-0 [lindex $pend 1] + } + + test {XACKDEL w/ ACKED is a no-op when target group already acked the message} { + r DEL testxadstream + r XADD testxadstream 1-0 msg hello + r XGROUP CREATE testxadstream grp1 0 + r XGROUP CREATE testxadstream grp2 0 + r XREADGROUP GROUP grp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + r XREADGROUP GROUP grp2 testxadcnsmr COUNT 1 STREAMS testxadstream > + # Target grp2 acks the message, dropping it from grp2's PEL. + r XACK testxadstream grp2 1-0 + + # grp2 no longer has 1-0 pending, so reply -1 and dont modify anything. + set ids [r XACKDEL testxadstream grp2 ACKED IDS 1 1-0] + assert_equal 1 [llength $ids] + assert_equal -1 [lindex $ids 0] + assert_equal {{1-0 {msg hello}}} [r XRANGE testxadstream 1-0 1-0] + + # grp1 still holds its PEL entry. + set pend [r XPENDING testxadstream grp1] + assert_equal 1-0 [lindex $pend 1] + } + + test {XACKDEL w/ mix of existing and non-existent messages} { + r DEL testxadstream + r XADD testxadstream 1 msg hello + r XGROUP CREATE testxadstream testxadgrp1 1 + r XGROUP CREATE testxadstream testxadgrp2 1 + r XADD testxadstream 2 msg2 hello2 + + # Setup consumer groups w/ message in both groups PEL + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + r XREADGROUP GROUP testxadgrp2 testxadcnsmr COUNT 1 STREAMS testxadstream > + + # Group 1 + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2-0 [lindex $pend 1] + set ids [r XACKDEL testxadstream testxadgrp1 IDS 4 2 10 99 234] + assert_equal 4 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal -1 [lindex $ids 1] + assert_equal -1 [lindex $ids 2] + assert_equal -1 [lindex $ids 3] + assert_equal {} [r xrange testxadstream 2 2] + + # Group 2 still has ref in PEL + set pend [r XPENDING testxadstream testxadgrp2] + assert_equal 2-0 [lindex $pend 1] + } + + test {XACKDEL multiple IDs some acked some not} { + r DEL testxadstream + r XADD testxadstream 1-0 f v1 + r XADD testxadstream 2-0 f v2 + r XADD testxadstream 3-0 f v3 + r XGROUP CREATE testxadstream testxadgrp1 0 + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 10 STREAMS testxadstream > + + # Delete 1-0 and 3-0; leave 2-0 in stream; 99-0 doesn't exist + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 3 1-0 3-0 99-0] + assert_equal 3 [llength $ids] + assert_equal 1 [lindex $ids 0] ;# 1-0 deleted + assert_equal 1 [lindex $ids 1] ;# 3-0 deleted + assert_equal -1 [lindex $ids 2] ;# 99-0 not found + assert_equal 1 [r XLEN testxadstream] + assert_equal 2-0 [lindex [lindex [r XRANGE testxadstream - +] 0] 0] + } + + test {XACKDEL w/ message not claimed does nothing} { + r DEL testxadstream + r XADD testxadstream 1 msg hello + r XGROUP CREATE testxadstream testxadgrp1 1 + r XGROUP CREATE testxadstream testxadgrp2 1 + r XADD testxadstream 2 msg2 hello2 + + # Group 1 + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 2 99 2] + assert_equal 2 [llength $ids] + assert_equal -1 [lindex $ids 0] + assert_equal -1 [lindex $ids 1] + assert_equal {{2-0 {msg2 hello2}}} [r xrange testxadstream 2 2] + + # Group 2 + set ids [r XACKDEL testxadstream testxadgrp2 ACKED IDS 2 2 99] + assert_equal 2 [llength $ids] + assert_equal -1 [lindex $ids 0] + assert_equal -1 [lindex $ids 1] + assert_equal {{2-0 {msg2 hello2}}} [r xrange testxadstream 2 2] + + # Check stream length + assert_equal 2 [r xlen testxadstream] + } + + test {XACKDEL run multiple times returns -1 after first time} { + r DEL testxadstream + r XADD testxadstream 1 msg hello + r XGROUP CREATE testxadstream testxadgrp1 1 + r XGROUP CREATE testxadstream testxadgrp2 1 + r XADD testxadstream 2 msg2 hello2 + + # Setup consumer groups w/ message in both groups PEL + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + r XREADGROUP GROUP testxadgrp2 testxadcnsmr COUNT 1 STREAMS testxadstream > + + # Group 1 w/ ACKED only ack's, doesn't delete b/c group2 still hasn't gotten there + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2-0 [lindex $pend 1] + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal 2 [lindex $ids 0] + assert_equal {{2-0 {msg2 hello2}}} [r xrange testxadstream 2 2] + + # Group 1 run again now returns -1 + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal -1 [lindex $ids 0] + assert_equal {{2-0 {msg2 hello2}}} [r xrange testxadstream 2 2] + + # Group 2 w/ ACKED both ack's and deletes now that all groups have ACK'd + set pend [r XPENDING testxadstream testxadgrp2] + assert_equal 2-0 [lindex $pend 1] + set ids [r XACKDEL testxadstream testxadgrp2 ACKED IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal {} [r xrange testxadstream 2 2] + } + + test {XACKDEL with non-existent stream and group} { + r DEL testxadstream + + # Missing stream and group + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal -1 [lindex $ids 0] + + # Missing Group + r XADD testxadstream 1 msg hello + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 2] + assert_equal 1 [llength $ids] + assert_equal -1 [lindex $ids 0] + } + + test {XACKDEL should fail if given an invalid stream ID} { + r DEL testxadstream + r XADD testxadstream 1 msg hello + r XGROUP CREATE testxadstream testxadgrp1 1 + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + assert_error "*Invalid stream ID*" {r XACKDEL testxadstream testxadgrp1 IDS 1 not-a-valid-id} + } + + test {XACKDEL should fail if called on a non-stream key} { + r DEL testxadstream + r SET testxadstream notastream + assert_error "*WRONGTYPE*" {r XACKDEL testxadstream testxadgrp1 IDS 1 1-0} + r DEL testxadstream + } + + test {XACKDEL should fail if given an unrecognized mode} { + r DEL testxadstream + r XADD testxadstream 1 msg hello + r XGROUP CREATE testxadstream testxadgrp1 1 + assert_error "*" {r XACKDEL testxadstream testxadgrp1 BADMODE IDS 1 1-0} + } + + test {XACKDEL IDS numids must be a positive integer} { + r DEL testxadstream + r XADD testxadstream 1-0 f v + r XGROUP CREATE testxadstream testxadgrp1 0 + assert_error {*Number of IDs must be a positive integer*} {r XACKDEL testxadstream testxadgrp1 IDS abc 1-0} + assert_error {*Number of IDs must be a positive integer*} {r XACKDEL testxadstream testxadgrp1 IDS 0 1-0} + assert_error {*Number of IDs must be a positive integer*} {r XACKDEL testxadstream testxadgrp1 IDS -5 1-0} + } + + test {XACKDEL IDS numids must match argument count} { + r DEL testxadstream + r XADD testxadstream 1-0 f v + r XGROUP CREATE testxadstream testxadgrp1 0 + assert_error {*syntax error*} {r XACKDEL testxadstream testxadgrp1 IDS 3 1-0 2-0} + assert_error {*syntax error*} {r XACKDEL testxadstream testxadgrp1 IDS 1 1-0 2-0} + } + + test {XACKDEL with more than 8 IDs exercises dynamic allocation} { + r DEL teststream + # STREAMID_STATIC_VECTOR_LEN is 8, use 10 to force zmalloc path + for {set i 1} {$i <= 10} {incr i} { + r XADD teststream $i msg hello + } + r XGROUP CREATE teststream testgrp 0 + r XREADGROUP GROUP testgrp consumer1 COUNT 10 STREAMS teststream > + set ids [r XACKDEL teststream testgrp IDS 10 1 2 3 4 5 6 7 8 9 10] + assert_equal 10 [llength $ids] + foreach id $ids { + assert_equal 1 $id + } + assert_equal 0 [r XLEN teststream] + } + + test {XACKDEL returns -1 for deleted entry not in group PEL} { + r DEL teststream + r XADD teststream 1-0 msg hello + r XGROUP CREATE teststream testgrp 0 + # Message never claimed, so not in any PEL. Delete it from the stream. + r XDEL teststream 1-0 + set res [r XACKDEL teststream testgrp IDS 1 1-0] + assert_equal 1 [llength $res] + assert_equal -1 [lindex $res 0] + } + + test {XACKDEL updates first_id via streamGetEdgeID when first entry is deleted but stream is non-empty} { + r DEL testxadstream + r XADD testxadstream 1-0 msg hello + r XADD testxadstream 2-0 msg2 hello2 + r XGROUP CREATE testxadstream testxadgrp1 0 + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 10 STREAMS testxadstream > + + # Delete 1-0 (the first entry) while 2-0 remains; triggers the streamGetEdgeID path + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 1-0] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + + # 2-0 must still be present and must now be the first entry (first_id updated) + assert_equal 1 [r XLEN testxadstream] + set entries [r XRANGE testxadstream - +] + assert_equal 1 [llength $entries] + assert_equal 2-0 [lindex [lindex $entries 0] 0] + } + + test {XACKDEL DELREF deletes entry when non-target group has not yet claimed it} { + r DEL testxadstream + r XADD testxadstream 1-0 msg hello + r XGROUP CREATE testxadstream testxadgrp1 0 + r XGROUP CREATE testxadstream testxadgrp2 0 + + # Only grp1 claims the message; grp2 has never read it (beyond grp2's last_id) + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + + set ids [r XACKDEL testxadstream testxadgrp1 DELREF IDS 1 1-0] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + + # Entry must be deleted from the stream even though grp2 never had a PEL entry for it + assert_equal 0 [r XLEN testxadstream] + assert_equal {} [r XRANGE testxadstream - +] + set pend2 [r XPENDING testxadstream testxadgrp2] + assert_equal 0 [lindex $pend2 0] + } + + test {XACKDEL ACKED returns 1 when stream entry was already removed via XDEL} { + r DEL testxadstream + r XADD testxadstream 1-0 f v + r XGROUP CREATE testxadstream testxadgrp1 0 + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + + # XDEL removes the stream entry but leaves the PEL entry intact + r XDEL testxadstream 1-0 + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 1 [lindex $pend 0] + + # XACKDEL clears the PEL and still returns 1 even though already deleted + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 1-0] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + + # Check that XACKDEL ACKED removes the dangling PEL entry left after XDEL + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 0 [lindex $pend 0] + } + + test {XACKDEL ACKED with a single consumer group deletes the entry} { + r DEL testxadstream + r XADD testxadstream 1-0 f v + r XGROUP CREATE testxadstream testxadgrp1 0 + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 1 STREAMS testxadstream > + + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 1-0] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal 0 [r XLEN testxadstream] + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 0 [lindex $pend 0] + } + + test {XACKDEL ACKED drains PEL despite XGROUP SETID moving last_id backward} { + r DEL testxadstream + r XADD testxadstream 1-0 f v1 + r XADD testxadstream 5-0 f v5 + r XGROUP CREATE testxadstream testxadgrp1 0 + + # Claim both entries: last_id advances to 5-0 and both land in the PEL. + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 10 STREAMS testxadstream > + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2 [lindex $pend 0] + assert_equal 5-0 [lindex $pend 2] + + # Move last_id back before the latest claimed entry. 5-0 stays in the + # PEL even though id > last_id now. + r XGROUP SETID testxadstream testxadgrp1 1-0 + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2 [lindex $pend 0] + + # ACKED must still drain the dangling 5-0 PEL entry and delete the msg. + set ids [r XACKDEL testxadstream testxadgrp1 ACKED IDS 1 5-0] + assert_equal 1 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal {} [r xrange testxadstream 5-0 5-0] + + # 5-0 should be gone from the PEL; 1-0 remains pending. + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 1 [lindex $pend 0] + assert_equal 1-0 [lindex $pend 1] + } + + test {XDELEX deletes items} { + r DEL teststream + r XADD teststream 1 msg helllo + r XADD teststream 2 msg helllo + r XADD teststream 3 msg helllo + set resp [ r XDELEX teststream IDS 1 1 ] + assert_equal 1 [llength $resp] + assert_equal 1 [lindex $resp 0] + + assert_equal 2 [ r XLEN teststream ] + } + + test {XDELEX on non-existent key returns -1 for each ID} { + r DEL nonexistent + set resp [r XDELEX nonexistent IDS 2 1 2] + assert_equal 2 [llength $resp] + assert_equal -1 [lindex $resp 0] + assert_equal -1 [lindex $resp 1] + } + + test {XDELEX on wrong key type returns error} { + r DEL testset + r SADD testset a b c + assert_error "*WRONGTYPE*" {r XDELEX testset IDS 1 1} + } + + test {XDELEX w/ ACKED only deletes acked items} { + r DEL teststream + r XADD teststream 1 msg helllo + r XADD teststream 2 msg helllo + r XADD teststream 3 msg helllo + + r XGROUP CREATE teststream testgrp1 0 + r XREADGROUP GROUP testgrp1 testconsumer COUNT 1 STREAMS teststream > + r XACK teststream testgrp1 1 + + set ids [ r XDELEX teststream ACKED IDS 3 1 2 99 ] + assert_equal 3 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal 2 [lindex $ids 1] + assert_equal -1 [lindex $ids 2] + + assert_equal 2 [ r XLEN teststream ] + } + + test {XDELEX ACKED blocks deletion for entries pending below a rewound last_id} { + r DEL testxadstream + r XADD testxadstream 1-0 f v1 + r XADD testxadstream 5-0 f v5 + r XGROUP CREATE testxadstream testxadgrp1 0 + + # Claim both entries: last_id advances to 5-0 and both land in the PEL. + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 10 STREAMS testxadstream > + + # Move last_id back before 5-0, which stays in the PEL (id > last_id). + r XGROUP SETID testxadstream testxadgrp1 1-0 + + # 5-0 is still pending in the group, so ACKED must not delete it. + set ids [r XDELEX testxadstream ACKED IDS 1 5-0] + assert_equal 2 [lindex $ids 0] + assert_equal 2 [r XLEN testxadstream] + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2 [lindex $pend 0] + + # After XACK drains the PEL entry, deletion stays blocked because the + # rewound delivery cursor means the group may still be served 5-0 by + # XREADGROUP "" (same fallback as XACKDEL ACKED). + r XACK testxadstream testxadgrp1 5-0 + set ids [r XDELEX testxadstream ACKED IDS 1 5-0] + assert_equal 2 [lindex $ids 0] + assert_equal 2 [r XLEN testxadstream] + + # Catching the cursor back up unblocks deletion. + r XGROUP SETID testxadstream testxadgrp1 5-0 + set ids [r XDELEX testxadstream ACKED IDS 1 5-0] + assert_equal 1 [lindex $ids 0] + assert_equal 1 [r XLEN testxadstream] + } + + test {XDELEX ACKED reports pending ref even when a clean group is iterated first} { + r DEL testxadstream + r XADD testxadstream 1-0 f v1 + r XADD testxadstream 5-0 f v5 + # agrp sorts before zgrp, so it is iterated first; it never claims 5-0. + r XGROUP CREATE testxadstream agrp 0 + r XGROUP CREATE testxadstream zgrp 0 + r XREADGROUP GROUP zgrp zcnsmr COUNT 10 STREAMS testxadstream > + + # Remove the stream entry, leaving 5-0 dangling only in zgrp's PEL. + r XDEL testxadstream 5-0 + + # The dangling pending ref must block ACKED with 2; the reply cannot + # depend on which consumer group the iteration reaches first. + set ids [r XDELEX testxadstream ACKED IDS 1 5-0] + assert_equal 2 [lindex $ids 0] + assert_equal 1 [r XLEN testxadstream] + set pend [r XPENDING testxadstream zgrp] + assert_equal 2 [lindex $pend 0] + } + + test {XDELEX ACKED returns -1 for deleted entry not referenced by any group} { + r DEL testxadstream + r XADD testxadstream 1-0 f v1 + r XADD testxadstream 5-0 f v5 + r XGROUP CREATE testxadstream testxadgrp1 0 + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 10 STREAMS testxadstream > + r XGROUP SETID testxadstream testxadgrp1 1-0 + r XACK testxadstream testxadgrp1 5-0 + + # Remove the entry; nothing references it anymore. The rewound last_id + # must not produce 2, since a deleted entry can never be re-delivered. + r XDEL testxadstream 5-0 + set ids [r XDELEX testxadstream ACKED IDS 1 5-0] + assert_equal -1 [lindex $ids 0] + assert_equal 1 [r XLEN testxadstream] + } + + test {XDELEX ACKED reports pending ref, not -1, when entry is deleted below a rewound last_id} { + r DEL testxadstream + r XADD testxadstream 1-0 f v1 + r XADD testxadstream 5-0 f v5 + r XGROUP CREATE testxadstream testxadgrp1 0 + + # Claim both entries, then rewind last_id below the still-pending 5-0. + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 10 STREAMS testxadstream > + r XGROUP SETID testxadstream testxadgrp1 1-0 + + # Remove the stream entry, leaving 5-0 dangling in the PEL. + r XDEL testxadstream 5-0 + + # The dangling pending ref must block ACKED (2), like when the group + # cursor is ahead of the ID, instead of reporting -1 (not found). + set ids [r XDELEX testxadstream ACKED IDS 1 5-0] + assert_equal 2 [lindex $ids 0] + assert_equal 1 [r XLEN testxadstream] + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 2 [lindex $pend 0] + } + + test {XDELEX DELREF clears PEL ref even when last_id was rewound below it} { + r DEL testxadstream + r XADD testxadstream 1-0 f v1 + r XADD testxadstream 5-0 f v5 + r XGROUP CREATE testxadstream testxadgrp1 0 + + # Claim both entries, then rewind last_id below the still-pending 5-0. + r XREADGROUP GROUP testxadgrp1 testxadcnsmr COUNT 10 STREAMS testxadstream > + r XGROUP SETID testxadstream testxadgrp1 1-0 + + # DELREF must remove the pending ref even though 5-0 > last_id. + set ids [r XDELEX testxadstream DELREF IDS 1 5-0] + assert_equal 1 [lindex $ids 0] + assert_equal 1 [r XLEN testxadstream] + + # Only 1-0 remains pending; the 5-0 ref must be gone. + set pend [r XPENDING testxadstream testxadgrp1] + assert_equal 1 [lindex $pend 0] + assert_equal 1-0 [lindex $pend 1] + assert_equal 1-0 [lindex $pend 2] + } + + test {XDELEX w/ KEEPREF deletes all but keeps refs in consumer group PELs} { + r DEL teststream + r XADD teststream 1 msg helllo + r XADD teststream 2 msg helllo + r XADD teststream 3 msg helllo + + r XGROUP CREATE teststream testgrp1 0 + r XREADGROUP GROUP testgrp1 testconsumer COUNT 1 STREAMS teststream > + + set ids [ r XDELEX teststream KEEPREF IDS 3 1 2 99 ] + assert_equal 3 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal 1 [lindex $ids 1] + assert_equal -1 [lindex $ids 2] + + assert_equal {1 1-0 1-0 {{testconsumer 1}}} [r XPENDING teststream testgrp1] + + assert_equal 1 [ r XLEN teststream ] + } + + test {XDELEX w/ DELREF deletes entries and clears consumer group PEL refs} { + r DEL teststream + r XADD teststream 1 msg helllo + r XADD teststream 2 msg helllo + r XADD teststream 3 msg helllo + + r XGROUP CREATE teststream testgrp1 0 + r XREADGROUP GROUP testgrp1 testconsumer COUNT 1 STREAMS teststream > + + set ids [ r XDELEX teststream DELREF IDS 3 1 2 99 ] + assert_equal 3 [llength $ids] + assert_equal 1 [lindex $ids 0] + assert_equal 1 [lindex $ids 1] + assert_equal -1 [lindex $ids 2] + + assert_equal {0 {} {} {}} [r XPENDING teststream testgrp1] + + assert_equal 1 [ r XLEN teststream ] + } + + test {XDELEX DELREF signals WATCH when removing an orphaned PEL ref} { + r DEL teststream + r XADD teststream 1-0 msg hello + r XGROUP CREATE teststream grp1 0 + r XREADGROUP GROUP grp1 consumer COUNT 1 STREAMS teststream > + + # Delete the entry but keep its (now orphaned) PEL reference. + r XDELEX teststream KEEPREF IDS 1 1-0 + assert_equal 0 [r XLEN teststream] + assert_equal {1 1-0 1-0 {{consumer 1}}} [r XPENDING teststream grp1] + + # DELREF clearing the orphaned NACK modifies the key even though no + # stream entry is deleted, so the pending MULTI/EXEC must abort and + # return the empty reply. + r WATCH teststream + r XDELEX teststream DELREF IDS 1 1-0 + r MULTI + r ping + assert_equal {} [r EXEC] + + assert_equal {0 {} {} {}} [r XPENDING teststream grp1] + } + + test {XDELEX on already-deleted ID returns -1} { + r DEL teststream + r XADD teststream 1 msg hello + r XADD teststream 2 msg hello + + # First delete succeeds + set resp [r XDELEX teststream IDS 1 1] + assert_equal 1 [lindex $resp 0] + + # Second delete on the same ID returns -1 + set resp [r XDELEX teststream IDS 1 1] + assert_equal -1 [lindex $resp 0] + } + + test {XDELEX should fail on invalid stream ID format} { + r DEL teststream + r XADD teststream 1 msg hello + assert_error "*Invalid stream ID specified*" {r XDELEX teststream IDS 1 not-a-valid-id} + } + + test {XDELEX w/ KEEPREF preserves second consumer group's PEL} { + r DEL teststream + r XADD teststream 1-0 msg hello + r XADD teststream 2-0 msg hello + r XGROUP CREATE teststream grp1 0 + r XGROUP CREATE teststream grp2 0 + r XREADGROUP GROUP grp1 consumer COUNT 1 STREAMS teststream > + r XREADGROUP GROUP grp2 consumer COUNT 1 STREAMS teststream > + + # KEEPREF: delete from stream but leave grp2's PEL reference intact + set ids [r XDELEX teststream KEEPREF IDS 1 1-0] + assert_equal 1 [lindex $ids 0] + + assert_equal {1 1-0 1-0 {{consumer 1}}} [r XPENDING teststream grp2] + assert_equal 1 [r XLEN teststream] + } + + test {XDELEX w/ DELREF clears all consumer groups' PELs} { + r DEL teststream + r XADD teststream 1-0 msg hello + r XADD teststream 2-0 msg hello + r XGROUP CREATE teststream grp1 0 + r XGROUP CREATE teststream grp2 0 + r XREADGROUP GROUP grp1 consumer COUNT 1 STREAMS teststream > + r XREADGROUP GROUP grp2 consumer COUNT 1 STREAMS teststream > + + # DELREF: delete from stream and wipe all groups' PEL references + set ids [r XDELEX teststream DELREF IDS 1 1-0] + assert_equal 1 [lindex $ids 0] + + assert_equal {0 {} {} {}} [r XPENDING teststream grp1] + assert_equal {0 {} {} {}} [r XPENDING teststream grp2] + assert_equal 1 [r XLEN teststream] + } + + test {XDELEX w/ ACKED waits for all consumer groups before deleting} { + r DEL teststream + r XADD teststream 1-0 msg hello + r XGROUP CREATE teststream grp1 0 + r XGROUP CREATE teststream grp2 0 + r XREADGROUP GROUP grp1 consumer COUNT 1 STREAMS teststream > + r XREADGROUP GROUP grp2 consumer COUNT 1 STREAMS teststream > + + # grp1 acks but grp2 still has it in PEL → no deletion + r XACK teststream grp1 1-0 + set ids [r XDELEX teststream ACKED IDS 1 1-0] + assert_equal 2 [lindex $ids 0] + assert_equal 1 [r XLEN teststream] + + # grp2 acks; all groups done → deletion occurs + r XACK teststream grp2 1-0 + set ids [r XDELEX teststream ACKED IDS 1 1-0] + assert_equal 1 [lindex $ids 0] + assert_equal 0 [r XLEN teststream] + } + + test {XDELEX returns syntax error when IDS token is missing after mode} { + r DEL teststream + r XADD teststream 1 msg hello + assert_error "*syntax error*" {r XDELEX teststream KEEPREF NOIDS 1 1} + assert_error "*syntax error*" {r XDELEX teststream DELREF NOIDS 1 1} + assert_error "*syntax error*" {r XDELEX teststream ACKED NOIDS 1 1} + } + + test {XDELEX returns error for unrecognized mode token} { + r DEL teststream + r XADD teststream 1 msg hello + assert_error "*syntax error*" {r XDELEX teststream BADMODE IDS 1 1} + } + + test {XDELEX returns error for invalid numids} { + r DEL teststream + r XADD teststream 1 msg hello + assert_error "*positive integer*" {r XDELEX teststream IDS 0 1} + assert_error "*positive integer*" {r XDELEX teststream IDS -1 1} + assert_error "*positive integer*" {r XDELEX teststream IDS abc 1} + } + + test {XDELEX returns error when numids does not match ID count} { + r DEL teststream + r XADD teststream 1 msg hello + r XADD teststream 2 msg hello + # Too few IDs provided for numids + assert_error "*syntax error*" {r XDELEX teststream IDS 3 1 2} + # Too many IDs provided for numids + assert_error "*syntax error*" {r XDELEX teststream IDS 1 1 2} + } + + test {XDELEX with more than 8 IDs exercises dynamic allocation} { + r DEL teststream + # STREAMID_STATIC_VECTOR_LEN is 8, use 10 to force zmalloc path + for {set i 1} {$i <= 10} {incr i} { + r XADD teststream $i msg hello + } + set ids [r XDELEX teststream IDS 10 1 2 3 4 5 6 7 8 9 10] + assert_equal 10 [llength $ids] + foreach id $ids { + assert_equal 1 $id + } + assert_equal 0 [r XLEN teststream] + } + + test {XDELEX with more than 8 IDs and ACKED mode exercises dynamic allocation} { + r DEL teststream + for {set i 1} {$i <= 10} {incr i} { + r XADD teststream $i msg hello + } + r XGROUP CREATE teststream grp 0 + # Read all entries into the group's PEL + r XREADGROUP GROUP grp consumer COUNT 10 STREAMS teststream > + # Ack all so ACKED mode can delete them + for {set i 1} {$i <= 10} {incr i} { + r XACK teststream grp $i + } + set ids [r XDELEX teststream ACKED IDS 10 1 2 3 4 5 6 7 8 9 10] + assert_equal 10 [llength $ids] + foreach id $ids { + assert_equal 1 $id + } + assert_equal 0 [r XLEN teststream] + } + test {XRANGE fuzzing} { set items [r XRANGE mystream{t} - +] set low_id [lindex $items 0 0] @@ -925,6 +1813,7 @@ start_server {tags {"stream needs:debug"} overrides {appendonly yes stream-node- test {XADD/XTRIM strip redundant LIMIT when rewriting for propagation} { set aof [get_last_incr_aof_path r] + set aof_offset [file size $aof] r config set stream-node-max-entries 10 for {set j 0} {$j < 100} {incr j} { @@ -942,6 +1831,7 @@ start_server {tags {"stream needs:debug"} overrides {appendonly yes stream-node- set fp [open $aof r] fconfigure $fp -translation binary + seek $fp $aof_offset set blob [read $fp] close $fp assert_equal -1 [string first "LIMIT" $blob] diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl index a9ea9822e..83b7e4c99 100644 --- a/tests/unit/type/string.tcl +++ b/tests/unit/type/string.tcl @@ -990,6 +990,32 @@ if {[string match {*jemalloc*} [s mem_allocator]]} { assert_error {ERR syntax error} {r set foo "new_value" ifeq "initial_value" nx} } + test {SET with IFNE conditional} { + r del foo + r set foo "initial_value" + assert_equal {OK} [r set foo "new_value" ifne "wrong_value"] + assert_equal "new_value" [r get foo] + assert_equal {} [r set foo "should_not_set" ifne "new_value"] + assert_equal "new_value" [r get foo] + } + + test {SET with IFNE conditional - with get} { + r del foo + assert_equal {} [r set foo "new_value" ifne "initial_value" get] + assert_equal "new_value" [r get foo] + r set foo "initial_value" + assert_equal "initial_value" [r set foo "new_value" ifne "wrong_value" get] + assert_equal "new_value" [r get foo] + assert_equal {new_value} [r set foo "should_not_set" ifne "new_value" get] + } + + test "SET with IFNE conditional - non string current value with get" { + r del foo + r sadd foo "some_set_value" + assert_error {WRONGTYPE Operation against a key holding the wrong kind of value} \ + {r set foo "new_value" ifne "initial_value" get} + } + test {Extended SET EX option} { r del foo r set foo bar ex 10 diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index c9336ab66..77f40b3cb 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -787,6 +787,27 @@ start_server {tags {"zset"}} { assert_equal 1 [r zlexcount zset (maxstring +] } + test "ZLEXCOUNT/ZREMRANGEBYLEX include empty-string member at - bound - $encoding" { + r del zset + r zadd zset 0 "" 0 a 0 b + assert_equal 3 [r zlexcount zset - +] + assert_equal 2 [r zlexcount zset - \[a] + assert_equal 1 [r zlexcount zset - (a] + assert_equal {{} a} [r zrangebylex zset - \[a] + assert_equal 2 [r zremrangebylex zset - \[a] + assert_equal {b} [r zrange zset 0 -1] + } + + test "ZRANGEBYLEX/ZREVRANGEBYLEX crossed sentinel bounds are empty - $encoding" { + create_default_lex_zset + assert_equal {} [r zrangebylex zset + \[c] + assert_equal {} [r zrangebylex zset + +] + assert_equal {} [r zrevrangebylex zset - \[c] + assert_equal {} [r zrevrangebylex zset - -] + assert_equal {} [r zrangebylex zset + \[c LIMIT 1 2] + assert_equal {} [r zrevrangebylex zset - \[c LIMIT 1 2] + } + test "ZRANGEBYLEX with LIMIT - $encoding" { create_default_lex_zset assert_equal {alpha bar} [r zrangebylex zset - \[cool LIMIT 0 2] @@ -1755,6 +1776,28 @@ start_server {tags {"zset"}} { r zmscore zmscoretest x } {10} + set original_max [lindex [r config get zset-max-listpack-entries] 1] + r config set zset-max-listpack-entries 0 + test {ZMSCORE uses hashtable batch lookup} { + r del zmscoretest + for {set i 1} {$i <= 128} {incr i} { + r zadd zmscoretest $i [format "m%02d" $i] + } + + assert_encoding btree zmscoretest + assert_equal {1} [r zmscore zmscoretest m01] + assert_equal {1 {} 1 4} [r zmscore zmscoretest m01 missing m01 m04] + + set members {missing} + set expected [list {}] + for {set i 1} {$i <= 19} {incr i} { + lappend members [format "m%02d" $i] + lappend expected $i + } + assert_equal $expected [r zmscore zmscoretest {*}$members] + } + r config set zset-max-listpack-entries $original_max + test {ZMSCORE retrieve requires one or more members} { r del zmscoretest r zadd zmscoretest 10 x @@ -1895,6 +1938,92 @@ start_server {tags {"zset"}} { assert_equal 0 $delta } + # The fuzzy tests above draw every score from [expr rand()], so no two + # members ever share a score. That distribution cannot produce a range + # boundary that lands *inside* a run of equal scores spanning more than + # one btree leaf, which is the shape that stresses boundary resolution. + # These two variants keep the same oracles but draw scores from a small + # discrete set, so each score run is far wider than one leaf. + test "ZCOUNT/ZRANGEBYSCORE fuzzy test with dense duplicate scores - $encoding" { + set err {} + set n 400 + set nscores 3 + # keep the arm's encoding at 400 members: listpack needs a raised + # limit, btree needs it pinned at 0 + set lp_entries [expr {$encoding eq "listpack" ? 100000 : 0}] + with_config zset-max-ziplist-entries $lp_entries { + r del zset + for {set i 0} {$i < $n} {incr i} { + r zadd zset [expr {int(rand() * $nscores) * 2 + 2}] "m$i" + } + assert_encoding $encoding zset + + # ~133 members per score: well beyond one 61-item leaf + for {set i 0} {$i < 60} {incr i} { + set a [expr {int(rand() * ($nscores + 2)) * 2}] + set b [expr {int(rand() * ($nscores + 2)) * 2}] + if {$a > $b} { set aux $a; set a $b; set b $aux } + foreach {min max} [list $a $b ($a $b $a ($b ($a ($b] { + set got [r zrangebyscore zset $min $max] + if {[r zcount zset $min $max] != [llength $got]} { + append err "zcount zset $min $max = [r zcount zset $min $max] but zrangebyscore returned [llength $got]\n" + } + } + } + } + assert_equal {} $err + } + + test "ZREMRANGEBYSCORE fuzzy test with dense duplicate scores - $encoding" { + set err {} + set n 400 + set nscores 3 + set lp_entries [expr {$encoding eq "listpack" ? 100000 : 0}] + with_config zset-max-ziplist-entries $lp_entries { + for {set i 0} {$i < 25} {incr i} { + r del zset + for {set j 0} {$j < $n} {incr j} { + r zadd zset [expr {int(rand() * $nscores) * 2 + 2}] "m$j" + } + assert_encoding $encoding zset + + set a [expr {int(rand() * ($nscores + 2)) * 2}] + set b [expr {int(rand() * ($nscores + 2)) * 2}] + if {$a > $b} { set aux $a; set a $b; set b $aux } + set variants [list [list $a $b] [list ($a $b] [list $a ($b] [list ($a ($b]] + set pick [lindex $variants [expr {int(rand() * 4)}]] + set min [lindex $pick 0] + set max [lindex $pick 1] + + # establish ground truth before mutating + set doomed [lsort [r zrangebyscore zset $min $max]] + set expected_count [llength $doomed] + set before [lsort [r zrange zset 0 -1]] + if {[r zcount zset $min $max] != $expected_count} { + append err "zcount disagrees with zrangebyscore for $min $max\n" + } + + set removed [r zremrangebyscore zset $min $max] + if {$removed != $expected_count} { + append err "zremrangebyscore zset $min $max removed $removed, expected $expected_count\n" + } + if {[r zcard zset] != [expr {$n - $expected_count}]} { + append err "zcard after zremrangebyscore $min $max is [r zcard zset], expected [expr {$n - $expected_count}]\n" + } + + set expected_survivors {} + foreach m $before { + if {[lsearch -exact -sorted $doomed $m] == -1} { lappend expected_survivors $m } + } + if {$expected_survivors ne [lsort [r zrange zset 0 -1]]} { + append err "surviving members wrong after zremrangebyscore $min $max\n" + } + if {$err ne {}} break + } + } + assert_equal {} $err + } + test "ZRANGEBYSCORE fuzzy test, 100 ranges in $elements element sorted set - $encoding" { set err {} r del zset @@ -1993,6 +2122,20 @@ start_server {tags {"zset"}} { set maxinc [randomInt 2] if {$mininc} {set cmin "\[$min"} else {set cmin "($min"} if {$maxinc} {set cmax "\[$max"} else {set cmax "($max"} + + # Sometimes replace a bound with an infinite sentinel so the + # special range items are exercised, including double and + # crossed sentinel combinations that must yield empty results. + # minlim/maxlim track the sentinel for the Tcl model: + # -1 = negatively infinite, 1 = positively infinite, 0 = none. + set minlim 0 + set maxlim 0 + if {[randomInt 10] == 0} { + if {[randomInt 2]} {set cmin -; set minlim -1} else {set cmin +; set minlim 1} + } + if {[randomInt 10] == 0} { + if {[randomInt 2]} {set cmax -; set maxlim -1} else {set cmax +; set maxlim 1} + } set rev [randomInt 2] if {$rev} { set cmd zrevrangebylex @@ -2014,25 +2157,32 @@ start_server {tags {"zset"}} { # Compute the same output via Tcl set o {} set copy $lexset - if {(!$rev && [string compare $min $max] > 0) || - ($rev && [string compare $max $min] > 0)} { - # Empty output when ranges are inverted. + if {$rev} { + # Invert the Tcl array using the server itself. + set copy [r zrevrange zset 0 -1] + # Invert min / max as well + lassign [list $min $max $mininc $maxinc $minlim $maxlim] \ + max min maxinc mininc maxlim minlim + } + if {$minlim == 1 || $maxlim == -1 || + ($minlim == 0 && $maxlim == 0 && [string compare $min $max] > 0)} { + # Empty output when the range is inverted, including a + # positively infinite min or negatively infinite max. } else { - if {$rev} { - # Invert the Tcl array using the server itself. - set copy [r zrevrange zset 0 -1] - # Invert min / max as well - lassign [list $min $max $mininc $maxinc] \ - max min maxinc mininc - } foreach e $copy { - set mincmp [string compare $e $min] - set maxcmp [string compare $e $max] - if { - ($mininc && $mincmp >= 0 || !$mininc && $mincmp > 0) - && - ($maxinc && $maxcmp <= 0 || !$maxinc && $maxcmp < 0) - } { + if {$minlim == -1} { + set minok 1 + } else { + set mincmp [string compare $e $min] + set minok [expr {$mininc ? $mincmp >= 0 : $mincmp > 0}] + } + if {$maxlim == 1} { + set maxok 1 + } else { + set maxcmp [string compare $e $max] + set maxok [expr {$maxinc ? $maxcmp <= 0 : $maxcmp < 0}] + } + if {$minok && $maxok} { lappend o $e } } @@ -2063,6 +2213,12 @@ start_server {tags {"zset"}} { if {$mininc} {set cmin "\[$min"} else {set cmin "($min"} if {$maxinc} {set cmax "\[$max"} else {set cmax "($max"} + # Sometimes replace a bound with an infinite sentinel so the + # special range items are exercised, including double and + # crossed sentinel combinations that must yield empty results. + if {[randomInt 10] == 0} {set cmin [lindex {- +} [randomInt 2]]} + if {[randomInt 10] == 0} {set cmax [lindex {- +} [randomInt 2]]} + # Make sure data is the same in both sides assert {[r zrange zset{t} 0 -1] eq $lexset} @@ -2443,6 +2599,39 @@ start_server {tags {"zset"}} { } } + test {ZSET btree lex range delete does not skip non-empty middle leaves} { + with_config zset-max-ziplist-entries 0 { + r del zk + # Build a multi-leaf btree with enough members that a lex range + # spans several leaves under the boundary leaves. + r zadd zk 0 {} + for {set i 1} {$i <= 300} {incr i} { + r zadd zk 0 [format "m%04d" $i] + } + assert_encoding btree zk + + # Trim the first leaf down to just the empty-string member: the + # start boundary of a later range will land in this now-mostly- + # empty leaf with no matching elements of its own (leaf-local + # "untouched"), while non-empty middle leaves still lie further + # to the right, inside the range about to be deleted. + r zremrangebylex zk \[m0001 \[m0060 + + # The exclusive lower bound "(" and the upper bound landing in a + # gap just past "m0121" (a value between two stored members) + # together make BOTH boundary leaves leaf-locally untouched, but + # the range still fully contains several middle leaves. The + # delete short-circuit must not treat this as an empty range. + set expected [r zlexcount zk \( \[m0121x] + assert {$expected > 0} + set removed [r zremrangebylex zk \( \[m0121x] + assert_equal $expected $removed + + # Nothing in the deleted range should remain. + assert_equal 0 [r zlexcount zk \( \[m0121x] + } + } + test {ZSET btree MEMORY USAGE reflects member sizes} { with_config zset-max-ziplist-entries 0 { r del zmem @@ -3098,6 +3287,51 @@ start_server {tags {"zset"}} { } } +start_server {config "minimal.conf" tags {"zset" "external:skip"} overrides {io-threads 4 io-threads-always-active yes zset-max-listpack-entries 0}} { + test "Zset nested prefetch - ZSCORE correctness with pipelined commands" { + for {set i 0} {$i < 200} {incr i} { + r zadd myzset $i "member:$i" + } + assert_encoding btree myzset + + set rd [valkey_deferring_client] + for {set i 0} {$i < 50} {incr i} { + $rd zscore myzset "member:$i" + } + $rd flush + for {set i 0} {$i < 50} {incr i} { + assert_equal $i [$rd read] + } + $rd close + } + + test "Zset nested prefetch - short members are looked up safely" { + # The zset hashtable stores packed [score][element] items, so a plain sds + # lookup key must be marked before the hash/compare callbacks read it. + # An unmarked key takes the packed path (sdslen - 8), which underflows for + # members shorter than the 8 byte score prefix. + foreach m {a bb ccc dddd eeeee ffffff ggggggg} { + r zadd shortzset [string length $m] $m + } + for {set i 0} {$i < 200} {incr i} { r zadd shortzset $i "member:$i" } + assert_encoding btree shortzset + + set clients {} + for {set c 0} {$c < 8} {incr c} { + set rd [valkey_deferring_client] + lappend clients $rd + foreach m {a bb ccc dddd eeeee ffffff ggggggg} { $rd zscore shortzset $m } + $rd flush + } + foreach rd $clients { + foreach m {a bb ccc dddd eeeee ffffff ggggggg} { + assert_equal [string length $m] [$rd read] + } + $rd close + } + } +} + start_server [list overrides [list save ""] tags {"zset needs:debug external:skip"}] { test {ZSET resize test - rehash more empty buckets in shrinking case} { if {[s arch_bits] != 64} { @@ -3461,6 +3695,143 @@ start_server {tags {"zset" "cluster:skip"}} { } } + # Regression tests for boundary resolution in the btree (fbtree) backend. + # A btree leaf holds NODE_SIZE (61) items, so a run of members sharing one + # score only spans multiple leaves once it exceeds that. A boundary resolved + # per leaf cannot express a bound that lands inside such a run, and getting it + # wrong yields bad counts and bad deletions while leaving the tree + # structurally valid. Every case below keeps the run well above one leaf so + # the multi-leaf path is always exercised. + test {ZCOUNT with a duplicate-score run spanning multiple btree leaves} { + with_config zset-max-ziplist-entries 0 { + r del zset + # 200 members per score: > 61, so each run spans several leaves + foreach score {2 4 6} { + for {set i 0} {$i < 200} {incr i} { + r zadd zset $score "s${score}:m$i" + } + } + assert_encoding btree zset + assert_equal 600 [r zcard zset] + + # ZCOUNT must agree with enumerating the same range + foreach {min max} {2 2 4 4 6 6 2 4 4 6 2 6 (2 (6 (2 6 2 (6 (2 +inf -inf (6 1 3 -inf +inf} { + assert_equal [llength [r zrangebyscore zset $min $max]] \ + [r zcount zset $min $max] "zcount zset $min $max" + } + + # explicit expectations, so the test still pins behaviour if + # ZRANGEBYSCORE ever regressed in the same way + assert_equal 200 [r zcount zset 2 2] + assert_equal 200 [r zcount zset 4 4] + assert_equal 400 [r zcount zset 2 4] + assert_equal 200 [r zcount zset (2 (6] + assert_equal 0 [r zcount zset (6 +inf] + } + } + + test {ZREMRANGEBYSCORE with duplicate-score runs spanning multiple btree leaves} { + with_config zset-max-ziplist-entries 0 { + foreach {min max expected_deleted} { + 2 2 200 + 4 4 200 + (2 (6 200 + 2 4 400 + (2 6 400 + 2 (6 400 + (6 +inf 0 + (1 (3 200 + } { + r del zset + foreach score {2 4 6} { + for {set i 0} {$i < 200} {incr i} { + r zadd zset $score "s${score}:m$i" + } + } + assert_encoding btree zset + + # the count must agree with the deletion, and the deletion must + # remove exactly the members the equivalent range enumerates + set doomed [lsort [r zrangebyscore zset $min $max]] + set survivors_before [lsort [r zrange zset 0 -1]] + assert_equal $expected_deleted [llength $doomed] "range $min $max" + assert_equal $expected_deleted [r zcount zset $min $max] "zcount $min $max" + + assert_equal $expected_deleted [r zremrangebyscore zset $min $max] \ + "zremrangebyscore zset $min $max" + assert_equal [expr {600 - $expected_deleted}] [r zcard zset] \ + "zcard after $min $max" + + # nothing outside the range may be touched + set expected_survivors {} + foreach m $survivors_before { + if {[lsearch -exact -sorted $doomed $m] == -1} { + lappend expected_survivors $m + } + } + assert_equal $expected_survivors [lsort [r zrange zset 0 -1]] \ + "survivors after $min $max" + } + } + } + + test {ZREMRANGEBYSCORE exclusive bound keeps members sitting on the bound} { + with_config zset-max-ziplist-entries 0 { + # Every member shares one score, so an exclusive bound on that score + # must match nothing at all. Resolving the bound per leaf instead + # deletes most of the set and leaves roughly one leaf behind. + r del zset + for {set i 0} {$i < 300} {incr i} { + r zadd zset 5 "m[format %04d $i]" + } + assert_encoding btree zset + + assert_equal 0 [r zcount zset (5 +inf] + assert_equal 0 [r zremrangebyscore zset (5 +inf] + assert_equal 300 [r zcard zset] + + assert_equal 0 [r zcount zset -inf (5] + assert_equal 0 [r zremrangebyscore zset -inf (5] + assert_equal 300 [r zcard zset] + + # the inclusive range still removes everything + assert_equal 300 [r zremrangebyscore zset 5 5] + assert_equal 0 [r zcard zset] + } + } + + test {btree range delete over a deep tree keeps the full set consistent} { + with_config zset-max-ziplist-entries 0 { + # >61*31 members forces a >=3-level tree, where a range delete can + # reduce an inner node to a single child -- the shape that can + # leave a stale prefix on that node. + r del zset + set n 4000 + for {set i 0} {$i < $n} {incr i} { + r zadd zset $i "m[format %05d $i]" + } + assert_encoding btree zset + + # delete an asymmetric interior range + assert_equal 1830 [r zremrangebyscore zset 1831 3660] + assert_equal [expr {$n - 1830}] [r zcard zset] + + # every surviving member must still be findable, correctly ranked, + # and enumerated in order + set expected {} + for {set i 0} {$i < $n} {incr i} { + if {$i < 1831 || $i > 3660} { lappend expected "m[format %05d $i]" } + } + assert_equal $expected [r zrange zset 0 -1] + assert_equal [llength $expected] [r zcount zset -inf +inf] + assert_equal 0 [r zrank zset [lindex $expected 0]] + assert_equal [expr {[llength $expected] - 1}] [r zrank zset [lindex $expected end]] + # a member inside the deleted range is really gone + assert_equal {} [r zscore zset "m[format %05d 2000]"] + assert_equal 0 [r zcount zset 1831 3660] + } + } + test {ZLEXCOUNT on btree-encoded set} { with_config zset-max-ziplist-entries 0 { diff --git a/utils/gen-test-certs.sh b/utils/gen-test-certs.sh index 02590e9ec..4f5bf3d82 100755 --- a/utils/gen-test-certs.sh +++ b/utils/gen-test-certs.sh @@ -6,19 +6,23 @@ # tests/tls/ca-{expired,notyet}.crt Self signed invalid CA certificates. # tests/tls/ca-expired/ Directory containing expired CA certificate. # tests/tls/ca-notyet/ Directory containing not-yet-valid CA certificate. +# tests/tls/ca-empty/ Empty directory for testing empty dir rejection. # tests/tls/ca-multi.crt CA bundle with multiple certs. # tests/tls/ca-dir/ CA directory with hashed links. -# tests/tls/valkey.{crt,key} A certificate with no key usage/policy restrictions. +# tests/tls/valkey{,-pw}.{crt,key} A certificate with no key usage/policy restrictions. With and without a passphrase. +# tests/tls/valkey-mldsa{,-pw}.{crt,key} A PQC certificate with no key usage/policy restrictions. With and without a passphrase. # tests/tls/client.{crt,key} A certificate restricted for SSL client usage. # tests/tls/client-{expired,notyet}.crt Invalid certificates restricted for SSL client usage. # tests/tls/server.{crt,key} A certificate restricted for SSL server usage. # tests/tls/server-{expired,notyet}.crt Invalid certificates restricted for SSL server usage. +# tests/tls/client-nul-cn.{crt,key} Client certificate whose CN contains an embedded NUL. # tests/tls/valkey.dh DH Params file. generate_cert() { local name=$1 local cn="$2" - local opts="$3" + local reqopts="$3" + local opts="$4" local keyfile=tests/tls/${name}.key local certfile=tests/tls/${name}.crt @@ -27,7 +31,8 @@ generate_cert() { openssl req \ -new -sha256 \ -subj "/O=Valkey Test/CN=$cn" \ - -key $keyfile | \ + -key "$keyfile" \ + $reqopts | \ openssl x509 \ -req -sha256 \ -CA tests/tls/ca.crt \ @@ -36,7 +41,7 @@ generate_cert() { -CAcreateserial \ -days 365 \ $opts \ - -out $certfile + -out "$certfile" } mkdir -p tests/tls @@ -63,9 +68,66 @@ subjectAltName = URI:urn:valkey:user:first, URI:urn:valkey:user:second subjectAltName = IP:127.0.0.1, IP:::1, DNS:localhost _END_ -generate_cert server "Server-only" "-extfile tests/tls/openssl.cnf -extensions server_cert" -generate_cert client "Client-only" "-extfile tests/tls/openssl.cnf -extensions client_cert" -generate_cert valkey "Generic-cert" "-extfile tests/tls/openssl.cnf -extensions generic_cert" +generate_cert server "Server-only" "" "-extfile tests/tls/openssl.cnf -extensions server_cert" +generate_cert client "Client-only" "" "-extfile tests/tls/openssl.cnf -extensions client_cert" +generate_cert valkey "Generic-cert" "" "-extfile tests/tls/openssl.cnf -extensions generic_cert" + +openssl genrsa -passout pass:1234 -aes256 -out tests/tls/valkey-pw.key 2048 +openssl ecparam -name prime256v1 -genkey -noout -out tests/tls/valkey-ec.key +openssl ecparam -name prime256v1 -genkey | openssl ec -passout pass:asdf -aes256 -out tests/tls/valkey-ec-pw.key + +generate_cert valkey-pw "Generic-cert-passworded" "-passin pass:1234" "-extfile tests/tls/openssl.cnf -extensions generic_cert" +generate_cert valkey-ec "EC-cert" "" "-extfile tests/tls/openssl.cnf -extensions generic_cert" +generate_cert valkey-ec-pw "EC-cert-passworded" "-passin pass:asdf" "-extfile tests/tls/openssl.cnf -extensions generic_cert" + +# A client certificate with the CN "Client-only\0attacker", which anything +# reading the CN as a C string sees as "Client-only". +# +# The openssl CLI will not put a NUL in a name, so issue with a placeholder +# byte, overwrite it with a NUL, and re-sign tbsCertificate. The signature is +# the same length, so the DER layout is unchanged. +generate_cert client-nul-cn "Client-only@attacker" "" "-extfile tests/tls/openssl.cnf -extensions client_cert" +python3 - tests/tls/client-nul-cn.crt tests/tls/ca.key 'Client-only@' <<'_PYEND_' +import base64, re, subprocess, sys + +cert_path, ca_key_path, marker = sys.argv[1:4] + +pem = open(cert_path).read() +der = bytearray(base64.b64decode(re.sub(r"-----[^-]*-----|\s", "", pem))) + +# Overwrite the last byte of the marker with a NUL. +der[der.index(marker.encode()) + len(marker) - 1] = 0 + +# tbsCertificate and signatureValue are direct children of Certificate. +fields = subprocess.run(["openssl", "asn1parse", "-in", cert_path], + capture_output=True, text=True, check=True).stdout.splitlines() + +def span(line): + """(element start, content start, content end)""" + off, hl, length = map(int, re.match(r"\s*(\d+):d=1\s+hl=\s*(\d+)\s+l=\s*(\d+)", line).groups()) + return off, off + hl, off + hl + length + +tbs_start, _, tbs_end = span(fields[1]) +_, sig_start, sig_end = span([f for f in fields if "BIT STRING" in f][-1]) + +# The signature covers tbsCertificate including its header. +open(cert_path + ".tbs", "wb").write(bytes(der[tbs_start:tbs_end])) +subprocess.run(["openssl", "dgst", "-sha256", "-sign", ca_key_path, + "-out", cert_path + ".sig", cert_path + ".tbs"], check=True) +sig = open(cert_path + ".sig", "rb").read() + +# Skip the BIT STRING's unused-bit count octet. +assert sig_end - sig_start - 1 == len(sig), "signature length changed" +der[sig_start + 1:sig_end] = sig + +b64 = base64.b64encode(bytes(der)).decode() +with open(cert_path, "w") as out: + out.write("-----BEGIN CERTIFICATE-----\n") + for i in range(0, len(b64), 64): + out.write(b64[i:i + 64] + "\n") + out.write("-----END CERTIFICATE-----\n") +_PYEND_ +rm -f tests/tls/client-nul-cn.crt.tbs tests/tls/client-nul-cn.crt.sig # Create a CA bundle and hashed CA directory used by TLS tests. # (ca-multi.crt and ca-dir/) @@ -212,6 +274,7 @@ openssl ca -batch -config "$CA_CONFIG" \ # Create CA certificate directories for testing tls-ca-cert-dir with invalid certs mkdir -p tests/tls/ca-expired mkdir -p tests/tls/ca-notyet +mkdir -p tests/tls/ca-empty cp tests/tls/ca-expired.crt tests/tls/ca-expired/ cp tests/tls/ca-notyet.crt tests/tls/ca-notyet/ @@ -219,6 +282,7 @@ cp tests/tls/ca-notyet.crt tests/tls/ca-notyet/ echo "Created CA certificate test directories:" echo " tests/tls/ca-expired/ (contains expired CA cert)" echo " tests/tls/ca-notyet/ (contains not-yet-valid CA cert)" +echo " tests/tls/ca-empty/ (empty, for testing empty dir rejection)" # Clean up temporary files rm -f tests/tls/*-expired.csr tests/tls/*-notyet.csr tests/tls/ca-expired.csr tests/tls/ca-notyet.csr diff --git a/utils/generate-command-code.py b/utils/generate-command-code.py index 63ebd1ae6..889c3e41f 100755 --- a/utils/generate-command-code.py +++ b/utils/generate-command-code.py @@ -6,6 +6,8 @@ ARG_TYPES = { "string": "ARG_TYPE_STRING", + "field": "ARG_TYPE_STRING", + "member": "ARG_TYPE_STRING", "integer": "ARG_TYPE_INTEGER", "double": "ARG_TYPE_DOUBLE", "key": "ARG_TYPE_KEY", @@ -16,6 +18,8 @@ "block": "ARG_TYPE_BLOCK", } +MEMBER_ARG_TYPES = {"field", "member"} + GROUPS = { "generic": "COMMAND_GROUP_GENERIC", "string": "COMMAND_GROUP_STRING", @@ -216,6 +220,33 @@ def verify_no_dup_names(container_fullname, args): exit(1) +def argument_width(arg): + """Return argv entries consumed by a fixed argument shape, or None.""" + if arg.desc.get("optional", False) or arg.desc.get("multiple", False): + return None + if arg.type == "pure-token": + return 1 + + token_width = 1 if arg.desc.get("token") else 0 + + if arg.type == "oneof": + subarg_widths = set() + for subarg in arg.subargs: + subarg_width = argument_width(subarg) + if subarg_width is None: + return None + subarg_widths.add(subarg_width) + return token_width + subarg_widths.pop() if len(subarg_widths) == 1 else None + if arg.type == "block": + width = token_width + for subarg in arg.subargs: + subarg_width = argument_width(subarg) + if subarg_width is None: + return None + width += subarg_width + return width + return token_width + 1 + class Argument(object): def __init__(self, parent_name, desc): self.parent_name = parent_name @@ -380,6 +411,27 @@ def __init__(self, name, desc): if "reply_schema" in self.desc: self.reply_schema = ReplySchema(self.reply_schema_name(), self.desc["reply_schema"]) + def infer_member_arg_index(self): + """Infer the first inner field/member argv position from argument metadata.""" + + def visit_args(arg_list, argv_index): + for arg in arg_list: + if arg.type in MEMBER_ARG_TYPES: + return argv_index + + if arg.type == "block": + subargv_index = argv_index + (1 if arg.desc.get("token") else 0) + target = visit_args(arg.subargs, subargv_index) + if target: + return target + + width = argument_width(arg) + if width is None: + return None + argv_index += width + + return visit_args(self.args, 1) + def fullname(self): return self.name.replace("-", "_").replace(":", "") @@ -496,6 +548,10 @@ def _doc_flags_code(): if self.args: s += ".args=%s," % self.arg_table_name() + member_arg_index = self.infer_member_arg_index() + if member_arg_index: + s += ".member_arg_index=%d," % member_arg_index + if self.reply_schema and args.with_reply_schema: s += ".reply_schema=&%s," % self.reply_schema_name() diff --git a/utils/reply-schema-linter/package-lock.json b/utils/reply-schema-linter/package-lock.json index 7b5888846..86383b5df 100644 --- a/utils/reply-schema-linter/package-lock.json +++ b/utils/reply-schema-linter/package-lock.json @@ -32,9 +32,9 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "node_modules/fast-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", - "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", + "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", "funding": [ { "type": "github", @@ -44,7 +44,8 @@ "type": "opencollective", "url": "https://opencollective.com/fastify" } - ] + ], + "license": "BSD-3-Clause" }, "node_modules/json-schema-traverse": { "version": "1.0.0", diff --git a/valkey.conf b/valkey.conf index dcb1be372..83a774fd8 100644 --- a/valkey.conf +++ b/valkey.conf @@ -209,7 +209,7 @@ tcp-keepalive 300 # tls-port 6379 # Configure a X.509 certificate and private key to use for authenticating the -# server to connected clients, primaries or cluster peers. These files should be +# server to connected clients, primaries or cluster peers. These files should be # PEM formatted. # # tls-cert-file valkey.crt @@ -219,6 +219,17 @@ tcp-keepalive 300 # as well. # # tls-key-file-pass secret +# +# You may also set up a secondary certificate and private key that use a +# different encryption algorithm in case a client, primary or cluster peer doesn't +# support the algorithm the above defined certificate uses (e.g. a PQC one). +# The ordering of the primary and secondary certificate does not matter as +# the most secure one will be chosen automatically if supported on both the +# client and server. +# +# tls-alt-cert-file valkey_rsa.crt +# tls-alt-key-file valkey_rsa.key +# tls-alt-key-file-pass secret # Normally the server uses the same certificate for both server functions (accepting # connections) and client functions (replicating from a primary, establishing @@ -556,6 +567,17 @@ locale-collate "" # # hash-seed example-seed-val +# Enable support for forkless snapshot operations by allocating metadata for each key. +# This is an immutable configuration that must be set at server startup and +# cannot be changed at runtime. +# +# When enabled, the server allocates 4 additional bytes per key. +# +# Note: This only enables the infrastructure support. The actual forkless save +# behavior is controlled separately by 'bgsave-default-method forkless'. +# +# forkless-infrastructure-enabled no + ################################ SNAPSHOTTING ################################ # Save the DB to disk. @@ -594,6 +616,13 @@ locale-collate "" # permissions, and so forth. stop-writes-on-bgsave-error yes +# Default method for background saves (BGSAVE and automatic periodic saves). +# +# 'forkless' can only be selected when the server was started with +# 'forkless-infrastructure-enabled yes'. Otherwise setting it is rejected. +# +# bgsave-default-method fork + # Control compression when dumping .rdb databases. # Supported values: # yes - use the default compression algorithm (currently lzf) @@ -602,9 +631,11 @@ stop-writes-on-bgsave-error yes # lz4 - streaming LZ4 frame compression for the entire RDB file. # Supported by Valkey 9.2 and later. # -# Streaming compression currently applies only to on-disk snapshots. -# Replication full synchronization continues to use the plain RDB format until -# compressed full-sync capability negotiation is implemented. +# This setting selects compression for regular RDB persistence and disk-based +# full sync. With lz4, a disk-based full sync is also LZ4 framed when every +# replica in the sync supports it. Diskless full sync is controlled by +# repl-compression. When AOF is enabled, a compressed sync snapshot is not +# reused as an AOF base and the replica runs BGREWRITEAOF instead. # # Before downgrading a server that may load an existing lz4 snapshot, # rewrite the snapshot in the legacy format and verify it with the older binary: @@ -830,6 +861,16 @@ repl-diskless-sync-max-replicas 0 # lose all your data. repl-diskless-load disabled +# Compress diskless full sync and the incremental replication stream. +# Compression is enabled when both the primary and replica enable it when the +# replica connects. +# Disk-based full sync follows rdbcompression. +# no - no compression (default) +# yes - compression with the current default algorithm (currently lz4) +# lz4 - LZ4 compression +# Supported by Valkey 9.2 and later. +repl-compression no + # This dual channel replication sync feature optimizes the full synchronization process # between a primary and its replicas. When enabled, it reduces both memory and CPU load # on the primary server. @@ -933,6 +974,13 @@ repl-disable-tcp-nodelay no # # repl-backlog-ttl 3600 +# When enabled, the primary throttles write-command clients when a replica's +# replication output buffer grows toward its configured soft limit, slowing writes so +# the replica can catch up instead of being disconnected (which would otherwise +# force a full resynchronization). Disabled by default. +# +repl-throttling-enabled no + # The replica priority is an integer number published by the server in the INFO # output. It is used by Sentinel in order to select a replica to promote # into a primary if the primary is no longer working correctly. @@ -1144,6 +1192,14 @@ replica-priority 100 # May be used with `,` for adding multiple IDs (e.g "db=1,2,3"). # alldbs Allow access to all databases. # resetdbs Flush the set of allowed database IDs. +# role= Assign the named roles to the user, replacing any role the user +# already had. May be used with `,` for assigning multiple roles +# (e.g "role=reader,writer"). At least one role has to be named. +# A role name may be any run of printable ASCII characters, +# except for the comma that separates the names here and the +# quotes and backslashes that the config parser would read back +# differently. See the "role" directive below. +# resetroles Remove every role from the user. # > Add this password to the list of valid password for the user. # For example >mypass will add "mypass" to the list. # This directive clears the "nopass" flag (see later). @@ -1229,6 +1285,28 @@ replica-priority 100 # For more information about ACL configuration please refer to # the Valkey web site at https://valkey.io/topics/acl +# ACL roles +# +# A role is a named, reusable set of permissions that can be assigned to any +# number of users, so a policy is written once instead of being repeated in +# every user. Roles are declared with the "role" directive, which takes the +# same ACL rules as "user" minus the ones that only make sense for a user +# (passwords, on/off, and role= itself): +# +# role reader ~app:* +@read +# role writer ~app:* +@write +# user alice on >alicepass role=reader +# user bob on >bobpass role=reader,writer +# +# A user's own permissions and those of each of its roles are OR'ed together, +# in the same way multiple selectors are, so a user can add permissions on top +# of a role but cannot take away what a role grants. Editing a role takes +# effect immediately for every user holding it. A role cannot be deleted while +# a user still has it, and roles cannot be nested. +# +# Roles are loaded before users, so the "role" and "user" lines may appear in +# any order in this file and in the ACL file. + # ACL LOG # # The ACL Log tracks failed commands and authentication events associated @@ -1245,7 +1323,7 @@ acllog-max-len 128 # ACL file, the server will refuse to start. # # The format of the external ACL user file is exactly the same as the -# format that is used inside valkey.conf to describe users. +# format that is used inside valkey.conf to describe users and roles. # # aclfile /etc/valkey/users.acl @@ -1838,13 +1916,18 @@ aof-timestamp-enabled no # the process exits immediately. This is the traditional behavior that # prioritizes configuration consistency. # -# - best-effort: Synchronously save the config file. If the save fails, +# - best-effort: Asynchronously save the config file using a background +# thread (BIO) to avoid blocking the main thread. If the save fails, # only log a warning and continue running. The node will retry saving # on the next configuration change. Passive exit may bring unexpected # effects, such as cluster down. This mode allows the node to survive # temporary disk failures, giving administrators time to address the -# issue without causing immediate service disruption. -# +# issue without causing immediate service disruption. The save status +# can be monitored via the "cluster_config_save_status" field in +# CLUSTER INFO output (returns "ok" or "err"). The unix time of the +# last successful save is exposed as "cluster_config_last_save_time", +# which can be used to detect persistence lag caused by disk issues. + # Note: The 'best-effort' mode is particularly useful in some environments. # However, if the disk issue persists and the node restarts, it may load # stale configuration data. Use with caution and ensure proper monitoring. @@ -1950,13 +2033,25 @@ aof-timestamp-enabled no # # cluster-require-full-coverage yes -# This option, when set to yes, prevents replicas from trying to failover its -# primary during primary failures. However the replica can still perform a -# manual failover, if forced to do so. -# -# This is useful in different scenarios, especially in the case of multiple -# data center operations, where we want one side to never be promoted if not -# in the case of a total DC failure. +# This option controls whether a replica is allowed to start an automatic +# failover of its primary. It accepts one of the following values: +# +# no - The replica may fail over its primary automatically (default). +# yes - The replica never starts an automatic failover. It can still +# perform a manual failover if forced to do so. This is useful in +# different scenarios, especially in the case of multiple data center +# operations, where we want one side to never be promoted if not in +# the case of a total DC failure. +# if-empty - The replica refuses to start an automatic failover only while it +# is empty, i.e. its replication offset is 0 because it has never +# completed an initial sync with its primary. Note that "empty" +# refers to the data received from the primary, not to the number +# of keys: a replica fully synced with an empty primary has a +# non-zero offset, so it is not considered empty. Promoting an +# empty replica would make an empty dataset the new primary and +# silently lose all data in the shard. This favors data safety +# over availability: the shard stays down until a replica with +# data is available or a manual failover is performed. # # cluster-replica-no-failover no @@ -2669,6 +2764,34 @@ rdb-save-incremental-fsync yes # # ignore-warnings ARM64-COW-BUG +# Limit the maximum memory used by cached Lua scripts (via EVAL). +# +# By default, Valkey caches up to 500 scripts loaded via EVAL using count-based +# LRU eviction. However, this can lead to memory abuse if a small number of EVAL +# scripts are very large. +# +# The maxmemory-scripts option provides memory-based eviction control for EVAL +# scripts, independent of the 500-script limit. When the configured memory limit +# is exceeded, existing EVAL scripts are evicted from the EVAL LRU list. Scripts +# loaded via SCRIPT LOAD are not evicted by this limit; their memory is subject +# to the global maxmemory limit and SCRIPT LOAD may return OOM. +# +# Configuration options: +# +# 1. Absolute memory value: +# Specify a fixed memory limit for script cache. +# Example: maxmemory-scripts 100mb +# +# 2. Percentage of maxmemory: +# Set the script memory limit as a percentage of maxmemory (1% to 100%). +# This is useful when you want script memory to scale with overall memory. +# Example: maxmemory-scripts 5% +# Note: This requires maxmemory to be set, otherwise no limit is applied. +# +# 3. Disabled (default): +# When set to 0, memory-based eviction is disabled. +# +# maxmemory-scripts 0 ########################### ACTIVE DEFRAGMENTATION ####################### # @@ -2740,3 +2863,122 @@ rdb-save-incremental-fsync yes # Jemalloc background thread for purging will be enabled by default jemalloc-bg-thread yes + +########################### HOT KEY DETECTION ################################# + +# Valkey can sample client key accesses and report the hottest keys through the +# HOTKEYS command. Detection is disabled by default. + +# Number of hottest keys to track (the "top-K"), which also acts as the on/off +# switch: 0 (the default) disables hot-key detection entirely and consumes no +# resources. Any positive value enables it; increase it if you expect a large +# number of hot keys and want a longer ranked list. +# +# hotkeys-top-k 0 + +# Percentage (1-100) of client key accesses that are sampled while detection is +# enabled; 100 samples every access. Lower values reduce the overhead on very +# high-throughput servers at the cost of some accuracy. +# +# hotkeys-sampling-percentage 1 + +# Length, in seconds, of the reporting window. Accesses accumulate in a live +# window; HOTKEYS GET reports the last completed (frozen) window. +# +# hotkeys-window-seconds 1 + +# How windows are closed, and when a window is dropped: rotation happens on the +# server's periodic task, so a window is closed at or shortly after its nominal +# boundary and therefore spans hotkeys-window-seconds plus a small lag. Reported +# QPS is divided by the span actually measured, not by the configured length, so +# that lag does not inflate the numbers; INFO hotkeys reports the measured span of +# the last completed window as hotkeys_last_window_duration_ms. +# +# If the server is stalled long enough that the open window ends up covering more +# than twice hotkeys-window-seconds, its counts describe too coarse an interval to +# report as "the last window" and are dropped instead. HOTKEYS GET then returns +# an empty result until the next window completes, rather than presenting a +# long-run average as if it were one window. This bounds how stale a report can +# be, at the cost of discarding the accesses seen during the stall. + +# Choosing the two knobs together: detection samples a fraction of accesses, so +# the smallest non-zero rate it can report (and the quantization step) is about +# 100 / (sampling-percentage * window-seconds) QPS, and the expected number of +# samples for a key in one window is qps * sampling-percentage/100 * +# window-seconds. With the defaults (1%, 1s) every reported value is a multiple +# of 100 QPS and only keys around 10k+ QPS have a trustworthy figure. Raise the +# sampling percentage and/or the window to resolve lower-rate keys more finely. + +# Note on RENAME / MOVE / SWAPDB: An entry is tracked by (key name, db) and +# its statistics are not moved when the key's name or database changes. The old +# entry stops accruing hits immediately and disappears when the window rotates, +# so HOTKEYS GET may report the previous name or database for up to +# hotkeys-window-seconds. + +# Hot-key state, both the live window and the last completed one, is cleared by +# HOTKEYS RESET and by anything that discards an entire database or slot range: +# FLUSHDB, FLUSHALL, a full sync or RDB reload that empties the dataset, a +# cluster reset, and dropping a slot. Ordinary key access and removal is treated +# as updates, so DEL and UNLINK count as accesses, and expiry and eviction +# do not clear state. + +################################ QUALITY OF SERVICE ############################### +# +# Valkey prioritizes system-critical internal connections (such as cluster bus +# heartbeats/gossip, primary-replica replication streams, and slot migration links) +# ahead of standard client traffic to maintain stability and +# prevent false failovers under heavy command workloads. +# +# When the main eventloop processes large batches of normal client events (such as +# heavy pipelined commands or multi-key operations), batch execution can take +# several milliseconds. To prevent head-of-line blocking and ensure system-critical +# internal events are not starved, Valkey periodically interrupts normal events +# processing to check and drain pending high-priority events. +# +# The `priority-preemptive-poll-interval-us` configuration defines the maximum elapsed +# time (in microseconds) between preemptive poll of the high-priority events +# while iterating through normal client events. +# +# Tuning guidelines: +# - Smaller values (e.g., 500 to 1000 us) ensure tighter latency bounds for cluster +# heartbeats and replication under intense client query bursts. +# - Larger values (e.g., 5000 to 10000 us) maximize client query batching throughput. +# - Setting this value to 0 disables mid-batch preemption entirely; high-priority +# events will only be processed at the start of each event loop cycle. +# +# Default: 2000 microseconds (2 milliseconds) +# priority-preemptive-poll-interval-us 2000 +# +# Reserve connection capacity for critical administrative traffic +# (such as controlplane or monitoring probes), so operators are not locked out +# when maxclients is reached. +# +# The `maxclients-reserved` configuration defines the minimum number of connections +# within maxclients guaranteed for administrative traffic originating from the +# priority-subnets. +# +# Note: When this configuration is applied or increased dynamically at runtime, +# the reservation guarantees capacity only for new priority-subnet connections; it +# does not evict or displace existing normal clients that are already holding those +# slots. Instead, incoming normal connections are rejected until existing normal +# clients disconnect and capacity drops below the normal client ceiling. +# +# Default: 0 (disabled) +# maxclients-reserved 0 +# +# The `priority-subnets` configuration is a comma- or space-separated list of +# IP addresses or CIDR subnets (IPv4 or IPv6) treated as prioritized connections. +# +# Default: "" (disabled) +# priority-subnets "" +# +# Example setup: +# +# maxclients 10000 +# priority-subnets "192.168.1.0/24 10.0.0.0/8" +# maxclients-reserved 128 +# +# Under this configuration: +# 1. Normal clients are capped at 9872 connections (maxclients - maxclients-reserved). +# 2. Administrative clients (from 192.168.1.0/24 or 10.0.0.0/8) can take up to maxclients (10000) if available. +# 3. Minimum of 128 (maxclients-reserved) connections guaranteed for administrative clients.