From 7bfc4266ff4facd446bedfce64c8091ade2656fc Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Wed, 3 Jun 2026 10:46:06 -0700 Subject: [PATCH 01/27] Steady-state replication throttling and throttling infrastructure Signed-off-by: harrylin98 --- cmake/Modules/SourceFiles.cmake | 6 +- src/Makefile | 6 +- src/blocked.c | 1 + src/config.c | 1 + src/monotonic.h | 12 + src/networking.c | 13 +- src/server.c | 20 ++ src/server.h | 11 + src/throttle.c | 459 ++++++++++++++++++++++++++++++++ src/throttle.h | 46 ++++ src/throttle_repl.c | 193 ++++++++++++++ src/throttle_repl.h | 28 ++ src/throttle_stat_calc.c | 148 ++++++++++ src/throttle_stat_calc.h | 47 ++++ src/throttle_token_bucket.c | 113 ++++++++ src/throttle_token_bucket.h | 28 ++ 16 files changed, 1128 insertions(+), 4 deletions(-) create mode 100644 src/throttle.c create mode 100644 src/throttle.h create mode 100644 src/throttle_repl.c create mode 100644 src/throttle_repl.h create mode 100644 src/throttle_stat_calc.c create mode 100644 src/throttle_stat_calc.h create mode 100644 src/throttle_token_bucket.c create mode 100644 src/throttle_token_bucket.h diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake index 46a7ea81d60..4181dd344fa 100644 --- a/cmake/Modules/SourceFiles.cmake +++ b/cmake/Modules/SourceFiles.cmake @@ -121,7 +121,11 @@ set(VALKEY_SERVER_SRCS ${CMAKE_SOURCE_DIR}/src/vset.c ${CMAKE_SOURCE_DIR}/src/fifo.c ${CMAKE_SOURCE_DIR}/src/mutexqueue.c - ${CMAKE_SOURCE_DIR}/src/queues.c) + ${CMAKE_SOURCE_DIR}/src/queues.c + ${CMAKE_SOURCE_DIR}/src/throttle_token_bucket.c + ${CMAKE_SOURCE_DIR}/src/throttle_stat_calc.c + ${CMAKE_SOURCE_DIR}/src/throttle_repl.c + ${CMAKE_SOURCE_DIR}/src/throttle.c) # valkey-cli diff --git a/src/Makefile b/src/Makefile index 66d6652858c..abb819f7acb 100644 --- a/src/Makefile +++ b/src/Makefile @@ -581,7 +581,11 @@ ENGINE_SERVER_OBJ = \ ziplist.o \ zipmap.o \ zmalloc.o \ - queues.o + queues.o \ + throttle_token_bucket.o \ + throttle_stat_calc.o \ + throttle_repl.o \ + throttle.o ENGINE_SERVER_OBJ+=$(ENGINE_TRACE_OBJ) ENGINE_CLI_NAME=$(ENGINE_NAME)-cli$(PROG_SUFFIX) ENGINE_CLI_OBJ = \ diff --git a/src/blocked.c b/src/blocked.c index d7c2a220983..94265d398d8 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -165,6 +165,7 @@ void processUnblockedClients(void) { c = ln->value; listDelNode(server.unblocked_clients, ln); c->flag.unblocked = 0; + serverAssert(!c->flag.throttled); if (c->flag.module) { if (!c->flag.blocked) { diff --git a/src/config.c b/src/config.c index 88ccc8a146d..02ace7fa1de 100644 --- a/src/config.c +++ b/src/config.c @@ -3274,6 +3274,7 @@ standardConfig static_configs[] = { createBoolConfig("repl-mptcp", NULL, IMMUTABLE_CONFIG, server.repl_mptcp, 0, isValidMptcp, NULL), createBoolConfig("repl-diskless-sync", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.repl_diskless_sync, 1, NULL, NULL), createBoolConfig("dual-channel-replication-enabled", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.dual_channel_replication, 0, NULL, NULL), + createBoolConfig("repl-throttle", NULL, MODIFIABLE_CONFIG, server.repl_throttle, 0, NULL, NULL), createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, 1, NULL, NULL), createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, 0, NULL, NULL), createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, 1, NULL, updateClusterState), diff --git a/src/monotonic.h b/src/monotonic.h index b465f90b109..77ebad58b5e 100644 --- a/src/monotonic.h +++ b/src/monotonic.h @@ -58,4 +58,16 @@ static inline uint64_t elapsedMs(monotime start_time) { return elapsedUs(start_time) / 1000; } +static inline uint64_t elapsedSec(monotime start_time) { + return elapsedUs(start_time) / 1000000; +} + +static inline uint64_t durationUs(monotime start_time, monotime end_time) { + return end_time - start_time; +} + +static inline uint64_t durationMs(monotime start_time, monotime end_time) { + return durationUs(start_time, end_time) / 1000; +} + #endif diff --git a/src/networking.c b/src/networking.c index ed0b0b135ae..43f1c2191b4 100644 --- a/src/networking.c +++ b/src/networking.c @@ -37,6 +37,8 @@ #include "fpconv_dtoa.h" #include "fmtargs.h" #include "io_threads.h" +#include "throttle.h" +#include "throttle_repl.h" #include "module.h" #include "connection.h" #include "zmalloc.h" @@ -2019,6 +2021,9 @@ void unlinkClient(client *c) { c->conn = NULL; } + /* Remove from throttle queue if needed. */ + throttle_removeClient(c); + /* Remove from the list of pending writes if needed. */ if (c->flag.pending_write) { serverAssert(server.clients_pending_write->len > 0); @@ -2213,6 +2218,7 @@ int freeClient(client *c) { if (c->lib_name) decrRefCount(c->lib_name); if (c->lib_ver) decrRefCount(c->lib_ver); freeClientMultiState(c); + if (c->cob_trend) zfree(c->cob_trend); sdsfree(c->peerid); sdsfree(c->sockname); zfree(c); @@ -3333,6 +3339,7 @@ void resetClient(client *c) { c->flag.replication_done = 0; c->flag.buffered_reply = 0; c->flag.keyspace_notified = 0; + c->flag.throttle_checked = 0; c->net_output_bytes_curr_cmd = 0; /* Make sure the duration has been recorded to some command. */ @@ -3840,7 +3847,7 @@ void commandProcessed(client *c) { * The client will be reset in unblockClient(). * 2. Don't update replication offset or propagate commands to replicas, * since we have not applied the command. */ - if (c->flag.blocked) return; + if (c->flag.blocked || c->flag.throttled) return; reqresAppendResponse(c); clusterSlotStatsAddNetworkBytesInForUserClient(c); @@ -4373,7 +4380,7 @@ int isClientConnIpV6(client *c) { * readable format, into the sds string 's'. */ sds catClientInfoString(sds s, client *client, int hide_user_data) { if (!server.crashed) waitForClientIO(client); - char flags[17], events[3], capa[9], conninfo[CONN_INFO_LEN], *p; + char flags[18], events[3], capa[9], conninfo[CONN_INFO_LEN], *p; p = flags; if (client->flag.replica) { @@ -4398,6 +4405,7 @@ sds catClientInfoString(sds s, client *client, int hide_user_data) { if (client->flag.readonly) *p++ = 'r'; if (client->flag.no_evict) *p++ = 'e'; if (client->flag.no_touch) *p++ = 'T'; + if (client->flag.throttled) *p++ = 'h'; if (client->flag.import_source) *p++ = 'I'; if (client->slot_migration_job && isImportSlotMigrationJob(client->slot_migration_job)) *p++ = 'i'; if (client->slot_migration_job && !isImportSlotMigrationJob(client->slot_migration_job)) *p++ = 'E'; @@ -6169,6 +6177,7 @@ int checkClientOutputBufferLimits(client *c) { } else { c->obuf_soft_limit_reached_time = 0; } + if ((soft || hard) && throttleRepl_isClientExempt(c)) return 0; return soft || hard; } diff --git a/src/server.c b/src/server.c index 8961d8f20a1..baabf89ed9b 100644 --- a/src/server.c +++ b/src/server.c @@ -51,6 +51,8 @@ #include "sds.h" #include "module.h" #include "scripting_engine.h" +#include "throttle.h" +#include "throttle_repl.h" #include "util.h" #include "eval.h" @@ -1698,6 +1700,8 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa run_with_period(1000) replicationCron(); } + run_with_period(100) throttleRepl_adjustThrottling(); + /* Run the Cluster cron. */ if (server.cluster_enabled) { run_with_period(CLUSTER_CRON_PERIOD_MS) clusterCron(); @@ -3115,6 +3119,7 @@ void initServer(void) { commandlogInit(); latencyMonitorInit(); + throttle_init(); initSharedQueryBuf(); /* Initialize ACL default password if it exists */ @@ -4686,6 +4691,9 @@ int processCommand(client *c) { return C_OK; } + /* Throttle framework: defer command if rate-limited. */ + if (throttle_deferCommand(c)) return C_OK; + /* Exec the command */ if (c->flag.multi && c->cmd->proc != execCommand && c->cmd->proc != discardCommand && c->cmd->proc != quitCommand && @@ -6068,6 +6076,7 @@ dict *genInfoSectionDict(robj **argv, int argc, char **defaults, int *out_all, i "errorstats", "cluster", "keyspace", + "throttle", NULL, }; if (!defaults) defaults = default_sections; @@ -6799,6 +6808,17 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { } } + /* Throttle */ + if (all_sections || (dictFind(section_dict, "throttle") != NULL)) { + if (sections++) info = sdscat(info, "\r\n"); + info = sdscat(info, "# Throttle\r\n"); + info = sdscatprintf(info, + "throttle_total_throttled_commands:%lld\r\n", + server.total_throttled_commands); + info = throttle_sdscatMetrics(info); + info = throttleRepl_sdscatMetrics(info); + } + /* Get info from modules. * Returned when the user asked for "everything", "modules", or a specific module section. * We're not aware of the module section names here, and we rather avoid the search when we can. diff --git a/src/server.h b/src/server.h index 66868a15f96..636aca0c403 100644 --- a/src/server.h +++ b/src/server.h @@ -1194,6 +1194,9 @@ typedef struct ClientFlags { uint64_t keyspace_notified : 1; /* Indicates that a keyspace notification was triggered during the execution of the current command. */ uint64_t argv_borrowed : 1; /* The argv array and its elements are borrowed from the caller (VM_CallArgv) and must not be freed. */ + uint64_t throttled : 1; /* Currently queued in a throttler */ + uint64_t throttle_checked : 1; /* Already passed throttle check for this command */ + uint64_t throttle_multi : 1; /* Matches multiple throttlers */ } ClientFlags; /* Ensure ClientFlags never silently grows beyond two uint64_t words. * If this fires, move a flag to a separate field or widen the limit. */ @@ -1398,6 +1401,11 @@ typedef struct client { list *deferred_reply; /* List of reply objects to be sent to the client, typically after the client has been unblocked. */ unsigned long long deferred_reply_bytes; /* Total bytes of objects in the blocked client pending list.*/ + /* Throttling */ + struct throttler *throttler; /* Current throttler this client is queued in, or NULL */ + listNode *throttle_node; /* Node in throttler's client_queue */ + monotime throttle_start_us; /* When this client was queued for throttling */ + struct trendCalculator *cob_trend; /* Per-replica COB size trend (NULL if not replica) */ #ifdef LOG_REQ_RES clientReqResInfo reqres; #endif @@ -2379,6 +2387,9 @@ struct valkeyServer { /* Local environment */ char *locale_collate; char *debug_context; /* A free-form string that has no impact on server except being included in a crash report. */ + /* Throttling */ + long long total_throttled_commands; /* Total commands deferred by the throttle framework */ + int repl_throttle; /* Enable replication throttle */ }; #define MAX_KEYS_BUFFER 256 diff --git a/src/throttle.c b/src/throttle.c new file mode 100644 index 00000000000..c63b6c4f922 --- /dev/null +++ b/src/throttle.c @@ -0,0 +1,459 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "throttle.h" +#include "throttle_token_bucket.h" +#include "throttle_stat_calc.h" +#include "hashtable.h" +#include "monotonic.h" + +#include +#include + +#define MAX_WAIT_TIME_MS 100 +#define MAX_UNTHROTTLE_PROCESSING_TIME_MS 10 +#define THROTTLE_CLEANUP_ID (-1) +#define THROTTLE_OPS_PER_MIN_GUARDRAIL 6 +#define TPS_WINDOW_SEC 5 +#define EPSILON 0.0001 +#define TOKENS_BURST_RATE_SEC 0.1 +#define MIN_ADJUST_AFTER_DISABLE 100.0 + +/* === Internal metrics (shared by name via hashtable) === */ + +typedef struct throttleInternalMetrics { + sds name; + int num_clients; + int total_throttled_commands; + tpsCalculator *incoming_tps; +} throttleInternalMetrics; + +/* Metrics hashtable callbacks. */ +static const void *metricsGetKey(const void *entry) { + return ((throttleInternalMetrics *)entry)->name; +} + +static void metricsDestructor(void *entry) { + throttleInternalMetrics *m = entry; + sdsfree(m->name); + tpsCalculator_free(m->incoming_tps); + zfree(m); +} + +static hashtableType metricsHashtableType = { + .entryGetKey = metricsGetKey, + .hashFunction = dictSdsHash, + .keyCompare = dictSdsKeyCompare, + .entryDestructor = metricsDestructor, +}; + +/* === Throttler instance === */ + +static int nextThrottlerId = 1; +static list *throttlerList = NULL; +static hashtable *metricsTable = NULL; + +typedef struct throttler { + int id; + throttleCriteriaProc *criteria_proc; + long long time_event_id; + void *priv_data; + tokenBucket *bucket; + list *client_queue; + listNode *ln; /* my node in throttlerList */ + monotime rate_below_guardrail_since; + throttleInternalMetrics *metrics; +} throttler; + +static int listMatchThrottler(void *ptr, void *id) { + return ((throttler *)ptr)->id == (long long)id; +} + +/* === Lookup === */ +static throttler *findThrottler(int id) { + listNode *ln = listSearchKey(throttlerList, (void *)(long long)id); + serverAssert(ln != NULL); + throttler *t = ln->value; + serverAssert(t->ln == ln); + return t; +} + +/* === Bucket sizing === */ +static double computeBucketSize(double tokens_per_sec, double burst_time_sec) { + return (tokens_per_sec < EPSILON) ? 0.0 + : 2.0 + tokens_per_sec * burst_time_sec; +} + +static void replenishTokens(throttler *t) { + if (t->id == THROTTLE_CLEANUP_ID) { + tokenBucket_setTokensPerSec(t->bucket, THROTTLE_UNLIMITED_RATE); + tokenBucket_add(t->bucket, THROTTLE_UNLIMITED_RATE); + return; + } + tokenBucket_replenish(t->bucket); + tokenBucket_capDebt(t->bucket, tokenBucket_getBucketSize(t->bucket)); +} + +static int waitTimeMs(throttler *t) { + serverAssert(listLength(t->client_queue) > 0); + double ms = tokenBucket_msUntilAvailable(t->bucket, 1.0); + if (ms < 0) return MAX_WAIT_TIME_MS; + return MIN(MAX_WAIT_TIME_MS, (int)ceil(ms)); +} + +static void freeThrottler(throttler *t) { + serverAssert(listLength(t->client_queue) == 0); + serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); + serverAssert(t->ln != NULL); + listDelNode(throttlerList, t->ln); + listRelease(t->client_queue); + tokenBucket_free(t->bucket); + /* metrics is shared — We do not free here */ + zfree(t); +} + +/* === Rate setting (with guardrail tracking) === */ + +static void setRate(throttler *t, double new_rate) { + if (new_rate < EPSILON) { + tokenBucket_halt(t->bucket); + } else { + if (new_rate > THROTTLE_UNLIMITED_RATE) new_rate = THROTTLE_UNLIMITED_RATE; + tokenBucket_setTokensPerSec(t->bucket, new_rate); + } + + double rate_per_min = tokenBucket_getTokensPerSec(t->bucket) * 60.0; + if (rate_per_min <= THROTTLE_OPS_PER_MIN_GUARDRAIL) { + if (t->rate_below_guardrail_since == 0) { + elapsedStart(&t->rate_below_guardrail_since); + } + } else { + t->rate_below_guardrail_since = 0; + } +} + +static void validateAlphaNumeric(const char *s) { + for (; *s; s++) { + serverAssert(isalnum(*s) || (*s == '_') || (*s == '-')); + } +} + +/* === Metrics lookup/create === */ + +static throttleInternalMetrics *findMetrics(const char *name) { + sds key = sdsnew(name); + void *found = NULL; + if (hashtableFind(metricsTable, key, &found)) { + sdsfree(key); + return (throttleInternalMetrics *)found; + } + throttleInternalMetrics *m = zmalloc(sizeof(throttleInternalMetrics)); + m->name = key; + m->num_clients = 0; + m->total_throttled_commands = 0; + m->incoming_tps = tpsCalculator_create(TPS_WINDOW_SEC); + hashtableAdd(metricsTable, m); + return m; +} + +/* === Public API === */ + +void throttle_init(void) { + if (throttlerList == NULL) { + throttlerList = listCreate(); + listSetMatchMethod(throttlerList, listMatchThrottler); + } + if (metricsTable == NULL) { + metricsTable = hashtableCreate(&metricsHashtableType); + } +} + +int throttle_register(throttleCriteriaProc *criteria_proc, + void *priv_data, + const char *metrics_name, + double ops_per_sec) { + serverAssert(criteria_proc != NULL); + serverAssert(metrics_name != NULL); + serverAssert(ops_per_sec >= 0); + validateAlphaNumeric(metrics_name); + serverAssert(nextThrottlerId > 0); + + throttler *t = zmalloc(sizeof(throttler)); + t->id = nextThrottlerId++; + t->criteria_proc = criteria_proc; + t->time_event_id = AE_DELETED_EVENT_ID; + t->priv_data = priv_data; + t->bucket = tokenBucket_create(ops_per_sec, TOKENS_BURST_RATE_SEC, computeBucketSize); + t->metrics = findMetrics(metrics_name); + t->client_queue = listCreate(); + t->rate_below_guardrail_since = 0; + setRate(t, ops_per_sec); + + listAddNodeTail(throttlerList, t); + t->ln = listLast(throttlerList); + return t->id; +} + +void throttle_deregister(int id) { + serverAssert(throttlerList != NULL && listLength(throttlerList) > 0); + throttler *t = findThrottler(id); + + if (listLength(t->client_queue) == 0) { + freeThrottler(t); + } else { + t->id = THROTTLE_CLEANUP_ID; + } +} + +void *throttle_setPrivData(int id, void *new_priv_data) { + throttler *t = findThrottler(id); + void *old = t->priv_data; + t->priv_data = new_priv_data; + return old; +} + +void throttle_setRate(int id, double ops_per_sec) { + serverAssert(ops_per_sec >= 0); + throttler *t = findThrottler(id); + setRate(t, ops_per_sec); +} + +double throttle_adjustRate(int id, double multiplier) { + serverAssert(multiplier >= 0.0 && multiplier <= 3.0); + throttler *t = findThrottler(id); + + double throttle_rate = tokenBucket_getTokensPerSec(t->bucket); + double new_rate; + + if (multiplier <= 1.0) { + new_rate = throttle_rate * multiplier; + double incoming_rate = tpsCalculator_averageTps(t->metrics->incoming_tps); + if (incoming_rate > EPSILON && new_rate < incoming_rate) { + new_rate = incoming_rate; + } + } else { + if (throttle_rate == THROTTLE_UNLIMITED_RATE) { + new_rate = throttle_rate; + } else if (throttle_rate < EPSILON) { + new_rate = MIN_ADJUST_AFTER_DISABLE; + } else { + double delta = throttle_rate * (multiplier - 1.0); + if (delta < 1.0) delta = 1.0; + new_rate = throttle_rate + delta; + } + } + + if (new_rate != throttle_rate) setRate(t, new_rate); + return tokenBucket_getTokensPerSec(t->bucket); +} + +const throttleMetrics *throttle_getMetrics(const char *metrics_name) { + static throttleMetrics result; + throttleInternalMetrics *m = findMetrics(metrics_name); + + result.num_clients = m->num_clients; + result.total_throttled_commands = m->total_throttled_commands; + result.incoming_tps = tpsCalculator_averageTps(m->incoming_tps); + result.ops_per_sec = 0.0; + result.oldest_client_delay_us = 0; + + /* Aggregate ops_per_sec and oldest_client from all throttlers sharing this metrics. */ + listNode *ln; + listIter li; + listRewind(throttlerList, &li); + while ((ln = listNext(&li))) { + throttler *t = ln->value; + if (t->metrics != m) continue; + result.ops_per_sec += tokenBucket_getTokensPerSec(t->bucket); + if (listLength(t->client_queue) > 0) { + client *oldest = listNodeValue(listFirst(t->client_queue)); + long delay_us = elapsedUs(oldest->throttle_start_us); + if (result.oldest_client_delay_us < delay_us) { + result.oldest_client_delay_us = delay_us; + } + } + } + return &result; +} + +/* === Multi-throttler token accounting === */ + +static void consumeOtherThrottlers(client *c, throttler *except) { + listNode *ln; + listIter li; + listRewind(throttlerList, &li); + while ((ln = listNext(&li))) { + throttler *t = ln->value; + if (t->id == THROTTLE_CLEANUP_ID) continue; + if (t == except) continue; + if (t->criteria_proc(c, t->priv_data)) { + tokenBucket_consume(t->bucket, 1.0); + } + } +} + +/* === Timer: drain the queue when tokens become available === */ + +static void processUnthrottledClient(client *c) { + serverAssert(c->argc > 0 && c->flag.pending_command && !c->flag.throttled); + if (c->conn && !connHasReadHandler(c->conn)) { + if (connSetReadHandler(c->conn, readQueryFromClient) == C_ERR) { + freeClient(c); + return; + } + } + if (processPendingCommandAndInputBuffer(c) == C_OK) beforeNextClient(c); +} + +static long long throttlerTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData) { + UNUSED(eventLoop); + UNUSED(id); + if (isPausedActionsWithUpdate(PAUSE_ACTIONS_CLIENT_ALL_SET)) return 1; + + throttler *t = (throttler *)clientData; + replenishTokens(t); + + monotime work_start; + elapsedStart(&work_start); + + while (tokenBucket_canConsume(t->bucket, 1.0) && + listLength(t->client_queue) > 0 && + elapsedMs(work_start) < MAX_UNTHROTTLE_PROCESSING_TIME_MS) { + tokenBucket_consume(t->bucket, 1.0); + client *c = listNodeValue(listFirst(t->client_queue)); + throttle_removeClient(c); + if (c->flag.throttle_multi) { + c->flag.throttle_multi = 0; + consumeOtherThrottlers(c, t); + } + processUnthrottledClient(c); + } + + if (listLength(t->client_queue) == 0) { + serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); // Already set in throttle_removeClient + if (t->id == THROTTLE_CLEANUP_ID) freeThrottler(t); + return AE_NOMORE; + } + return waitTimeMs(t); +} + +/* === Queue management === */ + +static void throttlerAddClient(throttler *t, client *c) { + serverAssert(c->throttler == NULL); + serverAssert(!c->flag.throttled); + elapsedStart(&c->throttle_start_us); + c->flag.throttled = 1; + listAddNodeTail(t->client_queue, c); + + if (c->conn) connSetReadHandler(c->conn, NULL); + + t->metrics->num_clients++; + t->metrics->total_throttled_commands++; + server.total_throttled_commands++; + c->throttler = t; + c->throttle_node = listLast(t->client_queue); + + if (listLength(t->client_queue) == 1) { + serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); + t->time_event_id = aeCreateTimeEvent(server.el, + waitTimeMs(t), + throttlerTimeProc, + t, NULL); + } +} + +void throttle_removeClient(client *c) { + if (!c->flag.throttled) return; + + c->flag.throttled = 0; + throttler *t = c->throttler; + serverAssert(t != NULL); + + listDelNode(t->client_queue, c->throttle_node); + + if (listLength(t->client_queue) == 0) { + serverAssert(t->time_event_id != AE_DELETED_EVENT_ID); + aeDeleteTimeEvent(server.el, t->time_event_id); + t->time_event_id = AE_DELETED_EVENT_ID; + } + t->metrics->num_clients--; + c->throttler = NULL; + c->throttle_node = NULL; + c->throttle_start_us = 0; +} + +/* === Per-command entry point === */ + +bool throttle_deferCommand(client *c) { + if (throttlerList == NULL || listLength(throttlerList) == 0) return false; + // Exempt all internal commands that has no connection from throttling. + if (!c->conn) return false; + if (c->flag.throttle_checked) return false; + c->flag.throttle_checked = 1; + + int match_count = 0; + throttler *strictest = NULL; + + listNode *ln; + listIter li; + listRewind(throttlerList, &li); + while ((ln = listNext(&li))) { + throttler *t = ln->value; + if (t->id == THROTTLE_CLEANUP_ID) continue; + + if (t->criteria_proc(c, t->priv_data)) { + match_count++; + tpsCalculator_record(t->metrics->incoming_tps, 1); + if (strictest == NULL || + tokenBucket_getTokensPerSec(t->bucket) < tokenBucket_getTokensPerSec(strictest->bucket)) { + strictest = t; + } + } + } + + if (strictest == NULL) return false; + + /* Fast path: queue empty AND a token is available. */ + if (listLength(strictest->client_queue) == 0) { + replenishTokens(strictest); + if (tokenBucket_canConsume(strictest->bucket, 1.0)) { + if (match_count > 1) consumeOtherThrottlers(c, strictest); + tokenBucket_consume(strictest->bucket, 1.0); + return false; + } + } + + if (match_count > 1) c->flag.throttle_multi = 1; + throttlerAddClient(strictest, c); + return true; +} + +/* === INFO output === */ +// Harry TODO: Should we do the info based on overall metrics or single throttler +sds throttle_sdscatMetrics(sds info) { + listNode *ln; + listIter li; + listRewind(throttlerList, &li); + while ((ln = listNext(&li))) { + throttler *t = ln->value; + if (t->rate_below_guardrail_since != 0) { + int secs = elapsedSec(t->rate_below_guardrail_since); + if (secs > 0) { + info = sdscatprintf(info, + "throttle_%s_guardrail_secs:%d\r\n", + t->metrics->name, secs); + } + } + } + return info; +} + +long throttle_getGuardrailSecs(int id) { + throttler *t = findThrottler(id); + if (t == NULL || t->rate_below_guardrail_since == 0) return 0; + return (long)elapsedSec(t->rate_below_guardrail_since); +} diff --git a/src/throttle.h b/src/throttle.h new file mode 100644 index 00000000000..3237a7a4fde --- /dev/null +++ b/src/throttle.h @@ -0,0 +1,46 @@ +#ifndef THROTTLE_H +#define THROTTLE_H + +#include "server.h" +#include + +static const double THROTTLE_UNLIMITED_RATE = 10000000.0; +static const int THROTTLE_INVALID_ID = -2; + +typedef bool throttleCriteriaProc(client *c, void *priv_data); + +typedef struct { + int num_clients; + int total_throttled_commands; + double ops_per_sec; + double incoming_tps; + long oldest_client_delay_us; +} throttleMetrics; + +/* Public API */ +void throttle_init(void); + +int throttle_register(throttleCriteriaProc *criteria_proc, + void *priv_data, + const char *metrics_name, + double ops_per_sec); + +void throttle_deregister(int id); + +void *throttle_setPrivData(int id, void *new_priv_data); + +void throttle_setRate(int id, double ops_per_sec); + +double throttle_adjustRate(int id, double multiplier); + +const throttleMetrics *throttle_getMetrics(const char *metrics_name); + +void throttle_removeClient(client *c); + +bool throttle_deferCommand(client *c); + +sds throttle_sdscatMetrics(sds info); + +long throttle_getGuardrailSecs(int id); + +#endif /* THROTTLE_H */ diff --git a/src/throttle_repl.c b/src/throttle_repl.c new file mode 100644 index 00000000000..70cbc5ba4f3 --- /dev/null +++ b/src/throttle_repl.c @@ -0,0 +1,193 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "server.h" +#include "throttle_repl.h" +#include "throttle.h" +#include "throttle_stat_calc.h" + +/* Configuration constants. */ +#define RATE_INCREASE_MULTIPLIER 1.05 +#define RATE_DECREASE_MULTIPLIER 0.95 +#define COB_TREND_WINDOW_SECS 2 +#define CONVERGENCE_SECS 30 +#define MAX_COB_TARGET (1024L * 1024 * 1024) /* 1GB */ +#define METRICS_NAME "ReplThrottle" + +typedef struct { + bool is_throttler_active; + double current_throttle_rate; + unsigned long throttle_activation_events; + unsigned long throttle_more_events; + unsigned long throttle_less_events; +} throttleReplMetrics; + +static throttleReplMetrics metrics = {0}; +static int throttle_id = 0; + +/* --- Internal helpers --- */ + +static bool isThrottleActive(void) { + return (throttle_id != 0); +} + +/* Criteria: throttle commands that generate replication traffic. */ +static bool criteriaProc(client *c, void *priv_data) { + UNUSED(priv_data); + if (c->cmd->flags & (CMD_WRITE | CMD_MAY_REPLICATE)) return true; + return false; +} + +static void installThrottler(void) { + serverAssert(!isThrottleActive()); + throttle_id = throttle_register(criteriaProc, NULL, METRICS_NAME, THROTTLE_UNLIMITED_RATE); + metrics.is_throttler_active = true; + metrics.current_throttle_rate = THROTTLE_UNLIMITED_RATE; + metrics.throttle_activation_events++; +} + +static void removeThrottler(void) { + serverAssert(isThrottleActive()); + throttle_deregister(throttle_id); + throttle_id = 0; + metrics.is_throttler_active = false; + metrics.current_throttle_rate = THROTTLE_UNLIMITED_RATE; +} + +static void adjustThrottleRate(bool reduceTrafficRate) { + if (isThrottleActive()) { + double rate; + if (reduceTrafficRate) { + rate = throttle_adjustRate(throttle_id, RATE_DECREASE_MULTIPLIER); + metrics.throttle_more_events++; + } else { + rate = throttle_adjustRate(throttle_id, RATE_INCREASE_MULTIPLIER); + metrics.throttle_less_events++; + if (rate >= THROTTLE_UNLIMITED_RATE) removeThrottler(); + } + metrics.current_throttle_rate = rate; + } else { + if (reduceTrafficRate) installThrottler(); + } +} + +/* Evaluate whether steady-state throttling is needed. + * Uses short-term COB trend to extrapolate future COB size. */ +static bool evaluateSteadyState(client *c, uint64_t cob_size) { + unsigned long cob_target = throttleRepl_getCobTargetSize(); // 50 % of target cob + uint64_t min_throttle = cob_target / 2; // Begin throttling at half the target, 25 % of max cob + + if (cob_size < min_throttle) return false; + + double short_trend = trendCalc_changePerSecShortTerm(c->cob_trend); + int64_t extrapolated = (int64_t)cob_size + (int64_t)(short_trend * CONVERGENCE_SECS); + + return (extrapolated > (int64_t)cob_target); +} + +/* --- Public API --- */ + +bool throttleRepl_isEnabled(void) { + return server.repl_throttle; +} + +bool throttleRepl_isClientExempt(client *c) { + if (!iAmPrimary()) return false; + if (!c->flag.replica) return false; + if (!isThrottleActive()) return false; + /* Throttle is actively working -- protect this replica from COB + * disconnect if its COB is above target (throttle needs time). */ + unsigned long cob = getClientOutputBufferMemoryUsage(c); + if (cob < throttleRepl_getCobTargetSize()) return false; + + /* Don't protect if throttle has been working too long without success. */ + time_t elapsed = server.unixtime - c->obuf_soft_limit_reached_time; + if (elapsed > 4 * CONVERGENCE_SECS) return false; + return true; +} + +unsigned long throttleRepl_getCobTargetSize(void) { + int64_t cob_target = server.client_obuf_limits[CLIENT_TYPE_REPLICA].soft_limit_bytes; + if (cob_target == 0) cob_target = server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes; + + cob_target /= 2; /* Target is half the limit. */ + + if (cob_target == 0 || cob_target > MAX_COB_TARGET) cob_target = MAX_COB_TARGET; + + return (unsigned long)cob_target; +} + +void throttleRepl_adjustThrottling(void) { + if (!iAmPrimary()) { + // Failover happened + if (isThrottleActive()) removeThrottler(); + return; + } + if (!throttleRepl_isEnabled() && !isThrottleActive()) return; + + bool reduce = false; + client *measured_replica = NULL; + uint64_t largest_cob = 0; + + /* Scan replicas, find steady-state replica with smallest COB. */ + listIter li; + listNode *ln; + listRewind(server.replicas, &li); + while ((ln = listNext(&li)) != NULL) { + client *c = ln->value; + if (!c->repl_data || c->repl_data->repl_state != REPLICA_STATE_ONLINE) continue; + + unsigned long cob_size = getClientOutputBufferMemoryUsage(c); + + /* Record trend per-replica. */ + if (c->cob_trend == NULL) c->cob_trend = newTrendCalc(COB_TREND_WINDOW_SECS); + trendCalc_recordMetric(c->cob_trend, cob_size); + + /* Ignore tiny COB (overhead only). */ + if (cob_size <= PROTO_REPLY_CHUNK_BYTES) cob_size = 0; + + // Find the largest cob size among replica clients + if (measured_replica == NULL || cob_size > largest_cob) { + measured_replica = c; + largest_cob = cob_size; + } + } + + if (measured_replica != NULL) { + reduce = evaluateSteadyState(measured_replica, largest_cob); + } + + adjustThrottleRate(reduce); +} + +sds throttleRepl_sdscatMetrics(sds info) { + info = sdscatprintf(info, + "repl_throttle_active:%d\r\n", + metrics.is_throttler_active ? 1 : 0); + + if (metrics.is_throttler_active) { + info = sdscatprintf(info, + "repl_throttle_rate:%.2f\r\n", + metrics.current_throttle_rate); + } + + const throttleMetrics *throttle_metrics = throttle_getMetrics(METRICS_NAME); + info = sdscatprintf(info, + "repl_throttle_activation_events:%lu\r\n" + "repl_throttle_more_events:%lu\r\n" + "repl_throttle_less_events:%lu\r\n" + "repl_throttle_below_guardrail_secs:%ld\r\n" + "repl_throttle_current_clients:%d\r\n" + "repl_throttle_total_commands:%d\r\n", + metrics.throttle_activation_events, + metrics.throttle_more_events, + metrics.throttle_less_events, + isThrottleActive() ? throttle_getGuardrailSecs(throttle_id) : 0L, + throttle_metrics->num_clients, + throttle_metrics->total_throttled_commands); + + return info; +} diff --git a/src/throttle_repl.h b/src/throttle_repl.h new file mode 100644 index 00000000000..4058c8596d1 --- /dev/null +++ b/src/throttle_repl.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef THROTTLE_REPL_H +#define THROTTLE_REPL_H + +#include "sds.h" + +/* Returns true if replication throttle is enabled. */ +bool throttleRepl_isEnabled(void); + +/* Get the COB target size for steady-state throttling. */ +unsigned long throttleRepl_getCobTargetSize(void); + +/* Returns true if the client should be exempt from COB disconnect limits + * because throttling is actively working to stabilize the replica. */ +bool throttleRepl_isClientExempt(client *c); + +/* Determine throttling needs and adjust rate. Called from serverCron. */ +void throttleRepl_adjustThrottling(void); + +/* Add repl throttle metrics to INFO string. */ +sds throttleRepl_sdscatMetrics(sds info); + +#endif /* THROTTLE_REPL_H */ diff --git a/src/throttle_stat_calc.c b/src/throttle_stat_calc.c new file mode 100644 index 00000000000..a12c2109e9f --- /dev/null +++ b/src/throttle_stat_calc.c @@ -0,0 +1,148 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include "throttle_stat_calc.h" +#include "server.h" +#include "monotonic.h" + +#define ONE_SECOND_IN_MICROS 1000000 + +struct tpsCalculator { + double window_secs; + double window_us; + double trans_per_window; + monotime last_update; + long update_freq_us; + long uncounted_trans; + bool is_new; +}; + +tpsCalculator *tpsCalculator_create(int window_secs) { + serverAssert(window_secs > 0); + tpsCalculator *calc = zmalloc(sizeof(tpsCalculator)); + calc->window_secs = (double)window_secs; + calc->window_us = (double)window_secs * 1000000.0; + calc->trans_per_window = 0.0; + calc->last_update = getMonotonicUs(); + /* Update at most 20 times per window for smooth results. */ + calc->update_freq_us = window_secs * (ONE_SECOND_IN_MICROS / 20); + calc->uncounted_trans = 0; + calc->is_new = true; + return calc; +} + +void tpsCalculator_free(tpsCalculator *calc) { + zfree(calc); +} + +void tpsCalculator_record(tpsCalculator *calc, unsigned long transactions) { + monotime now = getMonotonicUs(); + long elapsed_us = now - calc->last_update; + + if (elapsed_us < calc->update_freq_us) { + /* Accumulate until the update frequency is hit — updating too often + * increases metric lag. */ + calc->uncounted_trans += transactions; + return; + } + + double total = (double)(calc->uncounted_trans + transactions); + calc->uncounted_trans = 0; + calc->last_update = now; + + if (elapsed_us >= calc->window_us || calc->is_new) { + /* Elapsed >= window or first update: base entirely on this sample. */ + calc->trans_per_window = total * calc->window_us / elapsed_us; + calc->is_new = false; + } else { + /* Blend: decay existing by fraction of window elapsed, add new. */ + calc->trans_per_window = + (calc->trans_per_window * (calc->window_us - elapsed_us) / calc->window_us) + total; + } +} + +double tpsCalculator_averageTps(tpsCalculator *calc) { + /* Flush any pending samples so the value reflects "now". */ + tpsCalculator_record(calc, 0); + return calc->trans_per_window / calc->window_secs; +} + +#define DATA_POINTS 10 // Code assumes an even number +struct trendCalculator { + int windowSec; + monotime lastUpdate; + long updateFreqUs; + bool newCalculator; + long metrics[DATA_POINTS]; + long uncountedTotal; + int uncountedSamples; + double trend; + double trendShort; +}; + +trendCalculator *newTrendCalc(int windowSecs) { + trendCalculator *calc = zcalloc(sizeof(trendCalculator)); + calc->windowSec = windowSecs; + calc->lastUpdate = getMonotonicUs(); + calc->updateFreqUs = windowSecs * ONE_SECOND_IN_MICROS / DATA_POINTS; + calc->newCalculator = true; + return calc; +} + +void trendCalc_recordMetric(trendCalculator *calc, long metricValue) { + monotime now = getMonotonicUs(); + long elapsedUs = now - calc->lastUpdate; + + // When called more than updateFreqUs, just save values for later averaging. + calc->uncountedTotal += metricValue; + calc->uncountedSamples++; + + if (elapsedUs >= calc->updateFreqUs) { + long newValue = calc->uncountedTotal / calc->uncountedSamples; + calc->uncountedTotal = 0; + calc->uncountedSamples = 0; + calc->lastUpdate = now; + + if (calc->newCalculator) { + for (int i = 0; i < DATA_POINTS; i++) calc->metrics[i] = newValue; + calc->newCalculator = false; + } + + long olderTotal = 0; + for (int i = 0; i < DATA_POINTS / 2; i++) { + calc->metrics[i] = calc->metrics[i + 1]; + olderTotal += calc->metrics[i]; + } + long newerTotal = 0; + for (int i = DATA_POINTS / 2; i < DATA_POINTS - 1; i++) { + calc->metrics[i] = calc->metrics[i + 1]; + newerTotal += calc->metrics[i]; + } + calc->metrics[DATA_POINTS - 1] = newValue; + newerTotal += newValue; + + // Formula is the average of the newer data points, less the average of the older data + // points... this is the measured delta. But, the time is from the center of each half, + // resulting in half the window size (secs). So the formula is: + // (AveNewer - AveOlder) / (WindowSec/2) + double olderAvg = (double)olderTotal / (DATA_POINTS / 2); + double newerAvg = (double)newerTotal / (DATA_POINTS / 2); + double timeBetweenCenters = (double)calc->windowSec / 2.0; + calc->trend = (newerAvg - olderAvg) / timeBetweenCenters; + + // Compute short-term change-per-sec using the last 2 datapoints + long deltaShort = calc->metrics[DATA_POINTS - 1] - calc->metrics[DATA_POINTS - 2]; + double timeBetweenSlots = (double)calc->windowSec / DATA_POINTS; + calc->trendShort = deltaShort / timeBetweenSlots; + } +} + +double trendCalc_changePerSec(trendCalculator *calc) { + return calc->trend; +} + +double trendCalc_changePerSecShortTerm(trendCalculator *calc) { + return calc->trendShort; +} diff --git a/src/throttle_stat_calc.h b/src/throttle_stat_calc.h new file mode 100644 index 00000000000..64017d27049 --- /dev/null +++ b/src/throttle_stat_calc.h @@ -0,0 +1,47 @@ +#ifndef THROTTLE_STAT_CALC_H +#define THROTTLE_STAT_CALC_H + +/* Rolling-average TPS calculator over a configurable time window. + * + * Records transaction counts and reports a smoothed average TPS. + * The smoothing uses a blending approach: existing count decays + * proportional to elapsed time, new transactions are added on top. + * This produces a lagging average that converges over the window. + */ + +typedef struct tpsCalculator tpsCalculator; + +tpsCalculator *tpsCalculator_create(int window_secs); +void tpsCalculator_free(tpsCalculator *calc); + +void tpsCalculator_record(tpsCalculator *calc, unsigned long transactions); +double tpsCalculator_averageTps(tpsCalculator *calc); + +/* A trend calculator is used to compute the trend of data points over a specified time window. + * Periodically, values are added to the calculator. The calculator computes a "running trend" of + * the data over the given time window. The trend is reported as an average increase/decrease per + * second. Examples: + * - Data 1,2,1,2,1,2,1,2,1 - trend is essentially 0. A trend line would have 0 slope. + * - Data 0,0,0,10,10,10 - trend is approximately 3/sec + * This is similar to slope from a linear regression, but a simple speed-optimized algorithm. + */ +typedef struct trendCalculator trendCalculator; + +// Allocate a new trend calculator. Caller is responsible to deallocate with zfree(). +trendCalculator *newTrendCalc(int windowSecs); + +// Add a metric value to the trend calculator. This should be called at minimum 10 +// times over the window for best results. Note, if the metric is highly volatile, +// it is better to call more often - as a single outlier is less likely to skew results. +void trendCalc_recordMetric(trendCalculator *calc, long metricValue); + +// Retrieve the average trend over the calculator's window. Note: this value is updated +// approximately 10 times over the size of the window. So, with a 1 minute window, the +// reported trend will be updated roughly every 6 seconds. +double trendCalc_changePerSec(trendCalculator *calc); + +// Retrieve the trend using only the FINAL 10% of the calculator's window. This represents a +// short-term view of the trend. +double trendCalc_changePerSecShortTerm(trendCalculator *calc); + +#endif /* THROTTLE_STAT_CALC_H */ diff --git a/src/throttle_token_bucket.c b/src/throttle_token_bucket.c new file mode 100644 index 00000000000..bbfafb4d86f --- /dev/null +++ b/src/throttle_token_bucket.c @@ -0,0 +1,113 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include "throttle_token_bucket.h" +#include "server.h" +#include "monotonic.h" + +struct tokenBucket { + double tokens_per_sec; + double max_burst_time_secs; + double token_count; + monotime last_time_check; + bucketSizeFunc *bucket_size_func; +}; + +static double calcBucketSize(double tokens_per_sec, double max_burst_time_secs) { + return tokens_per_sec * max_burst_time_secs; +} + +static double trimTokenBucket(tokenBucket *bucket) { + double unused_tokens = 0; + double bucket_size = bucket->bucket_size_func(bucket->tokens_per_sec, bucket->max_burst_time_secs); + if (bucket->token_count > bucket_size) { + unused_tokens = bucket->token_count - bucket_size; + bucket->token_count = bucket_size; + } + return unused_tokens; +} + +tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs, bucketSizeFunc *bucket_size_func) { + serverAssert(tokens_per_sec >= 0); + serverAssert(max_burst_time_secs >= 0); + tokenBucket *bucket = zmalloc(sizeof(tokenBucket)); + bucket->tokens_per_sec = tokens_per_sec; + bucket->max_burst_time_secs = max_burst_time_secs; + bucket->bucket_size_func = bucket_size_func == NULL ? calcBucketSize : bucket_size_func; + bucket->token_count = bucket->bucket_size_func(bucket->tokens_per_sec, bucket->max_burst_time_secs); + bucket->last_time_check = getMonotonicUs(); + return bucket; +} + +void tokenBucket_free(tokenBucket *bucket) { + zfree(bucket); +} + +double tokenBucket_getTokenCount(tokenBucket *bucket) { + return bucket->token_count; +} + +double tokenBucket_getTokensPerSec(tokenBucket *bucket) { + return bucket->tokens_per_sec; +} + +double tokenBucket_getBucketSize(tokenBucket *bucket) { + return bucket->bucket_size_func(bucket->tokens_per_sec, bucket->max_burst_time_secs); +} + +double tokenBucket_getMaxBurstTime(tokenBucket *bucket) { + return bucket->max_burst_time_secs; +} + +void tokenBucket_setTokensPerSec(tokenBucket *bucket, double tokens_per_sec) { + serverAssert(tokens_per_sec >= 0); + bucket->tokens_per_sec = tokens_per_sec; + trimTokenBucket(bucket); +} + +void tokenBucket_setMaxBurstSec(tokenBucket *bucket, double max_burst_time_secs) { + serverAssert(max_burst_time_secs >= 0); + bucket->max_burst_time_secs = max_burst_time_secs; + trimTokenBucket(bucket); +} + +void tokenBucket_capDebt(tokenBucket *bucket, double max_debt) { + serverAssert(max_debt >= 0); + if (bucket->token_count < -max_debt) { + bucket->token_count = -max_debt; + } +} + +double tokenBucket_add(tokenBucket *bucket, double tokens) { + bucket->token_count += tokens; + return trimTokenBucket(bucket); +} + +double tokenBucket_replenish(tokenBucket *bucket) { + monotime now = getMonotonicUs(); + uint64_t delta_us = now - bucket->last_time_check; + bucket->last_time_check = now; + return tokenBucket_add(bucket, delta_us * bucket->tokens_per_sec / 1000000.0); +} + +void tokenBucket_consume(tokenBucket *bucket, double tokens) { + bucket->token_count -= tokens; +} + +bool tokenBucket_canConsume(tokenBucket *bucket, double tokens) { + return bucket->token_count >= tokens; +} + +double tokenBucket_msUntilAvailable(tokenBucket *bucket, double target_tokens) { + if (bucket->token_count >= target_tokens) return 0.0; + if (bucket->tokens_per_sec <= 0) return -1.0; /* halted — never available */ + double needed = target_tokens - bucket->token_count; + return needed / bucket->tokens_per_sec * 1000.0; +} + +void tokenBucket_halt(tokenBucket *bucket) { + bucket->token_count = 0; + bucket->tokens_per_sec = 0; +} diff --git a/src/throttle_token_bucket.h b/src/throttle_token_bucket.h new file mode 100644 index 00000000000..6a42ebc0248 --- /dev/null +++ b/src/throttle_token_bucket.h @@ -0,0 +1,28 @@ +#ifndef THROTTLE_TOKEN_BUCKET_H +#define THROTTLE_TOKEN_BUCKET_H + +#include + +typedef double bucketSizeFunc(double tokens_per_sec, double max_burst_time_secs); +typedef struct tokenBucket tokenBucket; + +// APIs +tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs, bucketSizeFunc *bucket_size_func); +void tokenBucket_free(tokenBucket *bucket); + +double tokenBucket_getTokenCount(tokenBucket *bucket); +double tokenBucket_getTokensPerSec(tokenBucket *bucket); +double tokenBucket_getBucketSize(tokenBucket *bucket); +double tokenBucket_getMaxBurstTime(tokenBucket *bucket); +void tokenBucket_setTokensPerSec(tokenBucket *bucket, double tokens_per_sec); +void tokenBucket_setMaxBurstSec(tokenBucket *bucket, double max_burst_time_secs); + +void tokenBucket_capDebt(tokenBucket *bucket, double max_debt); +double tokenBucket_add(tokenBucket *bucket, double tokens); +double tokenBucket_replenish(tokenBucket *bucket); +bool tokenBucket_canConsume(tokenBucket *bucket, double tokens); +void tokenBucket_consume(tokenBucket *bucket, double tokens); +double tokenBucket_msUntilAvailable(tokenBucket *bucket, double tokens); +void tokenBucket_halt(tokenBucket *bucket); + +#endif From a6c4a40fadcbbcdb4397567ef045fc465e1ed521 Mon Sep 17 00:00:00 2001 From: Harry Lin <49881386+harrylin98@users.noreply.github.com> Date: Wed, 27 May 2026 08:58:53 -0700 Subject: [PATCH 02/27] Set pending_command flag consistently across all command execution paths (#3600) The `pending_command` flag indicates that a client has a fully parsed command ready for execution. This update ensures that the flag is set/cleared consistently across different execution paths. --------- Signed-off-by: harrylin98 --- src/blocked.c | 12 +++++------- src/db.c | 2 ++ src/module.c | 7 +++++++ src/networking.c | 3 ++- src/replication.c | 10 ++++++++-- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 94265d398d8..55722aa3a50 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -479,10 +479,10 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo } } c->bstate->unblock_on_nokey = unblock_on_nokey; - /* Currently we assume key blocking will require reprocessing the command. - * However in case of modules, they have a different way to handle the reprocessing - * which does not require setting the pending command flag */ - if (btype != BLOCKED_MODULE) c->flag.pending_command = 1; + /* Key-blocked clients require pending_command for reprocessing on unblock. + * The caller must have set it (processInputBuffer for real clients, + * RM_Call for module fake clients). */ + serverAssert(c->flag.pending_command == 1); blockClient(c, btype); } @@ -683,8 +683,7 @@ void blockPostponeClient(client *c) { listAddNodeTail(server.postponed_clients, c); serverAssert(c->bstate->postponed_list_node == NULL); c->bstate->postponed_list_node = listLast(server.postponed_clients); - /* Mark this client to execute its command */ - c->flag.pending_command = 1; + serverAssert(c->flag.pending_command == 1); } /* Block client due to shutdown command */ @@ -715,7 +714,6 @@ static void unblockClientOnKey(client *c, robj *key) { /* In case this client was blocked on keys during command * we need to re process the command again */ if (c->flag.pending_command) { - c->flag.pending_command = 0; c->flag.reexecuting_command = 1; /* We want the command processing and the unblock handler (see RM_Call 'K' option) * to run atomically, this is why we must enter the execution unit here before diff --git a/src/db.c b/src/db.c index b171987ae30..85edb78c213 100644 --- a/src/db.c +++ b/src/db.c @@ -1483,6 +1483,8 @@ void shutdownCommand(client *c) { return; } + /* Clear pending_command to avoid re-execution. */ + c->flag.pending_command = 0; blockClientShutdown(c); if (prepareForShutdown(c, flags) == C_OK) exit(0); /* If we're here, then shutdown is ongoing (the client is still blocked) or diff --git a/src/module.c b/src/module.c index a9467c747df..29a4b86f5f8 100644 --- a/src/module.c +++ b/src/module.c @@ -6922,6 +6922,9 @@ static void moduleCallCommandHelper(ValkeyModuleCtx *ctx, client *c, robj **argv if (!(flags & VALKEYMODULE_CALL_ARGV_NO_AOF)) call_flags |= CMD_CALL_PROPAGATE_AOF; if (!(flags & VALKEYMODULE_CALL_ARGV_NO_REPLICAS)) call_flags |= CMD_CALL_PROPAGATE_REPL; } + /* Mirror processInputBuffer: set pending_command so that if the command + * blocks on keys, unblockClientOnKey will reprocess it on unblock. */ + c->flag.pending_command = 1; call(c, call_flags); /* Propagate database changes from the temporary client back to the context client @@ -8409,6 +8412,10 @@ ValkeyModuleBlockedClient *moduleBlockClient(ValkeyModuleCtx *ctx, c->bstate->timeout = timeout; blockClient(c, BLOCKED_MODULE); } + /* Module handles its own reply on unblock, so clear pending_command + * to prevent re-execution. Auth clients are the exception — they + * need re-execution after auth completes. */ + if (!auth_reply_callback) c->flag.pending_command = 0; /* Defer response until after being unblocked for a context originated from * keyspace notification events */ if (is_keyspace_notification) { diff --git a/src/networking.c b/src/networking.c index 7f7ba67f675..9b56d440ecb 100644 --- a/src/networking.c +++ b/src/networking.c @@ -3858,6 +3858,7 @@ void commandProcessed(client *c) { * since we have not applied the command. */ if (c->flag.blocked || c->flag.throttled) return; + c->flag.pending_command = 0; reqresAppendResponse(c); clusterSlotStatsAddNetworkBytesInForUserClient(c); resetClient(c); @@ -3937,7 +3938,6 @@ int processPendingCommandAndInputBuffer(client *c) { * So whenever we change the code here we need to consider if we need this change on module * blocked client as well */ if (c->flag.pending_command) { - c->flag.pending_command = 0; if (processCommandAndResetClient(c) == C_ERR) { return C_ERR; } @@ -4209,6 +4209,7 @@ int processInputBuffer(client *c) { } /* We are finally ready to execute the command. */ + c->flag.pending_command = 1; if (processCommandAndResetClient(c) == C_ERR) { /* If the client is no longer valid, we avoid exiting this * loop and trimming the client buffer later. So we return diff --git a/src/replication.c b/src/replication.c index 44ef22b7acd..696ad9481eb 100644 --- a/src/replication.c +++ b/src/replication.c @@ -5048,7 +5048,10 @@ void waitCommand(client *c) { } /* Otherwise block the client and put it into our list of clients - * waiting for ack from replicas. */ + * waiting for ack from replicas. WAIT handles its own reply in + * processClientsWaitingReplicas, so clear pending_command to avoid + * being mistaken for a command that needs re-execution. */ + c->flag.pending_command = 0; blockClientForReplicaAck(c, timeout, offset, numreplicas, 0); /* Make sure that the server will send an ACK request to all the replicas @@ -5090,7 +5093,10 @@ void waitaofCommand(client *c) { } /* Otherwise block the client and put it into our list of clients - * waiting for ack from replicas. */ + * waiting for ack from replicas. WAITAOF handles its own reply in + * processClientsWaitingReplicas, so clear pending_command to avoid + * being mistaken for a command that needs re-execution. */ + c->flag.pending_command = 0; blockClientForReplicaAck(c, timeout, offset, numreplicas, numlocal); /* Make sure that the server will send an ACK request to all the replicas From 1a60842b3c9f1bb77ea70ad7fda5873f9b840666 Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Tue, 23 Jun 2026 16:45:08 -0700 Subject: [PATCH 03/27] zombie connection detection from forkless branch Signed-off-by: harrylin98 --- src/connection.h | 12 +++++++++++- src/rdma.c | 1 + src/server.c | 19 +++++++++++++++++++ src/socket.c | 24 ++++++++++++++++++++++++ src/tls.c | 1 + src/unix.c | 1 + 6 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/connection.h b/src/connection.h index 44e15e8f1c1..0e07d60fe56 100644 --- a/src/connection.h +++ b/src/connection.h @@ -153,7 +153,8 @@ typedef struct ConnectionType { struct user *(*get_peer_user)(connection *conn, sds *cert_username); /* Miscellaneous */ - int (*connIntegrityChecked)(void); // return 1 if connection type has built-in integrity checks + int (*connIntegrityChecked)(void); // return 1 if connection type has built-in integrity checks + int (*is_closing)(connection *conn); // return 1 if connection is closed } ConnectionType; struct connection { @@ -390,6 +391,15 @@ static inline int connHasReadHandler(connection *conn) { return conn->read_handler != NULL; } +/* Check if the remote side has closed the connection. */ +static inline int connIsClosing(connection *conn) { + if (!conn->type->is_closing) return 0; + return conn->type->is_closing(conn); +} + +/* Shared is_closing implementation for socket-based connections. */ +int connSocketIsClosing(connection *conn); + /* Associate a private data pointer with the connection */ static inline void connSetPrivateData(connection *conn, void *data) { conn->private_data = data; diff --git a/src/rdma.c b/src/rdma.c index 8a51331a8b2..116238053d6 100644 --- a/src/rdma.c +++ b/src/rdma.c @@ -1863,6 +1863,7 @@ static ConnectionType CT_RDMA = { /* Miscellaneous */ .connIntegrityChecked = NULL, + .is_closing = NULL, }; ConnectionType *connectionTypeRdma(void) { diff --git a/src/server.c b/src/server.c index 3967ded7c50..05f310cad0c 100644 --- a/src/server.c +++ b/src/server.c @@ -1190,6 +1190,24 @@ void getExpensiveClientsInfo(size_t *in_usage, size_t *out_usage) { *out_usage = o; } +/* Detect and free zombie connections whose read handler was removed (e.g. + * BLOCKED_INUSE). Without a read handler the event loop won't notice the + * remote side closing, so these fds would leak until the fd limit is hit. */ +static bool clientsCronTcpIsClosing(client *c) { + if (!c->conn) return false; + + if (!connIsClosing(c->conn)) return false; + + if (server.verbosity <= LL_VERBOSE) { + sds client_info = catClientInfoString(sdsempty(), c, server.hide_user_data_from_log); + serverLog(LL_VERBOSE, "Client closed connection while blocked %s", client_info); + sdsfree(client_info); + } + + freeClientAsync(c); + return true; +} + /* This function is called by clientsTimeProc() and is used in order to perform * operations on clients that are important to perform constantly. For instance * we use this function in order to disconnect clients after a timeout, including @@ -1244,6 +1262,7 @@ static void clientsCron(int clients_this_cycle) { if (clientsCronResizeQueryBuffer(c)) continue; if (clientsCronResizeOutputBuffer(c, now)) continue; if (clientsCronTrackExpensiveClients(c, curr_peak_mem_usage_slot)) continue; + if (clientsCronTcpIsClosing(c)) continue; /* Iterating all the clients in getMemoryOverheadData() is too slow and * in turn would make the INFO command too slow. So we perform this diff --git a/src/socket.c b/src/socket.c index c9f9cae046e..55143e2d026 100644 --- a/src/socket.c +++ b/src/socket.c @@ -30,6 +30,10 @@ #include "server.h" #include "connhelpers.h" #include "io_threads.h" +#include +#ifdef __APPLE__ +#include +#endif /* The connections module provides a lean abstraction of network connections * to avoid direct socket and async event management across the server code base. @@ -418,6 +422,25 @@ static int connSocketGetType(void) { return CONN_TYPE_SOCKET; } +int connSocketIsClosing(connection *conn) { + if (aeGetFileEvents(server.el, conn->fd) != AE_NONE) return false; +#if defined(__linux__) + struct tcp_info info; + socklen_t infolen = sizeof(info); + if (getsockopt(conn->fd, IPPROTO_TCP, TCP_INFO, &info, &infolen) != 0 || infolen < sizeof(info)) return false; // Cannot retrieve TCP info + return (info.tcpi_state == TCP_CLOSE_WAIT || info.tcpi_state == TCP_CLOSE); +#elif defined(__APPLE__) + struct tcp_connection_info info; + socklen_t infolen = sizeof(info); + if (getsockopt(conn->fd, IPPROTO_TCP, TCP_CONNECTION_INFO, &info, &infolen) != 0 || infolen < sizeof(info)) return false; // Cannot retrieve TCP info + return (info.tcpi_state == TCPS_CLOSE_WAIT || info.tcpi_state == TCPS_CLOSED); +#else + /* Unsupported platform: zombie connection detection is not available. */ + UNUSED(conn); + return false; +#endif +} + static ConnectionType CT_Socket = { /* connection type */ .get_type = connSocketGetType, @@ -465,6 +488,7 @@ static ConnectionType CT_Socket = { /* Miscellaneous */ .connIntegrityChecked = NULL, + .is_closing = connSocketIsClosing, }; int connBlock(connection *conn) { diff --git a/src/tls.c b/src/tls.c index e75b6f0d407..e708d8db758 100644 --- a/src/tls.c +++ b/src/tls.c @@ -1950,6 +1950,7 @@ static ConnectionType CT_TLS = { /* Miscellaneous */ .connIntegrityChecked = connTLSIsIntegrityChecked, + .is_closing = connSocketIsClosing, }; diff --git a/src/unix.c b/src/unix.c index e5db7cbb9a1..e2b4ec656a4 100644 --- a/src/unix.c +++ b/src/unix.c @@ -214,6 +214,7 @@ static ConnectionType CT_Unix = { /* Miscellaneous */ .connIntegrityChecked = NULL, + .is_closing = NULL, }; int RedisRegisterConnectionTypeUnix(void) { From 5904359efe03550e4a6c0663e9961bdd24c03d8a Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Tue, 23 Jun 2026 16:58:54 -0700 Subject: [PATCH 04/27] Format and correct comments Signed-off-by: harrylin98 --- src/networking.c | 5 +- src/server.c | 1 - src/throttle.c | 187 ++++++++++++++++++--------------------- src/throttle.h | 2 +- src/throttle_repl.c | 14 +-- src/throttle_repl.h | 2 +- src/throttle_stat_calc.c | 12 +-- src/throttle_stat_calc.h | 17 +--- 8 files changed, 105 insertions(+), 135 deletions(-) diff --git a/src/networking.c b/src/networking.c index 9b56d440ecb..57dd5189410 100644 --- a/src/networking.c +++ b/src/networking.c @@ -352,6 +352,10 @@ client *createClient(connection *conn) { listSetFreeMethod(c->reply, freeClientReplyValue); listSetDupMethod(c->reply, dupClientReplyValue); c->repl_data = NULL; + c->throttler = NULL; + c->throttle_node = NULL; + c->throttle_start_us = 0; + c->cob_trend = NULL; c->bstate = NULL; c->pubsub_data = NULL; c->module_data = NULL; @@ -2022,7 +2026,6 @@ void unlinkClient(client *c) { c->conn = NULL; } - /* Remove from throttle queue if needed. */ throttle_removeClient(c); /* Remove from the list of pending writes if needed. */ diff --git a/src/server.c b/src/server.c index 05f310cad0c..f628bb6125e 100644 --- a/src/server.c +++ b/src/server.c @@ -4710,7 +4710,6 @@ int processCommand(client *c) { return C_OK; } - /* Throttle framework: defer command if rate-limited. */ if (throttle_deferCommand(c)) return C_OK; /* Exec the command */ diff --git a/src/throttle.c b/src/throttle.c index c63b6c4f922..718943eac77 100644 --- a/src/throttle.c +++ b/src/throttle.c @@ -23,7 +23,6 @@ #define MIN_ADJUST_AFTER_DISABLE 100.0 /* === Internal metrics (shared by name via hashtable) === */ - typedef struct throttleInternalMetrics { sds name; int num_clients; @@ -50,8 +49,6 @@ static hashtableType metricsHashtableType = { .entryDestructor = metricsDestructor, }; -/* === Throttler instance === */ - static int nextThrottlerId = 1; static list *throttlerList = NULL; static hashtable *metricsTable = NULL; @@ -69,19 +66,17 @@ typedef struct throttler { } throttler; static int listMatchThrottler(void *ptr, void *id) { - return ((throttler *)ptr)->id == (long long)id; + return ((throttler *)ptr)->id == (long)id; } -/* === Lookup === */ static throttler *findThrottler(int id) { - listNode *ln = listSearchKey(throttlerList, (void *)(long long)id); + listNode *ln = listSearchKey(throttlerList, (void *)(long)id); serverAssert(ln != NULL); throttler *t = ln->value; serverAssert(t->ln == ln); return t; } -/* === Bucket sizing === */ static double computeBucketSize(double tokens_per_sec, double burst_time_sec) { return (tokens_per_sec < EPSILON) ? 0.0 : 2.0 + tokens_per_sec * burst_time_sec; @@ -111,12 +106,10 @@ static void freeThrottler(throttler *t) { listDelNode(throttlerList, t->ln); listRelease(t->client_queue); tokenBucket_free(t->bucket); - /* metrics is shared — We do not free here */ + /* metrics is shared and do not free here */ zfree(t); } -/* === Rate setting (with guardrail tracking) === */ - static void setRate(throttler *t, double new_rate) { if (new_rate < EPSILON) { tokenBucket_halt(t->bucket); @@ -141,8 +134,6 @@ static void validateAlphaNumeric(const char *s) { } } -/* === Metrics lookup/create === */ - static throttleInternalMetrics *findMetrics(const char *name) { sds key = sdsnew(name); void *found = NULL; @@ -159,6 +150,87 @@ static throttleInternalMetrics *findMetrics(const char *name) { return m; } +static void consumeOtherThrottlers(client *c, throttler *except) { + listNode *ln; + listIter li; + listRewind(throttlerList, &li); + while ((ln = listNext(&li))) { + throttler *t = ln->value; + if (t->id == THROTTLE_CLEANUP_ID) continue; + if (t == except) continue; + if (t->criteria_proc(c, t->priv_data)) { + tokenBucket_consume(t->bucket, 1.0); + } + } +} + +static void processUnthrottledClient(client *c) { + serverAssert(c->argc > 0 && c->flag.pending_command && !c->flag.throttled); + if (c->conn && !connHasReadHandler(c->conn)) { + if (connSetReadHandler(c->conn, readQueryFromClient) == C_ERR) { + freeClient(c); + return; + } + } + if (processPendingCommandAndInputBuffer(c) == C_OK) beforeNextClient(c); +} + +static long long throttlerTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData) { + UNUSED(eventLoop); + UNUSED(id); + if (isPausedActionsWithUpdate(PAUSE_ACTIONS_CLIENT_ALL_SET)) return 1; + + throttler *t = (throttler *)clientData; + replenishTokens(t); + + monotime work_start; + elapsedStart(&work_start); + + while (tokenBucket_canConsume(t->bucket, 1.0) && + listLength(t->client_queue) > 0 && + elapsedMs(work_start) < MAX_UNTHROTTLE_PROCESSING_TIME_MS) { + tokenBucket_consume(t->bucket, 1.0); + client *c = listNodeValue(listFirst(t->client_queue)); + throttle_removeClient(c); + if (c->flag.throttle_multi) { + c->flag.throttle_multi = 0; + consumeOtherThrottlers(c, t); + } + processUnthrottledClient(c); + } + + if (listLength(t->client_queue) == 0) { + serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); // Already set in throttle_removeClient + if (t->id == THROTTLE_CLEANUP_ID) freeThrottler(t); + return AE_NOMORE; + } + return waitTimeMs(t); +} + +static void throttlerAddClient(throttler *t, client *c) { + serverAssert(c->throttler == NULL); + serverAssert(!c->flag.throttled); + elapsedStart(&c->throttle_start_us); + c->flag.throttled = 1; + listAddNodeTail(t->client_queue, c); + + if (c->conn) connSetReadHandler(c->conn, NULL); + + t->metrics->num_clients++; + t->metrics->total_throttled_commands++; + server.total_throttled_commands++; + c->throttler = t; + c->throttle_node = listLast(t->client_queue); + + if (listLength(t->client_queue) == 1) { + serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); + t->time_event_id = aeCreateTimeEvent(server.el, + waitTimeMs(t), + throttlerTimeProc, + t, NULL); + } +} + /* === Public API === */ void throttle_init(void) { @@ -279,93 +351,6 @@ const throttleMetrics *throttle_getMetrics(const char *metrics_name) { return &result; } -/* === Multi-throttler token accounting === */ - -static void consumeOtherThrottlers(client *c, throttler *except) { - listNode *ln; - listIter li; - listRewind(throttlerList, &li); - while ((ln = listNext(&li))) { - throttler *t = ln->value; - if (t->id == THROTTLE_CLEANUP_ID) continue; - if (t == except) continue; - if (t->criteria_proc(c, t->priv_data)) { - tokenBucket_consume(t->bucket, 1.0); - } - } -} - -/* === Timer: drain the queue when tokens become available === */ - -static void processUnthrottledClient(client *c) { - serverAssert(c->argc > 0 && c->flag.pending_command && !c->flag.throttled); - if (c->conn && !connHasReadHandler(c->conn)) { - if (connSetReadHandler(c->conn, readQueryFromClient) == C_ERR) { - freeClient(c); - return; - } - } - if (processPendingCommandAndInputBuffer(c) == C_OK) beforeNextClient(c); -} - -static long long throttlerTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData) { - UNUSED(eventLoop); - UNUSED(id); - if (isPausedActionsWithUpdate(PAUSE_ACTIONS_CLIENT_ALL_SET)) return 1; - - throttler *t = (throttler *)clientData; - replenishTokens(t); - - monotime work_start; - elapsedStart(&work_start); - - while (tokenBucket_canConsume(t->bucket, 1.0) && - listLength(t->client_queue) > 0 && - elapsedMs(work_start) < MAX_UNTHROTTLE_PROCESSING_TIME_MS) { - tokenBucket_consume(t->bucket, 1.0); - client *c = listNodeValue(listFirst(t->client_queue)); - throttle_removeClient(c); - if (c->flag.throttle_multi) { - c->flag.throttle_multi = 0; - consumeOtherThrottlers(c, t); - } - processUnthrottledClient(c); - } - - if (listLength(t->client_queue) == 0) { - serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); // Already set in throttle_removeClient - if (t->id == THROTTLE_CLEANUP_ID) freeThrottler(t); - return AE_NOMORE; - } - return waitTimeMs(t); -} - -/* === Queue management === */ - -static void throttlerAddClient(throttler *t, client *c) { - serverAssert(c->throttler == NULL); - serverAssert(!c->flag.throttled); - elapsedStart(&c->throttle_start_us); - c->flag.throttled = 1; - listAddNodeTail(t->client_queue, c); - - if (c->conn) connSetReadHandler(c->conn, NULL); - - t->metrics->num_clients++; - t->metrics->total_throttled_commands++; - server.total_throttled_commands++; - c->throttler = t; - c->throttle_node = listLast(t->client_queue); - - if (listLength(t->client_queue) == 1) { - serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); - t->time_event_id = aeCreateTimeEvent(server.el, - waitTimeMs(t), - throttlerTimeProc, - t, NULL); - } -} - void throttle_removeClient(client *c) { if (!c->flag.throttled) return; @@ -386,8 +371,6 @@ void throttle_removeClient(client *c) { c->throttle_start_us = 0; } -/* === Per-command entry point === */ - bool throttle_deferCommand(client *c) { if (throttlerList == NULL || listLength(throttlerList) == 0) return false; // Exempt all internal commands that has no connection from throttling. @@ -417,7 +400,6 @@ bool throttle_deferCommand(client *c) { if (strictest == NULL) return false; - /* Fast path: queue empty AND a token is available. */ if (listLength(strictest->client_queue) == 0) { replenishTokens(strictest); if (tokenBucket_canConsume(strictest->bucket, 1.0)) { @@ -433,7 +415,6 @@ bool throttle_deferCommand(client *c) { } /* === INFO output === */ -// Harry TODO: Should we do the info based on overall metrics or single throttler sds throttle_sdscatMetrics(sds info) { listNode *ln; listIter li; diff --git a/src/throttle.h b/src/throttle.h index 3237a7a4fde..b941c01da03 100644 --- a/src/throttle.h +++ b/src/throttle.h @@ -43,4 +43,4 @@ sds throttle_sdscatMetrics(sds info); long throttle_getGuardrailSecs(int id); -#endif /* THROTTLE_H */ +#endif diff --git a/src/throttle_repl.c b/src/throttle_repl.c index 70cbc5ba4f3..3720db7fadc 100644 --- a/src/throttle_repl.c +++ b/src/throttle_repl.c @@ -9,7 +9,6 @@ #include "throttle.h" #include "throttle_stat_calc.h" -/* Configuration constants. */ #define RATE_INCREASE_MULTIPLIER 1.05 #define RATE_DECREASE_MULTIPLIER 0.95 #define COB_TREND_WINDOW_SECS 2 @@ -77,8 +76,8 @@ static void adjustThrottleRate(bool reduceTrafficRate) { /* Evaluate whether steady-state throttling is needed. * Uses short-term COB trend to extrapolate future COB size. */ static bool evaluateSteadyState(client *c, uint64_t cob_size) { - unsigned long cob_target = throttleRepl_getCobTargetSize(); // 50 % of target cob - uint64_t min_throttle = cob_target / 2; // Begin throttling at half the target, 25 % of max cob + unsigned long cob_target = throttleRepl_getCobTargetSize(); + uint64_t min_throttle = cob_target / 2; // 25 % of the cob limit if (cob_size < min_throttle) return false; @@ -98,8 +97,9 @@ bool throttleRepl_isClientExempt(client *c) { if (!iAmPrimary()) return false; if (!c->flag.replica) return false; if (!isThrottleActive()) return false; - /* Throttle is actively working -- protect this replica from COB - * disconnect if its COB is above target (throttle needs time). */ + + /* Throttle is actively working, protect this replica from COB + * disconnect if its COB is above target. */ unsigned long cob = getClientOutputBufferMemoryUsage(c); if (cob < throttleRepl_getCobTargetSize()) return false; @@ -122,7 +122,7 @@ unsigned long throttleRepl_getCobTargetSize(void) { void throttleRepl_adjustThrottling(void) { if (!iAmPrimary()) { - // Failover happened + /* Failover could happen before. */ if (isThrottleActive()) removeThrottler(); return; } @@ -142,7 +142,7 @@ void throttleRepl_adjustThrottling(void) { unsigned long cob_size = getClientOutputBufferMemoryUsage(c); - /* Record trend per-replica. */ + /* Record trend per replica. */ if (c->cob_trend == NULL) c->cob_trend = newTrendCalc(COB_TREND_WINDOW_SECS); trendCalc_recordMetric(c->cob_trend, cob_size); diff --git a/src/throttle_repl.h b/src/throttle_repl.h index 4058c8596d1..19bb76a6ba4 100644 --- a/src/throttle_repl.h +++ b/src/throttle_repl.h @@ -25,4 +25,4 @@ void throttleRepl_adjustThrottling(void); /* Add repl throttle metrics to INFO string. */ sds throttleRepl_sdscatMetrics(sds info); -#endif /* THROTTLE_REPL_H */ +#endif diff --git a/src/throttle_stat_calc.c b/src/throttle_stat_calc.c index a12c2109e9f..a738951ad13 100644 --- a/src/throttle_stat_calc.c +++ b/src/throttle_stat_calc.c @@ -9,6 +9,7 @@ #define ONE_SECOND_IN_MICROS 1000000 +/* ------------- TPS Calculator ------------- */ struct tpsCalculator { double window_secs; double window_us; @@ -42,8 +43,7 @@ void tpsCalculator_record(tpsCalculator *calc, unsigned long transactions) { long elapsed_us = now - calc->last_update; if (elapsed_us < calc->update_freq_us) { - /* Accumulate until the update frequency is hit — updating too often - * increases metric lag. */ + /* Accumulate until the update frequency is hit */ calc->uncounted_trans += transactions; return; } @@ -53,11 +53,10 @@ void tpsCalculator_record(tpsCalculator *calc, unsigned long transactions) { calc->last_update = now; if (elapsed_us >= calc->window_us || calc->is_new) { - /* Elapsed >= window or first update: base entirely on this sample. */ calc->trans_per_window = total * calc->window_us / elapsed_us; calc->is_new = false; } else { - /* Blend: decay existing by fraction of window elapsed, add new. */ + /* Decay existing by fraction of window elapsed, add new. */ calc->trans_per_window = (calc->trans_per_window * (calc->window_us - elapsed_us) / calc->window_us) + total; } @@ -69,7 +68,9 @@ double tpsCalculator_averageTps(tpsCalculator *calc) { return calc->trans_per_window / calc->window_secs; } -#define DATA_POINTS 10 // Code assumes an even number +/* ------------- Trend Calculator ------------- */ + +#define DATA_POINTS 10 struct trendCalculator { int windowSec; monotime lastUpdate; @@ -127,6 +128,7 @@ void trendCalc_recordMetric(trendCalculator *calc, long metricValue) { // points... this is the measured delta. But, the time is from the center of each half, // resulting in half the window size (secs). So the formula is: // (AveNewer - AveOlder) / (WindowSec/2) + double olderAvg = (double)olderTotal / (DATA_POINTS / 2); double newerAvg = (double)newerTotal / (DATA_POINTS / 2); double timeBetweenCenters = (double)calc->windowSec / 2.0; diff --git a/src/throttle_stat_calc.h b/src/throttle_stat_calc.h index 64017d27049..9763adea8a8 100644 --- a/src/throttle_stat_calc.h +++ b/src/throttle_stat_calc.h @@ -4,9 +4,6 @@ /* Rolling-average TPS calculator over a configurable time window. * * Records transaction counts and reports a smoothed average TPS. - * The smoothing uses a blending approach: existing count decays - * proportional to elapsed time, new transactions are added on top. - * This produces a lagging average that converges over the window. */ typedef struct tpsCalculator tpsCalculator; @@ -27,21 +24,9 @@ double tpsCalculator_averageTps(tpsCalculator *calc); */ typedef struct trendCalculator trendCalculator; -// Allocate a new trend calculator. Caller is responsible to deallocate with zfree(). trendCalculator *newTrendCalc(int windowSecs); - -// Add a metric value to the trend calculator. This should be called at minimum 10 -// times over the window for best results. Note, if the metric is highly volatile, -// it is better to call more often - as a single outlier is less likely to skew results. void trendCalc_recordMetric(trendCalculator *calc, long metricValue); - -// Retrieve the average trend over the calculator's window. Note: this value is updated -// approximately 10 times over the size of the window. So, with a 1 minute window, the -// reported trend will be updated roughly every 6 seconds. double trendCalc_changePerSec(trendCalculator *calc); - -// Retrieve the trend using only the FINAL 10% of the calculator's window. This represents a -// short-term view of the trend. double trendCalc_changePerSecShortTerm(trendCalculator *calc); -#endif /* THROTTLE_STAT_CALC_H */ +#endif From ec331acd179c103e4f2a50c7870095701c919bf7 Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Tue, 7 Jul 2026 14:51:20 -0700 Subject: [PATCH 05/27] Address comments Signed-off-by: harrylin98 --- cmake/Modules/SourceFiles.cmake | 2 +- src/Makefile | 2 +- src/config.c | 3 +- src/networking.c | 2 +- src/server.c | 2 +- src/server.h | 3 - src/{throttle_stat_calc.c => stat_calc.c} | 2 +- src/{throttle_stat_calc.h => stat_calc.h} | 10 ++- src/throttle.c | 96 ++++++++++------------- src/throttle.h | 17 +++- src/throttle_repl.c | 37 +++++---- src/throttle_repl.h | 12 +-- src/throttle_token_bucket.c | 92 +++++++--------------- src/throttle_token_bucket.h | 25 +++--- 14 files changed, 132 insertions(+), 173 deletions(-) rename src/{throttle_stat_calc.c => stat_calc.c} (99%) rename src/{throttle_stat_calc.h => stat_calc.h} (89%) diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake index 4181dd344fa..f8f743ceb72 100644 --- a/cmake/Modules/SourceFiles.cmake +++ b/cmake/Modules/SourceFiles.cmake @@ -123,7 +123,7 @@ set(VALKEY_SERVER_SRCS ${CMAKE_SOURCE_DIR}/src/mutexqueue.c ${CMAKE_SOURCE_DIR}/src/queues.c ${CMAKE_SOURCE_DIR}/src/throttle_token_bucket.c - ${CMAKE_SOURCE_DIR}/src/throttle_stat_calc.c + ${CMAKE_SOURCE_DIR}/src/stat_calc.c ${CMAKE_SOURCE_DIR}/src/throttle_repl.c ${CMAKE_SOURCE_DIR}/src/throttle.c) diff --git a/src/Makefile b/src/Makefile index abb819f7acb..ecdcf0df81c 100644 --- a/src/Makefile +++ b/src/Makefile @@ -583,7 +583,7 @@ ENGINE_SERVER_OBJ = \ zmalloc.o \ queues.o \ throttle_token_bucket.o \ - throttle_stat_calc.o \ + stat_calc.o \ throttle_repl.o \ throttle.o ENGINE_SERVER_OBJ+=$(ENGINE_TRACE_OBJ) diff --git a/src/config.c b/src/config.c index adcc1e56d1a..f5b37d16147 100644 --- a/src/config.c +++ b/src/config.c @@ -38,6 +38,7 @@ #include "cluster_migrateslots.h" #include "eval.h" #include "lrulfu.h" +#include "throttle_repl.h" #include #include @@ -3286,7 +3287,7 @@ standardConfig static_configs[] = { createBoolConfig("repl-mptcp", NULL, IMMUTABLE_CONFIG, server.repl_mptcp, 0, isValidMptcp, NULL), createBoolConfig("repl-diskless-sync", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.repl_diskless_sync, 1, NULL, NULL), createBoolConfig("dual-channel-replication-enabled", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.dual_channel_replication, 0, NULL, NULL), - createBoolConfig("repl-throttle", NULL, MODIFIABLE_CONFIG, server.repl_throttle, 0, NULL, NULL), + createBoolConfig("steady-state-repl-throttle-enabled", NULL, MODIFIABLE_CONFIG, throttle_repl_config.steady_state_repl_throttle_enabled, 0, NULL, NULL), createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, 1, NULL, NULL), createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, 0, NULL, NULL), createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, 1, NULL, updateClusterState), diff --git a/src/networking.c b/src/networking.c index 57dd5189410..e9ec8c163fb 100644 --- a/src/networking.c +++ b/src/networking.c @@ -6199,7 +6199,7 @@ int checkClientOutputBufferLimits(client *c) { } else { c->obuf_soft_limit_reached_time = 0; } - if ((soft || hard) && throttleRepl_isClientExempt(c)) return 0; + if ((soft || hard) && throttleRepl_isClientExemptFromCobLimits(c)) return 0; return soft || hard; } diff --git a/src/server.c b/src/server.c index f628bb6125e..ea66f6857fb 100644 --- a/src/server.c +++ b/src/server.c @@ -6835,7 +6835,7 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { info = sdscat(info, "# Throttle\r\n"); info = sdscatprintf(info, "throttle_total_throttled_commands:%lld\r\n", - server.total_throttled_commands); + throttle_framework_metrics.total_throttled_commands); info = throttle_sdscatMetrics(info); info = throttleRepl_sdscatMetrics(info); } diff --git a/src/server.h b/src/server.h index 39abf604212..c3da5b34dae 100644 --- a/src/server.h +++ b/src/server.h @@ -2390,9 +2390,6 @@ struct valkeyServer { /* Local environment */ char *locale_collate; char *debug_context; /* A free-form string that has no impact on server except being included in a crash report. */ - /* Throttling */ - long long total_throttled_commands; /* Total commands deferred by the throttle framework */ - int repl_throttle; /* Enable replication throttle */ }; #define MAX_KEYS_BUFFER 256 diff --git a/src/throttle_stat_calc.c b/src/stat_calc.c similarity index 99% rename from src/throttle_stat_calc.c rename to src/stat_calc.c index a738951ad13..c3163a7035d 100644 --- a/src/throttle_stat_calc.c +++ b/src/stat_calc.c @@ -3,7 +3,7 @@ * All rights reserved. * SPDX-License-Identifier: BSD-3-Clause */ -#include "throttle_stat_calc.h" +#include "stat_calc.h" #include "server.h" #include "monotonic.h" diff --git a/src/throttle_stat_calc.h b/src/stat_calc.h similarity index 89% rename from src/throttle_stat_calc.h rename to src/stat_calc.h index 9763adea8a8..74ec2dca48c 100644 --- a/src/throttle_stat_calc.h +++ b/src/stat_calc.h @@ -1,5 +1,11 @@ -#ifndef THROTTLE_STAT_CALC_H -#define THROTTLE_STAT_CALC_H +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef STAT_CALC_H +#define STAT_CALC_H /* Rolling-average TPS calculator over a configurable time window. * diff --git a/src/throttle.c b/src/throttle.c index 718943eac77..97cf546c668 100644 --- a/src/throttle.c +++ b/src/throttle.c @@ -6,7 +6,7 @@ #include "throttle.h" #include "throttle_token_bucket.h" -#include "throttle_stat_calc.h" +#include "stat_calc.h" #include "hashtable.h" #include "monotonic.h" @@ -22,6 +22,9 @@ #define TOKENS_BURST_RATE_SEC 0.1 #define MIN_ADJUST_AFTER_DISABLE 100.0 +/* Framework-level metrics */ +struct throttle_framework_metrics throttle_framework_metrics; + /* === Internal metrics (shared by name via hashtable) === */ typedef struct throttleInternalMetrics { sds name; @@ -65,8 +68,8 @@ typedef struct throttler { throttleInternalMetrics *metrics; } throttler; -static int listMatchThrottler(void *ptr, void *id) { - return ((throttler *)ptr)->id == (long)id; +static int listMatchThrottler(void *throttler_ptr, void *id) { + return ((throttler *)throttler_ptr)->id == (long)id; } static throttler *findThrottler(int id) { @@ -77,19 +80,10 @@ static throttler *findThrottler(int id) { return t; } -static double computeBucketSize(double tokens_per_sec, double burst_time_sec) { - return (tokens_per_sec < EPSILON) ? 0.0 - : 2.0 + tokens_per_sec * burst_time_sec; -} - static void replenishTokens(throttler *t) { if (t->id == THROTTLE_CLEANUP_ID) { - tokenBucket_setTokensPerSec(t->bucket, THROTTLE_UNLIMITED_RATE); - tokenBucket_add(t->bucket, THROTTLE_UNLIMITED_RATE); - return; + tokenBucket_setRate(t->bucket, THROTTLE_UNLIMITED_RATE); } - tokenBucket_replenish(t->bucket); - tokenBucket_capDebt(t->bucket, tokenBucket_getBucketSize(t->bucket)); } static int waitTimeMs(throttler *t) { @@ -112,13 +106,13 @@ static void freeThrottler(throttler *t) { static void setRate(throttler *t, double new_rate) { if (new_rate < EPSILON) { - tokenBucket_halt(t->bucket); + tokenBucket_setRate(t->bucket, 0); } else { if (new_rate > THROTTLE_UNLIMITED_RATE) new_rate = THROTTLE_UNLIMITED_RATE; - tokenBucket_setTokensPerSec(t->bucket, new_rate); + tokenBucket_setRate(t->bucket, new_rate); } - double rate_per_min = tokenBucket_getTokensPerSec(t->bucket) * 60.0; + double rate_per_min = tokenBucket_getRate(t->bucket) * 60.0; if (rate_per_min <= THROTTLE_OPS_PER_MIN_GUARDRAIL) { if (t->rate_below_guardrail_since == 0) { elapsedStart(&t->rate_below_guardrail_since); @@ -159,7 +153,7 @@ static void consumeOtherThrottlers(client *c, throttler *except) { if (t->id == THROTTLE_CLEANUP_ID) continue; if (t == except) continue; if (t->criteria_proc(c, t->priv_data)) { - tokenBucket_consume(t->bucket, 1.0); + tokenBucket_tryConsume(t->bucket, 1.0, true); } } } @@ -186,10 +180,9 @@ static long long throttlerTimeProc(struct aeEventLoop *eventLoop, long long id, monotime work_start; elapsedStart(&work_start); - while (tokenBucket_canConsume(t->bucket, 1.0) && - listLength(t->client_queue) > 0 && - elapsedMs(work_start) < MAX_UNTHROTTLE_PROCESSING_TIME_MS) { - tokenBucket_consume(t->bucket, 1.0); + while (listLength(t->client_queue) > 0 && + elapsedMs(work_start) < MAX_UNTHROTTLE_PROCESSING_TIME_MS && + tokenBucket_tryConsume(t->bucket, 1.0, false)) { client *c = listNodeValue(listFirst(t->client_queue)); throttle_removeClient(c); if (c->flag.throttle_multi) { @@ -218,7 +211,7 @@ static void throttlerAddClient(throttler *t, client *c) { t->metrics->num_clients++; t->metrics->total_throttled_commands++; - server.total_throttled_commands++; + throttle_framework_metrics.total_throttled_commands++; c->throttler = t; c->throttle_node = listLast(t->client_queue); @@ -245,11 +238,9 @@ void throttle_init(void) { int throttle_register(throttleCriteriaProc *criteria_proc, void *priv_data, - const char *metrics_name, - double ops_per_sec) { + const char *metrics_name) { serverAssert(criteria_proc != NULL); serverAssert(metrics_name != NULL); - serverAssert(ops_per_sec >= 0); validateAlphaNumeric(metrics_name); serverAssert(nextThrottlerId > 0); @@ -258,11 +249,11 @@ int throttle_register(throttleCriteriaProc *criteria_proc, t->criteria_proc = criteria_proc; t->time_event_id = AE_DELETED_EVENT_ID; t->priv_data = priv_data; - t->bucket = tokenBucket_create(ops_per_sec, TOKENS_BURST_RATE_SEC, computeBucketSize); + t->bucket = tokenBucket_create(THROTTLE_UNLIMITED_RATE, TOKENS_BURST_RATE_SEC); t->metrics = findMetrics(metrics_name); t->client_queue = listCreate(); t->rate_below_guardrail_since = 0; - setRate(t, ops_per_sec); + setRate(t, THROTTLE_UNLIMITED_RATE); listAddNodeTail(throttlerList, t); t->ln = listLast(throttlerList); @@ -280,13 +271,6 @@ void throttle_deregister(int id) { } } -void *throttle_setPrivData(int id, void *new_priv_data) { - throttler *t = findThrottler(id); - void *old = t->priv_data; - t->priv_data = new_priv_data; - return old; -} - void throttle_setRate(int id, double ops_per_sec) { serverAssert(ops_per_sec >= 0); throttler *t = findThrottler(id); @@ -296,30 +280,34 @@ void throttle_setRate(int id, double ops_per_sec) { double throttle_adjustRate(int id, double multiplier) { serverAssert(multiplier >= 0.0 && multiplier <= 3.0); throttler *t = findThrottler(id); + double current = tokenBucket_getRate(t->bucket); + + /* No change needed if already unlimited and trying to increase. */ + if (multiplier > 1.0 && current == THROTTLE_UNLIMITED_RATE) { + return current; + } - double throttle_rate = tokenBucket_getTokensPerSec(t->bucket); double new_rate; if (multiplier <= 1.0) { - new_rate = throttle_rate * multiplier; - double incoming_rate = tpsCalculator_averageTps(t->metrics->incoming_tps); - if (incoming_rate > EPSILON && new_rate < incoming_rate) { - new_rate = incoming_rate; + /* Decrease: plain multiply, but never drop below incoming TPS. */ + new_rate = current * multiplier; + double incoming = tpsCalculator_averageTps(t->metrics->incoming_tps); + if (incoming > EPSILON && new_rate < incoming) { + new_rate = incoming; } + } else if (current < EPSILON) { + /* Coming back from halted: jump to a sensible starting rate. */ + new_rate = MIN_ADJUST_AFTER_DISABLE; } else { - if (throttle_rate == THROTTLE_UNLIMITED_RATE) { - new_rate = throttle_rate; - } else if (throttle_rate < EPSILON) { - new_rate = MIN_ADJUST_AFTER_DISABLE; - } else { - double delta = throttle_rate * (multiplier - 1.0); - if (delta < 1.0) delta = 1.0; - new_rate = throttle_rate + delta; - } + /* Increase: proportional with minimum step of 1 ops/sec. */ + double delta = current * (multiplier - 1.0); + if (delta < 1.0) delta = 1.0; + new_rate = current + delta; } - if (new_rate != throttle_rate) setRate(t, new_rate); - return tokenBucket_getTokensPerSec(t->bucket); + if (new_rate != current) setRate(t, new_rate); + return tokenBucket_getRate(t->bucket); } const throttleMetrics *throttle_getMetrics(const char *metrics_name) { @@ -339,7 +327,7 @@ const throttleMetrics *throttle_getMetrics(const char *metrics_name) { while ((ln = listNext(&li))) { throttler *t = ln->value; if (t->metrics != m) continue; - result.ops_per_sec += tokenBucket_getTokensPerSec(t->bucket); + result.ops_per_sec += tokenBucket_getRate(t->bucket); if (listLength(t->client_queue) > 0) { client *oldest = listNodeValue(listFirst(t->client_queue)); long delay_us = elapsedUs(oldest->throttle_start_us); @@ -392,7 +380,7 @@ bool throttle_deferCommand(client *c) { match_count++; tpsCalculator_record(t->metrics->incoming_tps, 1); if (strictest == NULL || - tokenBucket_getTokensPerSec(t->bucket) < tokenBucket_getTokensPerSec(strictest->bucket)) { + tokenBucket_getRate(t->bucket) < tokenBucket_getRate(strictest->bucket)) { strictest = t; } } @@ -401,10 +389,8 @@ bool throttle_deferCommand(client *c) { if (strictest == NULL) return false; if (listLength(strictest->client_queue) == 0) { - replenishTokens(strictest); - if (tokenBucket_canConsume(strictest->bucket, 1.0)) { + if (tokenBucket_tryConsume(strictest->bucket, 1.0, false)) { if (match_count > 1) consumeOtherThrottlers(c, strictest); - tokenBucket_consume(strictest->bucket, 1.0); return false; } } diff --git a/src/throttle.h b/src/throttle.h index b941c01da03..d0bb5e4a10b 100644 --- a/src/throttle.h +++ b/src/throttle.h @@ -1,3 +1,9 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + #ifndef THROTTLE_H #define THROTTLE_H @@ -17,18 +23,21 @@ typedef struct { long oldest_client_delay_us; } throttleMetrics; +/* Framework-level metrics */ +struct throttle_framework_metrics { + long long total_throttled_commands; +}; +extern struct throttle_framework_metrics throttle_framework_metrics; + /* Public API */ void throttle_init(void); int throttle_register(throttleCriteriaProc *criteria_proc, void *priv_data, - const char *metrics_name, - double ops_per_sec); + const char *metrics_name); void throttle_deregister(int id); -void *throttle_setPrivData(int id, void *new_priv_data); - void throttle_setRate(int id, double ops_per_sec); double throttle_adjustRate(int id, double multiplier); diff --git a/src/throttle_repl.c b/src/throttle_repl.c index 3720db7fadc..e966732f6fa 100644 --- a/src/throttle_repl.c +++ b/src/throttle_repl.c @@ -7,7 +7,10 @@ #include "server.h" #include "throttle_repl.h" #include "throttle.h" -#include "throttle_stat_calc.h" +#include "stat_calc.h" + +/* Configuration instance. */ +struct throttle_repl_config throttle_repl_config; #define RATE_INCREASE_MULTIPLIER 1.05 #define RATE_DECREASE_MULTIPLIER 0.95 @@ -42,7 +45,7 @@ static bool criteriaProc(client *c, void *priv_data) { static void installThrottler(void) { serverAssert(!isThrottleActive()); - throttle_id = throttle_register(criteriaProc, NULL, METRICS_NAME, THROTTLE_UNLIMITED_RATE); + throttle_id = throttle_register(criteriaProc, NULL, METRICS_NAME); metrics.is_throttler_active = true; metrics.current_throttle_rate = THROTTLE_UNLIMITED_RATE; metrics.throttle_activation_events++; @@ -73,6 +76,17 @@ static void adjustThrottleRate(bool reduceTrafficRate) { } } +static unsigned long throttleRepl_getCobTargetSize(void) { + int64_t cob_target = server.client_obuf_limits[CLIENT_TYPE_REPLICA].soft_limit_bytes; + if (cob_target == 0) cob_target = server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes; + + cob_target /= 2; /* Target is half the limit. */ + + if (cob_target == 0 || cob_target > MAX_COB_TARGET) cob_target = MAX_COB_TARGET; + + return (unsigned long)cob_target; +} + /* Evaluate whether steady-state throttling is needed. * Uses short-term COB trend to extrapolate future COB size. */ static bool evaluateSteadyState(client *c, uint64_t cob_size) { @@ -89,11 +103,7 @@ static bool evaluateSteadyState(client *c, uint64_t cob_size) { /* --- Public API --- */ -bool throttleRepl_isEnabled(void) { - return server.repl_throttle; -} - -bool throttleRepl_isClientExempt(client *c) { +bool throttleRepl_isClientExemptFromCobLimits(client *c) { if (!iAmPrimary()) return false; if (!c->flag.replica) return false; if (!isThrottleActive()) return false; @@ -109,24 +119,13 @@ bool throttleRepl_isClientExempt(client *c) { return true; } -unsigned long throttleRepl_getCobTargetSize(void) { - int64_t cob_target = server.client_obuf_limits[CLIENT_TYPE_REPLICA].soft_limit_bytes; - if (cob_target == 0) cob_target = server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes; - - cob_target /= 2; /* Target is half the limit. */ - - if (cob_target == 0 || cob_target > MAX_COB_TARGET) cob_target = MAX_COB_TARGET; - - return (unsigned long)cob_target; -} - void throttleRepl_adjustThrottling(void) { if (!iAmPrimary()) { /* Failover could happen before. */ if (isThrottleActive()) removeThrottler(); return; } - if (!throttleRepl_isEnabled() && !isThrottleActive()) return; + if (!throttle_repl_config.steady_state_repl_throttle_enabled && !isThrottleActive()) return; bool reduce = false; client *measured_replica = NULL; diff --git a/src/throttle_repl.h b/src/throttle_repl.h index 19bb76a6ba4..45b3e2c4c1d 100644 --- a/src/throttle_repl.h +++ b/src/throttle_repl.h @@ -9,15 +9,15 @@ #include "sds.h" -/* Returns true if replication throttle is enabled. */ -bool throttleRepl_isEnabled(void); - -/* Get the COB target size for steady-state throttling. */ -unsigned long throttleRepl_getCobTargetSize(void); +/* Replication throttle configuration. */ +struct throttle_repl_config { + int steady_state_repl_throttle_enabled; +}; +extern struct throttle_repl_config throttle_repl_config; /* Returns true if the client should be exempt from COB disconnect limits * because throttling is actively working to stabilize the replica. */ -bool throttleRepl_isClientExempt(client *c); +bool throttleRepl_isClientExemptFromCobLimits(client *c); /* Determine throttling needs and adjust rate. Called from serverCron. */ void throttleRepl_adjustThrottling(void); diff --git a/src/throttle_token_bucket.c b/src/throttle_token_bucket.c index bbfafb4d86f..b16eec03536 100644 --- a/src/throttle_token_bucket.c +++ b/src/throttle_token_bucket.c @@ -12,31 +12,39 @@ struct tokenBucket { double max_burst_time_secs; double token_count; monotime last_time_check; - bucketSizeFunc *bucket_size_func; }; -static double calcBucketSize(double tokens_per_sec, double max_burst_time_secs) { - return tokens_per_sec * max_burst_time_secs; +#define BUCKET_EPSILON 0.0001 + +static double getBucketSize(tokenBucket *bucket) { + return (bucket->tokens_per_sec < BUCKET_EPSILON) ? 0.0 + : 2.0 + bucket->tokens_per_sec * bucket->max_burst_time_secs; +} + +static void trimTokenBucket(tokenBucket *bucket) { + double bucket_size = getBucketSize(bucket); + if (bucket->token_count > bucket_size) bucket->token_count = bucket_size; + if (bucket->token_count < -bucket_size) bucket->token_count = -bucket_size; } -static double trimTokenBucket(tokenBucket *bucket) { - double unused_tokens = 0; - double bucket_size = bucket->bucket_size_func(bucket->tokens_per_sec, bucket->max_burst_time_secs); - if (bucket->token_count > bucket_size) { - unused_tokens = bucket->token_count - bucket_size; - bucket->token_count = bucket_size; +static void tokenBucket_replenish(tokenBucket *bucket) { + monotime now = getMonotonicUs(); + uint64_t delta_us = now - bucket->last_time_check; + double tokens_to_add = delta_us * bucket->tokens_per_sec / 1000000.0; + if (tokens_to_add > 0) { + bucket->token_count += tokens_to_add; + trimTokenBucket(bucket); } - return unused_tokens; + bucket->last_time_check = now; } -tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs, bucketSizeFunc *bucket_size_func) { +tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs) { serverAssert(tokens_per_sec >= 0); serverAssert(max_burst_time_secs >= 0); tokenBucket *bucket = zmalloc(sizeof(tokenBucket)); bucket->tokens_per_sec = tokens_per_sec; bucket->max_burst_time_secs = max_burst_time_secs; - bucket->bucket_size_func = bucket_size_func == NULL ? calcBucketSize : bucket_size_func; - bucket->token_count = bucket->bucket_size_func(bucket->tokens_per_sec, bucket->max_burst_time_secs); + bucket->token_count = getBucketSize(bucket); bucket->last_time_check = getMonotonicUs(); return bucket; } @@ -45,59 +53,21 @@ void tokenBucket_free(tokenBucket *bucket) { zfree(bucket); } -double tokenBucket_getTokenCount(tokenBucket *bucket) { - return bucket->token_count; -} - -double tokenBucket_getTokensPerSec(tokenBucket *bucket) { +double tokenBucket_getRate(tokenBucket *bucket) { return bucket->tokens_per_sec; } -double tokenBucket_getBucketSize(tokenBucket *bucket) { - return bucket->bucket_size_func(bucket->tokens_per_sec, bucket->max_burst_time_secs); -} - -double tokenBucket_getMaxBurstTime(tokenBucket *bucket) { - return bucket->max_burst_time_secs; -} - -void tokenBucket_setTokensPerSec(tokenBucket *bucket, double tokens_per_sec) { - serverAssert(tokens_per_sec >= 0); - bucket->tokens_per_sec = tokens_per_sec; - trimTokenBucket(bucket); -} - -void tokenBucket_setMaxBurstSec(tokenBucket *bucket, double max_burst_time_secs) { - serverAssert(max_burst_time_secs >= 0); - bucket->max_burst_time_secs = max_burst_time_secs; +void tokenBucket_setRate(tokenBucket *bucket, double new_rate) { + serverAssert(new_rate >= 0); + bucket->tokens_per_sec = new_rate; trimTokenBucket(bucket); } -void tokenBucket_capDebt(tokenBucket *bucket, double max_debt) { - serverAssert(max_debt >= 0); - if (bucket->token_count < -max_debt) { - bucket->token_count = -max_debt; - } -} - -double tokenBucket_add(tokenBucket *bucket, double tokens) { - bucket->token_count += tokens; - return trimTokenBucket(bucket); -} - -double tokenBucket_replenish(tokenBucket *bucket) { - monotime now = getMonotonicUs(); - uint64_t delta_us = now - bucket->last_time_check; - bucket->last_time_check = now; - return tokenBucket_add(bucket, delta_us * bucket->tokens_per_sec / 1000000.0); -} - -void tokenBucket_consume(tokenBucket *bucket, double tokens) { +bool tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consume) { + tokenBucket_replenish(bucket); + if (!force_consume && bucket->token_count < tokens) return false; bucket->token_count -= tokens; -} - -bool tokenBucket_canConsume(tokenBucket *bucket, double tokens) { - return bucket->token_count >= tokens; + return true; } double tokenBucket_msUntilAvailable(tokenBucket *bucket, double target_tokens) { @@ -107,7 +77,3 @@ double tokenBucket_msUntilAvailable(tokenBucket *bucket, double target_tokens) { return needed / bucket->tokens_per_sec * 1000.0; } -void tokenBucket_halt(tokenBucket *bucket) { - bucket->token_count = 0; - bucket->tokens_per_sec = 0; -} diff --git a/src/throttle_token_bucket.h b/src/throttle_token_bucket.h index 6a42ebc0248..23b0b1c7fba 100644 --- a/src/throttle_token_bucket.h +++ b/src/throttle_token_bucket.h @@ -1,28 +1,23 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + #ifndef THROTTLE_TOKEN_BUCKET_H #define THROTTLE_TOKEN_BUCKET_H #include -typedef double bucketSizeFunc(double tokens_per_sec, double max_burst_time_secs); typedef struct tokenBucket tokenBucket; -// APIs -tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs, bucketSizeFunc *bucket_size_func); +tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs); void tokenBucket_free(tokenBucket *bucket); -double tokenBucket_getTokenCount(tokenBucket *bucket); -double tokenBucket_getTokensPerSec(tokenBucket *bucket); -double tokenBucket_getBucketSize(tokenBucket *bucket); -double tokenBucket_getMaxBurstTime(tokenBucket *bucket); -void tokenBucket_setTokensPerSec(tokenBucket *bucket, double tokens_per_sec); -void tokenBucket_setMaxBurstSec(tokenBucket *bucket, double max_burst_time_secs); +double tokenBucket_getRate(tokenBucket *bucket); +void tokenBucket_setRate(tokenBucket *bucket, double new_rate); -void tokenBucket_capDebt(tokenBucket *bucket, double max_debt); -double tokenBucket_add(tokenBucket *bucket, double tokens); -double tokenBucket_replenish(tokenBucket *bucket); -bool tokenBucket_canConsume(tokenBucket *bucket, double tokens); -void tokenBucket_consume(tokenBucket *bucket, double tokens); +bool tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consume); double tokenBucket_msUntilAvailable(tokenBucket *bucket, double tokens); -void tokenBucket_halt(tokenBucket *bucket); #endif From bfe7d9eca2396184602feecc44fa10315fe5e1e6 Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Fri, 10 Jul 2026 13:36:03 -0700 Subject: [PATCH 06/27] Add documentation Signed-off-by: harrylin98 --- src/networking.c | 3 +- src/server.c | 8 +- src/stat_calc.c | 83 +++++----- src/stat_calc.h | 44 ++++-- src/throttle.c | 291 ++++++++++++++++++------------------ src/throttle.h | 97 ++++++++++-- src/throttle_repl.c | 120 ++++++++------- src/throttle_repl.h | 22 ++- src/throttle_token_bucket.c | 17 ++- src/throttle_token_bucket.h | 25 ++++ 10 files changed, 429 insertions(+), 281 deletions(-) diff --git a/src/networking.c b/src/networking.c index 5d97504e1ba..f57c420859d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -39,6 +39,7 @@ #include "io_threads.h" #include "throttle.h" #include "throttle_repl.h" +#include "stat_calc.h" #include "module.h" #include "connection.h" #include "zmalloc.h" @@ -2222,7 +2223,7 @@ int freeClient(client *c) { if (c->lib_name) decrRefCount(c->lib_name); if (c->lib_ver) decrRefCount(c->lib_ver); freeClientMultiState(c); - if (c->cob_trend) zfree(c->cob_trend); + if (c->cob_trend) trendCalc_free(c->cob_trend); sdsfree(c->peerid); sdsfree(c->sockname); zfree(c); diff --git a/src/server.c b/src/server.c index 1d206746a63..1fe7a8eedaf 100644 --- a/src/server.c +++ b/src/server.c @@ -4722,7 +4722,7 @@ int processCommand(client *c) { return C_OK; } - if (throttle_deferCommand(c)) return C_OK; + if (throttleClientIfNeeded(c)) return C_OK; /* Exec the command */ if (c->flag.multi && c->cmd->proc != execCommand && c->cmd->proc != discardCommand && @@ -6847,9 +6847,9 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { info = sdscat(info, "# Throttle\r\n"); info = sdscatprintf(info, "throttle_total_throttled_commands:%lld\r\n", - throttle_framework_metrics.total_throttled_commands); - info = throttle_sdscatMetrics(info); - info = throttleRepl_sdscatMetrics(info); + throttle_getTotalThrottledCommands()); + info = throttle_sdscatInfoMetrics(info); + info = throttleRepl_sdscatInfoMetrics(info); } /* Get info from modules. diff --git a/src/stat_calc.c b/src/stat_calc.c index c3163a7035d..562aa4376e8 100644 --- a/src/stat_calc.c +++ b/src/stat_calc.c @@ -92,53 +92,58 @@ trendCalculator *newTrendCalc(int windowSecs) { return calc; } +void trendCalc_free(trendCalculator *calc) { + zfree(calc); +} + void trendCalc_recordMetric(trendCalculator *calc, long metricValue) { monotime now = getMonotonicUs(); long elapsedUs = now - calc->lastUpdate; - // When called more than updateFreqUs, just save values for later averaging. calc->uncountedTotal += metricValue; calc->uncountedSamples++; - if (elapsedUs >= calc->updateFreqUs) { - long newValue = calc->uncountedTotal / calc->uncountedSamples; - calc->uncountedTotal = 0; - calc->uncountedSamples = 0; - calc->lastUpdate = now; - - if (calc->newCalculator) { - for (int i = 0; i < DATA_POINTS; i++) calc->metrics[i] = newValue; - calc->newCalculator = false; - } - - long olderTotal = 0; - for (int i = 0; i < DATA_POINTS / 2; i++) { - calc->metrics[i] = calc->metrics[i + 1]; - olderTotal += calc->metrics[i]; - } - long newerTotal = 0; - for (int i = DATA_POINTS / 2; i < DATA_POINTS - 1; i++) { - calc->metrics[i] = calc->metrics[i + 1]; - newerTotal += calc->metrics[i]; - } - calc->metrics[DATA_POINTS - 1] = newValue; - newerTotal += newValue; - - // Formula is the average of the newer data points, less the average of the older data - // points... this is the measured delta. But, the time is from the center of each half, - // resulting in half the window size (secs). So the formula is: - // (AveNewer - AveOlder) / (WindowSec/2) - - double olderAvg = (double)olderTotal / (DATA_POINTS / 2); - double newerAvg = (double)newerTotal / (DATA_POINTS / 2); - double timeBetweenCenters = (double)calc->windowSec / 2.0; - calc->trend = (newerAvg - olderAvg) / timeBetweenCenters; - - // Compute short-term change-per-sec using the last 2 datapoints - long deltaShort = calc->metrics[DATA_POINTS - 1] - calc->metrics[DATA_POINTS - 2]; - double timeBetweenSlots = (double)calc->windowSec / DATA_POINTS; - calc->trendShort = deltaShort / timeBetweenSlots; + if (elapsedUs < calc->updateFreqUs) return; + + long newValue = calc->uncountedTotal / calc->uncountedSamples; + calc->uncountedTotal = 0; + calc->uncountedSamples = 0; + calc->lastUpdate = now; + + if (calc->newCalculator) { + for (int i = 0; i < DATA_POINTS; i++) calc->metrics[i] = newValue; + calc->newCalculator = false; + } + + long olderTotal = 0; + for (int i = 0; i < DATA_POINTS / 2; i++) { + calc->metrics[i] = calc->metrics[i + 1]; + olderTotal += calc->metrics[i]; + } + long newerTotal = 0; + for (int i = DATA_POINTS / 2; i < DATA_POINTS - 1; i++) { + calc->metrics[i] = calc->metrics[i + 1]; + newerTotal += calc->metrics[i]; } + calc->metrics[DATA_POINTS - 1] = newValue; + newerTotal += newValue; + + /* Formula is the average of the newer data points, less the average of the older data + * points. The time is from the center of each half, + * resulting in half the window size (secs). So the formula is: + * (AveNewer - AveOlder) / (WindowSec/2) + * Where: + * AveNewer = newerTotal / (DATA_POINTS/2) + * AveOlder = olderTotal / (DATA_POINTS/2) */ + double olderAvg = (double)olderTotal / (DATA_POINTS / 2); + double newerAvg = (double)newerTotal / (DATA_POINTS / 2); + double timeBetweenCenters = (double)calc->windowSec / 2.0; + calc->trend = (newerAvg - olderAvg) / timeBetweenCenters; + + /* Short-term: rate of change between last 2 datapoints. */ + long deltaShort = calc->metrics[DATA_POINTS - 1] - calc->metrics[DATA_POINTS - 2]; + double timeBetweenSlots = (double)calc->windowSec / DATA_POINTS; + calc->trendShort = deltaShort / timeBetweenSlots; } double trendCalc_changePerSec(trendCalculator *calc) { diff --git a/src/stat_calc.h b/src/stat_calc.h index 74ec2dca48c..5c8b5290f01 100644 --- a/src/stat_calc.h +++ b/src/stat_calc.h @@ -2,37 +2,61 @@ * Copyright (c) Valkey Contributors * All rights reserved. * SPDX-License-Identifier: BSD-3-Clause + * + * Calculators for statistical values. */ #ifndef STAT_CALC_H #define STAT_CALC_H -/* Rolling-average TPS calculator over a configurable time window. - * - * Records transaction counts and reports a smoothed average TPS. +/* =========================== TPS Calculator =============================== */ + +/* A TPS calculator computes a rolling average TPS over a specified time window. + * This smooths jitter in the measurement, with the average slightly lagging + * instantaneous changes. This provides a stable measurement that is resilient + * to short-lived traffic spikes. */ typedef struct tpsCalculator tpsCalculator; tpsCalculator *tpsCalculator_create(int window_secs); + void tpsCalculator_free(tpsCalculator *calc); +/* Add a datapoint of new transactions to the calculator. This should be called at minimum 10 times + * over the window for smooth results. */ void tpsCalculator_record(tpsCalculator *calc, unsigned long transactions); + +/* Retrieve the average TPS over the calculator's window */ double tpsCalculator_averageTps(tpsCalculator *calc); -/* A trend calculator is used to compute the trend of data points over a specified time window. - * Periodically, values are added to the calculator. The calculator computes a "running trend" of - * the data over the given time window. The trend is reported as an average increase/decrease per - * second. Examples: - * - Data 1,2,1,2,1,2,1,2,1 - trend is essentially 0. A trend line would have 0 slope. - * - Data 0,0,0,10,10,10 - trend is approximately 3/sec - * This is similar to slope from a linear regression, but a simple speed-optimized algorithm. + +/* ========================== Trend Calculator ============================== */ + +/* A trend calculator computes the rate of change of a metric over a specified + * time window, reported as average increase/decrease per second. + * Examples: + * - Data 1,2,1,2,1,2 — trend ≈ 0 (oscillating, no net change) + * - Data 0,0,0,10,10,10 — trend ≈ 3/sec (step increase) + * Conceptually similar to the slope of a linear regression, but uses a + * lightweight approximation suitable for high-frequency sampling. */ typedef struct trendCalculator trendCalculator; trendCalculator *newTrendCalc(int windowSecs); + +void trendCalc_free(trendCalculator *calc); + +/* Add a datapoint to the calculator. Should be called at minimum 10 times + * over the window for smooth results. If the metric is highly volatile, + * calling more often reduces the impact of individual outliers. */ void trendCalc_recordMetric(trendCalculator *calc, long metricValue); + +/* Get the average rate of change over the full window. */ double trendCalc_changePerSec(trendCalculator *calc); + +/* Get the rate of change using only the final 10% of the window. + * More responsive to sudden changes but noisier than the full-window trend. */ double trendCalc_changePerSecShortTerm(trendCalculator *calc); #endif diff --git a/src/throttle.c b/src/throttle.c index 97cf546c668..9a46ca804e6 100644 --- a/src/throttle.c +++ b/src/throttle.c @@ -11,36 +11,47 @@ #include "monotonic.h" #include -#include -#define MAX_WAIT_TIME_MS 100 -#define MAX_UNTHROTTLE_PROCESSING_TIME_MS 10 -#define THROTTLE_CLEANUP_ID (-1) -#define THROTTLE_OPS_PER_MIN_GUARDRAIL 6 -#define TPS_WINDOW_SEC 5 -#define EPSILON 0.0001 -#define TOKENS_BURST_RATE_SEC 0.1 -#define MIN_ADJUST_AFTER_DISABLE 100.0 +#define MAX_WAIT_TIME_MS 100 /* max ms before rescheduling timer */ +#define MAX_UNTHROTTLE_PROCESSING_TIME_MS 10 /* max ms spent unthrottling per timer fire */ +#define THROTTLE_CLEANUP_ID (-1) /* sentinel: throttler deregistered, draining queue */ +#define THROTTLE_OPS_PER_MIN_GUARDRAIL 6 /* 0.1 TPS - report when rate stays below this */ +#define TPS_WINDOW_SEC 5 /* rolling window for incoming TPS measurement */ +#define EPSILON 0.0001 /* values below this are treated as zero */ +#define TOKENS_BURST_RATE_SEC 0.1 /* burst capacity in seconds of sustained rate */ +#define MIN_ADJUST_AFTER_DISABLE 100.0 /* initial rate when recovering from halted state */ -/* Framework-level metrics */ -struct throttle_framework_metrics throttle_framework_metrics; +static int nextThrottlerId = 1; +static hashtable *metricsTable = NULL; +static list *throttlerList = NULL; -/* === Internal metrics (shared by name via hashtable) === */ -typedef struct throttleInternalMetrics { - sds name; - int num_clients; - int total_throttled_commands; +typedef struct metricsEntry { + sds throttler_type; + int num_clients_throttled; + int num_throttled_commands; tpsCalculator *incoming_tps; -} throttleInternalMetrics; +} metricsEntry; + +typedef struct throttler { + int id; + throttleCriteriaProc *criteria_proc; /* callback defining throttling criteria */ + long long time_event_id; /* timer event id for throttlerTimeProc */ + void *priv_data; /* private data for use by the criteria_proc */ + tokenBucket *bucket; /* token bucket: 1 token = 1 operation */ + list *client_queue; /* clients currently queued for throttling */ + listNode *ln; /* my node in throttlerList */ + monotime rate_below_guardrail_since; /* timestamp when rate dropped below guardrail, or 0 */ + metricsEntry *metrics; /* reference to the named metrics object */ +} throttler; /* Metrics hashtable callbacks. */ static const void *metricsGetKey(const void *entry) { - return ((throttleInternalMetrics *)entry)->name; + return ((metricsEntry *)entry)->throttler_type; } static void metricsDestructor(void *entry) { - throttleInternalMetrics *m = entry; - sdsfree(m->name); + metricsEntry *m = entry; + sdsfree(m->throttler_type); tpsCalculator_free(m->incoming_tps); zfree(m); } @@ -52,21 +63,24 @@ static hashtableType metricsHashtableType = { .entryDestructor = metricsDestructor, }; -static int nextThrottlerId = 1; -static list *throttlerList = NULL; -static hashtable *metricsTable = NULL; +static metricsEntry *findMetrics(const char *name) { + sds key = sdsnew(name); + void *found = NULL; + if (hashtableFind(metricsTable, key, &found)) { + sdsfree(key); + return (metricsEntry *)found; + } + metricsEntry *m = zmalloc(sizeof(metricsEntry)); + m->throttler_type = key; + m->num_clients_throttled = 0; + m->num_throttled_commands = 0; + m->incoming_tps = tpsCalculator_create(TPS_WINDOW_SEC); + hashtableAdd(metricsTable, m); + return m; +} -typedef struct throttler { - int id; - throttleCriteriaProc *criteria_proc; - long long time_event_id; - void *priv_data; - tokenBucket *bucket; - list *client_queue; - listNode *ln; /* my node in throttlerList */ - monotime rate_below_guardrail_since; - throttleInternalMetrics *metrics; -} throttler; +/* Framework-level metrics */ +static long long total_throttled_commands; static int listMatchThrottler(void *throttler_ptr, void *id) { return ((throttler *)throttler_ptr)->id == (long)id; @@ -80,19 +94,15 @@ static throttler *findThrottler(int id) { return t; } -static void replenishTokens(throttler *t) { - if (t->id == THROTTLE_CLEANUP_ID) { - tokenBucket_setRate(t->bucket, THROTTLE_UNLIMITED_RATE); - } -} - +/* Compute how long to wait before the next token becomes available. */ static int waitTimeMs(throttler *t) { serverAssert(listLength(t->client_queue) > 0); double ms = tokenBucket_msUntilAvailable(t->bucket, 1.0); - if (ms < 0) return MAX_WAIT_TIME_MS; - return MIN(MAX_WAIT_TIME_MS, (int)ceil(ms)); + if (ms < 0 || ms >= MAX_WAIT_TIME_MS) return MAX_WAIT_TIME_MS; + return (int)ceil(ms); } +/* Release throttler resources. Only called when client queue is fully drained. */ static void freeThrottler(throttler *t) { serverAssert(listLength(t->client_queue) == 0); serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); @@ -104,45 +114,6 @@ static void freeThrottler(throttler *t) { zfree(t); } -static void setRate(throttler *t, double new_rate) { - if (new_rate < EPSILON) { - tokenBucket_setRate(t->bucket, 0); - } else { - if (new_rate > THROTTLE_UNLIMITED_RATE) new_rate = THROTTLE_UNLIMITED_RATE; - tokenBucket_setRate(t->bucket, new_rate); - } - - double rate_per_min = tokenBucket_getRate(t->bucket) * 60.0; - if (rate_per_min <= THROTTLE_OPS_PER_MIN_GUARDRAIL) { - if (t->rate_below_guardrail_since == 0) { - elapsedStart(&t->rate_below_guardrail_since); - } - } else { - t->rate_below_guardrail_since = 0; - } -} - -static void validateAlphaNumeric(const char *s) { - for (; *s; s++) { - serverAssert(isalnum(*s) || (*s == '_') || (*s == '-')); - } -} - -static throttleInternalMetrics *findMetrics(const char *name) { - sds key = sdsnew(name); - void *found = NULL; - if (hashtableFind(metricsTable, key, &found)) { - sdsfree(key); - return (throttleInternalMetrics *)found; - } - throttleInternalMetrics *m = zmalloc(sizeof(throttleInternalMetrics)); - m->name = key; - m->num_clients = 0; - m->total_throttled_commands = 0; - m->incoming_tps = tpsCalculator_create(TPS_WINDOW_SEC); - hashtableAdd(metricsTable, m); - return m; -} static void consumeOtherThrottlers(client *c, throttler *except) { listNode *ln; @@ -150,14 +121,13 @@ static void consumeOtherThrottlers(client *c, throttler *except) { listRewind(throttlerList, &li); while ((ln = listNext(&li))) { throttler *t = ln->value; - if (t->id == THROTTLE_CLEANUP_ID) continue; - if (t == except) continue; - if (t->criteria_proc(c, t->priv_data)) { - tokenBucket_tryConsume(t->bucket, 1.0, true); - } + if (t->id == THROTTLE_CLEANUP_ID || t == except) continue; + if (t->criteria_proc(c, t->priv_data)) tokenBucket_tryConsume(t->bucket, 1.0, true); } } +/* Re-execute a client's deferred command after throttle release. + * Restores the read handler and processes the pending command and input buffer. */ static void processUnthrottledClient(client *c) { serverAssert(c->argc > 0 && c->flag.pending_command && !c->flag.throttled); if (c->conn && !connHasReadHandler(c->conn)) { @@ -169,13 +139,15 @@ static void processUnthrottledClient(client *c) { if (processPendingCommandAndInputBuffer(c) == C_OK) beforeNextClient(c); } +/* Timer event handler: releases queued clients at the token bucket rate. + * Processes clients until tokens are exhausted or time budget is spent. */ static long long throttlerTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData) { UNUSED(eventLoop); UNUSED(id); + // if the clients are paused, then return 1 ms so we wake up every ms if (isPausedActionsWithUpdate(PAUSE_ACTIONS_CLIENT_ALL_SET)) return 1; throttler *t = (throttler *)clientData; - replenishTokens(t); monotime work_start; elapsedStart(&work_start); @@ -209,9 +181,9 @@ static void throttlerAddClient(throttler *t, client *c) { if (c->conn) connSetReadHandler(c->conn, NULL); - t->metrics->num_clients++; - t->metrics->total_throttled_commands++; - throttle_framework_metrics.total_throttled_commands++; + t->metrics->num_clients_throttled++; + t->metrics->num_throttled_commands++; + total_throttled_commands++; c->throttler = t; c->throttle_node = listLast(t->client_queue); @@ -236,12 +208,15 @@ void throttle_init(void) { } } +/* In most cases, each throttler should have its own independent metrics_name. When the same + * throttler is instantiated multiple times (with different priv_data), they may share a single + * metrics object by using the same name. This allows statistics to be aggregated across related + * throttler instances. */ int throttle_register(throttleCriteriaProc *criteria_proc, void *priv_data, const char *metrics_name) { serverAssert(criteria_proc != NULL); serverAssert(metrics_name != NULL); - validateAlphaNumeric(metrics_name); serverAssert(nextThrottlerId > 0); throttler *t = zmalloc(sizeof(throttler)); @@ -253,7 +228,7 @@ int throttle_register(throttleCriteriaProc *criteria_proc, t->metrics = findMetrics(metrics_name); t->client_queue = listCreate(); t->rate_below_guardrail_since = 0; - setRate(t, THROTTLE_UNLIMITED_RATE); + throttle_setRate(t->id, THROTTLE_UNLIMITED_RATE); listAddNodeTail(throttlerList, t); t->ln = listLast(throttlerList); @@ -268,13 +243,29 @@ void throttle_deregister(int id) { freeThrottler(t); } else { t->id = THROTTLE_CLEANUP_ID; + tokenBucket_setRate(t->bucket, THROTTLE_UNLIMITED_RATE); } } void throttle_setRate(int id, double ops_per_sec) { serverAssert(ops_per_sec >= 0); throttler *t = findThrottler(id); - setRate(t, ops_per_sec); + + if (ops_per_sec < EPSILON) { + ops_per_sec = 0; + } else if (ops_per_sec > THROTTLE_UNLIMITED_RATE) { + ops_per_sec = THROTTLE_UNLIMITED_RATE; + } + tokenBucket_setRate(t->bucket, ops_per_sec); + + double rate_per_min = ops_per_sec * 60.0; + if (rate_per_min <= THROTTLE_OPS_PER_MIN_GUARDRAIL) { + if (t->rate_below_guardrail_since == 0) { + elapsedStart(&t->rate_below_guardrail_since); + } + } else { + t->rate_below_guardrail_since = 0; + } } double throttle_adjustRate(int id, double multiplier) { @@ -293,9 +284,7 @@ double throttle_adjustRate(int id, double multiplier) { /* Decrease: plain multiply, but never drop below incoming TPS. */ new_rate = current * multiplier; double incoming = tpsCalculator_averageTps(t->metrics->incoming_tps); - if (incoming > EPSILON && new_rate < incoming) { - new_rate = incoming; - } + if (incoming > EPSILON && new_rate < incoming) new_rate = incoming; } else if (current < EPSILON) { /* Coming back from halted: jump to a sensible starting rate. */ new_rate = MIN_ADJUST_AFTER_DISABLE; @@ -306,39 +295,10 @@ double throttle_adjustRate(int id, double multiplier) { new_rate = current + delta; } - if (new_rate != current) setRate(t, new_rate); + if (new_rate != current) throttle_setRate(t->id, new_rate); return tokenBucket_getRate(t->bucket); } -const throttleMetrics *throttle_getMetrics(const char *metrics_name) { - static throttleMetrics result; - throttleInternalMetrics *m = findMetrics(metrics_name); - - result.num_clients = m->num_clients; - result.total_throttled_commands = m->total_throttled_commands; - result.incoming_tps = tpsCalculator_averageTps(m->incoming_tps); - result.ops_per_sec = 0.0; - result.oldest_client_delay_us = 0; - - /* Aggregate ops_per_sec and oldest_client from all throttlers sharing this metrics. */ - listNode *ln; - listIter li; - listRewind(throttlerList, &li); - while ((ln = listNext(&li))) { - throttler *t = ln->value; - if (t->metrics != m) continue; - result.ops_per_sec += tokenBucket_getRate(t->bucket); - if (listLength(t->client_queue) > 0) { - client *oldest = listNodeValue(listFirst(t->client_queue)); - long delay_us = elapsedUs(oldest->throttle_start_us); - if (result.oldest_client_delay_us < delay_us) { - result.oldest_client_delay_us = delay_us; - } - } - } - return &result; -} - void throttle_removeClient(client *c) { if (!c->flag.throttled) return; @@ -348,27 +308,33 @@ void throttle_removeClient(client *c) { listDelNode(t->client_queue, c->throttle_node); + t->metrics->num_clients_throttled--; + if (listLength(t->client_queue) == 0) { serverAssert(t->time_event_id != AE_DELETED_EVENT_ID); aeDeleteTimeEvent(server.el, t->time_event_id); t->time_event_id = AE_DELETED_EVENT_ID; + if (t->id == THROTTLE_CLEANUP_ID) freeThrottler(t); } - t->metrics->num_clients--; c->throttler = NULL; c->throttle_node = NULL; c->throttle_start_us = 0; } -bool throttle_deferCommand(client *c) { +bool throttleClientIfNeeded(client *c) { if (throttlerList == NULL || listLength(throttlerList) == 0) return false; - // Exempt all internal commands that has no connection from throttling. - if (!c->conn) return false; - if (c->flag.throttle_checked) return false; + + /* Skip internal clients and clients already checked for this command. + * Prevents re-throttling after unblocking. */ + if (!c->conn || c->flag.throttle_checked) return false; c->flag.throttle_checked = 1; + bool need_throttle = false; int match_count = 0; + /* Strictest throttler is the applicable throttler with the lowest rate. + * It is the most restrictive throttler the client needs to throttle at. + */ throttler *strictest = NULL; - listNode *ln; listIter li; listRewind(throttlerList, &li); @@ -379,29 +345,60 @@ bool throttle_deferCommand(client *c) { if (t->criteria_proc(c, t->priv_data)) { match_count++; tpsCalculator_record(t->metrics->incoming_tps, 1); - if (strictest == NULL || - tokenBucket_getRate(t->bucket) < tokenBucket_getRate(strictest->bucket)) { - strictest = t; - } + if (strictest == NULL || tokenBucket_getRate(t->bucket) < tokenBucket_getRate(strictest->bucket)) strictest = t; } } - if (strictest == NULL) return false; - - if (listLength(strictest->client_queue) == 0) { - if (tokenBucket_tryConsume(strictest->bucket, 1.0, false)) { + if (strictest != NULL) { + if (listLength(strictest->client_queue) == 0 && + tokenBucket_tryConsume(strictest->bucket, 1.0, false)) { + /* token available, consume and let command proceed. */ if (match_count > 1) consumeOtherThrottlers(c, strictest); - return false; + } else { + /* no token available, defer the command. */ + if (match_count > 1) c->flag.throttle_multi = 1; + throttlerAddClient(strictest, c); + need_throttle = true; } } - if (match_count > 1) c->flag.throttle_multi = 1; - throttlerAddClient(strictest, c); - return true; + return need_throttle; +} + +/* === INFO metrics output === */ +long long throttle_getTotalThrottledCommands(void) { + return total_throttled_commands; +} + +const throttleMetrics *throttle_getMetrics(const char *metrics_name) { + static throttleMetrics result; + metricsEntry *m = findMetrics(metrics_name); + + result.num_clients_throttled = m->num_clients_throttled; + result.num_throttled_commands = m->num_throttled_commands; + result.incoming_tps = tpsCalculator_averageTps(m->incoming_tps); + result.ops_per_sec = 0.0; + result.oldest_client_delay_us = 0; + + /* Aggregate ops_per_sec and oldest_client from all throttlers sharing this metrics. */ + listNode *ln; + listIter li; + listRewind(throttlerList, &li); + while ((ln = listNext(&li))) { + throttler *t = ln->value; + if (t->metrics != m) continue; + result.ops_per_sec += tokenBucket_getRate(t->bucket); + if (listLength(t->client_queue) > 0) { + client *oldest = listNodeValue(listFirst(t->client_queue)); + long delay_us = elapsedUs(oldest->throttle_start_us); + result.oldest_client_delay_us = MAX(result.oldest_client_delay_us, delay_us); + } + } + return &result; } -/* === INFO output === */ -sds throttle_sdscatMetrics(sds info) { +sds throttle_sdscatInfoMetrics(sds info) { + // Check for any throttlers which are below guardrail. Report only offending throttlers. listNode *ln; listIter li; listRewind(throttlerList, &li); @@ -412,7 +409,7 @@ sds throttle_sdscatMetrics(sds info) { if (secs > 0) { info = sdscatprintf(info, "throttle_%s_guardrail_secs:%d\r\n", - t->metrics->name, secs); + t->metrics->throttler_type, secs); } } } diff --git a/src/throttle.h b/src/throttle.h index d0bb5e4a10b..4210dac5938 100644 --- a/src/throttle.h +++ b/src/throttle.h @@ -2,6 +2,19 @@ * Copyright (c) Valkey Contributors * All rights reserved. * SPDX-License-Identifier: BSD-3-Clause + * + * A generic client throttling framework using a token bucket algorithm. + * + * Plug-in evaluators register throttlers that control the rate at which client commands are + * processed. When a client's command matches a throttler's criteria, the client is queued and + * its commands are released at the configured rate. + * + * Design: + * Multiple throttlers can be registered simultaneously. When a client matches more than one, + * the most restrictive rate applies. Throttled clients have their read handler removed and + * are released via timer events at the configured rate. Throttling occurs in processCommand() + * before command execution. Once throttled, the client's command is deferred until tokens become + * available. */ #ifndef THROTTLE_H @@ -11,45 +24,97 @@ #include static const double THROTTLE_UNLIMITED_RATE = 10000000.0; + static const int THROTTLE_INVALID_ID = -2; +/* A throttleCriteriaProc checks a client's current command and decides if it meets the criteria + * for throttling. Returns true if the client meets the throttling criteria. + * + * priv_data - a private data structure provided during throttle_register. It can provide + * anything needed by the criteria proc, or NULL if unneeded. */ typedef bool throttleCriteriaProc(client *c, void *priv_data); +/* Metrics for a group of related throttlers sharing the same metrics_name. + * + * Note: Multiple related throttlers can share the same metrics by using the same metrics_name. + * A typical use case is multiple instantiations of the same throttler with different private + * data. */ typedef struct { - int num_clients; - int total_throttled_commands; - double ops_per_sec; - double incoming_tps; - long oldest_client_delay_us; + int num_clients_throttled; /* the backlog of currently throttled (queued) clients */ + int num_throttled_commands; /* total number of commands throttled through this metrics group */ + double ops_per_sec; /* the current throttling rate (summed across related throttlers) */ + double incoming_tps; /* average incoming TPS over a 5-second rolling window */ + long oldest_client_delay_us; /* delay in microseconds for the oldest throttled client */ } throttleMetrics; -/* Framework-level metrics */ -struct throttle_framework_metrics { - long long total_throttled_commands; -}; -extern struct throttle_framework_metrics throttle_framework_metrics; - -/* Public API */ void throttle_init(void); +/* Register a new throttler. + * criteria_proc - identifies clients whose commands meet the criteria for throttling + * priv_data - private data for passing to the criteria_proc (may be NULL) + * metrics_name - a string used to identify a shared metrics group + * + * Returns an integer ID of the new throttler. */ int throttle_register(throttleCriteriaProc *criteria_proc, void *priv_data, const char *metrics_name); +/* Deregisters the throttler such that: + * - No new clients will be throttled by this throttler. + * - Existing queued clients will be drained at unlimited rate until the queue is empty. */ void throttle_deregister(int id); void throttle_setRate(int id, double ops_per_sec); +/* A smart adjustment to the throttling rate. The multiplier is applied to the current rate, + * with consideration for the actual incoming traffic rate. + * multiplier - applied to current rate to determine new rate (range 0.0 .. 3.0) + * + * If multiplier > 1.0: increase rate (with minimum step of 1 ops/sec at low rates). + * If multiplier < 1.0: decrease rate (clamped to incoming TPS floor). + * If multiplier == 0.0: halt (rate set to 0). + * + * Returns the actual rate set after clamping and adjustment. + * + * Usage guidance: + * 1. Adjust throttling at a regular interval > 250ms. Adjusting the throttle too fast will + * result in large throttling swings before an observed metric has a chance to change. + * This can easily create a hysteresis problem. The current incoming rate is based on a + * 5-second window and will not update faster than 250ms. + * 2. Set a target for the observed metric. As the observed metric approaches the target, make + * progressively smaller changes to the rate. */ double throttle_adjustRate(int id, double multiplier); -const throttleMetrics *throttle_getMetrics(const char *metrics_name); - +/* Removes the client from the throttle queue. */ void throttle_removeClient(client *c); -bool throttle_deferCommand(client *c); +/* Check if the client's current command should be throttled. Called at the beginning of + * processCommand(). If any registered throttler's criteria matches, the client is queued + * and the most restrictive throttle rate applies. + * + * Returns true if the client has been throttled. + * Returns false if the client may proceed normally. + * + * Note: Even if a client matches throttling criteria, it might not be queued if tokens + * are available. Throttling is checked before blocking, so a throttled + * command cannot be blocked. Once a client is passed to this function, it will not be + * throttled again for the same command after unblocking. */ +bool throttleClientIfNeeded(client *c); + +/* Get the total number of commands throttled across all throttlers. */ +long long throttle_getTotalThrottledCommands(void); + +/* Get the metrics associated with a given metrics name. + * Memory is managed by the throttler. Do not free the returned pointer. + * Call this each time metrics are needed. Do not cache the pointer. */ +const throttleMetrics *throttle_getMetrics(const char *metrics_name); -sds throttle_sdscatMetrics(sds info); +/* Append framework-level throttle metrics to the INFO output string. + * Plug-in specific metrics are reported by their own sdscatInfoMetrics functions. */ +sds throttle_sdscatInfoMetrics(sds info); +/* Get the number of seconds the throttler's rate has been below the guardrail. + * Returns 0 if the rate is above the guardrail or the throttler is not active. */ long throttle_getGuardrailSecs(int id); #endif diff --git a/src/throttle_repl.c b/src/throttle_repl.c index e966732f6fa..efb9c853e94 100644 --- a/src/throttle_repl.c +++ b/src/throttle_repl.c @@ -14,17 +14,21 @@ struct throttle_repl_config throttle_repl_config; #define RATE_INCREASE_MULTIPLIER 1.05 #define RATE_DECREASE_MULTIPLIER 0.95 -#define COB_TREND_WINDOW_SECS 2 -#define CONVERGENCE_SECS 30 +#define COB_TREND_WINDOW_SECS 2 /* A 2-second window gives 20 data points at \ + * 100ms serverCron. Sufficient for a good \ + * measurement, while remaining short enough for \ + * throttling adjustments every 100ms. */ +#define STEADY_STATE_CONVERGENCE_SECS 30 /* projection horizon for COB extrapolation */ #define MAX_COB_TARGET (1024L * 1024 * 1024) /* 1GB */ -#define METRICS_NAME "ReplThrottle" +#define METRICS_NAME "ReplThrottle" /* shared metrics group name */ +/* Metrics for INFO output and operational visibility. */ typedef struct { bool is_throttler_active; - double current_throttle_rate; - unsigned long throttle_activation_events; - unsigned long throttle_more_events; - unsigned long throttle_less_events; + double current_throttle_rate; /* only valid if throttler is active */ + unsigned long throttle_activation_events; /* cumulative times throttler has been activated */ + unsigned long throttle_more_events; /* cumulative times we throttled more */ + unsigned long throttle_less_events; /* cumulative times we throttled less */ } throttleReplMetrics; static throttleReplMetrics metrics = {0}; @@ -32,7 +36,7 @@ static int throttle_id = 0; /* --- Internal helpers --- */ -static bool isThrottleActive(void) { +static bool isThrottlerActive(void) { return (throttle_id != 0); } @@ -44,23 +48,25 @@ static bool criteriaProc(client *c, void *priv_data) { } static void installThrottler(void) { - serverAssert(!isThrottleActive()); + serverAssert(!isThrottlerActive()); throttle_id = throttle_register(criteriaProc, NULL, METRICS_NAME); metrics.is_throttler_active = true; metrics.current_throttle_rate = THROTTLE_UNLIMITED_RATE; metrics.throttle_activation_events++; } -static void removeThrottler(void) { - serverAssert(isThrottleActive()); +static void uninstallThrottler(void) { + serverAssert(isThrottlerActive()); throttle_deregister(throttle_id); throttle_id = 0; metrics.is_throttler_active = false; metrics.current_throttle_rate = THROTTLE_UNLIMITED_RATE; } +/* Apply a rate change based on the evaluator's decision. Installs the throttler on first + * reduce request and removes it when rate reaches UNLIMITED. */ static void adjustThrottleRate(bool reduceTrafficRate) { - if (isThrottleActive()) { + if (isThrottlerActive()) { double rate; if (reduceTrafficRate) { rate = throttle_adjustRate(throttle_id, RATE_DECREASE_MULTIPLIER); @@ -68,70 +74,84 @@ static void adjustThrottleRate(bool reduceTrafficRate) { } else { rate = throttle_adjustRate(throttle_id, RATE_INCREASE_MULTIPLIER); metrics.throttle_less_events++; - if (rate >= THROTTLE_UNLIMITED_RATE) removeThrottler(); + if (rate >= THROTTLE_UNLIMITED_RATE) uninstallThrottler(); } metrics.current_throttle_rate = rate; } else { + /* Installing the throttler starts measurement of current traffic rate. + * Once the measurement is stable, rate adjustments will be meaningful. */ if (reduceTrafficRate) installThrottler(); } } -static unsigned long throttleRepl_getCobTargetSize(void) { - int64_t cob_target = server.client_obuf_limits[CLIENT_TYPE_REPLICA].soft_limit_bytes; - if (cob_target == 0) cob_target = server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes; +static int64_t getReplicaSteadyStateCobTargetSize(void) { + int64_t limit = server.client_obuf_limits[CLIENT_TYPE_REPLICA].soft_limit_bytes; + if (limit == 0) limit = server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes; - cob_target /= 2; /* Target is half the limit. */ + int64_t cob_target = limit / 2; /* Target is half the limit. */ if (cob_target == 0 || cob_target > MAX_COB_TARGET) cob_target = MAX_COB_TARGET; - return (unsigned long)cob_target; + return cob_target; } -/* Evaluate whether steady-state throttling is needed. - * Uses short-term COB trend to extrapolate future COB size. */ -static bool evaluateSteadyState(client *c, uint64_t cob_size) { - unsigned long cob_target = throttleRepl_getCobTargetSize(); - uint64_t min_throttle = cob_target / 2; // 25 % of the cob limit - - if (cob_size < min_throttle) return false; - +/* Steady-state throttling targets the replica with the largest COB to ensure all replicas + * maintain sync. Throttling begins at 25% of the configured soft limit (half the target COB size). + * The short-term COB trend is used to project when COB will intersect the target within the + * convergence window. This will result in a convergence to the desired target, rather than + * overshooting the target. */ +static bool evaluateSteadyStateThrottle(client *c, int64_t cob_size) { + int64_t cob_target = getReplicaSteadyStateCobTargetSize(); + int64_t throttle_threshold = cob_target / 2; + + if (cob_size < throttle_threshold) return false; + + /* Using the full window for COB trend shows greater hysteresis than using only the final + * datapoints. The short trend results in more jittery rate adjustments, but this is good + * as the up/down/up/down... type adjustments result in a smoother traffic rate than + * up/up/up/down/down/down... */ double short_trend = trendCalc_changePerSecShortTerm(c->cob_trend); - int64_t extrapolated = (int64_t)cob_size + (int64_t)(short_trend * CONVERGENCE_SECS); + int64_t extrapolated = cob_size + (int64_t)(short_trend * STEADY_STATE_CONVERGENCE_SECS); - return (extrapolated > (int64_t)cob_target); + return (extrapolated > cob_target); } /* --- Public API --- */ +/* In some cases, we want to protect replicas from being killed by the COB limits. When + * throttling hasn't had time to adjust and there is no severe memory condition, it makes + * sense to allow the replica to live until throttling can stabilize the situation. */ bool throttleRepl_isClientExemptFromCobLimits(client *c) { + if (!throttle_repl_config.steady_state_repl_throttle_enabled || !isThrottlerActive()) return false; if (!iAmPrimary()) return false; if (!c->flag.replica) return false; - if (!isThrottleActive()) return false; /* Throttle is actively working, protect this replica from COB * disconnect if its COB is above target. */ - unsigned long cob = getClientOutputBufferMemoryUsage(c); - if (cob < throttleRepl_getCobTargetSize()) return false; + int64_t client_cob_size = (int64_t)getClientOutputBufferMemoryUsage(c); + if (client_cob_size < getReplicaSteadyStateCobTargetSize()) return false; /* Don't protect if throttle has been working too long without success. */ time_t elapsed = server.unixtime - c->obuf_soft_limit_reached_time; - if (elapsed > 4 * CONVERGENCE_SECS) return false; + if (elapsed > 4 * STEADY_STATE_CONVERGENCE_SECS) return false; return true; } +/* Called from serverCron every 100ms. Evaluates the replica with the largest COB and + * adjusts throttling as needed. */ void throttleRepl_adjustThrottling(void) { if (!iAmPrimary()) { /* Failover could happen before. */ - if (isThrottleActive()) removeThrottler(); + if (isThrottlerActive()) uninstallThrottler(); return; } - if (!throttle_repl_config.steady_state_repl_throttle_enabled && !isThrottleActive()) return; + if (!throttle_repl_config.steady_state_repl_throttle_enabled && !isThrottlerActive()) return; - bool reduce = false; - client *measured_replica = NULL; - uint64_t largest_cob = 0; + bool reduceTrafficRate = false; + client *measured_steady_state_replica = NULL; + uint64_t largest_steady_state_cob = 0; - /* Scan replicas, find steady-state replica with smallest COB. */ + /* Scan replicas, find steady-state replica with largest COB. */ listIter li; listNode *ln; listRewind(server.replicas, &li); @@ -141,28 +161,26 @@ void throttleRepl_adjustThrottling(void) { unsigned long cob_size = getClientOutputBufferMemoryUsage(c); - /* Record trend per replica. */ if (c->cob_trend == NULL) c->cob_trend = newTrendCalc(COB_TREND_WINDOW_SECS); trendCalc_recordMetric(c->cob_trend, cob_size); - /* Ignore tiny COB (overhead only). */ + /* The COB size contains some overhead. Treat it as zero until we reach a minimum. */ if (cob_size <= PROTO_REPLY_CHUNK_BYTES) cob_size = 0; - // Find the largest cob size among replica clients - if (measured_replica == NULL || cob_size > largest_cob) { - measured_replica = c; - largest_cob = cob_size; + if (measured_steady_state_replica == NULL || cob_size > largest_steady_state_cob) { + measured_steady_state_replica = c; + largest_steady_state_cob = cob_size; } } - if (measured_replica != NULL) { - reduce = evaluateSteadyState(measured_replica, largest_cob); + if (measured_steady_state_replica != NULL) { + reduceTrafficRate = evaluateSteadyStateThrottle(measured_steady_state_replica, largest_steady_state_cob); } - adjustThrottleRate(reduce); + adjustThrottleRate(reduceTrafficRate); } -sds throttleRepl_sdscatMetrics(sds info) { +sds throttleRepl_sdscatInfoMetrics(sds info) { info = sdscatprintf(info, "repl_throttle_active:%d\r\n", metrics.is_throttler_active ? 1 : 0); @@ -184,9 +202,9 @@ sds throttleRepl_sdscatMetrics(sds info) { metrics.throttle_activation_events, metrics.throttle_more_events, metrics.throttle_less_events, - isThrottleActive() ? throttle_getGuardrailSecs(throttle_id) : 0L, - throttle_metrics->num_clients, - throttle_metrics->total_throttled_commands); + isThrottlerActive() ? throttle_getGuardrailSecs(throttle_id) : 0L, + throttle_metrics->num_clients_throttled, + throttle_metrics->num_throttled_commands); return info; } diff --git a/src/throttle_repl.h b/src/throttle_repl.h index 45b3e2c4c1d..994bf398b02 100644 --- a/src/throttle_repl.h +++ b/src/throttle_repl.h @@ -2,27 +2,35 @@ * Copyright (c) Valkey Contributors * All rights reserved. * SPDX-License-Identifier: BSD-3-Clause + * + * A replication throttler plug-in for the generic throttler (throttle.h). + * + * Throttles client traffic on the primary to establish and maintain healthy replica + * connections. It monitors replica COB (Client Output Buffer) growth and reduces the + * command processing rate when needed. + * + * Steady-state evaluator (normal replication): + * Throttles when the projected COB exceeds the target within the convergence window. + * Rate increases automatically once COB stabilizes. */ #ifndef THROTTLE_REPL_H #define THROTTLE_REPL_H #include "sds.h" - -/* Replication throttle configuration. */ struct throttle_repl_config { int steady_state_repl_throttle_enabled; }; extern struct throttle_repl_config throttle_repl_config; -/* Returns true if the client should be exempt from COB disconnect limits - * because throttling is actively working to stabilize the replica. */ +/* Returns true if the client should be exempt from COB disconnect limits because throttling + * is actively working to stabilize the replica. */ bool throttleRepl_isClientExemptFromCobLimits(client *c); -/* Determine throttling needs and adjust rate. Called from serverCron. */ +/* Determine throttling needs and adjust rate. Called from serverCron every 100ms. */ void throttleRepl_adjustThrottling(void); -/* Add repl throttle metrics to INFO string. */ -sds throttleRepl_sdscatMetrics(sds info); +/* Append replication throttle metrics to the INFO output string. */ +sds throttleRepl_sdscatInfoMetrics(sds info); #endif diff --git a/src/throttle_token_bucket.c b/src/throttle_token_bucket.c index b16eec03536..8a7fa6aa378 100644 --- a/src/throttle_token_bucket.c +++ b/src/throttle_token_bucket.c @@ -3,24 +3,30 @@ * All rights reserved. * SPDX-License-Identifier: BSD-3-Clause */ + #include "throttle_token_bucket.h" #include "server.h" #include "monotonic.h" struct tokenBucket { - double tokens_per_sec; - double max_burst_time_secs; - double token_count; - monotime last_time_check; + double tokens_per_sec; // Rate at which tokens are added to the bucket (tokens per second) + double max_burst_time_secs; // Maximum time for which tokens can accumulate in the bucket + double token_count; // Current number of tokens in the bucket (can be negative if force-consumed) + monotime last_time_check; // Last time the bucket was replenished (in microseconds) }; #define BUCKET_EPSILON 0.0001 +/* Bucket capacity scales with rate: higher rates allow larger bursts. + * The +2 guarantees the bucket can always hold at least 2 tokens, preventing + * permanent starvation at very low rates where rate * burst_time < 1. + * Returns 0 when rate is effectively zero. */ static double getBucketSize(tokenBucket *bucket) { return (bucket->tokens_per_sec < BUCKET_EPSILON) ? 0.0 - : 2.0 + bucket->tokens_per_sec * bucket->max_burst_time_secs; + : 2.0 + bucket->tokens_per_sec * bucket->max_burst_time_secs; } +/* Clamp token count to valid range [-bucket_size, bucket_size]. */ static void trimTokenBucket(tokenBucket *bucket) { double bucket_size = getBucketSize(bucket); if (bucket->token_count > bucket_size) bucket->token_count = bucket_size; @@ -76,4 +82,3 @@ double tokenBucket_msUntilAvailable(tokenBucket *bucket, double target_tokens) { double needed = target_tokens - bucket->token_count; return needed / bucket->tokens_per_sec * 1000.0; } - diff --git a/src/throttle_token_bucket.h b/src/throttle_token_bucket.h index 23b0b1c7fba..62b803ad9a6 100644 --- a/src/throttle_token_bucket.h +++ b/src/throttle_token_bucket.h @@ -2,6 +2,19 @@ * Copyright (c) Valkey Contributors * All rights reserved. * SPDX-License-Identifier: BSD-3-Clause + * + * The Token Bucket Algorithm is a traffic control method where tokens are added to a bucket at a fixed rate (up to a + * maximum capacity), and commands can be processed only if enough tokens are available. + * + * Terminology: + * Token: A permission unit required to process commands; a command can be processed only if enough tokens are available. + * Bucket: A logical storage that holds tokens until they are used. + * + * Working: + * 1. Tokens are added to the bucket at a constant rate and stored up to the maximum capacity. + * 2. When a command arrives, the system checks whether enough tokens are available in the bucket. + * 3. If enough tokens are available, the required number of tokens is removed from the bucket, and the command is processed. + * 4. If tokens are unavailable, the command is queued until new tokens are generated. */ #ifndef THROTTLE_TOKEN_BUCKET_H @@ -11,13 +24,25 @@ typedef struct tokenBucket tokenBucket; +/* Create a token bucket that starts full. + * max_burst_time_secs controls how many seconds of idle accumulation are + * allowed before the bucket is considered full. A larger value permits + * bigger bursts after idle periods. */ tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs); + void tokenBucket_free(tokenBucket *bucket); double tokenBucket_getRate(tokenBucket *bucket); + void tokenBucket_setRate(tokenBucket *bucket, double new_rate); +/* Attempt to consume tokens. Returns true if tokens were deducted. + * force_consume=false: only deducts if enough tokens are available. + * force_consume=true: always deducts (may drive count negative). */ bool tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consume); + +/* Estimate milliseconds until the requested tokens become available. + * Returns 0 if already available, or -1 if rate is 0 (never reached). */ double tokenBucket_msUntilAvailable(tokenBucket *bucket, double tokens); #endif From c1266dc92b8d6ccea2388b8a571dd5140d72646e Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Mon, 20 Jul 2026 16:58:04 -0700 Subject: [PATCH 07/27] Adding unit tests Signed-off-by: harrylin98 --- src/stat_calc.c | 2 +- src/stat_calc.h | 2 +- src/throttle.c | 5 +- src/throttle_token_bucket.c | 3 +- src/unit/test_stat_calc.cpp | 195 ++++++++++++++++++++ src/unit/test_throttle_repl.cpp | 308 ++++++++++++++++++++++++++++++++ src/unit/test_token_bucket.cpp | 145 +++++++++++++++ src/unit/wrappers.h | 14 ++ 8 files changed, 668 insertions(+), 6 deletions(-) create mode 100644 src/unit/test_stat_calc.cpp create mode 100644 src/unit/test_throttle_repl.cpp create mode 100644 src/unit/test_token_bucket.cpp diff --git a/src/stat_calc.c b/src/stat_calc.c index 562aa4376e8..976bb239014 100644 --- a/src/stat_calc.c +++ b/src/stat_calc.c @@ -20,7 +20,7 @@ struct tpsCalculator { bool is_new; }; -tpsCalculator *tpsCalculator_create(int window_secs) { +tpsCalculator *newTpsCalc(int window_secs) { serverAssert(window_secs > 0); tpsCalculator *calc = zmalloc(sizeof(tpsCalculator)); calc->window_secs = (double)window_secs; diff --git a/src/stat_calc.h b/src/stat_calc.h index 5c8b5290f01..7dc61d676a0 100644 --- a/src/stat_calc.h +++ b/src/stat_calc.h @@ -19,7 +19,7 @@ typedef struct tpsCalculator tpsCalculator; -tpsCalculator *tpsCalculator_create(int window_secs); +tpsCalculator *newTpsCalc(int window_secs); void tpsCalculator_free(tpsCalculator *calc); diff --git a/src/throttle.c b/src/throttle.c index 9a46ca804e6..e648f4ebd34 100644 --- a/src/throttle.c +++ b/src/throttle.c @@ -74,7 +74,7 @@ static metricsEntry *findMetrics(const char *name) { m->throttler_type = key; m->num_clients_throttled = 0; m->num_throttled_commands = 0; - m->incoming_tps = tpsCalculator_create(TPS_WINDOW_SEC); + m->incoming_tps = newTpsCalc(TPS_WINDOW_SEC); hashtableAdd(metricsTable, m); return m; } @@ -228,10 +228,9 @@ int throttle_register(throttleCriteriaProc *criteria_proc, t->metrics = findMetrics(metrics_name); t->client_queue = listCreate(); t->rate_below_guardrail_since = 0; - throttle_setRate(t->id, THROTTLE_UNLIMITED_RATE); - listAddNodeTail(throttlerList, t); t->ln = listLast(throttlerList); + throttle_setRate(t->id, THROTTLE_UNLIMITED_RATE); return t->id; } diff --git a/src/throttle_token_bucket.c b/src/throttle_token_bucket.c index 8a7fa6aa378..fadca4ca4c5 100644 --- a/src/throttle_token_bucket.c +++ b/src/throttle_token_bucket.c @@ -73,6 +73,7 @@ bool tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consu tokenBucket_replenish(bucket); if (!force_consume && bucket->token_count < tokens) return false; bucket->token_count -= tokens; + trimTokenBucket(bucket); /* bound debt at -bucket_size so recovery time stays bounded */ return true; } @@ -80,5 +81,5 @@ double tokenBucket_msUntilAvailable(tokenBucket *bucket, double target_tokens) { if (bucket->token_count >= target_tokens) return 0.0; if (bucket->tokens_per_sec <= 0) return -1.0; /* halted — never available */ double needed = target_tokens - bucket->token_count; - return needed / bucket->tokens_per_sec * 1000.0; + return needed * 1000.0 / bucket->tokens_per_sec; } diff --git a/src/unit/test_stat_calc.cpp b/src/unit/test_stat_calc.cpp new file mode 100644 index 00000000000..e5c88ad75d4 --- /dev/null +++ b/src/unit/test_stat_calc.cpp @@ -0,0 +1,195 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Unit tests for stat_calc.h. + */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "stat_calc.h" +static monotime fakeGetMonotonicUs(void); +static monotime (*origGetMonotonicUs)(void); +} + +#define ONE_SECOND_IN_MICROS 1000000 + +static monotime fakeMonotimeUs; + +static monotime fakeGetMonotonicUs(void) { + return fakeMonotimeUs; +} + +class StatCalcTest : public ::testing::Test { + protected: + tpsCalculator *tps; + trendCalculator *trend; + + static void SetUpTestSuite() { + origGetMonotonicUs = getMonotonicUs; + getMonotonicUs = fakeGetMonotonicUs; + } + + static void TearDownTestSuite() { + getMonotonicUs = origGetMonotonicUs; + } + + void SetUp() override { + fakeMonotimeUs = 100; + tps = newTpsCalc(5); + trend = newTrendCalc(5); + } + + void TearDown() override { + tpsCalculator_free(tps); + trendCalc_free(trend); + } +}; + +/* ========================== TPS Calculator Tests ========================== */ + +TEST_F(StatCalcTest, TpsInitZero) { + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 0.0); +} + +TEST_F(StatCalcTest, TpsExtrapolateFromOneSecond) { + /* 1 second of data at 10 transactions, TPS should be 10 */ + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 10); + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 10.0); +} + +TEST_F(StatCalcTest, TpsInterpolateFromTenSeconds) { + /* 10 seconds of data at 10 transactions, TPS should be 1 */ + fakeMonotimeUs += 10 * ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 10); + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 1.0); +} + +TEST_F(StatCalcTest, TpsSuddenIncrease) { + /* Initialize at 10/sec */ + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 10); + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 10.0); + + /* Add 1 more second at 100/sec */ + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 100); + + /* Window: 4s at 10 TPS + 1s at 100 TPS = 140/5 = 28 TPS */ + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 28.0); +} + +TEST_F(StatCalcTest, TpsSuddenDecrease) { + /* Fill window at 100/sec */ + for (int i = 0; i < 5; i++) { + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 100); + } + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 100.0); + + /* One second at 0 */ + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + tpsCalculator_record(tps, 0); + + /* Window shifts: 4s at 100 TPS + 1s at 0 TPS = 400/5 = 80 */ + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 80.0); +} + +TEST_F(StatCalcTest, TpsMultipleRecordsInOneInterval) { + /* Two records before the update interval elapses accumulate (5 + 5); the + * next flush folds them in together as 10 transactions. */ + tpsCalculator_record(tps, 5); + tpsCalculator_record(tps, 5); + fakeMonotimeUs += ONE_SECOND_IN_MICROS; + EXPECT_DOUBLE_EQ(tpsCalculator_averageTps(tps), 10.0); +} + +/* ======================== Trend Calculator Tests ========================== */ + +/* Trend calc updates once per window/DATA_POINTS. For a 5s window and 10 data + * points, that is 500ms per datapoint. */ +static const monotime TREND_INTERVAL = 5 * ONE_SECOND_IN_MICROS / 10; + +TEST_F(StatCalcTest, TrendInitZero) { + EXPECT_DOUBLE_EQ(trendCalc_changePerSec(trend), 0.0); +} + +TEST_F(StatCalcTest, TrendSingleDatapointFlat) { + /* A single datapoint cannot establish a slope, so the trend stays flat. */ + fakeMonotimeUs += TREND_INTERVAL; + trendCalc_recordMetric(trend, 100); + EXPECT_DOUBLE_EQ(trendCalc_changePerSec(trend), 0.0); + EXPECT_DOUBLE_EQ(trendCalc_changePerSecShortTerm(trend), 0.0); +} + +TEST_F(StatCalcTest, TrendTwoPoint) { + fakeMonotimeUs += TREND_INTERVAL; + trendCalc_recordMetric(trend, 100); /* First point fills all 10 slots */ + + fakeMonotimeUs += TREND_INTERVAL; + trendCalc_recordMetric(trend, 0); + + /* Now we have 9 points at 100 and 1 point at 0. + * Left average is 100. Right average is 400/5 = 80. + * Trend has decreased 20 over 2.5 seconds, or 8/sec. */ + EXPECT_DOUBLE_EQ(trendCalc_changePerSec(trend), -8.0); + + /* The short-term view shows a decrease from 100 to 0 over 1/2 sec. */ + EXPECT_DOUBLE_EQ(trendCalc_changePerSecShortTerm(trend), -200.0); + + fakeMonotimeUs += TREND_INTERVAL; + trendCalc_recordMetric(trend, 0); + + /* Now we have 8 points at 100 and 2 points at 0. + * Left average is 100. Right average is 300/5 = 60. + * Trend has decreased 40 over 2.5 seconds, or 16/sec. */ + EXPECT_DOUBLE_EQ(trendCalc_changePerSec(trend), -16.0); + + /* The short-term view shows no change (0 to 0). */ + EXPECT_DOUBLE_EQ(trendCalc_changePerSecShortTerm(trend), 0.0); +} + +TEST_F(StatCalcTest, TrendIntervalGating) { + fakeMonotimeUs += TREND_INTERVAL; + trendCalc_recordMetric(trend, 100); /* First point fills all 10 slots */ + + fakeMonotimeUs += TREND_INTERVAL - 1; /* Not at the collection interval yet */ + trendCalc_recordMetric(trend, 0); + + /* The datapoint was not collected, so the trend should not have changed. */ + EXPECT_DOUBLE_EQ(trendCalc_changePerSec(trend), 0.0); + EXPECT_DOUBLE_EQ(trendCalc_changePerSecShortTerm(trend), 0.0); +} + +TEST_F(StatCalcTest, TrendRising) { + /* Metric increases by 10 per 100ms. + * Batch averages: 20, 70, 120, ..., 470. + * olderAvg = (20+70+120+170+220)/5 = 120 + * newerAvg = (270+320+370+420+470)/5 = 370 + * trend = (370 - 120) / 2.5 = 100.0 + * Short-term: last two slots (420 -> 470) over 0.5s = 100.0 */ + for (int i = 0; i < 50; i++) { + fakeMonotimeUs += ONE_SECOND_IN_MICROS / 10; + trendCalc_recordMetric(trend, (long)(i * 10)); + } + EXPECT_DOUBLE_EQ(trendCalc_changePerSec(trend), 100.0); + EXPECT_DOUBLE_EQ(trendCalc_changePerSecShortTerm(trend), 100.0); +} + +TEST_F(StatCalcTest, TrendFalling) { + /* Metric decreases by 10 per 100ms. + * Batch averages: 480, 430, 380, ..., 30. + * olderAvg = (480+430+380+330+280)/5 = 380 + * newerAvg = (230+180+130+80+30)/5 = 130 + * trend = (130 - 380) / 2.5 = -100.0 + * Short-term: last two slots (80 -> 30) over 0.5s = -100.0 */ + for (int i = 0; i < 50; i++) { + fakeMonotimeUs += ONE_SECOND_IN_MICROS / 10; + trendCalc_recordMetric(trend, (long)(500 - i * 10)); + } + EXPECT_DOUBLE_EQ(trendCalc_changePerSec(trend), -100.0); + EXPECT_DOUBLE_EQ(trendCalc_changePerSecShortTerm(trend), -100.0); +} diff --git a/src/unit/test_throttle_repl.cpp b/src/unit/test_throttle_repl.cpp new file mode 100644 index 00000000000..063576ea9d6 --- /dev/null +++ b/src/unit/test_throttle_repl.cpp @@ -0,0 +1,308 @@ + +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Unit tests for throttle_repl.h + */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "stat_calc.h" +#include "throttle_repl.h" +static monotime fakeGetMonotonicUs(void); +static monotime (*origGetMonotonicUs)(void); +} + +static monotime fakeMonotimeUs; + +static monotime fakeGetMonotonicUs(void) { + return fakeMonotimeUs; +} + +class ThrottleReplTest : public ::testing::Test { + protected: + MockValkey mock; + RealValkey real; + static const unsigned long long COB_LIMIT = 10 * 1024 * 1024; /* 10 MB */ + client *replica_steady = nullptr; + static inline throttleMetrics fakeMetrics = {0}; + + static void SetUpTestSuite() { + /* Server set up */ + memset(&server, 0, sizeof(valkeyServer)); + server.hz = CONFIG_DEFAULT_HZ; + server.replicas = listCreate(); + server.client_obuf_limits[CLIENT_TYPE_REPLICA].soft_limit_bytes = COB_LIMIT; + server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes = COB_LIMIT; + + /* throttle_repl set up */ + throttle_repl_config.steady_state_repl_throttle_enabled = 1; + + /* monotonic set up */ + origGetMonotonicUs = getMonotonicUs; + getMonotonicUs = fakeGetMonotonicUs; + } + + static void TearDownTestSuite() { + getMonotonicUs = origGetMonotonicUs; + listRelease(server.replicas); + server.replicas = NULL; + } + + void SetUp() override { + replica_steady = createFakeReplicaClient(1); + replica_steady->repl_data->repl_state = REPLICA_STATE_ONLINE; + EXPECT_CALL(mock, throttle_getMetrics(_)).WillRepeatedly(Return(&fakeMetrics)); + EXPECT_CALL(mock, throttle_getGuardrailSecs(_)).WillRepeatedly(Return(0L)); + } + + void TearDown() override { + freeFakeReplicaClient(replica_steady); + replica_steady = NULL; + } + + client *createFakeReplicaClient(int client_id) { + client *c = (client *)zcalloc(sizeof(client)); + c->id = client_id; + c->flag.replica = 1; + c->repl_data = (ClientReplicationData *)zcalloc(sizeof(ClientReplicationData)); + listAddNodeTail(server.replicas, c); + return c; + } + + void freeFakeReplicaClient(client *c) { + ASSERT_TRUE(c->flag.throttled == 0); + ASSERT_TRUE(c->throttler == NULL); + ASSERT_TRUE(c->throttle_node == NULL); + if (c->cob_trend) trendCalc_free(c->cob_trend); + if (c->repl_data) zfree(c->repl_data); + listNode *ln = listSearchKey(server.replicas, c); + if (ln) listDelNode(server.replicas, ln); + zfree(c); + } + + bool isReplThrottlerActive() { + return (long)readMetric("repl_throttle_active") == 1; + } + + double getThrottlerRate() { + return readMetric("repl_throttle_rate"); + } + + bool verifyThrottleEvent(long activation_events, long more, long less) { + return (long)readMetric("repl_throttle_activation_events") == activation_events && + (long)readMetric("repl_throttle_more_events") == more && + (long)readMetric("repl_throttle_less_events") == less; + } + + private: + /* Snapshot the INFO output and return one field's numeric value (-1 if absent). */ + double readMetric(const char *key) { + sds info = throttleRepl_sdscatInfoMetrics(sdsempty()); + char needle[128]; + snprintf(needle, sizeof(needle), "%s:", key); + char *p = strstr(info, needle); + double v = p ? strtod(p + strlen(needle), NULL) : -1.0; + sdsfree(info); + return v; + } +}; + +TEST_F(ThrottleReplTest, NoReplicasNoThrottle) { + listEmpty(server.replicas); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); +} + +TEST_F(ThrottleReplTest, steadyStateNoThrottleCases) { + /* Test cases for steady-state replica that throttler will not enabled. */ + + /* For cob size < 1/4 soft limit, throttler should not be enabled regardless of trend. */ + /* cob size < 1/4 soft limit, trend is 0 */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 8)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + EXPECT_TRUE(replica_steady->cob_trend != NULL); + + /* cob size < 1/4 soft limit, huge trend: still no throttle (below threshold, so the trend is not considered) */ + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + + /* For cob size >= 1/4 soft limit, throttler should be enabled if the extrapolated cob size exceeds the cob target (1/2 soft limit). */ + /* cob size >= 1/4 soft limit but < 1/2 soft limit, trend is decreasing */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(-1.0)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + + /* cob size >= 1/4 soft limit but < 1/2 soft limit, trend is slowly increasing */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(1.0)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + + /* cob size >1/2 soft limit, trend is decreasing and extrapolated value below target */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 2 + 1)); + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(-1.0)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); +} + +TEST_F(ThrottleReplTest, steadyStateThrottleIncreasingTrend) { + /* Test case for steady-state replica above threshold (1/4 cob soft limit), + * for increasing trend, throttle could happen when the extrapolated value exceed the 1/2 cob soft limit. */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); + + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(1)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(isReplThrottlerActive()); + EXPECT_TRUE(verifyThrottleEvent(1, 0, 0)); // Throttler activated, no more/less events yets + + EXPECT_CALL(mock, throttle_adjustRate(_, 0.95)).WillOnce(Return(1.0)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(verifyThrottleEvent(1, 1, 0)); // Reduce traffic, throttle more traffic. + + EXPECT_CALL(mock, throttle_adjustRate(_, 0.95)).WillOnce(Return(1.0)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(verifyThrottleEvent(1, 2, 0)); // Reduce traffic, throttles more traffic. + + // Now mock traffic trend is slowed down more, throttler should be deregistered + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(1.0)); + EXPECT_CALL(mock, throttle_adjustRate(_, 1.05)).WillOnce(Return(10000000.0)); + EXPECT_CALL(mock, throttle_deregister(_)).Times(1); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + EXPECT_TRUE(verifyThrottleEvent(1, 2, 1)); +} + +TEST_F(ThrottleReplTest, steadyStateThrottleDecreasingTrend) { + /* Test case for steady-state replica above threshold (1/4 cob soft limit), + * for decreasing trend, throttle could happen when the extrapolated value exceed the 1/2 cob soft limit. */ + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 2 + 40)); + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(-1.0)); + + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(1)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(isReplThrottlerActive()); + EXPECT_TRUE(verifyThrottleEvent(1, 0, 0)); // Throttler activated, no more/less events yets + + EXPECT_CALL(mock, throttle_adjustRate(_, 0.95)).WillOnce(Return(1.0)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(verifyThrottleEvent(1, 1, 0)); // Reduce traffic, throttle more traffic. + + EXPECT_CALL(mock, throttle_adjustRate(_, 0.95)).WillOnce(Return(1.0)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(verifyThrottleEvent(1, 2, 0)); // Reduce traffic, throttles more traffic. + + // Now mock traffic trend is slowed down more, throttler should be deregistered + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(-2.0)); + EXPECT_CALL(mock, throttle_adjustRate(_, 1.05)).WillOnce(Return(10000000.0)); + EXPECT_CALL(mock, throttle_deregister(_)).Times(1); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + EXPECT_TRUE(verifyThrottleEvent(1, 2, 1)); +} + +TEST_F(ThrottleReplTest, steadyStateThrottleBasedOnLargestCob) { + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 8)); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); + + /* Add a second replica with a large COB. The scan tracks the largest + * COB, so the decision is driven by this replica and throttler activates. */ + client *dummy_replica = createFakeReplicaClient(2); + dummy_replica->repl_data->repl_state = REPLICA_STATE_ONLINE; + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(dummy_replica)).WillRepeatedly(Return(COB_LIMIT)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(1)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(isReplThrottlerActive()); + freeFakeReplicaClient(dummy_replica); +} + +TEST_F(ThrottleReplTest, disabledConfigNoNewThrottle) { + throttle_repl_config.steady_state_repl_throttle_enabled = 0; + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); + + throttleRepl_adjustThrottling(); + + /* Should not activate when config disabled */ + EXPECT_FALSE(isReplThrottlerActive()); +} + +TEST_F(ThrottleReplTest, throttlerRemovedAfterFailover) { + /* Simulate active throttler then failover (become replica) */ + int throttler_id = 42; + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(throttler_id)); + throttleRepl_adjustThrottling(); + EXPECT_TRUE(isReplThrottlerActive()); + + server.primary_host = (char *)"127.0.0.1"; /* now a replica */ + + EXPECT_CALL(mock, throttle_deregister(throttler_id)).Times(1); + throttleRepl_adjustThrottling(); + EXPECT_FALSE(isReplThrottlerActive()); +} + +TEST_F(ThrottleReplTest, insaneCobLimitConfig) { + /* If the configured soft limit is absurd, the target is capped at MAX_COB_TARGET (1GB). + * So a COB far below the configured limit can still exceed the capped target and throttle. */ + server.client_obuf_limits[CLIENT_TYPE_REPLICA].soft_limit_bytes = 10LL * 1024 * 1024 * 1024; /* 10 GB */ + server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes = 10LL * 1024 * 1024 * 1024; /* 10 GB */ + const long max_cob_target = 1024L * 1024 * 1024; /* 1 GB ceiling */ + + EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(max_cob_target + 1)); + EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(1.0)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(1)); + + throttleRepl_adjustThrottling(); + EXPECT_TRUE(isReplThrottlerActive()); +} + +/* ====================== COB Exemption Tests ============================== */ + +// TEST_F(ThrottleReplTest, CobExemptWhenThrottlerActive) { +// /* Throttler active, COB above target, within timeout */ +// isReplThrottlerActive() = true; +// replica_steady.obuf_soft_limit_reached_time = server.unixtime - 10; /* 10s ago */ + +// EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(&replica_steady)) +// .WillRepeatedly(Return(300 * 1024 * 1024)); + +// EXPECT_TRUE(throttleRepl_isClientExemptFromCobLimits(&replica_steady)); +// } + +// TEST_F(ThrottleReplTest, CobExemptExpiresAfterTimeout) { +// /* Throttler active but exceeded 4x convergence timeout (120s) */ +// isReplThrottlerActive() = true; +// replica_steady.obuf_soft_limit_reached_time = server.unixtime - 130; /* 130s > 120s */ + +// EXPECT_CALL(redis, getClientOutputBufferMemoryUsage(&replica_steady)) +// .WillRepeatedly(Return(300 * 1024 * 1024)); + +// EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(&replica_steady)); +// } + +// TEST_F(ThrottleReplTest, CobExemptFalseWhenNotReplica) { +// replica_steady.flag.replica = 0; +// isReplThrottlerActive() = true; +// EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(&replica_steady)); +// } + +// TEST_F(ThrottleReplTest, CobExemptFalseWhenNotPrimary) { +// server.masterhost = (char *)"127.0.0.1"; /* I'm a replica */ +// isReplThrottlerActive() = true; +// EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(&replica_steady)); +// } + +// TEST_F(ThrottleReplTest, CobExemptFalseWhenThrottlerInactive) { +// isReplThrottlerActive() = false; +// EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(&replica_steady)); +// } diff --git a/src/unit/test_token_bucket.cpp b/src/unit/test_token_bucket.cpp new file mode 100644 index 00000000000..775c7a38dd0 --- /dev/null +++ b/src/unit/test_token_bucket.cpp @@ -0,0 +1,145 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Unit tests for throttle_token_bucket.h (token bucket algorithm). + */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "throttle_token_bucket.h" +static monotime fakeGetMonotonicUs(void); +static monotime (*origGetMonotonicUs)(void); +} + +static monotime fakeMonotimeUs; + +static monotime fakeGetMonotonicUs(void) { + return fakeMonotimeUs; +} + +class TokenBucketTest : public ::testing::Test { + protected: + tokenBucket *bucket; + + static void SetUpTestSuite() { + origGetMonotonicUs = getMonotonicUs; + getMonotonicUs = fakeGetMonotonicUs; + } + + static void TearDownTestSuite() { + getMonotonicUs = origGetMonotonicUs; + } + + void SetUp() override { + fakeMonotimeUs = 1000000; /* start at 1 second */ + bucket = tokenBucket_create(100.0, 0.1); /* 100 tokens/sec, 0.1s burst */ + } + + void TearDown() override { + tokenBucket_free(bucket); + } +}; + +TEST_F(TokenBucketTest, BucketCreation) { + /* Bucket starts full and can consume up to bucket capacity */ + EXPECT_DOUBLE_EQ(tokenBucket_getRate(bucket), 100.0); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 10.0), 0.0); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); +} + +TEST_F(TokenBucketTest, HaltedBucket) { + /* Set the rate to zero, this will also empty the bucket. */ + tokenBucket_setRate(bucket, 0.0); + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), -1.0); + + fakeMonotimeUs += 1000000; /* advance 1 second */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, true)); /* Force consume should work */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), -1.0); /* Never available */ + + /* Now set the rate back to a positive value */ + tokenBucket_setRate(bucket, 100.0); + fakeMonotimeUs += 1000000; /* advance 1 second, now the token bucket is refilled */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 11.0), 0.0); /* We should still have 11 tokens available */ +} + +TEST_F(TokenBucketTest, ConsumeTokens_normal) { + /* Drain all tokens (bucket size = rate * burst_time + 2 = 100*0.1+2 = 12) */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 12.0, false)); + /* Now empty */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + + fakeMonotimeUs += 10000; // Advance 10ms -> 1 token replenished + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.1, false)); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); /* Now empty */ + + fakeMonotimeUs += 1000000; /* advance 1 second */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 12.1, false)); /* Cannot consume tokens over bucket capacity */ + for (int i = 0; i < 12; ++i) { + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 0.0); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); + } + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); /* Now empty */ +} + +TEST_F(TokenBucketTest, ConsumeTokens_force) { + /* Drain all tokens (bucket size = rate * burst_time + 2 = 100*0.1+2 = 12) */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 12.0, true)); + /* Now empty */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); + /* Force consume should work even when empty */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, true)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 20.0); /* Now we need to wait for 2 tokens to be available */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + + /* Force consume should work even when the token count is negative */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, true)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 30.0); /* Now we need to wait for 3 tokens to be available */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + + fakeMonotimeUs += 30000; + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.1, false)); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); + + /* Force consume can drop tokens below zero, but not below the minimum capacity (- bucket size)*/ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 100.0, true)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 130.0); + + fakeMonotimeUs += 130000; + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.1, false)); /* replenish -12+13=1; 1 < 1.1 */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); + + fakeMonotimeUs += 1000000; /* advance 1 second, refill bucket */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, true)); /* Force consume 1, 11 should be left*/ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 12.0), 10.0); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 11.0), 0.0); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 11.0, true)); + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); +} + +TEST_F(TokenBucketTest, SetRateChangesRate) { + tokenBucket_setRate(bucket, 200.0); + EXPECT_DOUBLE_EQ(tokenBucket_getRate(bucket), 200.0); + + fakeMonotimeUs += 1000000; /* replenish caps at the new 22 */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 22.0, false)); /* larger capacity is reachable */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); /* now empty */ + + fakeMonotimeUs += 1000000; + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 1.0, false)); /* replenish to 22, 21 left */ + tokenBucket_setRate(bucket, 10.0); + EXPECT_DOUBLE_EQ(tokenBucket_getRate(bucket), 10.0); + /* only 3 remain after trim */ + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 21.0, false)); + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 3.0, false)); + EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); /* now empty */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 100.0); +} diff --git a/src/unit/wrappers.h b/src/unit/wrappers.h index a576814c28c..f5136ddae08 100644 --- a/src/unit/wrappers.h +++ b/src/unit/wrappers.h @@ -45,6 +45,7 @@ extern "C" { #include "ae.h" #include "server.h" +#include "throttle.h" /** * The list of wrapper methods defined. Each wrapper method must @@ -60,6 +61,19 @@ extern "C" { * Example: serverLog(int level, const char *fmt, ...) should NOT be mocked. */ long long __wrap_aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, aeTimeProc *proc, void *clientData, aeEventFinalizerProc *finalizerProc); +size_t __wrap_getClientOutputBufferMemoryUsage(client *c); + +/* Throttler mocks */ +int __wrap_throttle_register(throttleCriteriaProc *criteria_proc, void *priv_data, const char *metrics_name); +void __wrap_throttle_deregister(int id); +double __wrap_throttle_adjustRate(int id, double multiplier); +const throttleMetrics *__wrap_throttle_getMetrics(const char *metrics_name); +long __wrap_throttle_getGuardrailSecs(int id); + +/* Statcalc mocks */ +trendCalculator *__wrap_newTrendCalc(int windowSecs); +void __wrap_trendCalc_recordMetric(trendCalculator *calc, long metricValue); +double __wrap_trendCalc_changePerSecShortTerm(trendCalculator *calc); #undef protected #undef _Bool #undef typename From 762ed686c6c67574a3ea9120eda9d6387d77d3da Mon Sep 17 00:00:00 2001 From: Harry Lin <49881386+harrylin98@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:51:00 -0700 Subject: [PATCH 08/27] Apply suggestions from code review Co-authored-by: Jim Brunner Signed-off-by: Harry Lin <49881386+harrylin98@users.noreply.github.com> --- src/throttle_token_bucket.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/throttle_token_bucket.h b/src/throttle_token_bucket.h index 62b803ad9a6..d4d1ef8608e 100644 --- a/src/throttle_token_bucket.h +++ b/src/throttle_token_bucket.h @@ -4,17 +4,17 @@ * SPDX-License-Identifier: BSD-3-Clause * * The Token Bucket Algorithm is a traffic control method where tokens are added to a bucket at a fixed rate (up to a - * maximum capacity), and commands can be processed only if enough tokens are available. + * maximum capacity), and tokens can be requested from the bucket as needed (if tokens are available). * * Terminology: - * Token: A permission unit required to process commands; a command can be processed only if enough tokens are available. + * Token: A permission unit required to perform some metered work; the caller will only perform the work if tokens are available. * Bucket: A logical storage that holds tokens until they are used. * * Working: * 1. Tokens are added to the bucket at a constant rate and stored up to the maximum capacity. - * 2. When a command arrives, the system checks whether enough tokens are available in the bucket. - * 3. If enough tokens are available, the required number of tokens is removed from the bucket, and the command is processed. - * 4. If tokens are unavailable, the command is queued until new tokens are generated. + * 2. When a caller needs to perform work, an attempt is made to get one or more tokens. + * 3. If enough tokens are available, the required number of tokens is removed from the bucket, and the caller may proceed with the intended work. + * 4. If tokens are unavailable, the caller must wait until sufficient tokens are available. */ #ifndef THROTTLE_TOKEN_BUCKET_H From 2c029c5e9c1a2e3f0eb0a942de9736ab0877bf85 Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Mon, 3 Aug 2026 11:47:51 -0700 Subject: [PATCH 09/27] Address comments second batch Signed-off-by: harrylin98 --- src/config.c | 2 +- src/connection.h | 4 +- src/networking.c | 2 +- src/server.c | 9 +- src/server.h | 2 +- src/socket.c | 5 +- src/stat_calc.c | 11 +- src/throttle.c | 163 +++++------ src/throttle.h | 65 +++-- src/throttle_repl.c | 51 ++-- src/throttle_repl.h | 2 +- src/throttle_token_bucket.c | 12 +- src/throttle_token_bucket.h | 3 + src/tls.c | 2 +- src/unit/test_stat_calc.cpp | 2 +- src/unit/test_throttle.cpp | 480 ++++++++++++++++++++++++++------ src/unit/test_throttle_repl.cpp | 35 ++- src/unit/test_token_bucket.cpp | 15 + src/unit/wrappers.h | 14 +- 19 files changed, 598 insertions(+), 281 deletions(-) diff --git a/src/config.c b/src/config.c index b8fc228b869..0e76a5ce667 100644 --- a/src/config.c +++ b/src/config.c @@ -3357,7 +3357,7 @@ standardConfig static_configs[] = { createBoolConfig("repl-mptcp", NULL, IMMUTABLE_CONFIG, server.repl_mptcp, 0, isValidMptcp, NULL), createBoolConfig("repl-diskless-sync", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.repl_diskless_sync, 1, NULL, NULL), createBoolConfig("dual-channel-replication-enabled", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.dual_channel_replication, 0, NULL, NULL), - createBoolConfig("steady-state-repl-throttle-enabled", NULL, MODIFIABLE_CONFIG, throttle_repl_config.steady_state_repl_throttle_enabled, 0, NULL, NULL), + createBoolConfig("repl-throttle-steady-state-enabled", NULL, MODIFIABLE_CONFIG, throttle_repl_config.repl_throttle_steady_state_enabled, 0, NULL, NULL), createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, 1, NULL, NULL), createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, 0, NULL, NULL), createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, 1, NULL, updateClusterState), diff --git a/src/connection.h b/src/connection.h index 0e07d60fe56..5b2bd21b211 100644 --- a/src/connection.h +++ b/src/connection.h @@ -397,8 +397,8 @@ static inline int connIsClosing(connection *conn) { return conn->type->is_closing(conn); } -/* Shared is_closing implementation for socket-based connections. */ -int connSocketIsClosing(connection *conn); +/* Shared is_closing implementation for TCP socket-based connections. */ +int connTcpSocketIsClosing(connection *conn); /* Associate a private data pointer with the connection */ static inline void connSetPrivateData(connection *conn, void *data) { diff --git a/src/networking.c b/src/networking.c index f974caa13be..318e055db88 100644 --- a/src/networking.c +++ b/src/networking.c @@ -361,7 +361,7 @@ client *createClient(connection *conn) { c->repl_data = NULL; c->throttler = NULL; c->throttle_node = NULL; - c->throttle_start_us = 0; + c->throttle_start = 0; c->cob_trend = NULL; c->bstate = NULL; c->pubsub_data = NULL; diff --git a/src/server.c b/src/server.c index 1929328121c..d034d2f0b2e 100644 --- a/src/server.c +++ b/src/server.c @@ -1187,6 +1187,9 @@ void getExpensiveClientsInfo(size_t *in_usage, size_t *out_usage) { static bool clientsCronTcpIsClosing(client *c) { if (!c->conn) return false; + /* If the fd is still watched by the event loop, it detects the close and frees the client itself. */ + if (connHasReadHandler(c->conn) || connHasWriteHandler(c->conn)) return false; + if (!connIsClosing(c->conn)) return false; if (server.verbosity <= LL_VERBOSE) { @@ -1250,10 +1253,10 @@ static void clientsCron(int clients_this_cycle) { * The protocol is that they return non-zero if the client was * terminated. */ if (clientsCronHandleTimeout(c, now)) continue; + if (clientsCronTcpIsClosing(c)) continue; if (clientsCronResizeQueryBuffer(c)) continue; if (clientsCronResizeOutputBuffer(c, now)) continue; if (clientsCronTrackExpensiveClients(c, curr_peak_mem_usage_slot)) continue; - if (clientsCronTcpIsClosing(c)) continue; /* Iterating all the clients in getMemoryOverheadData() is too slow and * in turn would make the INFO command too slow. So we perform this @@ -6100,7 +6103,6 @@ dict *genInfoSectionDict(robj **argv, int argc, char **defaults, int *out_all, i "errorstats", "cluster", "keyspace", - "throttle", NULL, }; if (!defaults) defaults = default_sections; @@ -6836,9 +6838,6 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { if (all_sections || (dictFind(section_dict, "throttle") != NULL)) { if (sections++) info = sdscat(info, "\r\n"); info = sdscat(info, "# Throttle\r\n"); - info = sdscatprintf(info, - "throttle_total_throttled_commands:%lld\r\n", - throttle_getTotalThrottledCommands()); info = throttle_sdscatInfoMetrics(info); info = throttleRepl_sdscatInfoMetrics(info); } diff --git a/src/server.h b/src/server.h index 2f7269bddcb..a0b30fc5876 100644 --- a/src/server.h +++ b/src/server.h @@ -1409,7 +1409,7 @@ typedef struct client { /* Throttling */ struct throttler *throttler; /* Current throttler this client is queued in, or NULL */ listNode *throttle_node; /* Node in throttler's client_queue */ - monotime throttle_start_us; /* When this client was queued for throttling */ + monotime throttle_start; /* When this client was queued for throttling */ struct trendCalculator *cob_trend; /* Per-replica COB size trend (NULL if not replica) */ #ifdef LOG_REQ_RES clientReqResInfo reqres; diff --git a/src/socket.c b/src/socket.c index 55143e2d026..93533827cd3 100644 --- a/src/socket.c +++ b/src/socket.c @@ -422,8 +422,7 @@ static int connSocketGetType(void) { return CONN_TYPE_SOCKET; } -int connSocketIsClosing(connection *conn) { - if (aeGetFileEvents(server.el, conn->fd) != AE_NONE) return false; +int connTcpSocketIsClosing(connection *conn) { #if defined(__linux__) struct tcp_info info; socklen_t infolen = sizeof(info); @@ -488,7 +487,7 @@ static ConnectionType CT_Socket = { /* Miscellaneous */ .connIntegrityChecked = NULL, - .is_closing = connSocketIsClosing, + .is_closing = connTcpSocketIsClosing, }; int connBlock(connection *conn) { diff --git a/src/stat_calc.c b/src/stat_calc.c index 976bb239014..561d6e4cfdd 100644 --- a/src/stat_calc.c +++ b/src/stat_calc.c @@ -7,7 +7,7 @@ #include "server.h" #include "monotonic.h" -#define ONE_SECOND_IN_MICROS 1000000 +static const long ONE_SECOND_IN_MICROS = 1000000; /* ------------- TPS Calculator ------------- */ struct tpsCalculator { @@ -42,13 +42,10 @@ void tpsCalculator_record(tpsCalculator *calc, unsigned long transactions) { monotime now = getMonotonicUs(); long elapsed_us = now - calc->last_update; - if (elapsed_us < calc->update_freq_us) { - /* Accumulate until the update frequency is hit */ - calc->uncounted_trans += transactions; - return; - } + calc->uncounted_trans += transactions; + if (elapsed_us < calc->update_freq_us) return; /* accumulate until update frequency is hit */ - double total = (double)(calc->uncounted_trans + transactions); + double total = (double)calc->uncounted_trans; calc->uncounted_trans = 0; calc->last_update = now; diff --git a/src/throttle.c b/src/throttle.c index e648f4ebd34..b1078bca75c 100644 --- a/src/throttle.c +++ b/src/throttle.c @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include "server.h" #include "throttle.h" #include "throttle_token_bucket.h" #include "stat_calc.h" @@ -12,28 +13,26 @@ #include -#define MAX_WAIT_TIME_MS 100 /* max ms before rescheduling timer */ -#define MAX_UNTHROTTLE_PROCESSING_TIME_MS 10 /* max ms spent unthrottling per timer fire */ -#define THROTTLE_CLEANUP_ID (-1) /* sentinel: throttler deregistered, draining queue */ -#define THROTTLE_OPS_PER_MIN_GUARDRAIL 6 /* 0.1 TPS - report when rate stays below this */ -#define TPS_WINDOW_SEC 5 /* rolling window for incoming TPS measurement */ -#define EPSILON 0.0001 /* values below this are treated as zero */ -#define TOKENS_BURST_RATE_SEC 0.1 /* burst capacity in seconds of sustained rate */ -#define MIN_ADJUST_AFTER_DISABLE 100.0 /* initial rate when recovering from halted state */ +static const int MAX_WAIT_TIME_MS = 100; /* max ms before rescheduling timer */ +static const uint64_t MAX_UNTHROTTLE_PROCESSING_TIME_MS = 10; /* max ms spent unthrottling per timer fire */ +static const double THROTTLE_OPS_PER_SEC_GUARDRAIL = 0.1; /* report when rate stays below this TPS */ +static const int TPS_WINDOW_SEC = 5; /* rolling window for incoming TPS measurement */ +static const double EPSILON = 0.0001; /* values below this are treated as zero */ +static const double TOKENS_BURST_RATE_SEC = 0.1; /* burst capacity in seconds of sustained rate */ +static const double MIN_ADJUST_AFTER_DISABLE = 100.0; /* initial rate when recovering from halted state */ -static int nextThrottlerId = 1; static hashtable *metricsTable = NULL; static list *throttlerList = NULL; typedef struct metricsEntry { sds throttler_type; int num_clients_throttled; - int num_throttled_commands; + int num_commands_throttled; tpsCalculator *incoming_tps; } metricsEntry; typedef struct throttler { - int id; + bool cleanup; /* true once deregistered; freed when its queue drains */ throttleCriteriaProc *criteria_proc; /* callback defining throttling criteria */ long long time_event_id; /* timer event id for throttlerTimeProc */ void *priv_data; /* private data for use by the criteria_proc */ @@ -65,15 +64,13 @@ static hashtableType metricsHashtableType = { static metricsEntry *findMetrics(const char *name) { sds key = sdsnew(name); - void *found = NULL; - if (hashtableFind(metricsTable, key, &found)) { + metricsEntry *found; + if (hashtableFind(metricsTable, key, (void **)&found)) { sdsfree(key); - return (metricsEntry *)found; + return found; } - metricsEntry *m = zmalloc(sizeof(metricsEntry)); + metricsEntry *m = zcalloc(sizeof(metricsEntry)); m->throttler_type = key; - m->num_clients_throttled = 0; - m->num_throttled_commands = 0; m->incoming_tps = newTpsCalc(TPS_WINDOW_SEC); hashtableAdd(metricsTable, m); return m; @@ -82,21 +79,8 @@ static metricsEntry *findMetrics(const char *name) { /* Framework-level metrics */ static long long total_throttled_commands; -static int listMatchThrottler(void *throttler_ptr, void *id) { - return ((throttler *)throttler_ptr)->id == (long)id; -} - -static throttler *findThrottler(int id) { - listNode *ln = listSearchKey(throttlerList, (void *)(long)id); - serverAssert(ln != NULL); - throttler *t = ln->value; - serverAssert(t->ln == ln); - return t; -} - /* Compute how long to wait before the next token becomes available. */ static int waitTimeMs(throttler *t) { - serverAssert(listLength(t->client_queue) > 0); double ms = tokenBucket_msUntilAvailable(t->bucket, 1.0); if (ms < 0 || ms >= MAX_WAIT_TIME_MS) return MAX_WAIT_TIME_MS; return (int)ceil(ms); @@ -114,6 +98,20 @@ static void freeThrottler(throttler *t) { zfree(t); } +/* Remove a throttled client from its throttler's queue and clear its throttle state. */ +static void dequeueThrottledClient(client *c) { + serverAssert(c->flag.throttled); + c->flag.throttled = 0; + throttler *t = c->throttler; + serverAssert(t != NULL); + + listDelNode(t->client_queue, c->throttle_node); + t->metrics->num_clients_throttled--; + + c->throttler = NULL; + c->throttle_node = NULL; + c->throttle_start = 0; +} static void consumeOtherThrottlers(client *c, throttler *except) { listNode *ln; @@ -121,7 +119,7 @@ static void consumeOtherThrottlers(client *c, throttler *except) { listRewind(throttlerList, &li); while ((ln = listNext(&li))) { throttler *t = ln->value; - if (t->id == THROTTLE_CLEANUP_ID || t == except) continue; + if (t->cleanup || t == except) continue; if (t->criteria_proc(c, t->priv_data)) tokenBucket_tryConsume(t->bucket, 1.0, true); } } @@ -136,6 +134,7 @@ static void processUnthrottledClient(client *c) { return; } } + /* Only call beforeNextClient if the client is not freed (did not return C_ERR). */ if (processPendingCommandAndInputBuffer(c) == C_OK) beforeNextClient(c); } @@ -144,8 +143,6 @@ static void processUnthrottledClient(client *c) { static long long throttlerTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData) { UNUSED(eventLoop); UNUSED(id); - // if the clients are paused, then return 1 ms so we wake up every ms - if (isPausedActionsWithUpdate(PAUSE_ACTIONS_CLIENT_ALL_SET)) return 1; throttler *t = (throttler *)clientData; @@ -156,7 +153,7 @@ static long long throttlerTimeProc(struct aeEventLoop *eventLoop, long long id, elapsedMs(work_start) < MAX_UNTHROTTLE_PROCESSING_TIME_MS && tokenBucket_tryConsume(t->bucket, 1.0, false)) { client *c = listNodeValue(listFirst(t->client_queue)); - throttle_removeClient(c); + dequeueThrottledClient(c); if (c->flag.throttle_multi) { c->flag.throttle_multi = 0; consumeOtherThrottlers(c, t); @@ -165,8 +162,9 @@ static long long throttlerTimeProc(struct aeEventLoop *eventLoop, long long id, } if (listLength(t->client_queue) == 0) { - serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); // Already set in throttle_removeClient - if (t->id == THROTTLE_CLEANUP_ID) freeThrottler(t); + t->time_event_id = AE_DELETED_EVENT_ID; + /* This throttler is drained and ready to be freed. */ + if (t->cleanup) freeThrottler(t); return AE_NOMORE; } return waitTimeMs(t); @@ -175,20 +173,20 @@ static long long throttlerTimeProc(struct aeEventLoop *eventLoop, long long id, static void throttlerAddClient(throttler *t, client *c) { serverAssert(c->throttler == NULL); serverAssert(!c->flag.throttled); - elapsedStart(&c->throttle_start_us); + elapsedStart(&c->throttle_start); c->flag.throttled = 1; listAddNodeTail(t->client_queue, c); if (c->conn) connSetReadHandler(c->conn, NULL); t->metrics->num_clients_throttled++; - t->metrics->num_throttled_commands++; + t->metrics->num_commands_throttled++; total_throttled_commands++; c->throttler = t; c->throttle_node = listLast(t->client_queue); - if (listLength(t->client_queue) == 1) { - serverAssert(t->time_event_id == AE_DELETED_EVENT_ID); + if (t->time_event_id == AE_DELETED_EVENT_ID) { + serverAssert(listLength(t->client_queue) == 1); t->time_event_id = aeCreateTimeEvent(server.el, waitTimeMs(t), throttlerTimeProc, @@ -201,7 +199,6 @@ static void throttlerAddClient(throttler *t, client *c) { void throttle_init(void) { if (throttlerList == NULL) { throttlerList = listCreate(); - listSetMatchMethod(throttlerList, listMatchThrottler); } if (metricsTable == NULL) { metricsTable = hashtableCreate(&metricsHashtableType); @@ -212,15 +209,14 @@ void throttle_init(void) { * throttler is instantiated multiple times (with different priv_data), they may share a single * metrics object by using the same name. This allows statistics to be aggregated across related * throttler instances. */ -int throttle_register(throttleCriteriaProc *criteria_proc, +throttler *throttle_register(throttleCriteriaProc *criteria_proc, void *priv_data, const char *metrics_name) { serverAssert(criteria_proc != NULL); serverAssert(metrics_name != NULL); - serverAssert(nextThrottlerId > 0); throttler *t = zmalloc(sizeof(throttler)); - t->id = nextThrottlerId++; + t->cleanup = false; t->criteria_proc = criteria_proc; t->time_event_id = AE_DELETED_EVENT_ID; t->priv_data = priv_data; @@ -230,25 +226,23 @@ int throttle_register(throttleCriteriaProc *criteria_proc, t->rate_below_guardrail_since = 0; listAddNodeTail(throttlerList, t); t->ln = listLast(throttlerList); - throttle_setRate(t->id, THROTTLE_UNLIMITED_RATE); - return t->id; + throttle_setRate(t, THROTTLE_UNLIMITED_RATE); + return t; } -void throttle_deregister(int id) { - serverAssert(throttlerList != NULL && listLength(throttlerList) > 0); - throttler *t = findThrottler(id); +void throttle_deregister(throttler *t) { + serverAssert(t != NULL); if (listLength(t->client_queue) == 0) { freeThrottler(t); } else { - t->id = THROTTLE_CLEANUP_ID; + t->cleanup = true; tokenBucket_setRate(t->bucket, THROTTLE_UNLIMITED_RATE); } } -void throttle_setRate(int id, double ops_per_sec) { +void throttle_setRate(throttler *t, double ops_per_sec) { serverAssert(ops_per_sec >= 0); - throttler *t = findThrottler(id); if (ops_per_sec < EPSILON) { ops_per_sec = 0; @@ -257,8 +251,7 @@ void throttle_setRate(int id, double ops_per_sec) { } tokenBucket_setRate(t->bucket, ops_per_sec); - double rate_per_min = ops_per_sec * 60.0; - if (rate_per_min <= THROTTLE_OPS_PER_MIN_GUARDRAIL) { + if (ops_per_sec <= THROTTLE_OPS_PER_SEC_GUARDRAIL) { if (t->rate_below_guardrail_since == 0) { elapsedStart(&t->rate_below_guardrail_since); } @@ -267,23 +260,24 @@ void throttle_setRate(int id, double ops_per_sec) { } } -double throttle_adjustRate(int id, double multiplier) { +double throttle_adjustRate(throttler *t, double multiplier) { serverAssert(multiplier >= 0.0 && multiplier <= 3.0); - throttler *t = findThrottler(id); double current = tokenBucket_getRate(t->bucket); /* No change needed if already unlimited and trying to increase. */ - if (multiplier > 1.0 && current == THROTTLE_UNLIMITED_RATE) { - return current; - } + if (multiplier >= 1.0 && current == THROTTLE_UNLIMITED_RATE) return current; double new_rate; - if (multiplier <= 1.0) { - /* Decrease: plain multiply, but never drop below incoming TPS. */ + if (multiplier < 1.0) { + /* Decrease: apply the multiplier to the current rate. If the result still exceeds the + * measured incoming TPS, reduce it directly to that rate. */ new_rate = current * multiplier; double incoming = tpsCalculator_averageTps(t->metrics->incoming_tps); - if (incoming > EPSILON && new_rate < incoming) new_rate = incoming; + /* If there is no incoming rate, it's possible that the tpsCalc hasn't been populated with + * data yet. Otherwise, if there's actually no incoming traffic, it doesn't matter if the + * rate is adjusted. */ + if (incoming > EPSILON && new_rate > incoming) new_rate = incoming; } else if (current < EPSILON) { /* Coming back from halted: jump to a sensible starting rate. */ new_rate = MIN_ADJUST_AFTER_DISABLE; @@ -294,30 +288,22 @@ double throttle_adjustRate(int id, double multiplier) { new_rate = current + delta; } - if (new_rate != current) throttle_setRate(t->id, new_rate); + if (new_rate != current) throttle_setRate(t, new_rate); return tokenBucket_getRate(t->bucket); } void throttle_removeClient(client *c) { if (!c->flag.throttled) return; - c->flag.throttled = 0; throttler *t = c->throttler; - serverAssert(t != NULL); - - listDelNode(t->client_queue, c->throttle_node); - - t->metrics->num_clients_throttled--; + dequeueThrottledClient(c); if (listLength(t->client_queue) == 0) { serverAssert(t->time_event_id != AE_DELETED_EVENT_ID); aeDeleteTimeEvent(server.el, t->time_event_id); t->time_event_id = AE_DELETED_EVENT_ID; - if (t->id == THROTTLE_CLEANUP_ID) freeThrottler(t); + if (t->cleanup) freeThrottler(t); } - c->throttler = NULL; - c->throttle_node = NULL; - c->throttle_start_us = 0; } bool throttleClientIfNeeded(client *c) { @@ -339,7 +325,7 @@ bool throttleClientIfNeeded(client *c) { listRewind(throttlerList, &li); while ((ln = listNext(&li))) { throttler *t = ln->value; - if (t->id == THROTTLE_CLEANUP_ID) continue; + if (t->cleanup) continue; if (t->criteria_proc(c, t->priv_data)) { match_count++; @@ -365,19 +351,14 @@ bool throttleClientIfNeeded(client *c) { } /* === INFO metrics output === */ -long long throttle_getTotalThrottledCommands(void) { - return total_throttled_commands; -} - -const throttleMetrics *throttle_getMetrics(const char *metrics_name) { - static throttleMetrics result; +void throttle_getMetrics(const char *metrics_name, throttleMetrics *metrics) { metricsEntry *m = findMetrics(metrics_name); - result.num_clients_throttled = m->num_clients_throttled; - result.num_throttled_commands = m->num_throttled_commands; - result.incoming_tps = tpsCalculator_averageTps(m->incoming_tps); - result.ops_per_sec = 0.0; - result.oldest_client_delay_us = 0; + metrics->num_clients_throttled = m->num_clients_throttled; + metrics->num_commands_throttled = m->num_commands_throttled; + metrics->incoming_tps = tpsCalculator_averageTps(m->incoming_tps); + metrics->ops_per_sec = 0.0; + metrics->oldest_client_delay_us = 0; /* Aggregate ops_per_sec and oldest_client from all throttlers sharing this metrics. */ listNode *ln; @@ -386,17 +367,18 @@ const throttleMetrics *throttle_getMetrics(const char *metrics_name) { while ((ln = listNext(&li))) { throttler *t = ln->value; if (t->metrics != m) continue; - result.ops_per_sec += tokenBucket_getRate(t->bucket); + metrics->ops_per_sec += tokenBucket_getRate(t->bucket); if (listLength(t->client_queue) > 0) { client *oldest = listNodeValue(listFirst(t->client_queue)); - long delay_us = elapsedUs(oldest->throttle_start_us); - result.oldest_client_delay_us = MAX(result.oldest_client_delay_us, delay_us); + long delay_us = elapsedUs(oldest->throttle_start); + metrics->oldest_client_delay_us = MAX(metrics->oldest_client_delay_us, delay_us); } } - return &result; } sds throttle_sdscatInfoMetrics(sds info) { + info = sdscatprintf(info, "throttle_total_throttled_commands:%lld\r\n", total_throttled_commands); + // Check for any throttlers which are below guardrail. Report only offending throttlers. listNode *ln; listIter li; @@ -415,8 +397,7 @@ sds throttle_sdscatInfoMetrics(sds info) { return info; } -long throttle_getGuardrailSecs(int id) { - throttler *t = findThrottler(id); +long throttle_getGuardrailSecs(throttler *t) { if (t == NULL || t->rate_below_guardrail_since == 0) return 0; return (long)elapsedSec(t->rate_below_guardrail_since); } diff --git a/src/throttle.h b/src/throttle.h index 4210dac5938..113a216a552 100644 --- a/src/throttle.h +++ b/src/throttle.h @@ -20,33 +20,40 @@ #ifndef THROTTLE_H #define THROTTLE_H -#include "server.h" +#include "sds.h" #include +typedef struct client client; +typedef struct throttler throttler; static const double THROTTLE_UNLIMITED_RATE = 10000000.0; -static const int THROTTLE_INVALID_ID = -2; - /* A throttleCriteriaProc checks a client's current command and decides if it meets the criteria * for throttling. Returns true if the client meets the throttling criteria. * + * The criteria proc should base decisions only on the state of the client, not considering + * the question of the current requirements for throttling. If this returns true, the client + * MIGHT be throttled. + * * priv_data - a private data structure provided during throttle_register. It can provide * anything needed by the criteria proc, or NULL if unneeded. */ typedef bool throttleCriteriaProc(client *c, void *priv_data); -/* Metrics for a group of related throttlers sharing the same metrics_name. +/* Metrics for a throttler or group of related throttlers. The metrics name allows the metrics to + * persist even after the throttler(s) is deregistered. Metrics collection will continue (under the + * same name) if/when the throttler is registered again. * * Note: Multiple related throttlers can share the same metrics by using the same metrics_name. - * A typical use case is multiple instantiations of the same throttler with different private - * data. */ + * A typical use case is multiple instantiations of the same throttler with different private data. */ typedef struct { int num_clients_throttled; /* the backlog of currently throttled (queued) clients */ - int num_throttled_commands; /* total number of commands throttled through this metrics group */ + int num_commands_throttled; /* total number of commands throttled through this metrics group */ double ops_per_sec; /* the current throttling rate (summed across related throttlers) */ double incoming_tps; /* average incoming TPS over a 5-second rolling window */ long oldest_client_delay_us; /* delay in microseconds for the oldest throttled client */ } throttleMetrics; +/* Initialize the throttling framework. Must be called once at startup before any + * throttler is registered. Idempotent: safe to call more than once. */ void throttle_init(void); /* Register a new throttler. @@ -54,36 +61,42 @@ void throttle_init(void); * priv_data - private data for passing to the criteria_proc (may be NULL) * metrics_name - a string used to identify a shared metrics group * - * Returns an integer ID of the new throttler. */ -int throttle_register(throttleCriteriaProc *criteria_proc, + * Returns the registered throttler. */ +throttler *throttle_register(throttleCriteriaProc *criteria_proc, void *priv_data, const char *metrics_name); /* Deregisters the throttler such that: * - No new clients will be throttled by this throttler. * - Existing queued clients will be drained at unlimited rate until the queue is empty. */ -void throttle_deregister(int id); +void throttle_deregister(throttler *t); -void throttle_setRate(int id, double ops_per_sec); +/* Set the absolute throttling rate for the given throttler. + * ops_per_sec - target rate in operations per second (must be >= 0) + * + * The rate is clamped: values below EPSILON are treated as 0, + * and values above THROTTLE_UNLIMITED_RATE are capped at that ceiling. */ +void throttle_setRate(throttler *t, double ops_per_sec); /* A smart adjustment to the throttling rate. The multiplier is applied to the current rate, * with consideration for the actual incoming traffic rate. * multiplier - applied to current rate to determine new rate (range 0.0 .. 3.0) * - * If multiplier > 1.0: increase rate (with minimum step of 1 ops/sec at low rates). - * If multiplier < 1.0: decrease rate (clamped to incoming TPS floor). - * If multiplier == 0.0: halt (rate set to 0). + * If multiplier >= 1.0: increase the rate. If currently halted (rate ~0), jump to a + * starting rate; otherwise increase proportionally with a minimum + * step of 1 ops/sec. + * If multiplier < 1.0: decrease the rate proportionally. If the rate is far above the current + * incoming rate, immediately adjusts down to the incoming rate. * * Returns the actual rate set after clamping and adjustment. * * Usage guidance: - * 1. Adjust throttling at a regular interval > 250ms. Adjusting the throttle too fast will - * result in large throttling swings before an observed metric has a chance to change. - * This can easily create a hysteresis problem. The current incoming rate is based on a - * 5-second window and will not update faster than 250ms. - * 2. Set a target for the observed metric. As the observed metric approaches the target, make - * progressively smaller changes to the rate. */ -double throttle_adjustRate(int id, double multiplier); + * 1. Size each step to your call frequency: the more often you call this, the smaller + * each step should be. The driving metrics are smoothed and update slowly, so a large + * step applied at high frequency overshoots and causes hysteresis. + * 2. Prefer a small constant step, as a constant multiplicative step already tapers in + * absolute terms as the rate nears the target. */ +double throttle_adjustRate(throttler *t, double multiplier); /* Removes the client from the throttle queue. */ void throttle_removeClient(client *c); @@ -101,13 +114,9 @@ void throttle_removeClient(client *c); * throttled again for the same command after unblocking. */ bool throttleClientIfNeeded(client *c); -/* Get the total number of commands throttled across all throttlers. */ -long long throttle_getTotalThrottledCommands(void); - /* Get the metrics associated with a given metrics name. - * Memory is managed by the throttler. Do not free the returned pointer. - * Call this each time metrics are needed. Do not cache the pointer. */ -const throttleMetrics *throttle_getMetrics(const char *metrics_name); + * The caller provides the metrics structure. */ +void throttle_getMetrics(const char *metrics_name, throttleMetrics *metrics); /* Append framework-level throttle metrics to the INFO output string. * Plug-in specific metrics are reported by their own sdscatInfoMetrics functions. */ @@ -115,6 +124,6 @@ sds throttle_sdscatInfoMetrics(sds info); /* Get the number of seconds the throttler's rate has been below the guardrail. * Returns 0 if the rate is above the guardrail or the throttler is not active. */ -long throttle_getGuardrailSecs(int id); +long throttle_getGuardrailSecs(throttler *t); #endif diff --git a/src/throttle_repl.c b/src/throttle_repl.c index e3df121866f..16d198019f8 100644 --- a/src/throttle_repl.c +++ b/src/throttle_repl.c @@ -12,15 +12,14 @@ /* Configuration instance. */ struct throttle_repl_config throttle_repl_config; -#define RATE_INCREASE_MULTIPLIER 1.05 -#define RATE_DECREASE_MULTIPLIER 0.95 -#define COB_TREND_WINDOW_SECS 2 /* A 2-second window gives 20 data points at \ - * 100ms serverCron. Sufficient for a good \ - * measurement, while remaining short enough for \ - * throttling adjustments every 100ms. */ -#define STEADY_STATE_CONVERGENCE_SECS 30 /* projection horizon for COB extrapolation */ -#define MAX_COB_TARGET (1024L * 1024 * 1024) /* 1GB */ -#define METRICS_NAME "ReplThrottle" /* shared metrics group name */ +/* A 2-second window gives 20 data points at 100ms serverCron. Sufficient for a good + * measurement, while remaining short enough for throttling adjustments every 100ms. */ +static const int COB_TREND_WINDOW_SECS = 2; +static const double RATE_INCREASE_MULTIPLIER = 1.05; +static const double RATE_DECREASE_MULTIPLIER = 0.95; +static const int STEADY_STATE_CONVERGENCE_SECS = 30; /* projection horizon for COB extrapolation */ +static const long MAX_COB_TARGET = 1024L * 1024 * 1024; /* 1GB */ +static const char *const METRICS_NAME = "ReplThrottle"; /* shared metrics group name */ /* Metrics for INFO output and operational visibility. */ typedef struct { @@ -32,12 +31,12 @@ typedef struct { } throttleReplMetrics; static throttleReplMetrics metrics = {0}; -static int throttle_id = 0; +static throttler *repl_throttler = NULL; /* --- Internal helpers --- */ static bool isThrottlerActive(void) { - return (throttle_id != 0); + return (repl_throttler != NULL); } /* Criteria: throttle commands that generate replication traffic. */ @@ -49,7 +48,8 @@ static bool criteriaProc(client *c, void *priv_data) { static void installThrottler(void) { serverAssert(!isThrottlerActive()); - throttle_id = throttle_register(criteriaProc, NULL, METRICS_NAME); + repl_throttler = throttle_register(criteriaProc, NULL, METRICS_NAME); + serverAssert(repl_throttler != NULL); metrics.is_throttler_active = true; metrics.current_throttle_rate = THROTTLE_UNLIMITED_RATE; metrics.throttle_activation_events++; @@ -57,8 +57,8 @@ static void installThrottler(void) { static void uninstallThrottler(void) { serverAssert(isThrottlerActive()); - throttle_deregister(throttle_id); - throttle_id = 0; + throttle_deregister(repl_throttler); + repl_throttler = NULL; metrics.is_throttler_active = false; metrics.current_throttle_rate = THROTTLE_UNLIMITED_RATE; } @@ -69,10 +69,10 @@ static void adjustThrottleRate(bool reduceTrafficRate) { if (isThrottlerActive()) { double rate; if (reduceTrafficRate) { - rate = throttle_adjustRate(throttle_id, RATE_DECREASE_MULTIPLIER); + rate = throttle_adjustRate(repl_throttler, RATE_DECREASE_MULTIPLIER); metrics.throttle_more_events++; } else { - rate = throttle_adjustRate(throttle_id, RATE_INCREASE_MULTIPLIER); + rate = throttle_adjustRate(repl_throttler, RATE_INCREASE_MULTIPLIER); metrics.throttle_less_events++; if (rate >= THROTTLE_UNLIMITED_RATE) uninstallThrottler(); } @@ -122,13 +122,14 @@ static bool evaluateSteadyStateThrottle(client *c, int64_t cob_size) { * throttling hasn't had time to adjust and there is no severe memory condition, it makes * sense to allow the replica to live until throttling can stabilize the situation. */ bool throttleRepl_isClientExemptFromCobLimits(client *c) { - if (!throttle_repl_config.steady_state_repl_throttle_enabled || !isThrottlerActive()) return false; + if (!throttle_repl_config.repl_throttle_steady_state_enabled || !isThrottlerActive()) return false; if (!iAmPrimary()) return false; if (!c->flag.replica) return false; /* Throttle is actively working, protect this replica from COB * disconnect if its COB is above target. */ int64_t client_cob_size = (int64_t)getClientOutputBufferMemoryUsage(c); + /* There's no need to protect the replica if it's already using less than the target size. */ if (client_cob_size < getReplicaSteadyStateCobTargetSize()) return false; /* Don't exempt if server is over maxmemory. @@ -139,18 +140,19 @@ bool throttleRepl_isClientExemptFromCobLimits(client *c) { /* Don't protect if throttle has been working too long without success. */ time_t elapsed = server.unixtime - c->obuf_soft_limit_reached_time; if (elapsed > 4 * STEADY_STATE_CONVERGENCE_SECS) return false; + /* Otherwise, allow the replica to exceed the configured limits, giving the throttler time to correct. */ return true; } /* Called from serverCron every 100ms. Evaluates the replica with the largest COB and * adjusts throttling as needed. */ void throttleRepl_adjustThrottling(void) { - if (!iAmPrimary()) { - /* Failover could happen before. */ + /* If we're no longer the primary (e.g. after failover) or steady-state repl throttling + * was disabled, tear down any active throttler and stop. */ + if (!iAmPrimary() || !throttle_repl_config.repl_throttle_steady_state_enabled) { if (isThrottlerActive()) uninstallThrottler(); return; } - if (!throttle_repl_config.steady_state_repl_throttle_enabled && !isThrottlerActive()) return; bool reduceTrafficRate = false; client *measured_steady_state_replica = NULL; @@ -196,7 +198,8 @@ sds throttleRepl_sdscatInfoMetrics(sds info) { metrics.current_throttle_rate); } - const throttleMetrics *throttle_metrics = throttle_getMetrics(METRICS_NAME); + throttleMetrics throttle_metrics; + throttle_getMetrics(METRICS_NAME, &throttle_metrics); info = sdscatprintf(info, "repl_throttle_activation_events:%lu\r\n" "repl_throttle_more_events:%lu\r\n" @@ -207,9 +210,9 @@ sds throttleRepl_sdscatInfoMetrics(sds info) { metrics.throttle_activation_events, metrics.throttle_more_events, metrics.throttle_less_events, - isThrottlerActive() ? throttle_getGuardrailSecs(throttle_id) : 0L, - throttle_metrics->num_clients_throttled, - throttle_metrics->num_throttled_commands); + isThrottlerActive() ? throttle_getGuardrailSecs(repl_throttler) : 0L, + throttle_metrics.num_clients_throttled, + throttle_metrics.num_commands_throttled); return info; } diff --git a/src/throttle_repl.h b/src/throttle_repl.h index 994bf398b02..276778ecffa 100644 --- a/src/throttle_repl.h +++ b/src/throttle_repl.h @@ -19,7 +19,7 @@ #include "sds.h" struct throttle_repl_config { - int steady_state_repl_throttle_enabled; + int repl_throttle_steady_state_enabled; }; extern struct throttle_repl_config throttle_repl_config; diff --git a/src/throttle_token_bucket.c b/src/throttle_token_bucket.c index fadca4ca4c5..3c68e0fb52c 100644 --- a/src/throttle_token_bucket.c +++ b/src/throttle_token_bucket.c @@ -5,7 +5,7 @@ */ #include "throttle_token_bucket.h" -#include "server.h" +#include "zmalloc.h" #include "monotonic.h" struct tokenBucket { @@ -37,16 +37,12 @@ static void tokenBucket_replenish(tokenBucket *bucket) { monotime now = getMonotonicUs(); uint64_t delta_us = now - bucket->last_time_check; double tokens_to_add = delta_us * bucket->tokens_per_sec / 1000000.0; - if (tokens_to_add > 0) { - bucket->token_count += tokens_to_add; - trimTokenBucket(bucket); - } + bucket->token_count += tokens_to_add; + trimTokenBucket(bucket); bucket->last_time_check = now; } tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs) { - serverAssert(tokens_per_sec >= 0); - serverAssert(max_burst_time_secs >= 0); tokenBucket *bucket = zmalloc(sizeof(tokenBucket)); bucket->tokens_per_sec = tokens_per_sec; bucket->max_burst_time_secs = max_burst_time_secs; @@ -64,7 +60,6 @@ double tokenBucket_getRate(tokenBucket *bucket) { } void tokenBucket_setRate(tokenBucket *bucket, double new_rate) { - serverAssert(new_rate >= 0); bucket->tokens_per_sec = new_rate; trimTokenBucket(bucket); } @@ -78,6 +73,7 @@ bool tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consu } double tokenBucket_msUntilAvailable(tokenBucket *bucket, double target_tokens) { + tokenBucket_replenish(bucket); if (bucket->token_count >= target_tokens) return 0.0; if (bucket->tokens_per_sec <= 0) return -1.0; /* halted — never available */ double needed = target_tokens - bucket->token_count; diff --git a/src/throttle_token_bucket.h b/src/throttle_token_bucket.h index d4d1ef8608e..dd1bee8e871 100644 --- a/src/throttle_token_bucket.h +++ b/src/throttle_token_bucket.h @@ -30,10 +30,13 @@ typedef struct tokenBucket tokenBucket; * bigger bursts after idle periods. */ tokenBucket *tokenBucket_create(double tokens_per_sec, double max_burst_time_secs); +/* Free a token bucket and its resources. */ void tokenBucket_free(tokenBucket *bucket); +/* Return the current refill rate in tokens per second. */ double tokenBucket_getRate(tokenBucket *bucket); +/* Update the refill rate. Tokens are clamped to the new bucket capacity. */ void tokenBucket_setRate(tokenBucket *bucket, double new_rate); /* Attempt to consume tokens. Returns true if tokens were deducted. diff --git a/src/tls.c b/src/tls.c index 283e265a406..3727dd98c5c 100644 --- a/src/tls.c +++ b/src/tls.c @@ -1998,7 +1998,7 @@ static ConnectionType CT_TLS = { /* Miscellaneous */ .connIntegrityChecked = connTLSIsIntegrityChecked, - .is_closing = connSocketIsClosing, + .is_closing = connTcpSocketIsClosing, }; diff --git a/src/unit/test_stat_calc.cpp b/src/unit/test_stat_calc.cpp index e5c88ad75d4..7c0a201e14a 100644 --- a/src/unit/test_stat_calc.cpp +++ b/src/unit/test_stat_calc.cpp @@ -14,7 +14,7 @@ static monotime fakeGetMonotonicUs(void); static monotime (*origGetMonotonicUs)(void); } -#define ONE_SECOND_IN_MICROS 1000000 +static const long ONE_SECOND_IN_MICROS = 1000000; static monotime fakeMonotimeUs; diff --git a/src/unit/test_throttle.cpp b/src/unit/test_throttle.cpp index 8884d4944b8..2703ca860f7 100644 --- a/src/unit/test_throttle.cpp +++ b/src/unit/test_throttle.cpp @@ -75,6 +75,7 @@ class ThrottleTest : public ::testing::Test { c->conn->type = &dummyConnType; c->conn->read_handler = (ConnectionCallbackFunc)1; c->flag.pending_command = 1; + c->argc = 1; c->cmd = write_command ? &set_cmd : &get_cmd; return c; } @@ -94,24 +95,28 @@ class ThrottleTest : public ::testing::Test { EXPECT_NE(c->throttler, nullptr); EXPECT_NE(c->throttle_node, nullptr); EXPECT_EQ(c->flag.throttle_checked, 1ULL); - EXPECT_NE(c->throttle_start_us, 0ULL); + EXPECT_NE(c->throttle_start, 0ULL); } else { EXPECT_EQ(c->throttler, nullptr); EXPECT_EQ(c->throttle_node, nullptr); - EXPECT_EQ(c->throttle_start_us, 0ULL); + EXPECT_EQ(c->throttle_start, 0ULL); + EXPECT_EQ(c->flag.throttle_multi, 0ULL); } return throttled; } void verifyThrottler(const char *metric_name, int clients_throttled, int cmds_throttled) { - const throttleMetrics *m = throttle_getMetrics(metric_name); - EXPECT_EQ(m->num_clients_throttled, clients_throttled); - EXPECT_EQ(m->num_throttled_commands, cmds_throttled); + throttleMetrics m; + throttle_getMetrics(metric_name, &m); + EXPECT_EQ(m.num_clients_throttled, clients_throttled); + EXPECT_EQ(m.num_commands_throttled, cmds_throttled); } }; using ThrottleDeathTest = ThrottleTest; +/* ---- throttleClientIfNeeded tests ---- */ + TEST_F(ThrottleTest, noThrottlerPassesThrough) { client *c = createFakeClient(1, true); /* No throttler registered yet, nothing to throttle. */ @@ -121,55 +126,71 @@ TEST_F(ThrottleTest, noThrottlerPassesThrough) { } TEST_F(ThrottleTest, throttleHappyCase) { - int id = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); client *c = createFakeClient(1, true); - throttle_setRate(id, 0.0); // This will empty the bucket + throttle_setRate(t, 0.0); // This will empty the bucket - EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).WillOnce(Return(1)); + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); EXPECT_TRUE(throttleClientIfNeeded(c)); EXPECT_TRUE(clientIsThrottled(c)); EXPECT_FALSE(throttleClientIfNeeded(c)); // We don't throttle client if it's already throttled verifyThrottler("fake_throttler", 1, 1); - EXPECT_CALL(mock, aeDeleteTimeEvent(_, _)).WillOnce(Return(1)); - throttle_removeClient(c); + /* Drain via timeProc */ + throttle_setRate(t, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_FALSE(clientIsThrottled(c)); verifyThrottler("fake_throttler", 0, 1); - throttle_deregister(id); + throttle_deregister(t); freeFakeClient(c); } TEST_F(ThrottleTest, criteriaMismatchPassesThrough) { - int id = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); client *c = createFakeClient(1, false); // client with read command + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).Times(0); EXPECT_FALSE(throttleClientIfNeeded(c)); EXPECT_FALSE(clientIsThrottled(c)); verifyThrottler("fake_throttler", 0, 0); - throttle_deregister(id); + throttle_deregister(t); freeFakeClient(c); } TEST_F(ThrottleTest, tokenAvailablePassesThrough) { - int id = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); /* starts at UNLIMITED rate, full bucket */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); /* starts at UNLIMITED rate, full bucket */ client *c = createFakeClient(1, true); - /* Criteria matches, but tokens are available, consume token and proceed. */ + /* Criteria matches, tokens are available, consume token and proceed. */ + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).Times(0); EXPECT_FALSE(throttleClientIfNeeded(c)); EXPECT_FALSE(clientIsThrottled(c)); verifyThrottler("fake_throttler", 0, 0); - throttle_deregister(id); + throttle_deregister(t); freeFakeClient(c); } TEST_F(ThrottleTest, deregisteredThrottlerDrainsButDoesNotThrottleNewClients) { - int id = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); - throttle_setRate(id, 0.0); - EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).WillOnce(Return(1)); + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); /* Queue a client so the throttler cannot be freed on deregister. */ client *queued = createFakeClient(1, true); @@ -178,16 +199,21 @@ TEST_F(ThrottleTest, deregisteredThrottlerDrainsButDoesNotThrottleNewClients) { /* Deregister with a non-empty queue: the throttler stays alive in CLEANUP * state (still draining the queued client) but must not throttle new clients. */ - throttle_deregister(id); + throttle_deregister(t); /* A new matching write command passes through untouched. */ client *fresh = createFakeClient(2, true); EXPECT_FALSE(throttleClientIfNeeded(fresh)); EXPECT_FALSE(clientIsThrottled(fresh)); + verifyThrottler("fake_throttler", 1, 1); - /* Drain the original client; emptying the queue frees the CLEANUP throttler. */ - EXPECT_CALL(mock, aeDeleteTimeEvent(_, _)).WillOnce(Return(1)); - throttle_removeClient(queued); + /* Drain via timeProc — deregister already set rate to UNLIMITED. */ + fakeMonotimeUs += 1000000; + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(queued)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(queued)).Times(1); + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + verifyThrottler("fake_throttler", 0, 1); freeFakeClient(queued); freeFakeClient(fresh); @@ -197,10 +223,14 @@ TEST_F(ThrottleTest, strictestThrottlerThrottle) { /* Two throttlers both match a write command. The strictest (lowest rate) * wins: the client is queued under it and the multi-match flag is set so * the other bucket is charged on release. */ - int loose = throttle_register(fakeWriteCriteria, NULL, "loose"); /* UNLIMITED */ - int strict = throttle_register(fakeWriteCriteria, NULL, "strict"); + throttler *loose = throttle_register(fakeWriteCriteria, NULL, "loose"); /* UNLIMITED */ + throttler *strict = throttle_register(fakeWriteCriteria, NULL, "strict"); throttle_setRate(strict, 0.0); /* no tokens -> strictest */ - EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).WillOnce(Return(1)); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); client *c = createFakeClient(1, true); EXPECT_TRUE(throttleClientIfNeeded(c)); @@ -211,8 +241,17 @@ TEST_F(ThrottleTest, strictestThrottlerThrottle) { verifyThrottler("strict", 1, 1); verifyThrottler("loose", 0, 0); - EXPECT_CALL(mock, aeDeleteTimeEvent(_, _)).WillOnce(Return(1)); - throttle_removeClient(c); + /* Drain via timeProc. */ + throttle_setRate(strict, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, false)).WillOnce(Return(true)); + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, true)).WillOnce(Return(true)); + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_EQ(c->flag.throttle_multi, 0ULL); + throttle_deregister(loose); throttle_deregister(strict); freeFakeClient(c); @@ -222,8 +261,8 @@ TEST_F(ThrottleTest, strictestThrottlerNonThrottle) { /* Two throttlers both match, but the strictest still has a token, so the * client passes through (not throttled). Because it matched >1 throttler, * consumeOtherThrottlers also charges the other bucket on the pass path. */ - int loose = throttle_register(fakeWriteCriteria, NULL, "loose"); /* UNLIMITED */ - int strict = throttle_register(fakeWriteCriteria, NULL, "strict"); + throttler *loose = throttle_register(fakeWriteCriteria, NULL, "loose"); /* UNLIMITED */ + throttler *strict = throttle_register(fakeWriteCriteria, NULL, "strict"); throttle_setRate(strict, 1.0); client *c = createFakeClient(1, true); @@ -242,132 +281,405 @@ TEST_F(ThrottleTest, strictestThrottlerNonThrottle) { freeFakeClient(c); } +/* ---- throttler rate tests ---- */ + TEST_F(ThrottleTest, adjustRatePolicyIncrease) { - int id = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); /* starts UNLIMITED */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); /* starts UNLIMITED */ /* Increase while already UNLIMITED is a no-op. */ - EXPECT_DOUBLE_EQ(throttle_adjustRate(id, 2.0), THROTTLE_UNLIMITED_RATE); + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 2.0), THROTTLE_UNLIMITED_RATE); /* Recover from halted state jumps to the fixed restart rate (100 ops/sec). */ - throttle_setRate(id, 0.0); - EXPECT_DOUBLE_EQ(throttle_adjustRate(id, 2.0), 100.0); + throttle_setRate(t, 0.0); + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 2.0), 100.0); /* Normal increase: 100 * (2.0 - 1.0) = 200. */ - EXPECT_DOUBLE_EQ(throttle_adjustRate(id, 2.0), 200.0); + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 2.0), 200.0); /* Tiny multiplier still increases by the minimum step of 1 ops/sec. */ - EXPECT_DOUBLE_EQ(throttle_adjustRate(id, 1.00001), 201.0); + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 1.00001), 201.0); - throttle_deregister(id); + throttle_deregister(t); } TEST_F(ThrottleTest, adjustRatePolicyDecrease) { - int id = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); /* starts UNLIMITED */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); /* starts UNLIMITED */ - EXPECT_CALL(mock, tpsCalculator_averageTps(_)).WillRepeatedly(Return(500.0)); /* TPS floor */ + EXPECT_CALL(mock, tpsCalculator_averageTps(_)).WillRepeatedly(Return(500.0)); /* incoming TPS */ - /* Normal decrease, well above the floor. */ - EXPECT_DOUBLE_EQ(throttle_adjustRate(id, 0.5), THROTTLE_UNLIMITED_RATE * 0.5); + /* Decreasing a rate that is still above incoming snaps straight down to incoming. */ + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 0.95), 500.0); - /* Decrease clamped up to the incoming-TPS floor: 1000 * 0.1 = 100 < 500. */ - throttle_setRate(id, 1000.0); - EXPECT_DOUBLE_EQ(throttle_adjustRate(id, 0.1), 500.0); + /* Once at/below incoming, a further decrease goes below it (real throttling): + * 500 * 0.8 = 400 < 500. */ + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 0.8), 400.0); - /* With no incoming TPS the floor is disabled: 500 * 0.1 = 50. */ + /* With no measured incoming TPS the snap is disabled: 400 * 0.5 = 200. */ EXPECT_CALL(mock, tpsCalculator_averageTps(_)).WillRepeatedly(Return(0.0)); - EXPECT_DOUBLE_EQ(throttle_adjustRate(id, 0.1), 50.0); + EXPECT_DOUBLE_EQ(throttle_adjustRate(t, 0.5), 200.0); - throttle_deregister(id); + throttle_deregister(t); } TEST_F(ThrottleTest, setRate) { - int id = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttleMetrics m; /* A normal rate is stored as-is. */ - throttle_setRate(id, 1234.0); - EXPECT_DOUBLE_EQ(throttle_getMetrics("fake_throttler")->ops_per_sec, 1234.0); + throttle_setRate(t, 1234.0); + throttle_getMetrics("fake_throttler", &m); + EXPECT_DOUBLE_EQ(m.ops_per_sec, 1234.0); /* Above the unlimited ceiling is clamped down to THROTTLE_UNLIMITED_RATE. */ - throttle_setRate(id, THROTTLE_UNLIMITED_RATE * 2); - EXPECT_DOUBLE_EQ(throttle_getMetrics("fake_throttler")->ops_per_sec, THROTTLE_UNLIMITED_RATE); + throttle_setRate(t, THROTTLE_UNLIMITED_RATE * 2); + throttle_getMetrics("fake_throttler", &m); + EXPECT_DOUBLE_EQ(m.ops_per_sec, THROTTLE_UNLIMITED_RATE); /* Below epsilon collapses to zero. */ - throttle_setRate(id, 0.00001); - EXPECT_DOUBLE_EQ(throttle_getMetrics("fake_throttler")->ops_per_sec, 0.0); + throttle_setRate(t, 0.00001); + throttle_getMetrics("fake_throttler", &m); + EXPECT_DOUBLE_EQ(m.ops_per_sec, 0.0); - throttle_deregister(id); + throttle_deregister(t); } TEST_F(ThrottleTest, guardrailSecsTracking) { - int id = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); - /* 0.05 ops/sec == 3 ops/min, at or below the 6 ops/min guardrail. */ - throttle_setRate(id, 0.05); + /* 0.05 ops/sec, at or below the 0.1 ops/sec guardrail. */ + throttle_setRate(t, 0.05); fakeMonotimeUs += 3 * 1000000; /* advance 3 seconds */ - EXPECT_EQ(throttle_getGuardrailSecs(id), 3L); + EXPECT_EQ(throttle_getGuardrailSecs(t), 3L); /* Back above the guardrail resets the timer. */ - throttle_setRate(id, 1.0); /* 60 ops/min */ - EXPECT_EQ(throttle_getGuardrailSecs(id), 0L); + throttle_setRate(t, 1.0); /* above the 0.1 ops/sec guardrail */ + EXPECT_EQ(throttle_getGuardrailSecs(t), 0L); - throttle_deregister(id); + throttle_deregister(t); } -// /* ---- A-layer: metrics aggregation ---- */ +/* ---- metrics aggregation ---- */ TEST_F(ThrottleTest, metricsAggregateAcrossSharedName) { /* Two throttlers sharing one metrics group ("shared") aggregate their metrics */ - int a = throttle_register(fakeWriteCriteria, NULL, "shared"); - int b = throttle_register(fakeWriteCriteria, NULL, "shared"); + throttler *a = throttle_register(fakeWriteCriteria, NULL, "shared"); + throttler *b = throttle_register(fakeWriteCriteria, NULL, "shared"); /* ops_per_sec is the SUM of both throttlers' rates. */ throttle_setRate(a, 100.0); throttle_setRate(b, 250.0); - EXPECT_DOUBLE_EQ(throttle_getMetrics("shared")->ops_per_sec, 350.0); + throttleMetrics m; + throttle_getMetrics("shared", &m); + EXPECT_DOUBLE_EQ(m.ops_per_sec, 350.0); /* Empty both buckets. Both clients are writes matching both throttlers, so the * strictest (a, rate 0) wins and both queue under a; b stays empty. */ throttle_setRate(a, 0.0); throttle_setRate(b, 0.0); - EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).WillOnce(Return(1)); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); client *c1 = createFakeClient(1, true); - EXPECT_TRUE(throttleClientIfNeeded(c1)); /* throttle_start_us = 100 (fake clock) */ + EXPECT_TRUE(throttleClientIfNeeded(c1)); /* throttle_start = 100 (fake clock) */ fakeMonotimeUs += 5 * 1000000; /* +5s */ client *c2 = createFakeClient(2, true); - EXPECT_TRUE(throttleClientIfNeeded(c2)); /* throttle_start_us = 5,000,100 */ + EXPECT_TRUE(throttleClientIfNeeded(c2)); /* throttle_start = 5,000,100 */ - const throttleMetrics *m = throttle_getMetrics("shared"); + throttle_getMetrics("shared", &m); /* Both clients increment the shared metrics group. */ - EXPECT_EQ(m->num_clients_throttled, 2); - EXPECT_EQ(m->num_throttled_commands, 2); + EXPECT_EQ(m.num_clients_throttled, 2); + EXPECT_EQ(m.num_commands_throttled, 2); /* oldest_client_delay_us tracks the oldest queued client (c1, queued 5s ago). */ - EXPECT_EQ(m->oldest_client_delay_us, 5 * 1000000); + EXPECT_EQ(m.oldest_client_delay_us, 5 * 1000000); + + /* Drain via timeProc. */ + throttle_setRate(a, THROTTLE_UNLIMITED_RATE); + throttle_setRate(b, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + /* Both clients have throttle_multi set (matched both throttlers), so each + * release will also consume the other throttler's bucket. */ + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, false)).Times(2).WillRepeatedly(Return(true)); + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, true)).Times(2).WillRepeatedly(Return(true)); + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(_)).Times(2).WillRepeatedly(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(_)).Times(2); + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + + throttle_getMetrics("shared", &m); + verifyThrottler("shared", 0, 2); + EXPECT_EQ(m.oldest_client_delay_us, 0); - EXPECT_CALL(mock, aeDeleteTimeEvent(_, _)).WillOnce(Return(1)); - throttle_removeClient(c1); - throttle_removeClient(c2); throttle_deregister(a); throttle_deregister(b); freeFakeClient(c1); freeFakeClient(c2); } +/* ---- throttlerTimeProc tests ---- */ + +TEST_F(ThrottleTest, timeProcHappyCaseOneCall) { + /* When enough tokens are available, timeProc releases all queued clients in one call. */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c1 = createFakeClient(1, true); + client *c2 = createFakeClient(2, true); + EXPECT_TRUE(throttleClientIfNeeded(c1)); + EXPECT_TRUE(throttleClientIfNeeded(c2)); + + /* Set unlimited rate so both clients are released in one timeProc call. */ + throttle_setRate(t, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(_)).Times(2).WillRepeatedly(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(_)).Times(2); + + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + + EXPECT_FALSE(clientIsThrottled(c1)); + EXPECT_FALSE(clientIsThrottled(c2)); + EXPECT_NE(c1->conn->read_handler, nullptr); + EXPECT_NE(c2->conn->read_handler, nullptr); + verifyThrottler("fake_throttler", 0, 2); + + throttle_deregister(t); + freeFakeClient(c1); + freeFakeClient(c2); +} + +TEST_F(ThrottleTest, timeProcHappyCaseMultipleCall) { + /* When the timer fires but tokens run out before the queue is empty, it reschedules. */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c1 = createFakeClient(1, true); + client *c2 = createFakeClient(2, true); + EXPECT_TRUE(throttleClientIfNeeded(c1)); + EXPECT_TRUE(throttleClientIfNeeded(c2)); + + /* Set 1 ops/sec rate, so only 1 token available after refill. */ + throttle_setRate(t, 1.0); + fakeMonotimeUs += 1000000; + + /* Only the first client will be released. No aeDeleteTimeEvent since queue stays non-empty. */ + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c1)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c1)).Times(1); + + long long ret = timeProc(server.el, 1, clientData); + /* Should return a positive wait time (reschedule). */ + EXPECT_EQ(ret, 100); + + /* c1 released, c2 still throttled. */ + EXPECT_FALSE(clientIsThrottled(c1)); + EXPECT_NE(c1->conn->read_handler, nullptr); + EXPECT_TRUE(clientIsThrottled(c2)); + verifyThrottler("fake_throttler", 1, 2); + + /* Drain c2 for cleanup. */ + fakeMonotimeUs += 1000000; /* advance 1s, 1 token available */ + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c2)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c2)).Times(1); + ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_FALSE(clientIsThrottled(c2)); + EXPECT_NE(c2->conn->read_handler, nullptr); + verifyThrottler("fake_throttler", 0, 2); + + throttle_deregister(t); + freeFakeClient(c1); + freeFakeClient(c2); +} + +TEST_F(ThrottleTest, timeProcCallsFreeClientOnConnSetReadHandlerFailure) { + /* If connSetReadHandler returns C_ERR, the client is freed. */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c = createFakeClient(1, true); + EXPECT_TRUE(throttleClientIfNeeded(c)); + EXPECT_TRUE(clientIsThrottled(c)); + + /* Install a failing read handler to simulate connection error. */ + static ConnectionType failConnType = {0}; + failConnType.set_read_handler = [](connection *, ConnectionCallbackFunc) -> int { + return C_ERR; + }; + c->conn->type = &failConnType; + + throttle_setRate(t, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + + /* freeClient should be called because connSetReadHandler fails. */ + EXPECT_CALL(mock, freeClient(c)).WillOnce(Return(0)); + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(_)).Times(0); + + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_FALSE(clientIsThrottled(c)); + verifyThrottler("fake_throttler", 0, 1); + + throttle_deregister(t); + freeFakeClient(c); // We still need to call it since freeClient is mocked. +} + +TEST_F(ThrottleTest, timeProcProcessingFailureSkipsBeforeNextClient) { + /* If processPendingCommandAndInputBuffer returns C_ERR, beforeNextClient is NOT called. */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c = createFakeClient(1, true); + EXPECT_TRUE(throttleClientIfNeeded(c)); + + throttle_setRate(t, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_ERR)); + EXPECT_CALL(mock, beforeNextClient(_)).Times(0); + + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_FALSE(clientIsThrottled(c)); + throttle_deregister(t); + freeFakeClient(c); +} + +TEST_F(ThrottleTest, timeProcMultiThrottlerConsumesOtherBuckets) { + /* When a client matched multiple throttlers (throttle_multi flag), releasing it + * via the timer should also consume tokens from the other throttlers. */ + throttler *loose = throttle_register(fakeWriteCriteria, NULL, "share"); + throttler *strict = throttle_register(fakeWriteCriteria, NULL, "share"); + throttle_setRate(strict, 0.0); /* strict has no tokens and client queues here */ + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c = createFakeClient(1, true); + EXPECT_TRUE(throttleClientIfNeeded(c)); + EXPECT_EQ(c->flag.throttle_multi, 1ULL); + + /* Now release: set strict to high rate. */ + throttle_setRate(strict, THROTTLE_UNLIMITED_RATE); + fakeMonotimeUs += 1000000; + + /* The strict throttler's own bucket is consumed (force_consume=false) in the while loop, + * then consumeOtherThrottlers charges the loose throttler (force_consume=true). */ + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, false)).WillOnce(Return(true)); + EXPECT_CALL(mock, tokenBucket_tryConsume(_, _, true)).WillOnce(Return(true)); + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_FALSE(clientIsThrottled(c)); + verifyThrottler("share", 0, 1); + + throttle_deregister(loose); + throttle_deregister(strict); + freeFakeClient(c); +} + +TEST_F(ThrottleTest, timeProcCleanupThrottlerFreesOnDrain) { + /* A deregistered throttler (CLEANUP state) is freed when the timer drains it. */ + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + aeTimeProc *timeProc = NULL; + void *clientData = NULL; + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&timeProc), SaveArg<3>(&clientData), Return(1))); + + client *c = createFakeClient(1, true); + EXPECT_TRUE(throttleClientIfNeeded(c)); + + /* Deregister while client is still queued, enters CLEANUP state. + * deregister sets rate to THROTTLE_UNLIMITED_RATE internally. */ + throttle_deregister(t); + + fakeMonotimeUs += 1000000; + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + + long long ret = timeProc(server.el, 1, clientData); + EXPECT_EQ(ret, AE_NOMORE); + EXPECT_FALSE(clientIsThrottled(c)); + verifyThrottler("fake_throttler", 0, 1); + + freeFakeClient(c); +} + +/* ---- throttle_removeClient test ---- */ +TEST_F(ThrottleTest, removeClient) { + throttler *t = throttle_register(fakeWriteCriteria, NULL, "fake_throttler"); + throttle_setRate(t, 0.0); + + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).WillOnce(Return(1)); + + client *c1 = createFakeClient(1, true); + client *c2 = createFakeClient(2, true); + EXPECT_TRUE(throttleClientIfNeeded(c1)); + EXPECT_TRUE(throttleClientIfNeeded(c2)); + EXPECT_TRUE(clientIsThrottled(c1)); + EXPECT_TRUE(clientIsThrottled(c2)); + verifyThrottler("fake_throttler", 2, 2); + + /* Remove the client from the throttler queue. */ + throttle_removeClient(c1); + EXPECT_FALSE(clientIsThrottled(c1)); + EXPECT_EQ(c1->conn->read_handler, nullptr); + verifyThrottler("fake_throttler", 1, 2); + + EXPECT_CALL(mock, aeDeleteTimeEvent(_, _)).WillOnce(Return(AE_OK)); + throttle_removeClient(c2); + EXPECT_FALSE(clientIsThrottled(c2)); + EXPECT_EQ(c2->conn->read_handler, nullptr); + verifyThrottler("fake_throttler", 0, 2); + + throttle_deregister(t); + freeFakeClient(c1); + freeFakeClient(c2); +} + /* ---- Death tests ---- */ TEST_F(ThrottleDeathTest, deregisterThrottlerFail) { - /* deregister a non-existent throttler */ - EXPECT_DEATH(throttle_deregister(99999), ""); + /* deregister a NULL throttler */ + EXPECT_DEATH(throttle_deregister(NULL), ""); } TEST_F(ThrottleDeathTest, setRateNegativeAsserts) { - int id = throttle_register(fakeWriteCriteria, NULL, "neg_rate"); - EXPECT_DEATH(throttle_setRate(id, -1.0), ""); - throttle_deregister(id); + throttler *t = throttle_register(fakeWriteCriteria, NULL, "neg_rate"); + EXPECT_DEATH(throttle_setRate(t, -1.0), ""); + throttle_deregister(t); } TEST_F(ThrottleDeathTest, adjustRateOutOfRangeAsserts) { - int id = throttle_register(fakeWriteCriteria, NULL, "bad_mult"); - EXPECT_DEATH(throttle_adjustRate(id, 3.5), ""); /* multiplier must be <= 3.0 */ - throttle_deregister(id); + throttler *t = throttle_register(fakeWriteCriteria, NULL, "bad_mult"); + EXPECT_DEATH(throttle_adjustRate(t, 3.5), ""); /* multiplier must be <= 3.0 */ + throttle_deregister(t); } diff --git a/src/unit/test_throttle_repl.cpp b/src/unit/test_throttle_repl.cpp index bd09ada4acc..c4b660746dc 100644 --- a/src/unit/test_throttle_repl.cpp +++ b/src/unit/test_throttle_repl.cpp @@ -28,7 +28,7 @@ class ThrottleReplTest : public ::testing::Test { RealValkey real; static const unsigned long long COB_LIMIT = 10 * 1024 * 1024; /* 10 MB */ client *replica_steady = nullptr; - static inline throttleMetrics fakeMetrics = {0}; + throttler *dummy_throttler = (throttler *)1; static void SetUpTestSuite() { /* Server set up */ @@ -39,7 +39,7 @@ class ThrottleReplTest : public ::testing::Test { server.client_obuf_limits[CLIENT_TYPE_REPLICA].hard_limit_bytes = COB_LIMIT; /* throttle_repl set up */ - throttle_repl_config.steady_state_repl_throttle_enabled = 1; + throttle_repl_config.repl_throttle_steady_state_enabled = 1; /* monotonic set up */ origGetMonotonicUs = getMonotonicUs; @@ -55,7 +55,7 @@ class ThrottleReplTest : public ::testing::Test { void SetUp() override { replica_steady = createFakeReplicaClient(1); replica_steady->repl_data->repl_state = REPLICA_STATE_ONLINE; - EXPECT_CALL(mock, throttle_getMetrics(_)).WillRepeatedly(Return(&fakeMetrics)); + EXPECT_CALL(mock, throttle_getMetrics(_, _)).WillRepeatedly(SetArgPointee<1>(throttleMetrics{})); EXPECT_CALL(mock, throttle_getGuardrailSecs(_)).WillRepeatedly(Return(0L)); } @@ -102,10 +102,10 @@ class ThrottleReplTest : public ::testing::Test { /* Snapshot the INFO output and return one field's numeric value (-1 if absent). */ double readMetric(const char *key) { sds info = throttleRepl_sdscatInfoMetrics(sdsempty()); - char needle[128]; - snprintf(needle, sizeof(needle), "%s:", key); - char *p = strstr(info, needle); - double v = p ? strtod(p + strlen(needle), NULL) : -1.0; + char search_for[128]; + snprintf(search_for, sizeof(search_for), "%s:", key); + char *p = strstr(info, search_for); + double v = p ? strtod(p + strlen(search_for), NULL) : -1.0; sdsfree(info); return v; } @@ -158,7 +158,7 @@ TEST_F(ThrottleReplTest, steadyStateThrottleIncreasingTrend) { EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); - EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(1)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); throttleRepl_adjustThrottling(); EXPECT_TRUE(isReplThrottlerActive()); EXPECT_TRUE(verifyThrottleEvent(1, 0, 0)); // Throttler activated, no more/less events yets @@ -186,7 +186,7 @@ TEST_F(ThrottleReplTest, steadyStateThrottleDecreasingTrend) { EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 2 + 40)); EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(-1.0)); - EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(1)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); throttleRepl_adjustThrottling(); EXPECT_TRUE(isReplThrottlerActive()); EXPECT_TRUE(verifyThrottleEvent(1, 0, 0)); // Throttler activated, no more/less events yets @@ -218,14 +218,14 @@ TEST_F(ThrottleReplTest, steadyStateThrottleBasedOnLargestCob) { client *dummy_replica = createFakeReplicaClient(2); dummy_replica->repl_data->repl_state = REPLICA_STATE_ONLINE; EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(dummy_replica)).WillRepeatedly(Return(COB_LIMIT)); - EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(1)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); throttleRepl_adjustThrottling(); EXPECT_TRUE(isReplThrottlerActive()); freeFakeReplicaClient(dummy_replica); } TEST_F(ThrottleReplTest, disabledConfigNoNewThrottle) { - throttle_repl_config.steady_state_repl_throttle_enabled = 0; + throttle_repl_config.repl_throttle_steady_state_enabled = 0; EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); @@ -237,16 +237,15 @@ TEST_F(ThrottleReplTest, disabledConfigNoNewThrottle) { TEST_F(ThrottleReplTest, throttlerRemovedAfterFailover) { /* Simulate active throttler then failover (become replica) */ - int throttler_id = 42; EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 4 + 1)); EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); - EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(throttler_id)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); throttleRepl_adjustThrottling(); EXPECT_TRUE(isReplThrottlerActive()); server.primary_host = (char *)"127.0.0.1"; /* now a replica */ - EXPECT_CALL(mock, throttle_deregister(throttler_id)).Times(1); + EXPECT_CALL(mock, throttle_deregister(dummy_throttler)).Times(1); throttleRepl_adjustThrottling(); EXPECT_FALSE(isReplThrottlerActive()); } @@ -260,7 +259,7 @@ TEST_F(ThrottleReplTest, insaneCobLimitConfig) { EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(max_cob_target + 1)); EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(1.0)); - EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(1)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); throttleRepl_adjustThrottling(); EXPECT_TRUE(isReplThrottlerActive()); @@ -273,7 +272,7 @@ TEST_F(ThrottleReplTest, clientCobLimitsExempt) { /* Throttler now active */ EXPECT_CALL(mock, getClientOutputBufferMemoryUsage(replica_steady)).WillRepeatedly(Return(COB_LIMIT / 2 + 1)); EXPECT_CALL(mock, trendCalc_changePerSecShortTerm(_)).WillRepeatedly(Return(COB_LIMIT / 2)); - EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(1)); + EXPECT_CALL(mock, throttle_register(_, _, _)).WillOnce(Return(dummy_throttler)); throttleRepl_adjustThrottling(); EXPECT_TRUE(isReplThrottlerActive()); @@ -310,8 +309,8 @@ TEST_F(ThrottleReplTest, clientCobLimitsExempt) { EXPECT_TRUE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); /* If throttle repl disabled, not exempt. */ - throttle_repl_config.steady_state_repl_throttle_enabled = 0; + throttle_repl_config.repl_throttle_steady_state_enabled = 0; EXPECT_FALSE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); - throttle_repl_config.steady_state_repl_throttle_enabled = 1; + throttle_repl_config.repl_throttle_steady_state_enabled = 1; EXPECT_TRUE(throttleRepl_isClientExemptFromCobLimits(replica_steady)); } diff --git a/src/unit/test_token_bucket.cpp b/src/unit/test_token_bucket.cpp index 775c7a38dd0..1ac0961ac27 100644 --- a/src/unit/test_token_bucket.cpp +++ b/src/unit/test_token_bucket.cpp @@ -68,6 +68,21 @@ TEST_F(TokenBucketTest, HaltedBucket) { EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 11.0), 0.0); /* We should still have 11 tokens available */ } +TEST_F(TokenBucketTest, MsUntilAvailableReplenishes) { + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 12.0, false)); /* drain (size = 100*0.1+2 = 12) */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 10.0); /* empty right now */ + + fakeMonotimeUs += 1000000; /* advance 1s, bucket refills to full */ + + /* Bucket is full again */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 12.0), 0.0); + + /* Partial refill: drain again, advance only 5ms */ + EXPECT_TRUE(tokenBucket_tryConsume(bucket, 12.0, false)); + fakeMonotimeUs += 5000; /* 0.5 token accrued */ + EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 5.0); /* need 5ms more */ +} + TEST_F(TokenBucketTest, ConsumeTokens_normal) { /* Drain all tokens (bucket size = rate * burst_time + 2 = 100*0.1+2 = 12) */ EXPECT_TRUE(tokenBucket_tryConsume(bucket, 12.0, false)); diff --git a/src/unit/wrappers.h b/src/unit/wrappers.h index c08d605d0f8..0e9f3d45e8d 100644 --- a/src/unit/wrappers.h +++ b/src/unit/wrappers.h @@ -66,16 +66,20 @@ long long __wrap_aeCreateTimeEvent(aeEventLoop *eventLoop, long long millisecond int __wrap_aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id); size_t __wrap_getClientOutputBufferMemoryUsage(client *c); int __wrap_getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level); +int __wrap_processPendingCommandAndInputBuffer(client *c); +void __wrap_beforeNextClient(client *c); +int __wrap_freeClient(client *c); /* Throttler mocks */ -int __wrap_throttle_register(throttleCriteriaProc *criteria_proc, void *priv_data, const char *metrics_name); -void __wrap_throttle_deregister(int id); -double __wrap_throttle_adjustRate(int id, double multiplier); -const throttleMetrics *__wrap_throttle_getMetrics(const char *metrics_name); -long __wrap_throttle_getGuardrailSecs(int id); +throttler *__wrap_throttle_register(throttleCriteriaProc *criteria_proc, void *priv_data, const char *metrics_name); +void __wrap_throttle_deregister(throttler *t); +double __wrap_throttle_adjustRate(throttler *t, double multiplier); +void __wrap_throttle_getMetrics(const char *metrics_name, throttleMetrics *metrics); +long __wrap_throttle_getGuardrailSecs(throttler *t); /* Token bucket mocks */ bool __wrap_tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consume); +double __wrap_tokenBucket_msUntilAvailable(tokenBucket *bucket, double tokens); /* Statcalc mocks */ double __wrap_tpsCalculator_averageTps(tpsCalculator *calc); From 6f8ab3964aee4ce670f36703291b03b595803e55 Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Tue, 11 Aug 2026 11:58:11 -0700 Subject: [PATCH 10/27] Some nitpick comments from AI Signed-off-by: harrylin98 --- src/monotonic.h | 8 ---- src/networking.c | 6 ++- src/socket.c | 8 +++- src/stat_calc.c | 76 ++++++++++++++++----------------- src/stat_calc.h | 4 +- src/throttle.c | 8 ++-- src/throttle.h | 10 ++--- src/throttle_repl.c | 22 +++++----- src/throttle_token_bucket.c | 4 +- src/unit/test_throttle.cpp | 11 +++-- src/unit/test_throttle_repl.cpp | 1 - src/unit/test_token_bucket.cpp | 21 +++++++++ src/unit/wrappers.h | 3 -- 13 files changed, 103 insertions(+), 79 deletions(-) diff --git a/src/monotonic.h b/src/monotonic.h index 77ebad58b5e..69285f45a18 100644 --- a/src/monotonic.h +++ b/src/monotonic.h @@ -62,12 +62,4 @@ static inline uint64_t elapsedSec(monotime start_time) { return elapsedUs(start_time) / 1000000; } -static inline uint64_t durationUs(monotime start_time, monotime end_time) { - return end_time - start_time; -} - -static inline uint64_t durationMs(monotime start_time, monotime end_time) { - return durationUs(start_time, end_time) / 1000; -} - #endif diff --git a/src/networking.c b/src/networking.c index 48303c40123..20742af608a 100644 --- a/src/networking.c +++ b/src/networking.c @@ -5034,6 +5034,7 @@ static int validateClientFlagFilter(sds flag_filter) { case 'r': case 'e': case 'T': + case 'h': case 'I': case 'i': case 'E': @@ -5188,6 +5189,9 @@ static int clientMatchesFlagFilter(client *c, sds flag_filter) { case 'T': /* client will not touch the LRU/LFU of the keys it accesses */ if (!c->flag.no_touch) return 0; break; + case 'h': /* client is throttled */ + if (!c->flag.throttled) return 0; + break; case 'I': /* Import source flag */ if (!c->flag.import_source) return 0; break; @@ -5204,7 +5208,7 @@ static int clientMatchesFlagFilter(client *c, sds flag_filter) { c->flag.dirty_cas || c->flag.close_after_reply || c->flag.unblocked || c->flag.close_asap || c->flag.unix_socket || c->flag.readonly || - c->flag.no_evict || c->flag.no_touch || + c->flag.no_evict || c->flag.no_touch || c->flag.throttled || c->flag.import_source || c->slot_migration_job) { return 0; } diff --git a/src/socket.c b/src/socket.c index 93533827cd3..58cef717d04 100644 --- a/src/socket.c +++ b/src/socket.c @@ -426,12 +426,16 @@ int connTcpSocketIsClosing(connection *conn) { #if defined(__linux__) struct tcp_info info; socklen_t infolen = sizeof(info); - if (getsockopt(conn->fd, IPPROTO_TCP, TCP_INFO, &info, &infolen) != 0 || infolen < sizeof(info)) return false; // Cannot retrieve TCP info + if (getsockopt(conn->fd, IPPROTO_TCP, TCP_INFO, &info, &infolen) != 0 || + infolen < offsetof(struct tcp_info, tcpi_state) + sizeof(info.tcpi_state)) + return false; /* Cannot retrieve TCP info, or the state field was not returned. */ return (info.tcpi_state == TCP_CLOSE_WAIT || info.tcpi_state == TCP_CLOSE); #elif defined(__APPLE__) struct tcp_connection_info info; socklen_t infolen = sizeof(info); - if (getsockopt(conn->fd, IPPROTO_TCP, TCP_CONNECTION_INFO, &info, &infolen) != 0 || infolen < sizeof(info)) return false; // Cannot retrieve TCP info + if (getsockopt(conn->fd, IPPROTO_TCP, TCP_CONNECTION_INFO, &info, &infolen) != 0 || + infolen < offsetof(struct tcp_connection_info, tcpi_state) + sizeof(info.tcpi_state)) + return false; /* Cannot retrieve TCP info, or the state field was not returned. */ return (info.tcpi_state == TCPS_CLOSE_WAIT || info.tcpi_state == TCPS_CLOSED); #else /* Unsupported platform: zombie connection detection is not available. */ diff --git a/src/stat_calc.c b/src/stat_calc.c index 6fc421ec38b..f4255a88cd0 100644 --- a/src/stat_calc.c +++ b/src/stat_calc.c @@ -69,23 +69,23 @@ double tpsCalculator_averageTps(tpsCalculator *calc) { #define DATA_POINTS 10 struct trendCalculator { - int windowSec; - monotime lastUpdate; - long updateFreqUs; - bool newCalculator; + int window_sec; + monotime last_update; + long update_freq_us; + bool is_new; long metrics[DATA_POINTS]; - long uncountedTotal; - int uncountedSamples; + long uncounted_total; + int uncounted_samples; double trend; - double trendShort; + double trend_short; }; -trendCalculator *newTrendCalc(int windowSecs) { +trendCalculator *newTrendCalc(int window_secs) { trendCalculator *calc = zcalloc(sizeof(trendCalculator)); - calc->windowSec = windowSecs; - calc->lastUpdate = getMonotonicUs(); - calc->updateFreqUs = windowSecs * ONE_SECOND_IN_MICROS / DATA_POINTS; - calc->newCalculator = true; + calc->window_sec = window_secs; + calc->last_update = getMonotonicUs(); + calc->update_freq_us = window_secs * ONE_SECOND_IN_MICROS / DATA_POINTS; + calc->is_new = true; return calc; } @@ -93,37 +93,37 @@ void trendCalc_free(trendCalculator *calc) { zfree(calc); } -void trendCalc_recordMetric(trendCalculator *calc, long metricValue) { +void trendCalc_recordMetric(trendCalculator *calc, long metric_value) { monotime now = getMonotonicUs(); - long elapsedUs = now - calc->lastUpdate; + long elapsed_us = now - calc->last_update; - calc->uncountedTotal += metricValue; - calc->uncountedSamples++; + calc->uncounted_total += metric_value; + calc->uncounted_samples++; - if (elapsedUs < calc->updateFreqUs) return; + if (elapsed_us < calc->update_freq_us) return; - long newValue = calc->uncountedTotal / calc->uncountedSamples; - calc->uncountedTotal = 0; - calc->uncountedSamples = 0; - calc->lastUpdate = now; + long new_value = calc->uncounted_total / calc->uncounted_samples; + calc->uncounted_total = 0; + calc->uncounted_samples = 0; + calc->last_update = now; - if (calc->newCalculator) { - for (int i = 0; i < DATA_POINTS; i++) calc->metrics[i] = newValue; - calc->newCalculator = false; + if (calc->is_new) { + for (int i = 0; i < DATA_POINTS; i++) calc->metrics[i] = new_value; + calc->is_new = false; } - long olderTotal = 0; + long older_total = 0; for (int i = 0; i < DATA_POINTS / 2; i++) { calc->metrics[i] = calc->metrics[i + 1]; - olderTotal += calc->metrics[i]; + older_total += calc->metrics[i]; } - long newerTotal = 0; + long newer_total = 0; for (int i = DATA_POINTS / 2; i < DATA_POINTS - 1; i++) { calc->metrics[i] = calc->metrics[i + 1]; - newerTotal += calc->metrics[i]; + newer_total += calc->metrics[i]; } - calc->metrics[DATA_POINTS - 1] = newValue; - newerTotal += newValue; + calc->metrics[DATA_POINTS - 1] = new_value; + newer_total += new_value; /* Formula is the average of the newer data points, less the average of the older data * points. The time is from the center of each half, @@ -132,15 +132,15 @@ void trendCalc_recordMetric(trendCalculator *calc, long metricValue) { * Where: * AveNewer = newerTotal / (DATA_POINTS/2) * AveOlder = olderTotal / (DATA_POINTS/2) */ - double olderAvg = (double)olderTotal / (DATA_POINTS / 2); - double newerAvg = (double)newerTotal / (DATA_POINTS / 2); - double timeBetweenCenters = (double)calc->windowSec / 2.0; - calc->trend = (newerAvg - olderAvg) / timeBetweenCenters; + double older_avg = (double)older_total / (DATA_POINTS / 2); + double newer_avg = (double)newer_total / (DATA_POINTS / 2); + double time_between_centers = (double)calc->window_sec / 2.0; + calc->trend = (newer_avg - older_avg) / time_between_centers; /* Short-term: rate of change between last 2 datapoints. */ - long deltaShort = calc->metrics[DATA_POINTS - 1] - calc->metrics[DATA_POINTS - 2]; - double timeBetweenSlots = (double)calc->windowSec / DATA_POINTS; - calc->trendShort = deltaShort / timeBetweenSlots; + long delta_short = calc->metrics[DATA_POINTS - 1] - calc->metrics[DATA_POINTS - 2]; + double time_between_slots = (double)calc->window_sec / DATA_POINTS; + calc->trend_short = delta_short / time_between_slots; } double trendCalc_changePerSec(trendCalculator *calc) { @@ -148,5 +148,5 @@ double trendCalc_changePerSec(trendCalculator *calc) { } double trendCalc_changePerSecShortTerm(trendCalculator *calc) { - return calc->trendShort; + return calc->trend_short; } diff --git a/src/stat_calc.h b/src/stat_calc.h index 7dc61d676a0..d2943909632 100644 --- a/src/stat_calc.h +++ b/src/stat_calc.h @@ -43,14 +43,14 @@ double tpsCalculator_averageTps(tpsCalculator *calc); */ typedef struct trendCalculator trendCalculator; -trendCalculator *newTrendCalc(int windowSecs); +trendCalculator *newTrendCalc(int window_secs); void trendCalc_free(trendCalculator *calc); /* Add a datapoint to the calculator. Should be called at minimum 10 times * over the window for smooth results. If the metric is highly volatile, * calling more often reduces the impact of individual outliers. */ -void trendCalc_recordMetric(trendCalculator *calc, long metricValue); +void trendCalc_recordMetric(trendCalculator *calc, long metric_value); /* Get the average rate of change over the full window. */ double trendCalc_changePerSec(trendCalculator *calc); diff --git a/src/throttle.c b/src/throttle.c index bb2d8a9427a..21603a70edb 100644 --- a/src/throttle.c +++ b/src/throttle.c @@ -27,7 +27,7 @@ static list *throttlerList = NULL; typedef struct metricsEntry { sds throttler_type; int num_clients_throttled; - int num_commands_throttled; + long long num_commands_throttled; tpsCalculator *incoming_tps; } metricsEntry; @@ -365,7 +365,7 @@ void throttle_getMetrics(const char *metrics_name, throttleMetrics *metrics) { listRewind(throttlerList, &li); while ((ln = listNext(&li))) { throttler *t = ln->value; - if (t->metrics != m) continue; + if (t->metrics != m || t->cleanup) continue; metrics->ops_per_sec += tokenBucket_getRate(t->bucket); if (listLength(t->client_queue) > 0) { client *oldest = listNodeValue(listFirst(t->client_queue)); @@ -376,9 +376,9 @@ void throttle_getMetrics(const char *metrics_name, throttleMetrics *metrics) { } sds throttle_sdscatInfoMetrics(sds info) { - info = sdscatprintf(info, "throttle_total_throttled_commands:%lld\r\n", total_throttled_commands); + info = sdscatprintf(info, "total_throttled_commands:%lld\r\n", total_throttled_commands); - // Check for any throttlers which are below guardrail. Report only offending throttlers. + /* Check for any throttlers which are below guardrail. Report only offending throttlers. */ listNode *ln; listIter li; listRewind(throttlerList, &li); diff --git a/src/throttle.h b/src/throttle.h index be8efc3fa14..46b44739ded 100644 --- a/src/throttle.h +++ b/src/throttle.h @@ -45,11 +45,11 @@ typedef bool throttleCriteriaProc(client *c, void *priv_data); * Note: Multiple related throttlers can share the same metrics by using the same metrics_name. * A typical use case is multiple instantiations of the same throttler with different private data. */ typedef struct { - int num_clients_throttled; /* the backlog of currently throttled (queued) clients */ - int num_commands_throttled; /* total number of commands throttled through this metrics group */ - double ops_per_sec; /* the current throttling rate (summed across related throttlers) */ - double incoming_tps; /* average incoming TPS over a 5-second rolling window */ - long oldest_client_delay_us; /* delay in microseconds for the oldest throttled client */ + int num_clients_throttled; /* the backlog of currently throttled (queued) clients */ + long long num_commands_throttled; /* total number of commands throttled through this metrics group */ + double ops_per_sec; /* the current throttling rate (summed across related throttlers) */ + double incoming_tps; /* average incoming TPS over a 5-second rolling window */ + long oldest_client_delay_us; /* delay in microseconds for the oldest throttled client */ } throttleMetrics; /* Initialize the throttling framework. Must be called once at startup before any diff --git a/src/throttle_repl.c b/src/throttle_repl.c index 16d198019f8..363fa2e5147 100644 --- a/src/throttle_repl.c +++ b/src/throttle_repl.c @@ -17,9 +17,9 @@ struct throttle_repl_config throttle_repl_config; static const int COB_TREND_WINDOW_SECS = 2; static const double RATE_INCREASE_MULTIPLIER = 1.05; static const double RATE_DECREASE_MULTIPLIER = 0.95; -static const int STEADY_STATE_CONVERGENCE_SECS = 30; /* projection horizon for COB extrapolation */ -static const long MAX_COB_TARGET = 1024L * 1024 * 1024; /* 1GB */ -static const char *const METRICS_NAME = "ReplThrottle"; /* shared metrics group name */ +static const int STEADY_STATE_CONVERGENCE_SECS = 30; /* projection horizon for COB extrapolation */ +static const long MAX_COB_TARGET = 1024L * 1024 * 1024; /* 1GB */ +static const char *const METRICS_NAME = "repl_throttle"; /* shared metrics group name */ /* Metrics for INFO output and operational visibility. */ typedef struct { @@ -65,10 +65,10 @@ static void uninstallThrottler(void) { /* Apply a rate change based on the evaluator's decision. Installs the throttler on first * reduce request and removes it when rate reaches UNLIMITED. */ -static void adjustThrottleRate(bool reduceTrafficRate) { +static void adjustThrottleRate(bool reduce_traffic_rate) { if (isThrottlerActive()) { double rate; - if (reduceTrafficRate) { + if (reduce_traffic_rate) { rate = throttle_adjustRate(repl_throttler, RATE_DECREASE_MULTIPLIER); metrics.throttle_more_events++; } else { @@ -80,7 +80,7 @@ static void adjustThrottleRate(bool reduceTrafficRate) { } else { /* Installing the throttler starts measurement of current traffic rate. * Once the measurement is stable, rate adjustments will be meaningful. */ - if (reduceTrafficRate) installThrottler(); + if (reduce_traffic_rate) installThrottler(); } } @@ -124,7 +124,7 @@ static bool evaluateSteadyStateThrottle(client *c, int64_t cob_size) { bool throttleRepl_isClientExemptFromCobLimits(client *c) { if (!throttle_repl_config.repl_throttle_steady_state_enabled || !isThrottlerActive()) return false; if (!iAmPrimary()) return false; - if (!c->flag.replica) return false; + if (getClientType(c) != CLIENT_TYPE_REPLICA) return false; /* Throttle is actively working, protect this replica from COB * disconnect if its COB is above target. */ @@ -154,7 +154,7 @@ void throttleRepl_adjustThrottling(void) { return; } - bool reduceTrafficRate = false; + bool reduce_traffic_rate = false; client *measured_steady_state_replica = NULL; uint64_t largest_steady_state_cob = 0; @@ -181,10 +181,10 @@ void throttleRepl_adjustThrottling(void) { } if (measured_steady_state_replica != NULL) { - reduceTrafficRate = evaluateSteadyStateThrottle(measured_steady_state_replica, largest_steady_state_cob); + reduce_traffic_rate = evaluateSteadyStateThrottle(measured_steady_state_replica, largest_steady_state_cob); } - adjustThrottleRate(reduceTrafficRate); + adjustThrottleRate(reduce_traffic_rate); } sds throttleRepl_sdscatInfoMetrics(sds info) { @@ -206,7 +206,7 @@ sds throttleRepl_sdscatInfoMetrics(sds info) { "repl_throttle_less_events:%lu\r\n" "repl_throttle_below_guardrail_secs:%ld\r\n" "repl_throttle_current_clients:%d\r\n" - "repl_throttle_total_commands:%d\r\n", + "repl_throttle_total_commands:%lld\r\n", metrics.throttle_activation_events, metrics.throttle_more_events, metrics.throttle_less_events, diff --git a/src/throttle_token_bucket.c b/src/throttle_token_bucket.c index 76e142862cf..34ce69c4ed4 100644 --- a/src/throttle_token_bucket.c +++ b/src/throttle_token_bucket.c @@ -60,6 +60,7 @@ double tokenBucket_getRate(tokenBucket *bucket) { } void tokenBucket_setRate(tokenBucket *bucket, double new_rate) { + tokenBucket_replenish(bucket); bucket->tokens_per_sec = new_rate; trimTokenBucket(bucket); } @@ -75,7 +76,8 @@ bool tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consu double tokenBucket_msUntilAvailable(tokenBucket *bucket, double target_tokens) { tokenBucket_replenish(bucket); if (bucket->token_count >= target_tokens) return 0.0; - if (bucket->tokens_per_sec <= 0) return -1.0; /* halted — never available */ + /* Rates below BUCKET_EPSILON give zero capacity, so tokens never accumulate. */ + if (bucket->tokens_per_sec < BUCKET_EPSILON) return -1.0; /* halted -- never available */ double needed = target_tokens - bucket->token_count; return needed * 1000.0 / bucket->tokens_per_sec; } diff --git a/src/unit/test_throttle.cpp b/src/unit/test_throttle.cpp index f24f92c351a..0c371f8257c 100644 --- a/src/unit/test_throttle.cpp +++ b/src/unit/test_throttle.cpp @@ -68,6 +68,13 @@ class ThrottleTest : public ::testing::Test { return C_OK; } + /* A set_read_handler that always fails, to simulate a connection error. */ + static int failSetReadHandler(connection *conn, ConnectionCallbackFunc func) { + UNUSED(conn); + UNUSED(func); + return C_ERR; + } + client *createFakeClient(int client_id, bool write_command) { client *c = (client *)zcalloc(sizeof(client)); c->id = client_id; @@ -513,9 +520,7 @@ TEST_F(ThrottleTest, timeProcCallsFreeClientOnConnSetReadHandlerFailure) { /* Install a failing read handler to simulate connection error. */ static ConnectionType failConnType = {0}; - failConnType.set_read_handler = [](connection *, ConnectionCallbackFunc) -> int { - return C_ERR; - }; + failConnType.set_read_handler = failSetReadHandler; c->conn->type = &failConnType; throttle_setRate(t, THROTTLE_UNLIMITED_RATE); diff --git a/src/unit/test_throttle_repl.cpp b/src/unit/test_throttle_repl.cpp index c4b660746dc..392bf8cf757 100644 --- a/src/unit/test_throttle_repl.cpp +++ b/src/unit/test_throttle_repl.cpp @@ -1,4 +1,3 @@ - /* * Copyright (c) Valkey Contributors * All rights reserved. diff --git a/src/unit/test_token_bucket.cpp b/src/unit/test_token_bucket.cpp index 1ac0961ac27..62cfe67d076 100644 --- a/src/unit/test_token_bucket.cpp +++ b/src/unit/test_token_bucket.cpp @@ -158,3 +158,24 @@ TEST_F(TokenBucketTest, SetRateChangesRate) { EXPECT_FALSE(tokenBucket_tryConsume(bucket, 1.0, false)); /* now empty */ EXPECT_DOUBLE_EQ(tokenBucket_msUntilAvailable(bucket, 1.0), 100.0); } + +TEST_F(TokenBucketTest, SetRateSettlesElapsedAtOldRate) { + /* setRate must credit elapsed time at the OLD rate before switching. */ + tokenBucket *b = tokenBucket_create(1.0, 10.0); + EXPECT_TRUE(tokenBucket_tryConsume(b, 12.0, false)); /* drain to empty */ + EXPECT_FALSE(tokenBucket_tryConsume(b, 0.5, false)); + + fakeMonotimeUs += 5000000; /* 5 tokens should accrue */ + + tokenBucket_setRate(b, 1000.0); /* the 5 elapsed seconds belong to the OLD rate */ + + EXPECT_FALSE(tokenBucket_tryConsume(b, 6.0, false)); /* only 5 available, not thousands */ + EXPECT_TRUE(tokenBucket_tryConsume(b, 5.0, false)); + + /* Confirm the NEW rate now governs accrual: at 1000/s, 10ms yields ~10 tokens */ + fakeMonotimeUs += 10000; + EXPECT_TRUE(tokenBucket_tryConsume(b, 10.0, false)); + EXPECT_FALSE(tokenBucket_tryConsume(b, 0.1, false)); /* now empty */ + + tokenBucket_free(b); +} diff --git a/src/unit/wrappers.h b/src/unit/wrappers.h index baa66032d63..bf0e496d1cc 100644 --- a/src/unit/wrappers.h +++ b/src/unit/wrappers.h @@ -79,12 +79,9 @@ long __wrap_throttle_getGuardrailSecs(throttler *t); /* Token bucket mocks */ bool __wrap_tokenBucket_tryConsume(tokenBucket *bucket, double tokens, bool force_consume); -double __wrap_tokenBucket_msUntilAvailable(tokenBucket *bucket, double tokens); /* Statcalc mocks */ double __wrap_tpsCalculator_averageTps(tpsCalculator *calc); -trendCalculator *__wrap_newTrendCalc(int windowSecs); -void __wrap_trendCalc_recordMetric(trendCalculator *calc, long metricValue); double __wrap_trendCalc_changePerSecShortTerm(trendCalculator *calc); #undef protected From db7da3d1154baa795fad5cd2fd1f57e0a3c2435f Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:18:58 -0700 Subject: [PATCH 11/27] Resolve a key's slot from the key, not from the executing command (#4380) Fixes paths which used a client's cached slot id for keys not directly associated with the client. --------- Signed-off-by: Nitai Caro Co-authored-by: Nitai Caro --- src/db.c | 37 +++++++++++----------- src/pubsub.c | 4 +-- src/server.h | 4 +-- src/sort.c | 4 +-- tests/unit/cluster/misc.tcl | 62 +++++++++++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 24 deletions(-) diff --git a/src/db.c b/src/db.c index 36b7f78475f..df2e82ef8b3 100644 --- a/src/db.c +++ b/src/db.c @@ -80,7 +80,7 @@ static robj *dbFindWithDictIndex(serverDb *db, sds key, int dict_index); * expired on replicas even if the primary is lagging expiring our key via DELs * in the replication link. */ robj *lookupKey(serverDb *db, robj *key, int flags) { - int dict_index = getKVStoreIndexForKey(objectGetVal(key)); + int dict_index = getKVStoreIndexUsingCachedSlot(objectGetVal(key)); robj *val = dbFindWithDictIndex(db, objectGetVal(key), dict_index); if (val) { /* Forcing deletion of expired keys on a replica makes the replica @@ -201,7 +201,7 @@ void dbUpdateObjectWithVolatileItemsTracking(serverDb *db, robj *o) { * If the update_if_existing argument is false, the program is aborted * if the key already exists, otherwise, it can fall back to dbOverwrite. */ static void dbAddInternal(serverDb *db, robj *key, robj **valref, int update_if_existing) { - int dict_index = getKVStoreIndexForKey(objectGetVal(key)); + int dict_index = getKVStoreIndexUsingCachedSlot(objectGetVal(key)); void **oldref = NULL; if (update_if_existing) { oldref = kvstoreHashtableFindRef(db->keys, dict_index, objectGetVal(key)); @@ -230,15 +230,21 @@ void dbAdd(serverDb *db, robj *key, robj **valref) { dbAddInternal(db, key, valref, 0); } -/* Returns which dict index should be used with kvstore for a given key. */ +/* Returns which dict index should be used with kvstore for a given key, computed from the key itself. */ int getKVStoreIndexForKey(sds key) { - return server.cluster_enabled ? getKeySlot(key) : 0; + return server.cluster_enabled ? (int)keyHashSlot(key, (int)sdslen(key)) : 0; +} + +/* Same as getKVStoreIndexForKey(), but reuses the slot cached on the client rather than hashing the key. + * Only valid for keys the currently executing command declared, since the cached slot is that command's. */ +int getKVStoreIndexUsingCachedSlot(sds key) { + return server.cluster_enabled ? getCachedKeySlot(key) : 0; } -/* Returns the cluster hash slot for a given key, trying to use the cached slot that - * stored on the server.current_client first. If there is no cached value, it will compute the hash slot - * and then cache the value.*/ -int getKeySlot(sds key) { +/* Returns the slot cached on the client for the currently executing command, computing it from the key + * when no cached slot is available. Only valid for keys that command declared: any other key can hash to + * a different slot, so use keyHashSlot() or getKVStoreIndexForKey() for those. */ +int getCachedKeySlot(sds key) { serverAssert(server.cluster_enabled); /* This is performance optimization that uses pre-set slot id from the current command, * in order to avoid calculation of the key hash. @@ -320,7 +326,7 @@ int dbAddRDBLoad(serverDb *db, sds key, robj **valref) { static void dbSetValue(serverDb *db, robj *key, robj **valref, int overwrite, void **oldref) { robj *val = *valref; if (oldref == NULL) { - int dict_index = getKVStoreIndexForKey(objectGetVal(key)); + int dict_index = getKVStoreIndexUsingCachedSlot(objectGetVal(key)); oldref = kvstoreHashtableFindRef(db->keys, dict_index, objectGetVal(key)); } serverAssertWithInfo(NULL, key, oldref != NULL); @@ -366,7 +372,7 @@ static void dbSetValue(serverDb *db, robj *key, robj **valref, int overwrite, vo *oldref = new; /* Replace the old value at its location in the expire space. */ if (expire >= 0) { - int dict_index = getKVStoreIndexForKey(objectGetVal(key)); + int dict_index = getKVStoreIndexUsingCachedSlot(objectGetVal(key)); void **expireref = kvstoreHashtableFindRef(db->expires, dict_index, objectGetVal(key)); serverAssert(expireref != NULL); *expireref = new; @@ -1896,7 +1902,7 @@ void swapdbCommand(client *c) { *----------------------------------------------------------------------------*/ int removeExpire(serverDb *db, robj *key) { - int dict_index = getKVStoreIndexForKey(objectGetVal(key)); + int dict_index = getKVStoreIndexUsingCachedSlot(objectGetVal(key)); void *popped; if (kvstoreHashtablePop(db->expires, dict_index, objectGetVal(key), &popped)) { robj *val = popped; @@ -1922,7 +1928,7 @@ robj *setExpire(client *c, serverDb *db, robj *key, long long when) { /* Reuse the object from the main dict in the expire dict. When setting * expire in an robj, it's potentially reallocated. We need to updates the * pointer(s) to it. */ - int dict_index = getKVStoreIndexForKey(objectGetVal(key)); + int dict_index = getKVStoreIndexUsingCachedSlot(objectGetVal(key)); void **valref = kvstoreHashtableFindRef(db->keys, dict_index, objectGetVal(key)); serverAssertWithInfo(NULL, key, valref != NULL); val = *valref; @@ -1931,7 +1937,7 @@ robj *setExpire(client *c, serverDb *db, robj *key, long long when) { robj *newval = objectSetExpire(val, when); if (objectGetType(newval) == OBJ_HASH && hashTypeHasVolatileFields(newval)) { /* Replace the pointer in the keys_with_volatile_items table without accessing the old pointer. */ - int dict_index = getKVStoreIndexForKey(objectGetKey(newval)); + int dict_index = getKVStoreIndexUsingCachedSlot(objectGetKey(newval)); hashtable *volatile_items_ht = kvstoreGetHashtable(db->keys_with_volatile_items, dict_index); bool replaced = hashtableReplaceReallocatedEntry(volatile_items_ht, val, newval); serverAssert(replaced); @@ -1986,11 +1992,6 @@ void deleteExpiredKeyAndPropagateWithDictIndex(serverDb *db, robj *keyobj, int d server.stat_expiredkeys++; } -/* Delete the specified expired key and propagate expire. */ -void deleteExpiredKeyAndPropagate(serverDb *db, robj *keyobj) { - int dict_index = getKVStoreIndexForKey(objectGetVal(keyobj)); - deleteExpiredKeyAndPropagateWithDictIndex(db, keyobj, dict_index); -} /* Delete the specified expired key from overwriting and propagate the DEL or UNLINK. */ void deleteExpiredKeyFromOverwriteAndPropagate(client *c, robj *keyobj) { diff --git a/src/pubsub.c b/src/pubsub.c index 5e051e6a98b..76efaeebf50 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -302,7 +302,7 @@ int pubsubSubscribeChannel(client *c, robj *channel, pubsubtype type) { retval = 1; /* Add the client to the channel -> list of clients hash table */ if (server.cluster_enabled && type.shard) { - slot = getKeySlot(objectGetVal(channel)); + slot = getCachedKeySlot(objectGetVal(channel)); } hashtablePosition pos; @@ -344,7 +344,7 @@ int pubsubUnsubscribeChannel(client *c, robj *channel, int notify, pubsubtype ty retval = 1; /* Remove the client from the channel -> clients list hash table */ if (server.cluster_enabled && type.shard) { - /* Using keyHashSlot directly because we can't rely on the current_client's slot via getKeySlot() here, + /* Using keyHashSlot directly because we can't rely on the current_client's slot via getCachedKeySlot() here, * as it might differ from the channel's slot. */ slot = keyHashSlot(objectGetVal(channel), (int)sdslen(objectGetVal(channel))); } diff --git a/src/server.h b/src/server.h index 3c9f1302d85..2661ad451c9 100644 --- a/src/server.h +++ b/src/server.h @@ -3555,11 +3555,12 @@ long long getInstantaneousMetric(int metric); #define RESTART_SERVER_GRACEFULLY (1 << 0) /* Do proper shutdown. */ #define RESTART_SERVER_CONFIG_REWRITE (1 << 1) /* CONFIG REWRITE before restart.*/ int restartServer(client *c, int flags, mstime_t delay); -int getKeySlot(sds key); +int getCachedKeySlot(sds key); int calculateKeySlot(sds key); /* kvstore wrappers */ int getKVStoreIndexForKey(sds key); +int getKVStoreIndexUsingCachedSlot(sds key); int dbExpand(serverDb *db, uint64_t db_size, int try_expand); int dbExpandExpires(serverDb *db, uint64_t db_size, int try_expand); robj *dbFind(serverDb *db, sds key); @@ -3745,7 +3746,6 @@ int setModuleUnsignedNumericConfig(ModuleConfig *config, unsigned long long val, /* db.c -- Keyspace access API */ int removeExpire(serverDb *db, robj *key); -void deleteExpiredKeyAndPropagate(serverDb *db, robj *keyobj); void deleteExpiredKeyAndPropagateWithDictIndex(serverDb *db, robj *keyobj, int dict_index); void deleteExpiredKeyFromOverwriteAndPropagate(client *c, robj *keyobj); void propagateDeletion(serverDb *db, robj *key, int lazy, int slot); diff --git a/src/sort.c b/src/sort.c index ba51f67cde2..7286eadaf12 100644 --- a/src/sort.c +++ b/src/sort.c @@ -245,7 +245,7 @@ void sortCommandGeneric(client *c, int readonly) { * unless we can make sure the keys formed by the pattern are in the same slot * as the key to sort. */ if (server.cluster_enabled && - patternHashSlot(objectGetVal(sortby), sdslen(objectGetVal(sortby))) != getKeySlot(objectGetVal(c->argv[1]))) { + patternHashSlot(objectGetVal(sortby), sdslen(objectGetVal(sortby))) != getCachedKeySlot(objectGetVal(c->argv[1]))) { addReplyError(c, "BY option of SORT denied in Cluster mode when " "keys formed by the pattern may be in different slots."); syntax_error++; @@ -266,7 +266,7 @@ void sortCommandGeneric(client *c, int readonly) { * as the key to sort. */ if (server.cluster_enabled && !isReturnSubstPattern(objectGetVal(c->argv[j + 1])) && - patternHashSlot(objectGetVal(c->argv[j + 1]), sdslen(objectGetVal(c->argv[j + 1]))) != getKeySlot(objectGetVal(c->argv[1]))) { + patternHashSlot(objectGetVal(c->argv[j + 1]), sdslen(objectGetVal(c->argv[j + 1]))) != getCachedKeySlot(objectGetVal(c->argv[1]))) { addReplyError(c, "GET option of SORT denied in Cluster mode when " "keys formed by the pattern may be in different slots."); syntax_error++; diff --git a/tests/unit/cluster/misc.tcl b/tests/unit/cluster/misc.tcl index 1fa25f1d0f3..7a6dfc3e2d8 100644 --- a/tests/unit/cluster/misc.tcl +++ b/tests/unit/cluster/misc.tcl @@ -39,6 +39,68 @@ start_cluster 1 1 {tags {external:skip cluster}} { assert_equal QUEUED [r get bar] assert_error {CROSSSLOT *} {r exec} } + + # Regression tests for WATCHed keys that hash to a different slot than the transaction's commands. EXEC + # committed such a transaction even when the WATCHed key had expired or had been removed. + test {WATCHed key in another slot that expired aborts EXEC} { + set watched_key "{tag1}watched" + set transaction_key "{tag2}written-by-exec" + assert {[R 0 cluster keyslot $watched_key] != [R 0 cluster keyslot $transaction_key]} + + # The TTL is set before WATCH, and the key is still in the keyspace once it elapses, so nothing but + # the key having expired can abort the transaction. + R 0 debug set-active-expire 0 + R 0 del $transaction_key + R 0 set $watched_key alive px 50 + R 0 watch $watched_key + set keys_before [R 0 dbsize] + after 100 + assert_equal $keys_before [R 0 dbsize] + + R 0 multi + R 0 set $transaction_key committed + set reply [R 0 exec] + R 0 debug set-active-expire 1 + + assert_equal {} $reply + assert_equal 0 [R 0 exists $transaction_key] + } + + test {WATCHed key in another slot that is untouched commits EXEC} { + set watched_key "{tag1}watched-alive" + set transaction_key "{tag2}written-by-exec" + assert {[R 0 cluster keyslot $watched_key] != [R 0 cluster keyslot $transaction_key]} + + R 0 set $watched_key alive + R 0 watch $watched_key + + R 0 multi + R 0 set $transaction_key committed + assert_equal {OK} [R 0 exec] + assert_equal {committed} [R 0 get $transaction_key] + } + + test {WATCHed key in another slot flushed by a transaction aborts the watcher's EXEC} { + set watched_key "{tag1}watched-flush" + set transaction_key "{tag2}written-with-flush" + assert {[R 0 cluster keyslot $watched_key] != [R 0 cluster keyslot $transaction_key]} + + set watcher [valkey_client 0] + $watcher set $watched_key alive + $watcher watch $watched_key + + R 0 multi + R 0 set $transaction_key committed + R 0 flushall + assert_equal {OK OK} [R 0 exec] + + $watcher multi + $watcher set $transaction_key committed-by-watcher + set reply [$watcher exec] + $watcher close + + assert_equal {} $reply + } } # Create a folder called "nodes.conf" to trigger temp nodes.conf rename From 875696ffec88b34493fb931251c1711d779dcad4 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Wed, 12 Aug 2026 12:04:15 -0700 Subject: [PATCH 12/27] Clear the redaction bitmap between commands in a transaction (#4323) In an earlier commit we didn't properly reset the redaction bitmap during exec, and used the wrong one for lua scripts. This fixes that to properly redact commands. Added new regression tests and all existing tests still pass. --------- Signed-off-by: Madelyn Olson --- src/commandlog.c | 29 ++++++++++++----------------- src/server.c | 8 ++++++++ tests/unit/commandlog.tcl | 29 +++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/commandlog.c b/src/commandlog.c index a4ed1b33c9e..a0d2a7df473 100644 --- a/src/commandlog.c +++ b/src/commandlog.c @@ -69,8 +69,11 @@ static commandlogEntry *commandlogCreateEntry(client *c, robj **argv, int argc, ce->time = time(NULL); ce->value = value; ce->id = server.commandlog[type].entry_id++; - ce->peerid = sdsnew(getClientPeerId(c)); - ce->cname = c->name ? sdsnew(objectGetVal(c->name)) : sdsempty(); + /* For commands executed from a script, the executing client is a fake + * client with no connection, so attribute the entry to the calling client. */ + client *caller = scriptIsRunning() ? scriptGetCaller() : c; + ce->peerid = sdsnew(getClientPeerId(caller)); + ce->cname = caller->name ? sdsnew(objectGetVal(caller->name)) : sdsempty(); return ce; } @@ -154,21 +157,13 @@ void commandlogPushCurrentCommand(client *c, struct serverCommand *cmd) { robj **argv = c->original_argv ? c->original_argv : c->argv; int argc = c->original_argv ? c->original_argc : c->argc; - /* In script, client will be replaced with its caller, so commandlog needs to use the metrics - * of the client that currently executing the command. */ - long duration = c->duration; - unsigned long long net_input_bytes_curr_cmd = c->net_input_bytes_curr_cmd; - unsigned long long net_output_bytes_curr_cmd = c->net_output_bytes_curr_cmd; - - /* If a script is currently running, the client passed in is a - * fake client. Or the client passed in is the original client - * if this is a EVAL or alike, doesn't matter. In this case, - * use the original client to get the client information. */ - c = scriptIsRunning() ? scriptGetCaller() : c; - - commandlogPushEntryIfNeeded(c, argv, argc, duration, COMMANDLOG_TYPE_SLOW); - commandlogPushEntryIfNeeded(c, argv, argc, net_input_bytes_curr_cmd, COMMANDLOG_TYPE_LARGE_REQUEST); - commandlogPushEntryIfNeeded(c, argv, argc, net_output_bytes_curr_cmd, COMMANDLOG_TYPE_LARGE_REPLY); + /* 'c' is the client that executed the command: for a command called from a + * script this is the fake client, whose argv, metrics and redaction bitmap + * describe the executed command. Entry creation resolves the calling client + * for the connection identity fields. */ + commandlogPushEntryIfNeeded(c, argv, argc, c->duration, COMMANDLOG_TYPE_SLOW); + commandlogPushEntryIfNeeded(c, argv, argc, c->net_input_bytes_curr_cmd, COMMANDLOG_TYPE_LARGE_REQUEST); + commandlogPushEntryIfNeeded(c, argv, argc, c->net_output_bytes_curr_cmd, COMMANDLOG_TYPE_LARGE_REPLY); } /* The SLOWLOG command. Implements all the subcommands needed to handle the diff --git a/src/server.c b/src/server.c index c41655942f1..5973819d831 100644 --- a/src/server.c +++ b/src/server.c @@ -3924,6 +3924,14 @@ void call(client *c, int flags) { c->flag.force_repl = 0; c->flag.prevent_prop = 0; + /* The redaction bitmap describes the argv of the command about to execute and + * is set on demand by the command itself. Clearing it here covers every case + * where one client executes several commands without an intervening + * resetClient(): the queued commands of a MULTI, RM_Call sequences issued on a + * reused module temp client, and the server.call() chain of a script. Stale + * bits would otherwise redact the wrong argument of a later command. */ + c->redact_arg_bitmap = 0; + /* The server core is in charge of propagation when the first entry point * of call() is processCommand(). * The only other option to get to call() without having processCommand diff --git a/tests/unit/commandlog.tcl b/tests/unit/commandlog.tcl index 4db492c3d13..1e8a5efceba 100644 --- a/tests/unit/commandlog.tcl +++ b/tests/unit/commandlog.tcl @@ -198,6 +198,35 @@ start_server {tags {"commandlog"} overrides {commandlog-execution-slower-than 10 assert_match {* key 9 5000 AUTH2 (redacted) (redacted)} [lindex [lindex $slowlog_resp 0] 3] } {} {needs:repl} + test {COMMANDLOG slow - Redaction does not leak to later commands in a MULTI} { + r config set commandlog-execution-slower-than 0 + r commandlog reset slow + r multi + r acl setuser commandlog-test-user +get + r set foo bar + r exec + r config set commandlog-execution-slower-than -1 + set slowlog_resp [r commandlog get -1 slow] + + # The ACL SETUSER redaction must not carry over to the following SET + assert_equal {set foo bar} [lindex [lindex $slowlog_resp 0] 3] + r acl deluser commandlog-test-user + } + + test {COMMANDLOG slow - Redaction is applied to commands executed from scripts} { + r config set commandlog-execution-slower-than 0 + r commandlog reset slow + # MIGRATE on a missing key returns NOKEY before connecting anywhere, + # but redacts its AUTH2 arguments while parsing them. + r eval {server.call('migrate', '127.0.0.1', '9999', 'missingkey', '9', '100', 'AUTH2', 'user', 'password')} 0 + r config set commandlog-execution-slower-than -1 + set slowlog_resp [r commandlog get -1 slow] + + # Entry 0 is the EVAL itself, entry 1 is the MIGRATE the script executed + assert_equal {migrate 127.0.0.1 9999 missingkey 9 100 AUTH2 (redacted) (redacted)} \ + [lindex [lindex $slowlog_resp 1] 3] + } + test {COMMANDLOG slow - Rewritten commands are logged as their original command} { r config set commandlog-execution-slower-than 0 From 975cfc1020fc9adaa6681c551e6a7362b74a4b85 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:39:44 -0400 Subject: [PATCH 13/27] Correct spelling in module cleanup, command docs, source comments, and Tcl tests (#2243) Fix spelling and grammar issues across 17 files This is a subset of #2183. --------- Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> Co-authored-by: Sarthak Aggarwal --- src/bitops.c | 4 ++-- src/commands/README.md | 4 ++-- src/defrag.c | 2 +- src/lolwut5.c | 2 +- src/module.c | 8 ++++---- src/replication.c | 2 +- src/rio.c | 2 +- src/server.c | 4 ++-- src/siphash.c | 6 +++--- tests/support/test.tcl | 2 +- tests/unit/acl-v2.tcl | 2 +- tests/unit/acl.tcl | 8 ++++---- tests/unit/functions.tcl | 4 ++-- tests/unit/info-command.tcl | 2 +- tests/unit/moduleapi/datatype.tcl | 2 +- tests/unit/moduleapi/infotest.tcl | 4 ++-- valkey.conf | 4 ++-- 17 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index 9af36b66493..c0802a1c7c6 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -1122,8 +1122,8 @@ void bitposCommand(client *c) { return; } - /* For empty ranges (start > end) we return -1 as an empty range does - * not contain a 0 nor a 1. */ + /* For empty ranges (start > end) we return -1 as an empty range contains + * neither 0 nor 1. */ if (start > end) { addReplyLongLong(c, -1); } else { diff --git a/src/commands/README.md b/src/commands/README.md index ee400b87e76..79e7e80285b 100644 --- a/src/commands/README.md +++ b/src/commands/README.md @@ -78,7 +78,7 @@ following keys. To be safe, assume all of them are optional. * `"TOUCHES_ARBITRARY_KEYS"` * `"WRITE"` * `"acl_categories"`: A list of ACL categories in uppercase. Note that the - effective ACL categies include "implicit ACL categories" explained below. + effective ACL categories include "implicit ACL categories" explained below. * `"ADMIN"` * `"BITMAP"` * `"CONNECTION"` @@ -209,7 +209,7 @@ doesn't have an `"arguments"` key. Appendix -------- -How to list all the `group`, `command_flags` and `acl_categries`, etc. used in all these files: +How to list all the `group`, `command_flags` and `acl_categories`, etc. used in all these files: cat *.json | jq '.[].group' | grep -F '"' | sed 's/^ *//;s/, *$//;s/^/ * `/;s/$/`/' | sort | uniq cat *.json | jq '.[].command_flags' | grep -F '"' | sed 's/^ *//;s/, *$//;s/^/ * `/;s/$/`/' | sort | uniq diff --git a/src/defrag.c b/src/defrag.c index bed753adbce..4debda161e8 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -741,7 +741,7 @@ static void defragPubsubScanCallback(void *privdata, void *elemref) { *(robj **)hashtableMetadata(clients) = newchannel; /* The channel name is shared by the client's pubsub(shard) and server's - * pubsub(shard), after defraging the channel name, we need to update + * pubsub(shard), after defragging the channel name, we need to update * the reference in the clients' dictionary. */ hashtableIterator iter; hashtableInitIterator(&iter, clients, 0); diff --git a/src/lolwut5.c b/src/lolwut5.c index deb7d0e78bc..7b1e2c9993d 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -63,7 +63,7 @@ void lwTranslatePixelsGroup(int byte, char *output) { /* Schotter, the output of LOLWUT of Redis OSS 5, is a computer graphic art piece * generated by Georg Nees in the 60s. It explores the relationship between - * caos and order. + * chaos and order. * * The function creates the canvas itself, depending on the columns available * in the output display and the number of squares per row and per column diff --git a/src/module.c b/src/module.c index aa72794cfa8..47d8337e421 100644 --- a/src/module.c +++ b/src/module.c @@ -7244,12 +7244,12 @@ moduleType *moduleTypeLookupModuleByNameInternal(const char *name, int ignore_ca } return NULL; } -/* Search all registered modules by name, and name is case sensitive */ +/* Search all registered modules by name, and name is case-sensitive */ moduleType *moduleTypeLookupModuleByName(const char *name) { return moduleTypeLookupModuleByNameInternal(name, 0); } -/* Search all registered modules by name, but case insensitive */ +/* Search all registered modules by name, but case-insensitive */ moduleType *moduleTypeLookupModuleByNameIgnoreCase(const char *name) { return moduleTypeLookupModuleByNameInternal(name, 1); } @@ -13173,7 +13173,7 @@ void moduleRemoveConfigs(ValkeyModule *module) { } /* Remove ACL categories added by the module when it fails to load. */ -void moduleRemoveCateogires(ValkeyModule *module) { +void moduleRemoveCategories(ValkeyModule *module) { if (module->num_acl_categories_added) { ACLCleanupCategoriesOnFailure(module->num_acl_categories_added); } @@ -13404,7 +13404,7 @@ static int moduleInitPostOnLoadResolved(ModuleLoadFunc onload, serverLog(LL_WARNING, "%sModule %s initialization failed. Module not loaded.", is_static ? "Static " : "", display_name); moduleUnregisterCleanup(ctx.module); - moduleRemoveCateogires(ctx.module); + moduleRemoveCategories(ctx.module); moduleFreeModuleStructure(ctx.module); if (errmsg) *errmsg = "module initialization failed"; } else { diff --git a/src/replication.c b/src/replication.c index 62360af82fd..6afbee33c0c 100644 --- a/src/replication.c +++ b/src/replication.c @@ -3917,7 +3917,7 @@ int syncWithPrimaryHandleSendHandshakeState(connection *conn) { * Inform the primary of this capa only during diskless sync * using a connection that has integrity checks (such as TLS). * In non-diskless sync, or non-integrity-checked connection, there is more - * concern for data corruprion so we keep this extra layer of detection. + * concern for data corruption so we keep this extra layer of detection. * * The primary will ignore capabilities it does not understand. */ diff --git a/src/rio.c b/src/rio.c index b0142672cd2..21ef128e554 100644 --- a/src/rio.c +++ b/src/rio.c @@ -537,7 +537,7 @@ static size_t rioConnsetWrite(rio *r, const void *buf, size_t len) { len = sdslen(r->io.connset.buf); } - /* Write in little chunchs so that when there are big writes we + /* Write in little chunks so that when there are big writes we * parallelize while the kernel is sending data in background to * the TCP socket. */ while (len) { diff --git a/src/server.c b/src/server.c index 5973819d831..481e537b370 100644 --- a/src/server.c +++ b/src/server.c @@ -386,7 +386,7 @@ int dictSdsKeyCompare(const void *key1, const void *key2) { return memcmp(key1, key2, l1) == 0; } -/* A case insensitive version used for the command lookup table and other +/* A case-insensitive version used for the command lookup table and other * places where case insensitive non binary-safe comparison is needed. */ int dictSdsKeyCaseCompare(const void *key1, const void *key2) { return strcasecmp(key1, key2) == 0; @@ -467,7 +467,7 @@ int dictCStrKeyCompare(const void *key1, const void *key2) { return strcmp(key1, key2) == 0; } -/* Dict case insensitive compare function for null terminated string */ +/* Dict case-insensitive compare function for null terminated string */ int dictCStrKeyCaseCompare(const void *key1, const void *key2) { return strcasecmp(key1, key2) == 0; } diff --git a/src/siphash.c b/src/siphash.c index 75271762181..906e6117047 100644 --- a/src/siphash.c +++ b/src/siphash.c @@ -30,9 +30,9 @@ returns an uint64_t value, the hash itself, instead of receiving an output buffer. This also means that the output size is set to 8 bytes and the 16 bytes output code handling was removed. - 4. Provide a case insensitive variant to be used when hashing strings that + 4. Provide a case-insensitive variant to be used when hashing strings that must be considered identical by the hash table regardless of the case. - If we don't have directly a case insensitive hash function, we need to + If we don't have directly a case-insensitive hash function, we need to perform a text transformation in some temporary buffer, which is costly. 5. Remove debugging code. 6. Modified the original test.c file to be a stand-alone function testing @@ -341,7 +341,7 @@ int siphash_test(void) { } } - /* Run a few basic tests with the case insensitive version. */ + /* Run a few basic tests with the case-insensitive version. */ uint64_t h1, h2; h1 = siphash((uint8_t*)"hello world",11,(uint8_t*)"1234567812345678"); h2 = siphash_nocase((uint8_t*)"hello world",11,(uint8_t*)"1234567812345678"); diff --git a/tests/support/test.tcl b/tests/support/test.tcl index 3b7a35b1188..b0e0649a90f 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -129,7 +129,7 @@ proc assert_refcount_morethan {key ref} { # Wait for the specified condition to be true, with the specified number of # max retries and delay between retries. Otherwise, the 'elsescript' is # executed. If 'debugscript' is provided, it is executed after failure of -# the confition (before the retry delay). +# the condition (before the retry delay). proc wait_for_condition {maxtries delay e _else_ elsescript {_debug_ ""} {debugscript ""}} { while {[incr maxtries -1] >= 0} { set errcode [catch {uplevel 1 [list expr $e]} result] diff --git a/tests/unit/acl-v2.tcl b/tests/unit/acl-v2.tcl index 6b477f7f9fc..767818aa661 100644 --- a/tests/unit/acl-v2.tcl +++ b/tests/unit/acl-v2.tcl @@ -534,7 +534,7 @@ start_server {tags {"acl external:skip"}} { # Unlike existence test commands, intersection cardinality commands process the data # between keys and return an aggregated cardinality. therefore they have the access # requirement. - test {Intersection cardinaltiy commands are access commands} { + test {Intersection cardinality commands are access commands} { assert_equal "OK" [r ACL DRYRUN command-test SINTERCARD 2 read read] assert_match {*has no permissions to access the 'write' key*} [r ACL DRYRUN command-test SINTERCARD 2 write read] assert_match {*has no permissions to access the 'nothing' key*} [r ACL DRYRUN command-test SINTERCARD 2 nothing read] diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index 8f3c5c24580..e7d0814fe22 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -588,11 +588,11 @@ start_server {tags {"acl external:skip"}} { r ACL SETUSER adv-test -@string -@slow +@all assert_equal "+@all" [dict get [r ACL getuser adv-test] commands] - # Make sure categories are case insensitive + # Make sure categories are case-insensitive r ACL SETUSER adv-test -@all +@HASH +@hash +@HaSh assert_equal "-@all +@hash" [dict get [r ACL getuser adv-test] commands] - # Make sure commands are case insensitive + # Make sure commands are case-insensitive r ACL SETUSER adv-test -@all +HGET +hget +hGeT assert_equal "-@all +hget" [dict get [r ACL getuser adv-test] commands] @@ -1415,11 +1415,11 @@ tags {acl external:skip} { r ACL SETUSER adv-test +@hash assert_equal "+@all -@slow +hget +@hash" [dict get [r ACL getuser adv-test] commands] - # Make sure categories are case insensitive + # Make sure categories are case-insensitive r ACL SETUSER adv-test -@all +@HASH +@hash +@HaSh assert_equal "-@all +@hash" [dict get [r ACL getuser adv-test] commands] - # Make sure commands are case insensitive + # Make sure commands are case-insensitive r ACL SETUSER adv-test -@all +HGET +hget +hGeT assert_equal "-@all +hget" [dict get [r ACL getuser adv-test] commands] diff --git a/tests/unit/functions.tcl b/tests/unit/functions.tcl index a008e754d1e..970dec65273 100644 --- a/tests/unit/functions.tcl +++ b/tests/unit/functions.tcl @@ -26,7 +26,7 @@ start_server {tags {"scripting"}} { set _ $e } {*already exists*} - test {FUNCTION - Create an already exiting library raise error (case insensitive)} { + test {FUNCTION - Create an already exiting library raise error (case-insensitive)} { catch { r function load [get_function_code LUA test test {return 'hello1'}] } e @@ -59,7 +59,7 @@ start_server {tags {"scripting"}} { r fcall test 0 } {hello1} - test {FUNCTION - test function case insensitive} { + test {FUNCTION - test function case-insensitive} { r fcall TEST 0 } {hello1} diff --git a/tests/unit/info-command.tcl b/tests/unit/info-command.tcl index bc480d6aff3..3f0a2864684 100644 --- a/tests/unit/info-command.tcl +++ b/tests/unit/info-command.tcl @@ -36,7 +36,7 @@ start_server {tags {"info and its relative command"}} { assert { ![string match "*sentinel_tilt*" $info] } assert { ![string match "*used_memory*" $info] } - set info [r info commandSTATS] ;# test case insensitive compare + set info [r info commandSTATS] ;# test case-insensitive compare assert { ![string match "*used_memory*" $info] } assert { [string match "*rejected_calls*" $info] } } diff --git a/tests/unit/moduleapi/datatype.tcl b/tests/unit/moduleapi/datatype.tcl index 28bbf5c0f25..80e67f5ecce 100644 --- a/tests/unit/moduleapi/datatype.tcl +++ b/tests/unit/moduleapi/datatype.tcl @@ -114,7 +114,7 @@ start_server {tags {"modules"}} { assert_equal 1 [llength $keys] } - test {SCAN module datatype with case sensitive} { + test {SCAN module datatype with case-sensitive} { r flushdb populate 1000 r datatype.set foo 111 bar diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl index ccd8c4ecbe2..2659d43d20c 100644 --- a/tests/unit/moduleapi/infotest.tcl +++ b/tests/unit/moduleapi/infotest.tcl @@ -72,7 +72,7 @@ start_server {tags {"modules"}} { } test {module info one module} { - set info [r info INFOtest] ;# test case insensitive compare + set info [r info INFOtest] ;# test case-insensitive compare # info all does not contain modules assert { [string match "*Spanish*" $info] } assert { ![string match "*used_memory*" $info] } @@ -80,7 +80,7 @@ start_server {tags {"modules"}} { } {-2} test {module info one section} { - set info [r info INFOtest_SpanisH] ;# test case insensitive compare + set info [r info INFOtest_SpanisH] ;# test case-insensitive compare assert { ![string match "*used_memory*" $info] } assert { ![string match "*Italian*" $info] } assert { ![string match "*infotest_global*" $info] } diff --git a/valkey.conf b/valkey.conf index e944520fa73..272a394000b 100644 --- a/valkey.conf +++ b/valkey.conf @@ -15,7 +15,7 @@ # 1g => 1000000000 bytes # 1gb => 1024*1024*1024 bytes # -# units are case insensitive so 1GB 1Gb 1gB are all the same. +# units are case-insensitive so 1GB 1Gb 1gB are all the same. ################################## INCLUDES ################################### @@ -295,7 +295,7 @@ tcp-keepalive 300 # By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended # that older formally deprecated versions are kept disabled to reduce the attack surface. # You can explicitly specify TLS versions to support. -# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", +# Allowed values are case-insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", # "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. # To enable only TLSv1.2 and TLSv1.3, use: # From 967c7cc8e8d116c65474250258f1f8a0db77a003 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Wed, 12 Aug 2026 14:51:01 -0700 Subject: [PATCH 14/27] Harden stream validation on RDB load against crafted metadata (#3922) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Two crafted-`RESTORE` crashes in stream loading. In both, the payload passes the existing structural validation but violates an invariant downstream code relies on. **Any client with `RESTORE` access can remotely crash the server.** ### 1. Length vs. tombstones A stream can claim a positive `length` while every listpack entry is a tombstone (`STREAM_ITEM_FLAG_DELETED`). The length is loaded directly from the payload and only checked against the rax being non-empty. `streamLastValidID()` then finds no non-tombstone entry while `s->length` is non-zero and aborts: ``` serverPanic("Corrupt stream, length is %llu, but no max id", ...) // t_stream.c ``` Triggered by `XSETID` / `XADD` / `XREADGROUP`. Confirmed: a 2-entry stream with both entries flagged `DELETED` and `length=1` loads OK, then `XSETID` panics. ### 2. Negative field counts A master entry (or a per-entry field count for non-`SAMEFIELDS` entries) can declare a **negative** number of fields. The validator only checked `lpGetIntegerIfValid()`'s success flag, not the sign. The negative count drives listpack traversal in `streamIteratorGetID()`, walking past the listpack and asserting (`lpAssertValidEntry`) on `XRANGE` and similar reads. Confirmed: crafted payload loads OK, then `XRANGE` aborts at `listpack.c`. ## Fix 1. `streamValidateListpackIntegrity()` already parses each listpack's master entry count (live entries). Sum it across listpacks via a new out-parameter and reject the payload if it does not match the loaded length. This reuses the assertion-safe parsing rather than iterating the stream with `streamIteratorGetID()`, which can itself hit entry-level assertions on *other* malformed payloads (an earlier iterate-based version regressed three existing corrupt-dump tests). 2. Reject negative `primary_fields` and per-entry `fields` counts during validation. ## Testing - Two `RESTORE`-path integration tests in `tests/integration/corrupt-dump.tcl`. - Both verified to **fail pre-fix** (panic / assert) and **pass post-fix**. - Confirmed legitimate streams — including ones with real tombstones (5 entries, 2 deleted) and multi-field entries — still load and read correctly. - Full `integration/corrupt-dump` suite: 75 passed, 0 failed (including the three stream consumer-group tests an earlier iterate-based approach broke). > [!NOTE] > Found via structure-aware fuzzing + code review of the RESTORE path. This issue was generated by AI but verified, with love, by a human. --------- Signed-off-by: Madelyn Olson --- src/rdb.c | 19 +++++++++++++- src/stream.h | 2 +- src/t_stream.c | 21 ++++++++++----- tests/integration/corrupt-dump.tcl | 41 ++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 5dd9424573d..e603c0356b8 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2614,6 +2614,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd return NULL; } + /* Sum of non-deleted entries across all listpacks, used below to + * validate the stream length loaded from the payload. */ + uint64_t valid_entries = 0; + while (listpacks--) { /* Get the primary ID, the one we'll use as key of the radix tree * node: the entries inside the listpack itself are delta-encoded @@ -2642,7 +2646,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd return NULL; } server.stat_dump_payload_sanitizations++; - if (!streamValidateListpackIntegrity(lp, lp_size)) { + if (!streamValidateListpackIntegrity(lp, lp_size, &valid_entries)) { rdbReportCorruptRDB("Stream listpack integrity check failed."); sdsfree(nodekey); decrRefCount(o); @@ -2715,6 +2719,19 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error, int rd return NULL; } + /* Validate that the loaded length matches the number of non-deleted + * entries actually present in the listpacks. 's->length' comes straight + * from the payload; a crafted payload can claim a positive length while + * every listpack entry is a tombstone, which later makes + * streamLastValidID() panic ("length is N, but no max id") when a + * command such as XSETID, XADD or XREADGROUP looks up the last valid + * ID. valid_entries was accumulated during listpack validation above. */ + if (s->length != valid_entries) { + rdbReportCorruptRDB("Stream length inconsistent with the number of valid entries"); + decrRefCount(o); + return NULL; + } + /* Consumer groups loading */ uint64_t cgroups_count = rdbLoadLen(rdb, NULL); if (cgroups_count == RDB_LENERR) { diff --git a/src/stream.h b/src/stream.h index 7a1217ec8d7..b522984731a 100644 --- a/src/stream.h +++ b/src/stream.h @@ -148,7 +148,7 @@ int streamIncrID(streamID *id); int streamDecrID(streamID *id); void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds consumername); robj *streamDup(robj *o); -int streamValidateListpackIntegrity(unsigned char *lp, size_t size); +int streamValidateListpackIntegrity(unsigned char *lp, size_t size, uint64_t *valid_count); int streamParseID(const robj *o, streamID *id); robj *createObjectFromStreamID(streamID *id); int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id, int seq_given); diff --git a/src/t_stream.c b/src/t_stream.c index 99feef6d8de..abf8e976d2b 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -3987,8 +3987,13 @@ void xinfoCommand(client *c) { /* Validate the integrity stream listpack entries structure. Both in term of a * valid listpack, but also that the structure of the entries matches a valid - * stream. return 1 if valid 0 if not valid. */ -int streamValidateListpackIntegrity(unsigned char *lp, size_t size) { + * stream. return 1 if valid 0 if not valid. + * + * If 'valid_count' is not NULL, the number of non-deleted (non-tombstone) + * entries in this listpack is added to it. Callers use this to validate the + * stream length loaded from an RDB payload against the entries actually + * present. */ +int streamValidateListpackIntegrity(unsigned char *lp, size_t size, uint64_t *valid_count) { int valid_record; unsigned char *p, *next; @@ -4001,19 +4006,23 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size) { /* entry count */ int64_t entry_count = lpGetIntegerIfValid(p, &valid_record); - if (!valid_record) return 0; + if (!valid_record || entry_count < 0) return 0; p = next; if (!lpValidateNext(lp, &next, size)) return 0; + /* The master entry count is the number of live (non-deleted) entries in + * this listpack. Accumulate it so the caller can verify the stream length. */ + if (valid_count) *valid_count += entry_count; + /* deleted */ int64_t deleted_count = lpGetIntegerIfValid(p, &valid_record); - if (!valid_record) return 0; + if (!valid_record || deleted_count < 0) return 0; p = next; if (!lpValidateNext(lp, &next, size)) return 0; /* num-of-fields */ int64_t primary_fields = lpGetIntegerIfValid(p, &valid_record); - if (!valid_record) return 0; + if (!valid_record || primary_fields < 0) return 0; p = next; if (!lpValidateNext(lp, &next, size)) return 0; @@ -4051,7 +4060,7 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size) { if (!(flags & STREAM_ITEM_FLAG_SAMEFIELDS)) { /* num-of-fields */ fields = lpGetIntegerIfValid(p, &valid_record); - if (!valid_record) return 0; + if (!valid_record || fields < 0) return 0; p = next; if (!lpValidateNext(lp, &next, size)) return 0; diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl index 871c8cf87ce..f1371addf98 100644 --- a/tests/integration/corrupt-dump.tcl +++ b/tests/integration/corrupt-dump.tcl @@ -740,6 +740,19 @@ test {corrupt payload: zset listpack with NAN score} { } } +test {corrupt payload: stream length inconsistent with valid entries} { + # A stream whose listpack entries are all tombstones, but whose loaded + # length claims a positive value, must be rejected on load. Otherwise the + # mismatch makes streamLastValidID() panic ("length is N, but no max id") + # when a command such as XSETID looks up the last valid ID. + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r debug set-skip-checksum-validation 1 + catch {r restore _tomb_stream 0 "\x15\x01\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x01\x28\x28\x00\x00\x00\x0F\x00\x00\x01\x02\x01\x01\x01\x81\x66\x02\x00\x01\x03\x01\x00\x01\x00\x01\x81\x76\x02\x04\x01\x03\x01\x01\x01\x01\x01\x81\x76\x02\x04\x01\xFF\x01\x02\x02\x01\x01\x00\x00\x02\x00\x50\x00\x3F\xD1\x7E\xA1\xC7\x54\x12\x57"} err + assert_match "*Bad data format*" $err + assert_equal [r ping] "PONG" + } +} + test {corrupt payload: zset ziplist with NAN score} { # Same as the listpack case but for the legacy ziplist format, which is # converted to a listpack on load. A NAN score must be rejected so it can @@ -753,6 +766,34 @@ test {corrupt payload: zset ziplist with NAN score} { } } +test {corrupt payload: stream listpack with negative field count} { + # A stream master entry that declares a negative number of fields must be + # rejected on load. The field count drives listpack traversal in + # streamIteratorGetID(), so a negative value walks past the listpack and + # asserts when a command such as XRANGE reads the stream. + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r debug set-skip-checksum-validation 1 + catch {r restore _neg_fields 0 "\x15\x01\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x01\x18\x18\x00\x00\x00\x08\x00\x01\x01\x00\x01\xDF\xFD\x02\x00\x01\x02\x01\x00\x01\x00\x01\x00\x01\xFF\x01\x01\x01\x01\x01\x00\x00\x01\x00\x0B\x00\x00\x00\x00\x00\x00\x00\x00\x00"} err + assert_match "*Bad data format*" $err + assert_equal [r ping] "PONG" + } +} + +test {corrupt payload: stream listpack with negative deleted count} { + # A master entry that declares a negative number of deleted entries must be + # rejected on load. The deleted count is added to the entry count to bound + # the entry validation loop, so a negative value lets a listpack claim more + # live entries (3 here) than it actually contains (1) while still walking + # cleanly to the end of the listpack. Without the sign check the payload is + # accepted, leaving the master entry count inconsistent with the entries. + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r debug set-skip-checksum-validation 1 + catch {r restore _neg_deleted 0 "\x15\x01\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x01\x18\x18\x00\x00\x00\x08\x00\x03\x01\xDF\xFE\x02\x00\x01\x00\x01\x02\x01\x00\x01\x00\x01\x03\x01\xFF\x03\x01\x01\x01\x01\x00\x00\x03\x00\x0B\x00\x00\x00\x00\x00\x00\x00\x00\x00"} err + assert_match "*Bad data format*" $err + assert_equal [r ping] "PONG" + } +} + test {corrupt payload: fuzzer findings - streamLastValidID panic} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { r debug set-skip-checksum-validation 1 From 6cdff5ef06f820aaa244ef9f0272ecc60bad386f Mon Sep 17 00:00:00 2001 From: Shun Takahashi Date: Thu, 13 Aug 2026 07:24:25 +0900 Subject: [PATCH 15/27] Fix grammar errors in code comments (#3729) Grammatical errors are fixed in documentation Signed-off-by: takahashi shun Co-authored-by: Sarthak Aggarwal --- src/aof.c | 2 +- src/cluster_legacy.c | 2 +- src/db.c | 2 +- src/entry.c | 4 ++-- src/eval.c | 2 +- src/latency.c | 2 +- src/module.c | 4 ++-- src/t_set.c | 2 +- src/t_stream.c | 2 +- src/util.c | 4 ++-- src/valkey-check-aof.c | 2 +- src/valkeymodule.h | 2 +- src/ziplist.c | 2 +- 13 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/aof.c b/src/aof.c index 5b3f4f483d4..f38384d1136 100644 --- a/src/aof.c +++ b/src/aof.c @@ -602,7 +602,7 @@ int persistAofManifest(aofManifest *am) { return ret; } -/* Called in `loadAppendOnlyFiles` when we upgrade from a old version of the server. +/* Called in `loadAppendOnlyFiles` when we upgrade from an old version of the server. * * 1) Create AOF directory use 'server.aof_dirname' as the name. * 2) Use 'server.aof_filename' to construct a BASE type aofInfo and add it to diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index fadd4235969..5d4003aba6a 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -3358,7 +3358,7 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc serverLog(LL_NOTICE, "My last slot was migrated to node %.40s (%s) in shard %.40s. I am now an empty primary.", sender->name, humanNodename(sender), sender->shard_id); - /* We may still have dirty slots when we became a empty primary due to + /* We may still have dirty slots when we became an empty primary due to * a bad migration. * * In order to maintain a consistent state between keys and slots diff --git a/src/db.c b/src/db.c index df2e82ef8b3..23019505e0b 100644 --- a/src/db.c +++ b/src/db.c @@ -1915,7 +1915,7 @@ int removeExpire(serverDb *db, robj *key) { } /* Set an expire to the specified key. If the expire is set in the context - * of an user calling a command 'c' is the client, otherwise 'c' is set + * of a user calling a command 'c' is the client, otherwise 'c' is set * to NULL. The 'when' parameter is the absolute unix time in milliseconds * after which the key will no longer be considered valid. * diff --git a/src/entry.c b/src/entry.c index d28996722c4..d0535f29426 100644 --- a/src/entry.c +++ b/src/entry.c @@ -490,7 +490,7 @@ entry *entryUpdate(entry *e, sds value, mstime_t expiry) { return new_entry; } -/* Returns memory usage of a entry, including all allocations owned by +/* Returns memory usage of an entry, including all allocations owned by * the entry. */ size_t entryMemUsage(entry *entry) { size_t mem = 0; @@ -511,7 +511,7 @@ size_t entryMemUsage(entry *entry) { return mem; } -/* Defragments a entry (field-value pair) if needed, using the +/* Defragments an entry (field-value pair) if needed, using the * provided defrag functions. The defrag functions return NULL if the allocation * was not moved, otherwise they return a pointer to the new memory location. * A separate sds defrag function is needed because of the unique memory layout diff --git a/src/eval.c b/src/eval.c index 8e2839ae8a3..1ec2affe4b4 100644 --- a/src/eval.c +++ b/src/eval.c @@ -495,7 +495,7 @@ static void evalGenericCommand(client *c, int evalsha) { dictEntry *entry = dictFind(evalCtx.scripts, sha); if (evalsha && entry == NULL) { - /* Calling EVALSHA using an hash that was never added to the scripts + /* Calling EVALSHA using a hash that was never added to the scripts * cache. */ addReplyErrorObject(c, shared.noscripterr); return; diff --git a/src/latency.c b/src/latency.c index d597340a92f..e91d130c5b7 100644 --- a/src/latency.c +++ b/src/latency.c @@ -676,7 +676,7 @@ sds latencyCommandGenSparkeline(char *event, struct latencyTimeSeries *ts) { * LATENCY DOCTOR: returns a human readable analysis of instance latency. * LATENCY GRAPH: provide an ASCII graph of the latency of the specified event. * LATENCY RESET: reset data of a specified event or all the data if no event provided. - * LATENCY HISTOGRAM: return a cumulative distribution of latencies in the format of an histogram for the specified + * LATENCY HISTOGRAM: return a cumulative distribution of latencies in the format of a histogram for the specified * command names. */ void latencyCommand(client *c) { diff --git a/src/module.c b/src/module.c index 47d8337e421..bc4b5c88d58 100644 --- a/src/module.c +++ b/src/module.c @@ -49,7 +49,7 @@ * (with the exception of a ----- line which can appear first). Other comment * blocks, which are not intended for the modules API user, such as this comment * block, do NOT start with a markdown level 2 heading, so they are included in - * the generated a API documentation. + * the generated API documentation. * * The documentation comments may contain markdown formatting. Some automatic * replacements are done, such as the replacement of RM with ValkeyModule in @@ -11460,7 +11460,7 @@ int moduleUnregisterSharedAPI(ValkeyModule *module) { return count; } -/* Remove the specified module as an user of APIs of ever other module. +/* Remove the specified module as a user of APIs of every other module. * This is usually called when a module is unloaded. * * Returns the number of modules this module was using APIs from. */ diff --git a/src/t_set.c b/src/t_set.c index 807832167b5..9a6b9582173 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -407,7 +407,7 @@ sds setTypeNextObject(setTypeIterator *si) { /* Return random element from a non empty set. * The returned element can be an int64_t value if the set is encoded - * as an "intset" blob of integers, or an string. + * as an "intset" blob of integers, or a string. * * The caller provides three pointers to be populated with the right * object. The return value of the function is the object->encoding diff --git a/src/t_stream.c b/src/t_stream.c index abf8e976d2b..4bfbcdce1e2 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1762,7 +1762,7 @@ size_t streamReplyWithRange(client *c, /* Try to add a new NACK. Most of the time this will work and * will not require extra lookups. We'll fix the problem later - * if we find that there is already a entry for this ID. */ + * if we find that there is already an entry for this ID. */ streamNACK *nack = streamCreateNACK(consumer); int group_inserted = raxTryInsert(group->pel, buf, sizeof(buf), nack, NULL); int consumer_inserted = raxTryInsert(consumer->pel, buf, sizeof(buf), nack, NULL); diff --git a/src/util.c b/src/util.c index 261c80821f2..fa349a56419 100644 --- a/src/util.c +++ b/src/util.c @@ -435,9 +435,9 @@ int ull2string(char *dst, size_t dstlen, unsigned long long value) { #define MULTIPLIER_10E16 10000000000000000ULL /** - * Convert a string into an signed 64-bit integer using AVX-512 instructions. + * Convert a string into a signed 64-bit integer using AVX-512 instructions. * - * This function parses a string of digits and converts it into an signed + * This function parses a string of digits and converts it into a signed * 64-bit integer. It leverages AVX-512 SIMD instructions for optimized * processing and performs strict validation to ensure the input string * represents a valid signed integer. diff --git a/src/valkey-check-aof.c b/src/valkey-check-aof.c index 1a7fa1dc411..1a6bc15619f 100644 --- a/src/valkey-check-aof.c +++ b/src/valkey-check-aof.c @@ -334,7 +334,7 @@ int checkSingleAof(char *aof_filename, char *aof_filepath, int last_file, int fi return AOF_CHECK_OK; } -/* Used to determine whether the file is a RDB file. These two possibilities: +/* Used to determine whether the file is an RDB file. These two possibilities: * 1. The file is an old style RDB-preamble AOF * 2. The file is a BASE AOF in Multi Part AOF * */ diff --git a/src/valkeymodule.h b/src/valkeymodule.h index c284e63d866..ed7b2922294 100644 --- a/src/valkeymodule.h +++ b/src/valkeymodule.h @@ -577,7 +577,7 @@ typedef void (*ValkeyModuleEventCallback)(struct ValkeyModuleCtx *ctx, * 2 * } * The reason for that is forward-compatibility: We want that module that - * compiled with a new valkeymodule.h to be able to work with a old server, + * compiled with a new valkeymodule.h to be able to work with an old server, * unless the author explicitly decided to use the newer event type. */ static const ValkeyModuleEvent ValkeyModuleEvent_ReplicationRoleChanged = {VALKEYMODULE_EVENT_REPLICATION_ROLE_CHANGED, diff --git a/src/ziplist.c b/src/ziplist.c index 5e478168334..83d2f3b1f3f 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -744,7 +744,7 @@ unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) { size_t firstentrylen; /* Used to handle insert at head. */ size_t rawlen, curlen = intrev32ifbe(ZIPLIST_BYTES(zl)); size_t extra = 0, cnt = 0, offset; - size_t delta = 4; /* Extra bytes needed to update a entry's prevlen (5-1). */ + size_t delta = 4; /* Extra bytes needed to update an entry's prevlen (5-1). */ unsigned char *tail = zl + intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)); /* Empty ziplist */ From 2c38467d3bed09c9e3b8aa7476cde83b3e5c15f3 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:29:23 -0400 Subject: [PATCH 16/27] Fixes grammatical and spelling errors (#2249) Fixes grammatical and spelling errors --------- Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> Co-authored-by: Sarthak Aggarwal --- src/cluster_legacy.c | 2 +- src/commands/README.md | 2 +- src/functions.c | 12 ++++++------ src/geohash_helper.c | 2 +- src/lzf_d.c | 2 +- src/module.c | 2 +- src/quicklist.c | 2 +- src/rax.c | 2 +- src/rio.c | 4 ++-- src/server.c | 2 +- src/t_stream.c | 2 +- tests/instances.tcl | 2 +- tests/unit/functions.tcl | 2 +- tests/unit/tracking.tcl | 2 +- tests/unit/type/stream-cgroups.tcl | 2 +- tests/unit/type/stream.tcl | 2 +- valkey.conf | 2 +- 17 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index 5d4003aba6a..e76eced13f2 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -1065,7 +1065,7 @@ int clusterLoadConfig(char *filename) { * of the POSIX filesystem semantics, so that if the server is stopped * or crashes during the write, we'll end with either the old file or the * new one. Since we have the full payload to write available we can use - * a single write to write the whole file. If the pre-existing file was + * a single write to write the whole file. If the preexisting file was * bigger we pad our payload with newlines that are anyway ignored and truncate * the file afterward. */ int clusterSaveConfig(int do_fsync) { diff --git a/src/commands/README.md b/src/commands/README.md index 79e7e80285b..ba4c123ce1f 100644 --- a/src/commands/README.md +++ b/src/commands/README.md @@ -183,7 +183,7 @@ Each element in this array is an object with the following keys: command line. The first key is the argument after the keyword. * `{"unknown": null}`: Finding the keys of this command is too complicated to explain. -* `"find_keys"`: How to find the remainnig keys of this key spec. It's an object +* `"find_keys"`: How to find the remaining keys of this key spec. It's an object on one of these forms: * `{"range": {"lastkey": LAST, "step": STEP, "limit": LIMIT}}`: A range of keys. * LAST: If LAST is positive, it's the index of the last key relative to the diff --git a/src/functions.c b/src/functions.c index 971afed4444..9f4120ef840 100644 --- a/src/functions.c +++ b/src/functions.c @@ -772,7 +772,7 @@ void functionRestoreCommand(client *c) { return; } - restorePolicy restore_replicy = restorePolicy_Append; /* default policy: APPEND */ + restorePolicy restore_policy = restorePolicy_Append; /* default policy: APPEND */ sds data = objectGetVal(c->argv[2]); size_t data_len = sdslen(data); rio payload; @@ -781,11 +781,11 @@ void functionRestoreCommand(client *c) { if (c->argc == 4) { const char *restore_policy_str = objectGetVal(c->argv[3]); if (!strcasecmp(restore_policy_str, "append")) { - restore_replicy = restorePolicy_Append; + restore_policy = restorePolicy_Append; } else if (!strcasecmp(restore_policy_str, "replace")) { - restore_replicy = restorePolicy_Replace; + restore_policy = restorePolicy_Replace; } else if (!strcasecmp(restore_policy_str, "flush")) { - restore_replicy = restorePolicy_Flush; + restore_policy = restorePolicy_Flush; } else { addReplyError(c, "Wrong restore policy given, value should be either FLUSH, APPEND or REPLACE."); return; @@ -824,11 +824,11 @@ void functionRestoreCommand(client *c) { } } - if (restore_replicy == restorePolicy_Flush) { + if (restore_policy == restorePolicy_Flush) { functionsLibCtxSwapWithCurrent(functions_lib_ctx, server.lazyfree_lazy_user_flush); functions_lib_ctx = NULL; /* avoid releasing the f_ctx in the end */ } else { - if (libraryJoin(curr_functions_lib_ctx, functions_lib_ctx, restore_replicy == restorePolicy_Replace, &err) != + if (libraryJoin(curr_functions_lib_ctx, functions_lib_ctx, restore_policy == restorePolicy_Replace, &err) != C_OK) { goto load_error; } diff --git a/src/geohash_helper.c b/src/geohash_helper.c index b64799c0315..1a420ee680c 100644 --- a/src/geohash_helper.c +++ b/src/geohash_helper.c @@ -349,7 +349,7 @@ int geohashGetDistanceIfInRectangle(double width_m, * The Polygon's centroid's lon lat coordinates are `centroidLon` and `centroidLat`. * The algorithm is based on PNPOLY - Point Inclusion in Polygon Test by W. Randolph Franklin (WRF). * See: https://wrfranklin.org/Research/Short_Notes/pnpoly.html - * Returns 1 if inside the polyon and returns 0 otherwise. */ + * Returns 1 if inside the polygon and returns 0 otherwise. */ int geohashGetDistanceIfInPolygon(double centroidLon, double centroidLat, double *point, double (*vertices)[2], int num_vertices, double *distance) { int i, j; int inside = 0; diff --git a/src/lzf_d.c b/src/lzf_d.c index ff32be892a3..b936771d28c 100644 --- a/src/lzf_d.c +++ b/src/lzf_d.c @@ -160,7 +160,7 @@ lzf_decompress (const void *const in_data, size_t in_len, } else { - /* overlapping, use octte by octte copying */ + /* overlapping, use octet by octet copying */ do *op++ = *ref++; while (--len); diff --git a/src/module.c b/src/module.c index bc4b5c88d58..366d366a0b3 100644 --- a/src/module.c +++ b/src/module.c @@ -8771,7 +8771,7 @@ ValkeyModuleBlockedClient *VM_BlockClientOnAuth(ValkeyModuleCtx *ctx, return bc; } -/* Get the private data that was previusely set on a blocked client */ +/* Get the private data that was previously set on a blocked client */ void *VM_BlockClientGetPrivateData(ValkeyModuleBlockedClient *blocked_client) { return blocked_client->privdata; } diff --git a/src/quicklist.c b/src/quicklist.c index 67ffe4e17bf..7eb4a9e4c13 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -1468,7 +1468,7 @@ void quicklistRotate(quicklist *quicklist) { /* If quicklist has only one node, the head listpack is also the * tail listpack and PushHead() could have reallocated our single listpack, - * which would make our pre-existing 'p' unusable. */ + * which would make our preexisting 'p' unusable. */ if (quicklist->len == 1) { p = lpSeek(quicklist->tail->entry, -1); } diff --git a/src/rax.c b/src/rax.c index eaeebccc82b..e46972d4c40 100644 --- a/src/rax.c +++ b/src/rax.c @@ -335,7 +335,7 @@ raxNode *raxAddChild(raxNode *n, unsigned char c, raxNode **childptr, raxNode ** /* Move the pointers to the left of the insertion position as well. Often * we don't need to do anything if there was already some padding to use. In * that case the final destination of the pointers will be the same, however - * in our example there was no pre-existing padding, so we added one byte + * in our example there was no preexisting padding, so we added one byte * plus three bytes of padding. After the next memmove() things will look * like that: * diff --git a/src/rio.c b/src/rio.c index 21ef128e554..eaa665edc96 100644 --- a/src/rio.c +++ b/src/rio.c @@ -335,10 +335,10 @@ static size_t rioFdWrite(rio *r, const void *buf, size_t len) { /* For small writes, we rather keep the data in user-space buffer, and flush * it only when it grows. however for larger writes, we prefer to flush - * any pre-existing buffer, and write the new one directly without reallocs + * any preexisting buffer, and write the new one directly without reallocs * and memory copying. */ if (len > PROTO_IOBUF_LEN) { - /* First, flush any pre-existing buffered data. */ + /* First, flush any preexisting buffered data. */ if (sdslen(r->io.fd.buf)) { if (rioFdWrite(r, NULL, 0) == 0) return 0; } diff --git a/src/server.c b/src/server.c index 481e537b370..e69e26b5a6c 100644 --- a/src/server.c +++ b/src/server.c @@ -5007,7 +5007,7 @@ int finishShutdown(void) { rsiptr = rdbPopulateSaveInfo(&rsi); /* Keep the page cache since it's likely to restart soon */ if (rdbSave(REPLICA_REQ_NONE, server.rdb_filename, rsiptr, RDBFLAGS_KEEP_CACHE) != C_OK) { - /* Ooops.. error saving! The best we can do is to continue + /* Oops.. error saving! The best we can do is to continue * operating. Note that if there was a background saving process, * in the next cron() the server will be notified that the background * saving aborted, handling special stuff like replicas pending for diff --git a/src/t_stream.c b/src/t_stream.c index 4bfbcdce1e2..3a613c6f42e 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -3314,7 +3314,7 @@ void xclaimCommand(client *c) { * by the caller is satisfied by this entry. * * Note that the nack could be created by FORCE, in this - * case there was no pre-existing entry and minidle should + * case there was no preexisting entry and minidle should * be ignored, but in that case nack->consumer is NULL. */ if (nack->consumer && minidle) { mstime_t this_idle = now - nack->delivery_time; diff --git a/tests/instances.tcl b/tests/instances.tcl index 14fb4674e8a..9e23af30004 100644 --- a/tests/instances.tcl +++ b/tests/instances.tcl @@ -173,7 +173,7 @@ proc spawn_instance {type base_port count {conf {}} {base_conf_file ""}} { if {[server_is_up $::host $port 100] == 0} { set logfile [file join $dirname log.txt] puts [exec tail $logfile] - abort_sentinel_test "Problems starting $type #$instance_id: ping timeout, maybe server start failed, check $logfile" + abort_sentinel_test "Problem starting $type #$instance_id: ping timeout, maybe server start failed, check $logfile" } # Push the instance into the right list diff --git a/tests/unit/functions.tcl b/tests/unit/functions.tcl index 970dec65273..dacd4d6c99f 100644 --- a/tests/unit/functions.tcl +++ b/tests/unit/functions.tcl @@ -1039,7 +1039,7 @@ start_server {tags {"scripting"}} { r config set maxmemory 0 } {OK} {needs:config-maxmemory} - test {FUNCTION - verify allow-omm allows running any command} { + test {FUNCTION - verify allow-oom allows running any command} { r FUNCTION load replace {#!lua name=f1 server.register_function{ function_name='f1', diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl index b4c29ac4b4c..5daaa749f36 100644 --- a/tests/unit/tracking.tcl +++ b/tests/unit/tracking.tcl @@ -265,7 +265,7 @@ start_server {tags {"tracking network logreqres:skip"}} { assert_equal "PONG" [r ping] } - test {RESP3 Client gets tracking-redir-broken push message after cached key changed when rediretion client is terminated} { + test {RESP3 Client gets tracking-redir-broken push message after cached key changed when redirection client is terminated} { # make sure r is working resp 3 r HELLO 3 r CLIENT TRACKING on REDIRECT $redir_id diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index fe61d5de6c1..047defecf74 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -47,7 +47,7 @@ start_server { r XADD mystream * a 1 r XADD mystream * b 2 # XREADGROUP should return only the new elements "a 1" "b 1" - # and not the element "foo bar" which was pre existing in the + # and not the element "foo bar" which was preexisting in the # stream (see previous test) set reply [ r XREADGROUP GROUP mygroup consumer-1 STREAMS mystream ">" diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 89196f374d0..9126abebddc 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -1081,7 +1081,7 @@ start_server {tags {"stream"}} { assert_equal [dict get $reply max-deleted-entry-id] "2-0" } - test {XADD with artial ID with maximal seq} { + test {XADD with partial ID with maximal seq} { r DEL x r XADD x 1-18446744073709551615 f1 v1 assert_error {*The ID specified in XADD is equal or smaller*} {r XADD x 1-* f2 v2} diff --git a/valkey.conf b/valkey.conf index 272a394000b..3b795f66519 100644 --- a/valkey.conf +++ b/valkey.conf @@ -706,7 +706,7 @@ dir ./ # is still in progress, the replica can act in two different ways: # # 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will -# still reply to client requests, possibly with out of date data, or the +# still reply to client requests, possibly with out-of-date data, or the # data set may just be empty if this is the first synchronization. # # 2) If replica-serve-stale-data is set to 'no' the replica will reply with error From d9ce598b07c084270582abe6e85488e217271755 Mon Sep 17 00:00:00 2001 From: Recoordinate <296084221+latent-9@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:46:05 +1200 Subject: [PATCH 17/27] Remove duplicate unreachable "command" branch in fuzzer arg generator (#4368) In `generateStringArgValue`, the `argName` dispatch chain contains two identical `else if (strcmp(argName, "command") == 0)` branches. The earlier branch already handles every `argName == "command"` case, so the second one, whose body is identical, is unreachable dead code. This removes the duplicate branch; behavior is unchanged. Signed-off-by: latent-9 <296084221+latent-9@users.noreply.github.com> --- src/fuzzer_command_generator.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/fuzzer_command_generator.c b/src/fuzzer_command_generator.c index 71f313e6edf..300828a11e3 100644 --- a/src/fuzzer_command_generator.c +++ b/src/fuzzer_command_generator.c @@ -1548,8 +1548,6 @@ static void generateStringArgValue(FuzzerCommand *cmd, const char *argName, Comm appendArg(cmd, sdscatprintf(sdsempty(), "module-%d", rand() % 100)); } else if (strcmp(argName, "arg") == 0 || strcmp(argName, "args") == 0) { appendArg(cmd, sdscatprintf(sdsempty(), "arg%d", rand() % 10)); - } else if (strcmp(argName, "command") == 0) { - appendArg(cmd, sdsnew(commands[rand() % (sizeof(commands) / sizeof(commands[0]))])); } else if (strcmp(argName, "threshold") == 0) { appendArg(cmd, sdscatprintf(sdsempty(), "%d", rand() % 30)); } else if (strcmp(argName, "metric") == 0) { From bb3b927f200e47fa8c73fdc226789c80904a4651 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:46:29 -0400 Subject: [PATCH 18/27] Improve spelling and grammar (#2253) Improves spelling and grammar across codebase --------- Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- src/crccombine.c | 2 +- src/geo.c | 2 +- src/module.c | 2 +- src/modules/lua/script_lua.c | 2 +- src/rdb.c | 4 ++-- src/rdma.c | 2 +- src/replication.c | 2 +- tests/integration/aof-multi-part.tcl | 6 +++--- tests/integration/replication.tcl | 2 +- tests/modules/basics.c | 2 +- tests/unit/cluster/failure-marking.tcl | 2 +- tests/unit/expire.tcl | 2 +- tests/unit/multi.tcl | 2 +- tests/unit/tracking.tcl | 2 +- valkey.conf | 2 +- 15 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/crccombine.c b/src/crccombine.c index 0913fa9b96c..ddf3e292dc6 100644 --- a/src/crccombine.c +++ b/src/crccombine.c @@ -219,7 +219,7 @@ void init_combine_cache(uint64_t poly, uint8_t dim) { * WARNING: if you enable USE_STATIC_COMBINE_CACHE to make this fast, you MUST * ALWAYS USE THE SAME POLYNOMIAL, otherwise you will get the wrong results. * You MAY bzero() the even/odd static arrays, which will induce a re-cache on - * next call as a work-around, but ... maybe just parameterize the cached + * next call as a workaround, but ... maybe just parameterize the cached * models at that point like Mark Adler does in modern crcany/crc.c . */ diff --git a/src/geo.c b/src/geo.c index 439b4b2c22b..8b10a98fb61 100644 --- a/src/geo.c +++ b/src/geo.c @@ -525,7 +525,7 @@ void geoaddCommand(client *c) { * [COUNT count [ANY]] [STORE key|STOREDIST key] * GEORADIUSBYMEMBER key member radius unit ... options ... * GEOSEARCH key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit] - * [BYBOX width height unit] [WITHCOORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC] + * [BYBOX width height unit] [WITHCOORD] [WITHDIST] [WITHHASH] [COUNT count [ANY]] [ASC|DESC] * GEOSEARCHSTORE dest_key src_key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit] * [BYBOX width height unit] [COUNT count [ANY]] [ASC|DESC] [STOREDIST] * */ diff --git a/src/module.c b/src/module.c index 366d366a0b3..42e122bcb08 100644 --- a/src/module.c +++ b/src/module.c @@ -13099,7 +13099,7 @@ void moduleInitModulesSystem(void) { /* Create a pipe for module threads to be able to wake up the server main thread. * Make the pipe non blocking. This is just a best effort aware mechanism - * and we do not want to block not in the read nor in the write half. + * and we want to avoid blocking in both the read and write halves. * Enable close-on-exec flag on pipes in case of the fork-exec system calls in * sentinels or servers. */ if (anetPipe(server.module_pipe, O_CLOEXEC | O_NONBLOCK, O_CLOEXEC | O_NONBLOCK) == -1) { diff --git a/src/modules/lua/script_lua.c b/src/modules/lua/script_lua.c index 91f762be847..44af4f56b03 100644 --- a/src/modules/lua/script_lua.c +++ b/src/modules/lua/script_lua.c @@ -2053,7 +2053,7 @@ void luaCallFunction(ValkeyModuleCtx *ctx, int delhook = 0; /* We must set it before we set the Lua hook, theoretically the - * Lua hook might be called wheneven we run any Lua instruction + * Lua hook might be called whenever we run any Lua instruction * such as 'luaSetGlobalArray' and we want the run_ctx to be available * each time the Lua hook is invoked. */ diff --git a/src/rdb.c b/src/rdb.c index e603c0356b8..391ae7e19d8 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1871,7 +1871,7 @@ static int _listZiplistEntryConvertAndValidate(unsigned char *p, unsigned int he return 1; } -/* callback for to check the listpack doesn't have duplicate records */ +/* callback to check the listpack doesn't have duplicate records */ static int _lpEntryValidation(unsigned char *p, unsigned int head_count, void *userdata) { struct { int pairs; @@ -3111,7 +3111,7 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { * message on failure. * * The lib_ctx argument is also optional. If NULL is given, only verify rdb - * structure with out performing the actual functions loading. */ + * structure without performing the actual functions loading. */ int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx *lib_ctx, int rdbflags, sds *err) { UNUSED(ver); sds error = NULL; diff --git a/src/rdma.c b/src/rdma.c index 50f75fb9a10..c0def54d5cd 100644 --- a/src/rdma.c +++ b/src/rdma.c @@ -732,7 +732,7 @@ static void connRdmaEventHandler(struct aeEventLoop *el, int fd, void *clientDat return; } - /* uplayer should read all */ + /* up layer should read all */ while (!(rdma_conn->postpone_mask & CONN_POSTPONE_READ) && ctx->rx.pos < ctx->rx.offset) { /* When an IO-thread read completed but processClientIOReadsDone has not run yet, * readQueryFromClient cannot consume RDMA RX; without this break the read_handler diff --git a/src/replication.c b/src/replication.c index 6afbee33c0c..0e3b15b8155 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2294,7 +2294,7 @@ void disklessLoadDiscardTempDb(serverDb **tempDb) { discardTempDb(tempDb); } -/* Helper function for to initialize temp function lib context. +/* Helper function to initialize temp function lib context. * The temp ctx may be populated by functionsLibCtxSwapWithCurrent or * freed by disklessLoadDiscardFunctionsLibCtx later. */ functionsLibCtx *disklessLoadFunctionsLibCtxCreate(void) { diff --git a/tests/integration/aof-multi-part.tcl b/tests/integration/aof-multi-part.tcl index 02948401668..a0b3093ebd2 100644 --- a/tests/integration/aof-multi-part.tcl +++ b/tests/integration/aof-multi-part.tcl @@ -627,7 +627,7 @@ tags {"external:skip"} { clean_aof_persistence $aof_dirpath } - test {Multi Part AOF can upgrade when when two servers share the same server dir} { + test {Multi Part AOF can upgrade when two servers share the same server dir} { create_aof $server_path $aof_old_name_old_path { append_to_aof [formatCommand set k1 v1] append_to_aof [formatCommand set k2 v2] @@ -646,7 +646,7 @@ tags {"external:skip"} { start_server [list overrides [list dir $server_path appendonly yes appendfilename appendonly.aof2]] { set valkey2 [valkey [srv host] [srv port] 0 $::tls] - test "Multi Part AOF can upgrade when when two servers share the same server dir (server1)" { + test "Multi Part AOF can upgrade when two servers share the same server dir (server1)" { wait_done_loading $valkey1 assert_equal v1 [$valkey1 get k1] assert_equal v2 [$valkey1 get k2] @@ -677,7 +677,7 @@ tags {"external:skip"} { assert {$d1 eq $d2} } - test "Multi Part AOF can upgrade when when two servers share the same server dir (server2)" { + test "Multi Part AOF can upgrade when two servers share the same server dir (server2)" { wait_done_loading $valkey2 assert_equal 0 [$valkey2 exists k1] diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 038560469fd..238c0366faa 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -1461,7 +1461,7 @@ test {replica can handle EINTR if use diskless load} { set res [wait_for_log_messages -1 {"*Loading DB in memory*"} 0 200 10] set loglines [lindex $res 1] - # Wait till we see the watchgod log line AFTER the loading started + # Wait till we see the watchdog log line AFTER the loading started wait_for_log_messages -1 {"*WATCHDOG TIMER EXPIRED*"} $loglines 200 10 # Make sure we're still loading, and that there was just one full sync attempt diff --git a/tests/modules/basics.c b/tests/modules/basics.c index bc0bd7e7ae6..4b961a3d490 100644 --- a/tests/modules/basics.c +++ b/tests/modules/basics.c @@ -484,7 +484,7 @@ int TestCallResp3Set(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) /* * The order of elements on sets are not promised so we just - * veridy that the reply matches one of the elements. + * verify that the reply matches one of the elements. */ if (!TestMatchReply(val0,"v1") && !TestMatchReply(val0,"v2")) goto fail; if (!TestMatchReply(val1,"v1") && !TestMatchReply(val1,"v2")) goto fail; diff --git a/tests/unit/cluster/failure-marking.tcl b/tests/unit/cluster/failure-marking.tcl index 8552680a865..8cf6108007d 100644 --- a/tests/unit/cluster/failure-marking.tcl +++ b/tests/unit/cluster/failure-marking.tcl @@ -47,7 +47,7 @@ start_cluster 2 1 {tags {external:skip cluster}} { wait_node_marked_pfail 0 $replica1_instance_id - # Resume other primary and wait for to show replica as failed + # Resume other primary and wait for the replica to show as failed resume_process $primary2_pid wait_node_marked_fail 0 $replica1_instance_id diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl index 9a2f6b27142..89d4badc046 100644 --- a/tests/unit/expire.tcl +++ b/tests/unit/expire.tcl @@ -791,7 +791,7 @@ start_server {tags {"expire"}} { assert_equal [r TTL foo] -2 } {} - test {EXPIRE with negative expiry on a non-valitale key} { + test {EXPIRE with negative expiry on a non-volatile key} { r SET foo bar assert_equal [r EXPIRE foo -10 LT] 1 assert_equal [r TTL foo] -2 diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 2db5da889fb..0b3f38e63f1 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -867,7 +867,7 @@ start_server {tags {"multi"}} { r XADD mystream * foo3 bar3 r XGROUP CREATE mystream mygroup 0 - # make sure the XCALIM (propagated by XREADGROUP) is indeed inside MULTI/EXEC + # make sure the XCLAIM (propagated by XREADGROUP) is indeed inside MULTI/EXEC r multi r XREADGROUP GROUP mygroup consumer1 COUNT 2 STREAMS mystream ">" r XREADGROUP GROUP mygroup consumer1 STREAMS mystream ">" diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl index 5daaa749f36..77937895b32 100644 --- a/tests/unit/tracking.tcl +++ b/tests/unit/tracking.tcl @@ -166,7 +166,7 @@ start_server {tags {"tracking network logreqres:skip"}} { test {Tracking gets notification of lazy expired keys} { r CLIENT TRACKING off r CLIENT TRACKING on BCAST REDIRECT $redir_id NOLOOP - # Use multi-exec to expose a race where the key gets an two invalidations + # Use multi-exec to expose a race where the key gets two invalidations # in the same event loop, once by the client so filtered by NOLOOP, and # the second one by the lazy expire r MULTI diff --git a/valkey.conf b/valkey.conf index 3b795f66519..2992101b4e2 100644 --- a/valkey.conf +++ b/valkey.conf @@ -1981,7 +1981,7 @@ aof-timestamp-enabled no # # This is useful for two cases. The first case is for when an application # doesn't require consistency of data during node failures or network partitions. -# One example of this is a cache, where as long as the node has the data it +# One example of this is a cache, where, as long as the node has the data, it # should be able to serve it. # # The second use case is for configurations that don't meet the recommended From 7b374fcf7b7bf2a93bb08e6e356b504206456d8b Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Thu, 13 Aug 2026 00:41:42 -0400 Subject: [PATCH 19/27] Grammar corrections in comments (#2239) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grammar corrections in comments --------- Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> Signed-off-by: Viktor Söderqvist Co-authored-by: Viktor Söderqvist --- sentinel.conf | 2 +- src/db.c | 2 +- src/debug.c | 2 +- src/fmacros.h | 2 +- src/hyperloglog.c | 2 +- src/listpack.c | 2 +- src/module.c | 2 +- src/modules/helloworld.c | 2 +- src/rax.h | 2 +- src/replication.c | 6 +++--- src/sentinel.c | 4 ++-- src/tracking.c | 2 +- src/unit/test_rax.cpp | 2 +- src/valkey-cli.c | 2 +- tests/instances.tcl | 2 +- tests/support/cluster.tcl | 6 +++--- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/sentinel.conf b/sentinel.conf index 3db10fe8b48..53cb011d7e6 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -242,7 +242,7 @@ sentinel failover-timeout mymaster 180000 # If script exits with "1" the execution is retried later (up to a maximum # number of times currently set to 10). # -# If script exits with "2" (or an higher value) the script execution is +# If script exits with "2" (or a higher value) the script execution is # not retried. # # If script terminates because it receives a signal the behavior is the same diff --git a/src/db.c b/src/db.c index 23019505e0b..2670d386dfd 100644 --- a/src/db.c +++ b/src/db.c @@ -793,7 +793,7 @@ void signalFlushedDb(int dbid, int async) { /* Return the set of flags to use for the emptyData() call for FLUSHALL * and FLUSHDB commands. * - * sync: flushes the database in an sync manner. + * sync: flushes the database in a sync manner. * async: flushes the database in an async manner. * no option: determine sync or async according to the value of lazyfree-lazy-user-flush. * diff --git a/src/debug.c b/src/debug.c index 649624a4cf9..05864e5166a 100644 --- a/src/debug.c +++ b/src/debug.c @@ -820,7 +820,7 @@ void debugCommand(client *c) { addReplyStatus(c, d); sdsfree(d); } else if (!strcasecmp(objectGetVal(c->argv[1]), "digest-value") && c->argc >= 2) { - /* DEBUG DIGEST-VALUE key key key ... key. */ + /* DEBUG DIGEST-VALUE key key ... key. */ addReplyArrayLen(c, c->argc - 2); for (int j = 2; j < c->argc; j++) { unsigned char digest[20]; diff --git a/src/fmacros.h b/src/fmacros.h index af2e9235354..6a46090dad4 100644 --- a/src/fmacros.h +++ b/src/fmacros.h @@ -54,7 +54,7 @@ #define _XOPEN_SOURCE 700 /* * On NetBSD, _XOPEN_SOURCE undefines _NETBSD_SOURCE and - * thus hides inet_aton etc. + * thus hides inet_aton, etc. */ #elif !defined(__NetBSD__) #define _XOPEN_SOURCE diff --git a/src/hyperloglog.c b/src/hyperloglog.c index 1a4d71c1a4f..4372df0e5cc 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -1660,7 +1660,7 @@ int isHLLObjectOrReply(client *c, robj *o) { return C_ERR; } -/* PFADD var ele ele ele ... ele => :0 or :1 */ +/* PFADD var ele ele ... ele => :0 or :1 */ void pfaddCommand(client *c) { robj *o = lookupKeyWrite(c->db, c->argv[1]); struct hllhdr *hdr; diff --git a/src/listpack.c b/src/listpack.c index efe184713e2..5668f19a8e4 100644 --- a/src/listpack.c +++ b/src/listpack.c @@ -490,7 +490,7 @@ unsigned long lpLength(unsigned char *lp) { * If the function is called against a badly encoded ziplist, so that there * is no valid way to parse it, the function returns like if there was an * integer encoded with value 12345678900000000 + , this may - * be an hint to understand that something is wrong. To crash in this case is + * be a hint to understand that something is wrong. To crash in this case is * not sensible because of the different requirements of the application using * this lib. * diff --git a/src/module.c b/src/module.c index 42e122bcb08..82d83b2092d 100644 --- a/src/module.c +++ b/src/module.c @@ -11900,7 +11900,7 @@ size_t VM_MallocSizeDict(ValkeyModuleDict *dict) { return size; } -/* Return the a number between 0 to 1 indicating the amount of memory +/* Return a number between 0 to 1 indicating the amount of memory * currently used, relative to the server "maxmemory" configuration. * * * 0 - No memory limit configured. diff --git a/src/modules/helloworld.c b/src/modules/helloworld.c index d61c6d03db7..9939a06394f 100644 --- a/src/modules/helloworld.c +++ b/src/modules/helloworld.c @@ -70,7 +70,7 @@ int HelloPushNative_ValkeyCommand(ValkeyModuleCtx *ctx, ValkeyModuleString **arg return VALKEYMODULE_OK; } -/* HELLO.PUSH.CALL implements RPUSH using an higher level approach, calling +/* HELLO.PUSH.CALL implements RPUSH using a higher level approach, calling * a command instead of working with the key in a low level way. This * approach is useful when you need to call commands that are not * available as low level APIs, or when you don't need the maximum speed diff --git a/src/rax.h b/src/rax.h index 2d0c940698a..0d285eab07f 100644 --- a/src/rax.h +++ b/src/rax.h @@ -142,7 +142,7 @@ typedef struct rax { * field for space concerns, so we use the auxiliary stack when needed. */ #define RAX_STACK_STATIC_ITEMS 32 typedef struct raxStack { - void **stack; /* Points to static_items or an heap allocated array. */ + void **stack; /* Points to static_items or a heap allocated array. */ size_t items, maxitems; /* Number of items contained and total space. */ /* Up to RAXSTACK_STACK_ITEMS items we avoid to allocate on the heap * and use this static array of pointers instead. */ diff --git a/src/replication.c b/src/replication.c index 0e3b15b8155..86d9d628449 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1418,7 +1418,7 @@ void freeClientReplicationData(client *c) { * The replica reports its version. * * - rdb-channel <1|0> - * Used to identify the client as a replica's rdb connection in an dual channel + * Used to identify the client as a replica's rdb connection in a dual channel * sync session. * * - set-rdb-client-id @@ -4301,7 +4301,7 @@ void syncWithPrimary(connection *conn) { } } - /* If the primary is in an transient error, we should try to PSYNC + /* If the primary is in a transient error, we should try to PSYNC * from scratch later, so go to the error path. This happens when * the server is loading the dataset or is not connected with its * primary and so forth. */ @@ -5401,7 +5401,7 @@ void replicationCron(void) { /* Second, send a newline to all the replicas in pre-synchronization * stage, that is, replicas waiting for the primary to create the RDB file. * - * Also send the a newline to all the chained replicas we have, if we lost + * Also send a newline to all the chained replicas we have, if we lost * connection from our primary, to keep the replicas aware that their * primary is online. This is needed since sub-replicas only receive proxied * data from top-level primaries, so there is no explicit pinging in order diff --git a/src/sentinel.c b/src/sentinel.c index 5ea89857e2d..d48999794c0 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1482,7 +1482,7 @@ sentinelValkeyInstance *getSentinelValkeyInstanceByAddrAndRunID(dict *instances, serverAssert(addr || runid); /* User must pass at least one search param. */ if (addr != NULL) { /* Try to resolve addr. If hostnames are used, we're accepting an ri_addr - * that contains an hostname only and can still be matched based on that. + * that contains a hostname only and can still be matched based on that. */ ri_addr = createSentinelAddr(addr, port, 1); if (!ri_addr) return NULL; @@ -3778,7 +3778,7 @@ void sentinelCommand(client *c) { " failover.", "CONFIG SET param value [param value ...]", " Set a global Sentinel configuration parameter.", - "CONFIG GET [param param param ...]", + "CONFIG GET param [param ...]", " Get global Sentinel configuration parameter.", "DEBUG [ ...]", " Show a list of configurable time parameters and their values (milliseconds).", diff --git a/src/tracking.c b/src/tracking.c index 532beb7df92..00f030bd924 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -45,7 +45,7 @@ rax *TrackingTable = NULL; rax *PrefixTable = NULL; uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across the whole tracking table. This gives - an hint about the total memory we + a hint about the total memory we are using server side for CSC. */ robj *TrackingChannelName; diff --git a/src/unit/test_rax.cpp b/src/unit/test_rax.cpp index eace50ddd7a..ef3051f61d7 100644 --- a/src/unit/test_rax.cpp +++ b/src/unit/test_rax.cpp @@ -358,7 +358,7 @@ int fuzzTestCluster(size_t count, double addprob, double remprob) { /* Generate a random key by altering our template key. */ /* With a given probability, let's use a common prefix so that there - * is a subset of keys that have an higher percentage of probability + * is a subset of keys that have a higher percentage of probability * of being hit again and again. */ size_t commonprefix = genrand64_int64() & 0xf; if (commonprefix == 0) memcpy(key + 10, "2e68e5", 6); diff --git a/src/valkey-cli.c b/src/valkey-cli.c index b5bb23cb183..30560c10f8a 100644 --- a/src/valkey-cli.c +++ b/src/valkey-cli.c @@ -3775,7 +3775,7 @@ clusterManagerCommandDef clusterManagerCommands[] = { {"add-node", clusterManagerCommandAddNode, 2, "new_host:new_port existing_host:existing_port", "replica,primaries-id "}, {"del-node", clusterManagerCommandDeleteNode, 2, "host:port node_id", NULL}, - {"call", clusterManagerCommandCall, -2, "host:port command arg arg .. arg", "only-primaries,only-replicas"}, + {"call", clusterManagerCommandCall, -2, "host:port command arg arg ... arg", "only-primaries,only-replicas"}, {"set-timeout", clusterManagerCommandSetTimeout, 2, "host:port milliseconds", NULL}, {"import", clusterManagerCommandImport, 1, "host:port", "from ,from-user ,from-pass ,from-askpass,copy,replace"}, diff --git a/tests/instances.tcl b/tests/instances.tcl index 9e23af30004..02698796e7a 100644 --- a/tests/instances.tcl +++ b/tests/instances.tcl @@ -614,7 +614,7 @@ proc end_tests {} { # The "S" command is used to interact with the N-th Sentinel. # The general form is: # -# S command arg arg arg ... +# S command arg [arg ...] # # Example to ping the Sentinel 0 (first instance): S 0 PING proc S {n args} { diff --git a/tests/support/cluster.tcl b/tests/support/cluster.tcl index be946d8ea73..490c2087263 100644 --- a/tests/support/cluster.tcl +++ b/tests/support/cluster.tcl @@ -51,9 +51,9 @@ proc valkey_cluster {nodes {tls -1}} { # Totally reset the slots / nodes state for the client, calls # CLUSTER NODES in the first startup node available, populates the -# list of nodes ::valkey_cluster::nodes($id) with an hash mapping node +# list of nodes ::valkey_cluster::nodes($id) with a hash mapping node # ip:port to a representation of the node (another hash), and finally -# maps ::valkey_cluster::slots($id) with an hash mapping slot numbers +# maps ::valkey_cluster::slots($id) with a hash mapping slot numbers # to node IDs. # # This function is called when a new Cluster client is initialized @@ -116,7 +116,7 @@ proc ::valkey_cluster::__method__refresh_nodes_map {id} { set tls $::valkey_cluster::tls($id) catch {set link [valkey $host $port 0 $tls]} - # Build this node description as an hash. + # Build this node description as a hash. set node [dict create \ id $nodeid \ internal_id $id \ From 8cec1092b92ee2cbecb5101465ba36c1708059bc Mon Sep 17 00:00:00 2001 From: Roshan Khatri <117414976+roshkhatri@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:03:56 -0700 Subject: [PATCH 20/27] Validate stream listpack live and deleted record counts on load (#4381) Stream listpack master entries store separate live and deleted record counts, but integrity validation only verified their sum. A corrupted payload could therefore preserve the total while understating the live count, causing XLEN to disagree with the stored records and allowing XDEL to discard records not accounted for by the header. Count live and deleted records independently while validating stream listpacks and reject payloads when either count differs from its declared value. Signed-off-by: Roshan Khatri --- src/t_stream.c | 13 +++++ src/unit/test_t_stream.cpp | 90 ++++++++++++++++++++++++++++++ tests/integration/corrupt-dump.tcl | 32 +++++++++++ 3 files changed, 135 insertions(+) diff --git a/src/t_stream.c b/src/t_stream.c index 3a613c6f42e..f7aed1492e4 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -4038,12 +4038,24 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, uint64_t *va p = next; if (!lpValidateNext(lp, &next, size)) return 0; + /* Only the sum of the two header counts is validated by the traversal + * below, so count the live and deleted records actually present and + * reconcile both. streamIteratorRemoveEntry() frees the whole node once the + * declared live count reaches 1, so a header that understates it makes XDEL + * destroy the records it failed to account for. */ + int64_t declared_live = entry_count, declared_deleted = deleted_count; + uint64_t live_records = 0, deleted_records = 0; + entry_count += deleted_count; while (entry_count--) { if (!p) return 0; int64_t fields = primary_fields, extra_fields = 3; int64_t flags = lpGetIntegerIfValid(p, &valid_record); if (!valid_record) return 0; + if (flags & STREAM_ITEM_FLAG_DELETED) + deleted_records++; + else + live_records++; p = next; if (!lpValidateNext(lp, &next, size)) return 0; @@ -4088,6 +4100,7 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, uint64_t *va } if (next) return 0; + if (live_records != (uint64_t)declared_live || deleted_records != (uint64_t)declared_deleted) return 0; return 1; } diff --git a/src/unit/test_t_stream.cpp b/src/unit/test_t_stream.cpp index 84b0438495b..3d92451c807 100644 --- a/src/unit/test_t_stream.cpp +++ b/src/unit/test_t_stream.cpp @@ -9,9 +9,99 @@ #include extern "C" { +#include "listpack.h" #include "stream.h" } +/* Mirrors of the record flags private to t_stream.c. */ +#define TEST_STREAM_ITEM_FLAG_DELETED (1 << 0) +#define TEST_STREAM_ITEM_FLAG_SAMEFIELDS (1 << 1) + +/* Build a structurally valid stream listpack holding two same-fields records. + * The declared header counts and the per-record deleted flags are supplied + * separately so a caller can describe a header whose live/deleted split + * disagrees with the records that actually follow. */ +static unsigned char *buildTwoRecordStreamListpack(long long declared_live, + long long declared_deleted, + int first_deleted, + int second_deleted) { + unsigned char *lp = lpNew(0); + + /* Primary entry: count, deleted, num-primary-fields, field, terminator. */ + lp = lpAppendInteger(lp, declared_live); + lp = lpAppendInteger(lp, declared_deleted); + lp = lpAppendInteger(lp, 1); + lp = lpAppend(lp, (unsigned char *)"f", 1); + lp = lpAppendInteger(lp, 0); + + /* Two records, each reusing the primary field, so lp-count is 1 field + * plus the three fixed elements. */ + for (int i = 0; i < 2; i++) { + int deleted = i == 0 ? first_deleted : second_deleted; + long long flags = TEST_STREAM_ITEM_FLAG_SAMEFIELDS; + if (deleted) flags |= TEST_STREAM_ITEM_FLAG_DELETED; + + lp = lpAppendInteger(lp, flags); + lp = lpAppendInteger(lp, 0); /* ms diff */ + lp = lpAppendInteger(lp, i); /* seq diff */ + lp = lpAppend(lp, (unsigned char *)(i == 0 ? "v1" : "v2"), 2); + lp = lpAppendInteger(lp, 4); + } + + return lp; +} + +class StreamListpackIntegrityTest : public ::testing::Test {}; + +/* Control: the header split matches the records, so the payload is accepted. + * This keeps the mismatch tests below honest by proving the hand-built + * fixture is otherwise structurally valid. */ +TEST_F(StreamListpackIntegrityTest, TestAcceptsMatchingLiveAndDeletedCounts) { + unsigned char *lp = buildTwoRecordStreamListpack(2, 0, 0, 0); + uint64_t valid_count = 0; + ASSERT_EQ(lpLength(lp), 15u); + ASSERT_EQ(streamValidateListpackIntegrity(lp, lpBytes(lp), &valid_count), 1); + ASSERT_EQ(valid_count, 2u); + lpFree(lp); +} + +TEST_F(StreamListpackIntegrityTest, TestAcceptsMatchingDeletedRecord) { + unsigned char *lp = buildTwoRecordStreamListpack(1, 1, 0, 1); + uint64_t valid_count = 0; + ASSERT_EQ(streamValidateListpackIntegrity(lp, lpBytes(lp), &valid_count), 1); + ASSERT_EQ(valid_count, 1u); + lpFree(lp); +} + +/* Both records are live, but the header claims one live and one deleted. The + * total still equals two, so a total-only check accepts this payload while the + * live count is understated. XDEL then sees a declared live count of 1, frees + * the whole node and destroys the record the header did not account for. */ +TEST_F(StreamListpackIntegrityTest, TestRejectsUnderstatedLiveCount) { + unsigned char *lp = buildTwoRecordStreamListpack(1, 1, 0, 0); + uint64_t valid_count = 0; + ASSERT_EQ(lpLength(lp), 15u); + ASSERT_EQ(streamValidateListpackIntegrity(lp, lpBytes(lp), &valid_count), 0); + lpFree(lp); +} + +/* The mirrored case: one record is flagged deleted while the header claims + * two live records and no deleted ones. The total again matches. */ +TEST_F(StreamListpackIntegrityTest, TestRejectsOverstatedLiveCount) { + unsigned char *lp = buildTwoRecordStreamListpack(2, 0, 0, 1); + uint64_t valid_count = 0; + ASSERT_EQ(streamValidateListpackIntegrity(lp, lpBytes(lp), &valid_count), 0); + lpFree(lp); +} + +/* Every record is flagged deleted while the header claims both are live. */ +TEST_F(StreamListpackIntegrityTest, TestRejectsAllRecordsDeletedWithLiveHeader) { + unsigned char *lp = buildTwoRecordStreamListpack(2, 0, 1, 1); + uint64_t valid_count = 0; + ASSERT_EQ(streamValidateListpackIntegrity(lp, lpBytes(lp), &valid_count), 0); + lpFree(lp); +} + class StreamIdTest : public ::testing::Test {}; TEST_F(StreamIdTest, TestStreamEncodeDecodeRoundtrip) { diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl index f1371addf98..a0983d99c4e 100644 --- a/tests/integration/corrupt-dump.tcl +++ b/tests/integration/corrupt-dump.tcl @@ -794,6 +794,38 @@ test {corrupt payload: stream listpack with negative deleted count} { } } +test {corrupt payload: stream listpack live and deleted counts do not match the records} { + # A master entry may declare a live/deleted split that disagrees with the + # records that follow. Only the sum is validated by the entry loop, and the + # stream length here matches the declared live count, so the payload passes + # both checks while understating the live records. XDEL then reads the + # declared live count of 1, treats the node as holding its last live record + # and frees the whole node, destroying the record the header did not account + # for. XLEN disagrees with XRANGE until that happens. + # + # Built from a valid two-live-record control by changing the master entry + # count from 2 to 1, its deleted count from 0 to 1 and the stream length + # from 2 to 1, then recomputing the DUMP CRC64. All three encode in the + # same byte width, so no other byte moves. + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + catch {r restore _split_mismatch 0 "\x15\x01\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x2D\x2D\x00\x00\x00\x11\x00\x01\x01\x01\x01\x01\x01\x81\x66\x02\x00\x01\x02\x01\x00\x01\x00\x01\x81\x76\x02\x04\x01\x00\x01\x01\x01\x00\x01\x01\x01\x81\x67\x02\x81\x77\x02\x06\x01\xFF\x01\x02\x00\x01\x00\x00\x00\x02\x00\x50\x00\x90\x7A\xE8\x68\xB6\x55\xB2\x22"} err + assert_match "*Bad data format*" $err + assert_equal 0 [r exists _split_mismatch] + verify_log_message 0 "*Stream listpack integrity check failed*" 0 + + # The control differs only in those three bytes, so it must still load + # and delete one record without disturbing the other. + set stream_valid "\x15\x01\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x2D\x2D\x00\x00\x00\x11\x00\x02\x01\x00\x01\x01\x01\x81\x66\x02\x00\x01\x02\x01\x00\x01\x00\x01\x81\x76\x02\x04\x01\x00\x01\x01\x01\x00\x01\x01\x01\x81\x67\x02\x81\x77\x02\x06\x01\xFF\x02\x02\x00\x01\x00\x00\x00\x02\x00\x50\x00\xA1\xDF\xE0\xE1\x48\xC5\xF8\x85" + assert_equal OK [r restore _control 0 $stream_valid] + assert_equal 2 [r xlen _control] + assert_equal {{1-0 {f v}} {2-0 {g w}}} [r xrange _control - +] + assert_equal 1 [r xdel _control 1-0] + assert_equal 1 [r xlen _control] + assert_equal {{2-0 {g w}}} [r xrange _control - +] + assert_equal [r ping] "PONG" + } +} + test {corrupt payload: fuzzer findings - streamLastValidID panic} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { r debug set-skip-checksum-validation 1 From 3f16ffa044c6e2dfe40cecfd3316df4617e1f703 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:15:36 -0400 Subject: [PATCH 21/27] Consolidating nonexistent spelling (#2247) Consolidating nonexistent spelling Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- src/acl.c | 2 +- src/bitops.c | 2 +- src/hyperloglog.c | 4 ++-- src/module.c | 4 ++-- src/networking.c | 2 +- src/rax.c | 2 +- src/t_set.c | 4 ++-- src/t_stream.c | 2 +- tests/unit/bitops.tcl | 8 ++++---- tests/unit/dump.tcl | 2 +- tests/unit/expire.tcl | 2 +- tests/unit/geo.tcl | 4 ++-- tests/unit/info.tcl | 2 +- tests/unit/introspection-2.tcl | 4 ++-- tests/unit/keyspace.tcl | 4 ++-- tests/unit/moduleapi/moduleauth.tcl | 4 ++-- tests/unit/type/hash.tcl | 20 +++++++++---------- tests/unit/type/incr.tcl | 4 ++-- tests/unit/type/list.tcl | 22 ++++++++++----------- tests/unit/type/set.tcl | 30 ++++++++++++++--------------- tests/unit/type/stream.tcl | 4 ++-- tests/unit/type/string.tcl | 4 ++-- tests/unit/type/zset.tcl | 12 ++++++------ 23 files changed, 74 insertions(+), 74 deletions(-) diff --git a/src/acl.c b/src/acl.c index ecc8772ccee..74a68224228 100644 --- a/src/acl.c +++ b/src/acl.c @@ -659,7 +659,7 @@ static void ACLChangeSelectorPerm(aclSelector *selector, struct serverCommand *c /* This is like ACLSetSelectorCommandBit(), but instead of setting the specified * ID, it will check all the commands in the category specified as argument, * and will set all the bits corresponding to such commands to the specified - * value. Since the category passed by the user may be non existing, the + * value. Since the category passed by the user may be nonexistent, the * function returns C_ERR if the category was not found, or C_OK if it was * found and the operation was performed. */ static void ACLSetSelectorCommandBitsForCategory(hashtable *commands, aclSelector *selector, uint64_t cflag, int value) { diff --git a/src/bitops.c b/src/bitops.c index c0802a1c7c6..475eafe759e 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -1011,7 +1011,7 @@ void bitcountCommand(client *c) { return; } - /* Return 0 for non existing keys. */ + /* Return 0 for nonexistent keys. */ if (o == NULL) { addReply(c, shared.czero); return; diff --git a/src/hyperloglog.c b/src/hyperloglog.c index 4372df0e5cc..230f61f85f3 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -1717,7 +1717,7 @@ void pfcountCommand(client *c) { for (j = 1; j < c->argc; j++) { /* Check type and size. */ robj *o = lookupKeyRead(c->db, c->argv[j]); - if (o == NULL) continue; /* Assume empty HLL for non existing var.*/ + if (o == NULL) continue; /* Assume empty HLL for nonexistent var.*/ if (isHLLObjectOrReply(c, o) != C_OK) return; /* Merge with this HLL with our 'max' HLL by setting max[i] @@ -1805,7 +1805,7 @@ void pfmergeCommand(client *c) { for (j = 1; j < c->argc; j++) { /* Check type and size. */ robj *o = lookupKeyRead(c->db, c->argv[j]); - if (o == NULL) continue; /* Assume empty HLL for non existing var. */ + if (o == NULL) continue; /* Assume empty HLL for nonexistent var. */ if (isHLLObjectOrReply(c, o) != C_OK) return; /* If at least one involved HLL is dense, use the dense representation diff --git a/src/module.c b/src/module.c index 82d83b2092d..c9d17c4d41c 100644 --- a/src/module.c +++ b/src/module.c @@ -763,7 +763,7 @@ void moduleReleaseTempClient(client *c) { /* Create an empty key of the specified type. `key` must point to a key object * opened for writing where the `.value` member is set to NULL because the - * key was found to be non existing. + * key was found to be nonexistent. * * On success VALKEYMODULE_OK is returned and the key is populated with * the value of the specified type. The function fails and returns @@ -776,7 +776,7 @@ void moduleReleaseTempClient(client *c) { int moduleCreateEmptyKey(ValkeyModuleKey *key, int type) { robj *obj; - /* The key must be open for writing and non existing to proceed. */ + /* The key must be open for writing and nonexistent to proceed. */ if (!(key->mode & VALKEYMODULE_WRITE) || key->value) return VALKEYMODULE_ERR; switch (type) { diff --git a/src/networking.c b/src/networking.c index 3351ecc709c..faa3f4e04d2 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2386,7 +2386,7 @@ void beforeNextClient(client *c) { /* Handle async frees */ /* Note: this doesn't make the server.clients_to_close list redundant because of * cases where we want an async free of a client other than myself. For example - * in ACL modifications we disconnect clients authenticated to non-existent + * in ACL modifications we disconnect clients authenticated to nonexistent * users (see ACL LOAD). */ if (c->flag.close_asap) { freeClient(c); diff --git a/src/rax.c b/src/rax.c index e46972d4c40..ebe60047ea6 100644 --- a/src/rax.c +++ b/src/rax.c @@ -1299,7 +1299,7 @@ void raxIteratorDelChars(raxIterator *it, size_t count) { * lexicographically smaller children, and the current node is already assumed * to be the parent of the last key node, so the first operation to go back to * the parent will be skipped. This option is used by raxSeek() when - * implementing seeking a non existing element with the ">" or "<" options: + * implementing seeking a nonexistent element with the ">" or "<" options: * the starting node is not a key in that particular case, so we start the scan * from a node that does not represent the key set. * diff --git a/src/t_set.c b/src/t_set.c index 9a6b9582173..26913eaadb4 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -1537,7 +1537,7 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum, robj *dstke /* Union is trivial, just add every element of every set to the * temporary set. */ for (j = 0; j < setnum; j++) { - if (!sets[j]) continue; /* non existing keys are like empty sets */ + if (!sets[j]) continue; /* nonexistent keys are like empty sets */ si = setTypeInitIterator(sets[j]); while ((encoding = setTypeNext(si, &str, &len, &llval)) != -1) { @@ -1578,7 +1578,7 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum, robj *dstke * This is O(N) where N is the sum of all the elements in every * set. */ for (j = 0; j < setnum; j++) { - if (!sets[j]) continue; /* non existing keys are like empty sets */ + if (!sets[j]) continue; /* nonexistent keys are like empty sets */ si = setTypeInitIterator(sets[j]); while ((encoding = setTypeNext(si, &str, &len, &llval)) != -1) { diff --git a/src/t_stream.c b/src/t_stream.c index f7aed1492e4..af92e6a0316 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -3146,7 +3146,7 @@ void xpendingCommand(client *c) { * Creates the pending message entry in the PEL even if certain * specified IDs are not already in the PEL assigned to a different * client. However the message must be exist in the stream, otherwise - * the IDs of non existing messages are ignored. + * the IDs of nonexistent messages are ignored. * * 5. JUSTID: * Return just an array of IDs of messages successfully claimed, diff --git a/tests/unit/bitops.tcl b/tests/unit/bitops.tcl index 3db119b7f81..848ebb8e006 100644 --- a/tests/unit/bitops.tcl +++ b/tests/unit/bitops.tcl @@ -56,7 +56,7 @@ start_server {tags {"bitops"}} { assert_error "*WRONGTYPE*" {r bitcount mylist -6 -15 bit} } - test {BITCOUNT returns 0 against non existing key} { + test {BITCOUNT returns 0 against nonexistent key} { r del no-key assert {[r bitcount no-key] == 0} assert {[r bitcount no-key 0 1000 bit] == 0} @@ -73,7 +73,7 @@ start_server {tags {"bitops"}} { assert {[r bitcount str -6 -7] == 0} assert {[r bitcount str -6 -15 bit] == 0} - # against non existing key + # against nonexistent key r del str assert {[r bitcount str -6 -7] == 0} assert {[r bitcount str -6 -15 bit] == 0} @@ -178,7 +178,7 @@ start_server {tags {"bitops"}} { r set s 1 assert_error {ERR *not an integer*} {r bitcount s a b} - # against non existing key + # against nonexistent key r del s assert_error {ERR *not an integer*} {r bitcount s a b} @@ -328,7 +328,7 @@ start_server {tags {"bitops"}} { assert_error {ERR *not an integer*} {r bitpos s a} assert_error {ERR *not an integer*} {r bitpos s 0 a b} - # against non existing key + # against nonexistent key r del s assert_error {ERR *not an integer*} {r bitpos s b} assert_error {ERR *not an integer*} {r bitpos s 0 a b} diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl index e03e30c1fb8..54fdb3e1b17 100644 --- a/tests/unit/dump.tcl +++ b/tests/unit/dump.tcl @@ -142,7 +142,7 @@ start_server {tags {"dump"}} { assert_equal {bar} [r get foo] } - test {DUMP of non existing key returns nil} { + test {DUMP of nonexistent key returns nil} { r dump nonexisting_key } {} diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl index 89d4badc046..b9fe5392523 100644 --- a/tests/unit/expire.tcl +++ b/tests/unit/expire.tcl @@ -67,7 +67,7 @@ start_server {tags {"expire"}} { list [r ttl x] [r persist x] [r ttl x] [r get x] } {50 1 -1 foo} - test {PERSIST returns 0 against non existing or non volatile keys} { + test {PERSIST returns 0 against nonexistent or non volatile keys} { r set x foo list [r persist foo] [r persist nokeyatall] } {0 0} diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl index 4a03f28b94b..324e5ae0c6c 100644 --- a/tests/unit/geo.tcl +++ b/tests/unit/geo.tcl @@ -143,7 +143,7 @@ start_server {tags {"geo"}} { verify_geo_edge_response_generic "WRONGTYPE*" } - test {GEO with non existing src key} { + test {GEO with nonexistent src key} { r del src{t} verify_geo_edge_response_bylonlat {} 0 @@ -157,7 +157,7 @@ start_server {tags {"geo"}} { verify_geo_edge_response_bylonlat {} 0 } - test {GEO BYMEMBER with non existing member} { + test {GEO BYMEMBER with nonexistent member} { r del src{t} r geoadd src{t} 13.361389 38.115556 "Palermo" 15.087269 37.502669 "Catania" diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl index 5437bea6459..5c876b08edf 100644 --- a/tests/unit/info.tcl +++ b/tests/unit/info.tcl @@ -429,7 +429,7 @@ start_server {tags {"info" "external:skip" "debug_defrag:skip"}} { assert_equal {1} [subscribe $rd2 {chan2}] set info [r info clients] assert_equal [getInfoProperty $info pubsub_clients] {2} - # unsubscribe non existing channel + # unsubscribe nonexistent channel assert_equal {1} [unsubscribe $rd2 {non-exist-chan}] set info [r info clients] assert_equal [getInfoProperty $info pubsub_clients] {2} diff --git a/tests/unit/introspection-2.tcl b/tests/unit/introspection-2.tcl index 549f886884a..157ebb37d78 100644 --- a/tests/unit/introspection-2.tcl +++ b/tests/unit/introspection-2.tcl @@ -233,7 +233,7 @@ start_server {tags {"introspection"}} { assert_not_equal [lsearch $commands "client|list"] -1 } - test "COMMAND LIST FILTERBY ACLCAT against non existing category" { + test "COMMAND LIST FILTERBY ACLCAT against nonexistent category" { assert_equal {} [r command list filterby aclcat non_existing_category] } @@ -271,7 +271,7 @@ start_server {tags {"introspection"}} { assert_equal {} [r command list filterby pattern non_exists*] } - test "COMMAND LIST FILTERBY MODULE against non existing module" { + test "COMMAND LIST FILTERBY MODULE against nonexistent module" { # This should be empty, the real one is in subcommands.tcl assert_equal {} [r command list filterby module non_existing_module] } diff --git a/tests/unit/keyspace.tcl b/tests/unit/keyspace.tcl index 38eda6b8bac..9fcc6ccf3cc 100644 --- a/tests/unit/keyspace.tcl +++ b/tests/unit/keyspace.tcl @@ -130,7 +130,7 @@ start_server {tags {"keyspace"}} { append res [r get mykey2{t}] } {foobar} - test {RENAME against non existing source key} { + test {RENAME against nonexistent source key} { catch {r rename nokey{t} foobar{t}} err format $err } {ERR*} @@ -145,7 +145,7 @@ start_server {tags {"keyspace"}} { r renamenx mykey mykey } {0} - test {RENAME where source and dest key are the same (non existing)} { + test {RENAME where source and dest key are the same (nonexistent)} { r del mykey catch {r rename mykey mykey} err format $err diff --git a/tests/unit/moduleapi/moduleauth.tcl b/tests/unit/moduleapi/moduleauth.tcl index 56748db97c7..04b46e32494 100644 --- a/tests/unit/moduleapi/moduleauth.tcl +++ b/tests/unit/moduleapi/moduleauth.tcl @@ -19,9 +19,9 @@ start_server {tags {"modules"}} { assert_equal {OK} [r testmoduleone.rm_register_auth_cb] } - test {test module AUTH for non existing / disabled users} { + test {test module AUTH for nonexistent / disabled users} { r config resetstat - # Validate that an error is thrown for non existing users. + # Validate that an error is thrown for nonexistent users. assert_error {*WRONGPASS*} {r AUTH foo pwd} assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth] # Validate that an error is thrown for disabled users. diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index a089d5bb9c7..f8bdfb0e72c 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -78,7 +78,7 @@ start_server {tags {"hash"}} { assert_error {*value is out of range*} {r hrandfield myhash -9223372036854775808} } {} - test "HRANDFIELD with against non existing key" { + test "HRANDFIELD with against nonexistent key" { r hrandfield nonexisting_key 100 } {} @@ -89,7 +89,7 @@ start_server {tags {"hash"}} { r hrandfield myhash 0 } {*0} - test "HRANDFIELD with against non existing key - emptyarray" { + test "HRANDFIELD with against nonexistent key - emptyarray" { r hrandfield nonexisting_key 100 } {*0} @@ -269,7 +269,7 @@ start_server {tags {"hash"}} { set _ $err } {} - test {HGET against non existing key} { + test {HGET against nonexistent key} { set rv {} lappend rv [r hget smallhash __123123123__] lappend rv [r hget bighash __123123123__] @@ -342,9 +342,9 @@ start_server {tags {"hash"}} { r hmset bighash {*}$args } {OK} - test {HMGET against non existing key and fields} { + test {HMGET against nonexistent key and fields} { set rv {} - lappend rv [r hmget doesntexist __123123123__ __456456456__] + lappend rv [r hmget nonexistent __123123123__ __456456456__] lappend rv [r hmget smallhash __123123123__ __456456456__] lappend rv [r hmget bighash __123123123__ __456456456__] set _ $rv @@ -567,7 +567,7 @@ start_server {tags {"hash"}} { r debug object smallhash } {*hashtable*} {needs:debug} - test {HINCRBY against non existing database key} { + test {HINCRBY against nonexistent database key} { r del htest list [r hincrby htest foo 2] } {2} @@ -579,7 +579,7 @@ start_server {tags {"hash"}} { assert_error "*value is not a*" {r hincrbyfloat incrhash field v} } - test {HINCRBY against non existing hash key} { + test {HINCRBY against nonexistent hash key} { set rv {} r hdel smallhash tmp r hdel bighash tmp @@ -643,12 +643,12 @@ start_server {tags {"hash"}} { set e } {*overflow*} - test {HINCRBYFLOAT against non existing database key} { + test {HINCRBYFLOAT against nonexistent database key} { r del htest list [r hincrbyfloat htest foo 2.5] } {2.5} - test {HINCRBYFLOAT against non existing hash key} { + test {HINCRBYFLOAT against nonexistent hash key} { set rv {} r hdel smallhash tmp r hdel bighash tmp @@ -739,7 +739,7 @@ start_server {tags {"hash"}} { set _ $err } {} - test {HSTRLEN against non existing field} { + test {HSTRLEN against nonexistent field} { set rv {} lappend rv [r hstrlen smallhash __123123123__] lappend rv [r hstrlen bighash __123123123__] diff --git a/tests/unit/type/incr.tcl b/tests/unit/type/incr.tcl index 45a3b6f0be1..7a6bfbf723e 100644 --- a/tests/unit/type/incr.tcl +++ b/tests/unit/type/incr.tcl @@ -1,5 +1,5 @@ start_server {tags {"incr"}} { - test {INCR against non existing key} { + test {INCR against nonexistent key} { set res {} append res [r incr novar] append res [r get novar] @@ -87,7 +87,7 @@ start_server {tags {"incr"}} { assert {$old eq $new} } {} {needs:debug} - test {INCRBYFLOAT against non existing key} { + test {INCRBYFLOAT against nonexistent key} { r del novar list [roundFloat [r incrbyfloat novar 1]] \ [roundFloat [r get novar]] \ diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index 375737de24a..79cdbe18fa1 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -516,7 +516,7 @@ foreach {type large} [array get largevalue] { assert {[r LPOS mylist c COUNT 2 RANK -1] == {7 6}} } - test {LPOS non existing key} { + test {LPOS nonexistent key} { assert {[r LPOS mylistxxx c COUNT 0 RANK 2] eq {}} } @@ -610,14 +610,14 @@ foreach {type large} [array get largevalue] { assert_equal {*0} [r rpop listcount 0] } - test "LPOP/RPOP against non existing key in RESP$resp" { + test "LPOP/RPOP against nonexistent key in RESP$resp" { r del non_existing_key verify_resp_response $resp [r lpop non_existing_key] {$-1} {_} verify_resp_response $resp [r rpop non_existing_key] {$-1} {_} } - test "LPOP/RPOP with against non existing key in RESP$resp" { + test "LPOP/RPOP with against nonexistent key in RESP$resp" { r del non_existing_key verify_resp_response $resp [r lpop non_existing_key 0] {*-1} {_} @@ -1494,7 +1494,7 @@ foreach {pop} {BLPOP BLMPOP_LEFT} { assert_error {WRONGTYPE Operation against a key holding the wrong kind of value*} {r linsert k1 after 0 0} } - test {LINSERT against non existing key} { + test {LINSERT against nonexistent key} { assert_equal 0 [r linsert not-a-key before 0 0] } @@ -1554,7 +1554,7 @@ foreach type {listpack quicklist} { assert_error WRONGTYPE* {r llen mylist} } - test {LLEN against non existing key} { + test {LLEN against nonexistent key} { assert_equal 0 [r llen not-a-key] } @@ -1562,7 +1562,7 @@ foreach type {listpack quicklist} { assert_error WRONGTYPE* {r lindex mylist 0} } - test {LINDEX against non existing key} { + test {LINDEX against nonexistent key} { assert_equal "" [r lindex not-a-key 10] } @@ -1684,7 +1684,7 @@ foreach type {listpack quicklist} { } } - test {RPOPLPUSH against non existing key} { + test {RPOPLPUSH against nonexistent key} { r del srclist{t} dstlist{t} assert_equal {} [r rpoplpush srclist{t} dstlist{t}] assert_equal 0 [r exists srclist{t}] @@ -1709,7 +1709,7 @@ foreach {type large} [array get largevalue] { } } - test {RPOPLPUSH against non existing src key} { + test {RPOPLPUSH against nonexistent src key} { r del srclist{t} dstlist{t} assert_equal {} [r rpoplpush srclist{t} dstlist{t}] } {} @@ -1922,7 +1922,7 @@ foreach {type large} [array get largevalue] { } } - test {LRANGE against non existing key} { + test {LRANGE against nonexistent key} { assert_equal {} [r lrange nosuchkey 0 1] } @@ -1973,7 +1973,7 @@ foreach {type large} [array get largevalue] { } } - test {LSET against non existing key} { + test {LSET against nonexistent key} { assert_error ERR*key* {r lset nosuchkey 10 foo} } @@ -1994,7 +1994,7 @@ foreach {type large} [array get largevalue] { assert_equal "$e foobar foobared zap test foo" [r lrange mylist 0 -1] } - test "LREM remove non existing element - $type" { + test "LREM remove nonexistent element - $type" { assert_equal 0 [r lrem mylist 1 nosuchelement] assert_equal "$e foobar foobared zap test foo" [r lrange mylist 0 -1] } diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index 3a2ddd4e621..da58618cba7 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -62,7 +62,7 @@ start_server { assert_error WRONGTYPE* {r scard mylist} } - test {SMISMEMBER SMEMBERS SCARD against non existing key} { + test {SMISMEMBER SMEMBERS SCARD against nonexistent key} { assert_equal {0} [r smismember myset1 foo] assert_equal {0 0} [r smismember myset1 foo bar] assert_equal {} [r smembers myset1] @@ -363,7 +363,7 @@ foreach type {single multiple single_multiple} { assert_equal [list 195 199 $large] [lsort [r smembers setres{t}]] } - test "SUNION with non existing keys - $type" { + test "SUNION with nonexistent keys - $type" { set expected [lsort -uniq "[r smembers set1{t}] [r smembers set2{t}]"] assert_equal $expected [lsort [r sunion nokey1{t} set1{t} set2{t} nokey2{t}]] } @@ -487,7 +487,7 @@ foreach type {single multiple single_multiple} { assert_error "WRONGTYPE*" {r sdiff set1{t} key1{t}} } - test "SDIFF should handle non existing key as empty" { + test "SDIFF should handle nonexistent key as empty" { r del set1{t} set2{t} set3{t} r sadd set1{t} a b c @@ -519,7 +519,7 @@ foreach type {single multiple single_multiple} { assert_equal {e} [lsort [r smembers set3{t}]] } - test "SDIFFSTORE should handle non existing key as empty" { + test "SDIFFSTORE should handle nonexistent key as empty" { r del set1{t} set2{t} set3{t} r set setres{t} xxx @@ -554,7 +554,7 @@ foreach type {single multiple single_multiple} { assert_error "WRONGTYPE*" {r sinter set1{t} key1{t}} } - test "SINTER should handle non existing key as empty" { + test "SINTER should handle nonexistent key as empty" { r del set1{t} set2{t} set3{t} r sadd set1{t} a b c r sadd set2{t} b c d @@ -594,7 +594,7 @@ foreach type {single multiple single_multiple} { assert_equal {e} [lsort [r smembers set3{t}]] } - test "SINTERSTORE against non existing keys should delete dstkey" { + test "SINTERSTORE against nonexistent keys should delete dstkey" { r del set1{t} set2{t} set3{t} r set setres{t} xxx @@ -627,7 +627,7 @@ foreach type {single multiple single_multiple} { assert_error "WRONGTYPE*" {r sunion set1{t} key1{t}} } - test "SUNION should handle non existing key as empty" { + test "SUNION should handle nonexistent key as empty" { r del set1{t} set2{t} set3{t} r sadd set1{t} a b c @@ -658,7 +658,7 @@ foreach type {single multiple single_multiple} { assert_equal {e} [lsort [r smembers set3{t}]] } - test "SUNIONSTORE should handle non existing key as empty" { + test "SUNIONSTORE should handle nonexistent key as empty" { r del set1{t} set2{t} set3{t} r set setres{t} xxx @@ -682,7 +682,7 @@ foreach type {single multiple single_multiple} { assert_equal {a b c} [lsort [r smembers set3{t}]] } - test "SUNIONSTORE against non existing keys should delete dstkey" { + test "SUNIONSTORE against nonexistent keys should delete dstkey" { r set setres{t} xxx assert_equal 0 [r sunionstore setres{t} foo111{t} bar222{t}] assert_equal 0 [r exists setres{t}] @@ -818,7 +818,7 @@ foreach type {single multiple single_multiple} { r srandmember myset 0 } {} - test "SRANDMEMBER with against non existing key" { + test "SRANDMEMBER with against nonexistent key" { r srandmember nonexisting_key 100 } {} @@ -834,7 +834,7 @@ foreach type {single multiple single_multiple} { r srandmember myset 0 } {*0} - test "SRANDMEMBER with against non existing key - emptyarray" { + test "SRANDMEMBER with against nonexistent key - emptyarray" { r srandmember nonexisting_key 100 } {*0} @@ -1062,7 +1062,7 @@ foreach type {single multiple single_multiple} { assert_equal {3 4} [lsort [r smembers myset2{t}]] } - test "SMOVE non existing key" { + test "SMOVE nonexistent key" { setup_move assert_equal 0 [r smove myset1{t} myset2{t} foo] assert_equal 0 [r smove myset1{t} myset1{t} foo] @@ -1070,13 +1070,13 @@ foreach type {single multiple single_multiple} { assert_equal {2 3 4} [lsort [r smembers myset2{t}]] } - test "SMOVE non existing src set" { + test "SMOVE nonexistent src set" { setup_move assert_equal 0 [r smove noset{t} myset2{t} foo] assert_equal {2 3 4} [lsort [r smembers myset2{t}]] } - test "SMOVE from regular set to non existing destination set" { + test "SMOVE from regular set to nonexistent destination set" { setup_move assert_equal 1 [r smove myset1{t} myset3{t} a] assert_equal {1 b} [lsort [r smembers myset1{t}]] @@ -1084,7 +1084,7 @@ foreach type {single multiple single_multiple} { assert_encoding listpack myset3{t} } - test "SMOVE from intset to non existing destination set" { + test "SMOVE from intset to nonexistent destination set" { setup_move assert_equal 1 [r smove myset2{t} myset3{t} 2] assert_equal {3 4} [lsort [r smembers myset2{t}]] diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 9126abebddc..73a6f6432aa 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -488,7 +488,7 @@ start_server { r XADD "\{lestream\}2" 2-0 k2 v5 r XADD "\{lestream\}2" 3-0 k3 v6 - # read last element from 3 streams (2 with entries, 1 non-existent) + # read last element from 3 streams (2 with entries, 1 nonexistent) # verify the last element from the two existing streams were returned set res [r XREAD STREAMS "\{lestream\}1" "\{lestream\}2" "\{lestream\}3" + + +] assert_equal $res {{{{lestream}1} {{3-0 {k3 v3}}}} {{{lestream}2} {{3-0 {k3 v6}}}}} @@ -971,7 +971,7 @@ start_server {tags {"stream"}} { set err } {ERR *smaller*} - test {XSETID cannot SETID on non-existent key} { + test {XSETID cannot SETID on nonexistent key} { catch {r XSETID stream 1-1} err set _ $err } {ERR no such key} diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl index b164936f5fb..a9ea9822e9e 100644 --- a/tests/unit/type/string.tcl +++ b/tests/unit/type/string.tcl @@ -172,7 +172,7 @@ start_server {tags {"string"}} { test "GETEX syntax errors" { set ex {} - catch {r getex foo non-existent-option} ex + catch {r getex foo nonexistent-option} ex set ex } {*syntax*} @@ -229,7 +229,7 @@ start_server {tags {"string"}} { r mget foo{t} bar{t} } {BAR FOO} - test {MGET against non existing key} { + test {MGET against nonexistent key} { r mget foo{t} baazz{t} bar{t} } {BAR {} FOO} diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 5fe11cca915..c9336ab665c 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -216,7 +216,7 @@ start_server {tags {"zset"}} { set err } {ERR*} - test "ZADD NX with non existing key - $encoding" { + test "ZADD NX with nonexistent key - $encoding" { r del ztmp r zadd ztmp nx 10 x 20 y 30 z assert {[r zcard ztmp] == 3} @@ -346,7 +346,7 @@ start_server {tags {"zset"}} { r del ztmp r zadd ztmp 10 a 20 b 30 c assert_equal 3 [r zcard ztmp] - assert_equal 0 [r zcard zdoesntexist] + assert_equal 0 [r zcard znonexistent] } test "ZREM removes key after last element is removed - $encoding" { @@ -1528,7 +1528,7 @@ start_server {tags {"zset"}} { r readraw 1 - # ZPOP against non existing key. + # ZPOP against nonexistent key. assert_equal {*0} [r zpopmin zset{t}] assert_equal {*0} [r zpopmin zset{t} 1] @@ -1596,7 +1596,7 @@ start_server {tags {"zset"}} { r readraw 1 - # ZMPOP against non existing key. + # ZMPOP against nonexistent key. verify_nil_response $resp [r zmpop 1 zset{t} min] verify_nil_response $resp [r zmpop 1 zset{t} max count 1] verify_nil_response $resp [r zmpop 2 zset{t} zset2{t} min] @@ -2815,7 +2815,7 @@ start_server {tags {"zset"}} { r zrandmember myzset 0 } {} - test "ZRANDMEMBER with against non existing key" { + test "ZRANDMEMBER with against nonexistent key" { r zrandmember nonexisting_key 100 } {} @@ -2833,7 +2833,7 @@ start_server {tags {"zset"}} { r zrandmember myzset 0 } {*0} - test "ZRANDMEMBER with against non existing key - emptyarray" { + test "ZRANDMEMBER with against nonexistent key - emptyarray" { r zrandmember nonexisting_key 100 } {*0} From abebdb02021a53f96724540e5f53d8a501c4f587 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Thu, 13 Aug 2026 10:45:23 -0700 Subject: [PATCH 22/27] Correct the `since` version on CONFIG INFO and MOVE REPLACE to 9.2.0 (#4402) `src/commands/config-info.json` and `src/commands/move.json` still carry `"since": "10.0.0"`. *This was generated by AI but verified, with love, by a human.* Signed-off-by: Madelyn Olson --- src/commands.def | 6 +++--- src/commands/config-info.json | 2 +- src/commands/move.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/commands.def b/src/commands.def index 2b822af0a20..ad401849799 100644 --- a/src/commands.def +++ b/src/commands.def @@ -2370,7 +2370,7 @@ struct COMMAND_ARG MIGRATE_Args[] = { #ifndef SKIP_CMD_HISTORY_TABLE /* MOVE history */ commandHistory MOVE_History[] = { -{"10.0.0","added REPLACE option."}, +{"9.2.0","added REPLACE option."}, }; #endif @@ -2390,7 +2390,7 @@ keySpec MOVE_Keyspecs[1] = { struct COMMAND_ARG MOVE_Args[] = { {MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("db",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, -{MAKE_ARG("replace",ARG_TYPE_PURE_TOKEN,-1,"REPLACE",NULL,"10.0.0",CMD_ARG_OPTIONAL,0,NULL)}, +{MAKE_ARG("replace",ARG_TYPE_PURE_TOKEN,-1,"REPLACE",NULL,"9.2.0",CMD_ARG_OPTIONAL,0,NULL)}, }; /********** OBJECT ENCODING ********************/ @@ -7670,7 +7670,7 @@ struct COMMAND_ARG CONFIG_SET_Args[] = { struct COMMAND_STRUCT CONFIG_Subcommands[] = { {MAKE_CMD("get","Returns the effective values of configuration parameters.","O(N) when N is the number of configuration parameters provided","2.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,CONFIG_GET_History,1,CONFIG_GET_Tips,0,configGetCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CONFIG_GET_Keyspecs,0,NULL,1),.args=CONFIG_GET_Args}, {MAKE_CMD("help","Returns helpful text about the different subcommands.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,CONFIG_HELP_History,0,CONFIG_HELP_Tips,0,configHelpCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_SLOW,NULL,CONFIG_HELP_Keyspecs,0,NULL,0)}, -{MAKE_CMD("info","Returns information about configuration parameters matching the given patterns.","O(N) when N is the number of configuration parameters provided","10.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,CONFIG_INFO_History,0,CONFIG_INFO_Tips,0,configInfoCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CONFIG_INFO_Keyspecs,0,NULL,1),.args=CONFIG_INFO_Args}, +{MAKE_CMD("info","Returns information about configuration parameters matching the given patterns.","O(N) when N is the number of configuration parameters provided","9.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,CONFIG_INFO_History,0,CONFIG_INFO_Tips,0,configInfoCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CONFIG_INFO_Keyspecs,0,NULL,1),.args=CONFIG_INFO_Args}, {MAKE_CMD("resetstat","Resets the server's statistics.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,CONFIG_RESETSTAT_History,0,CONFIG_RESETSTAT_Tips,2,configResetStatCommand,2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CONFIG_RESETSTAT_Keyspecs,0,NULL,0)}, {MAKE_CMD("rewrite","Persists the effective configuration to file.","O(1)","2.8.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,CONFIG_REWRITE_History,0,CONFIG_REWRITE_Tips,2,configRewriteCommand,2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CONFIG_REWRITE_Keyspecs,0,NULL,0)}, {MAKE_CMD("set","Sets configuration parameters in-flight.","O(N) when N is the number of configuration parameters provided","2.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,CONFIG_SET_History,1,CONFIG_SET_Tips,2,configSetCommand,-4,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CONFIG_SET_Keyspecs,0,NULL,1),.args=CONFIG_SET_Args}, diff --git a/src/commands/config-info.json b/src/commands/config-info.json index 6d29c2c9c3d..f177c63e740 100644 --- a/src/commands/config-info.json +++ b/src/commands/config-info.json @@ -3,7 +3,7 @@ "summary": "Returns information about configuration parameters matching the given patterns.", "complexity": "O(N) when N is the number of configuration parameters provided", "group": "server", - "since": "10.0.0", + "since": "9.2.0", "arity": -3, "container": "CONFIG", "function": "configInfoCommand", diff --git a/src/commands/move.json b/src/commands/move.json index ac4909ccfa1..cf15bb88bc7 100644 --- a/src/commands/move.json +++ b/src/commands/move.json @@ -8,7 +8,7 @@ "function": "moveCommand", "history": [ [ - "10.0.0", + "9.2.0", "added REPLACE option." ] ], @@ -57,7 +57,7 @@ "name": "replace", "token": "REPLACE", "type": "pure-token", - "since": "10.0.0", + "since": "9.2.0", "optional": true } ], From afabf680adc34a410409c1c32cc344544b3fb2cb Mon Sep 17 00:00:00 2001 From: Binbin Date: Fri, 14 Aug 2026 03:32:56 +0800 Subject: [PATCH 23/27] Fix use-after-free crash when cluster messages arrive after module unload (#4360) moduleUnregisterCleanup did not remove the module's cluster message receivers. The stale entries kept pointing at the freed ValkeyModule, so a later cluster message of that type dereferenced r->module in moduleCallClusterReceivers, causing a use-after-free. Added a MODULE UNLOAD test to verify the fix, and also added type=254 to allow us to verify the correctness of the loop logic. --------- Signed-off-by: Binbin --- src/module.c | 25 +++++++++++++++++++++++ tests/modules/cluster.c | 8 ++++++-- tests/unit/moduleapi/cluster.tcl | 35 ++++++++++++++++++++++++++++---- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/module.c b/src/module.c index c9d17c4d41c..666fc37872e 100644 --- a/src/module.c +++ b/src/module.c @@ -13330,6 +13330,30 @@ void moduleUnregisterCommands(struct ValkeyModule *module) { invalidateCommandCache(); } +/* Remove every cluster message receiver that belongs to the given module. */ +static void moduleUnregisterClusterReceivers(ValkeyModule *module) { + if (!server.cluster_enabled) return; + + for (int type = 0; type < UINT8_MAX; type++) { + moduleClusterReceiver *r = clusterReceivers[type], *prev = NULL; + while (r) { + if (r->module == module) { + /* Unlink the receiver node from the linked list. A module + * registers at most one receiver per type, so we can stop + * scanning this type as soon as we removed it. */ + if (prev) + prev->next = r->next; + else + clusterReceivers[type] = r->next; + zfree(r); + break; + } + prev = r; + r = r->next; + } + } +} + /* We parse argv to add sds "NAME VALUE" pairs to the server.module_configs_queue list of configs. * We also increment the module_argv pointer to just after ARGS if there are args, otherwise * we set it to NULL */ @@ -13382,6 +13406,7 @@ void moduleUnregisterCleanup(ValkeyModule *module) { moduleUnsubscribeAllServerEvents(module); moduleRemoveConfigs(module); moduleUnregisterAuthCBs(module); + moduleUnregisterClusterReceivers(module); } /* Common helper for moduleLoad and moduleLoadStatic. diff --git a/tests/modules/cluster.c b/tests/modules/cluster.c index a844774e2fb..7806ef4c70c 100644 --- a/tests/modules/cluster.c +++ b/tests/modules/cluster.c @@ -67,6 +67,7 @@ int test_cluster_shards(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int arg #define MSGTYPE_DING 1 #define MSGTYPE_DONG 2 #define MSGTYPE_TEST_UAF 3 +#define MSGTYPE_TEST_MAX 254 /* test.pingall */ int PingallCommand(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { @@ -94,6 +95,7 @@ int test_register_receiver(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int UNUSED(argv); UNUSED(argc); ValkeyModule_RegisterClusterMessageReceiver(ctx, MSGTYPE_TEST_UAF, DingReceiver); + ValkeyModule_RegisterClusterMessageReceiver(ctx, MSGTYPE_TEST_MAX, DingReceiver); return ValkeyModule_ReplyWithSimpleString(ctx, "OK"); } @@ -101,13 +103,15 @@ int test_unregister_receiver(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, in UNUSED(argv); UNUSED(argc); ValkeyModule_RegisterClusterMessageReceiver(ctx, MSGTYPE_TEST_UAF, NULL); + ValkeyModule_RegisterClusterMessageReceiver(ctx, MSGTYPE_TEST_MAX, NULL); return ValkeyModule_ReplyWithSimpleString(ctx, "OK"); } -int test_send_msg_type3(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { +int test_send_msg_uaf(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { UNUSED(argv); UNUSED(argc); ValkeyModule_SendClusterMessage(ctx, NULL, MSGTYPE_TEST_UAF, "TestUAF", 7); + ValkeyModule_SendClusterMessage(ctx, NULL, MSGTYPE_TEST_MAX, "TestMAX", 7); return ValkeyModule_ReplyWithSimpleString(ctx, "OK"); } @@ -135,7 +139,7 @@ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int arg return VALKEYMODULE_ERR; if (ValkeyModule_CreateCommand(ctx, "test.unregister_receiver", test_unregister_receiver, "", 0, 0, 0) == VALKEYMODULE_ERR) return VALKEYMODULE_ERR; - if (ValkeyModule_CreateCommand(ctx, "test.send_msg_type3", test_send_msg_type3, "", 0, 0, 0) == VALKEYMODULE_ERR) + if (ValkeyModule_CreateCommand(ctx, "test.send_msg_uaf", test_send_msg_uaf, "", 0, 0, 0) == VALKEYMODULE_ERR) return VALKEYMODULE_ERR; /* Register our handlers for different message types. */ diff --git a/tests/unit/moduleapi/cluster.tcl b/tests/unit/moduleapi/cluster.tcl index 095d50be1c4..0b0f8f55cf5 100644 --- a/tests/unit/moduleapi/cluster.tcl +++ b/tests/unit/moduleapi/cluster.tcl @@ -314,10 +314,10 @@ start_cluster 3 0 [list config_lines $modules] { } test "VM_RegisterClusterMessageReceiver - unregister head and re-register does not crash" { - # Register a receiver for type 3 on node1 + # Register the receivers on node1 assert_equal OK [$node1 test.register_receiver] - # Unregister it (this is the head of the list for type 3) + # Unregister it (the head of the list) assert_equal OK [$node1 test.unregister_receiver] # Re-register - on the buggy code this traverses freed memory and crashes @@ -325,14 +325,41 @@ start_cluster 3 0 [list config_lines $modules] { # Send from node2 so node1 receives it via the re-registered receiver R 0 CONFIG RESETSTAT - assert_equal OK [$node2 test.send_msg_type3] + assert_equal OK [$node2 test.send_msg_uaf] wait_for_condition 50 100 { - [CI 0 cluster_stats_messages_module_received] >= 1 + [CI 0 cluster_stats_messages_module_received] >= 2 } else { fail "node1 didn't receive cluster module message after re-registration" } verify_log_message 0 "*DING (type 3) RECEIVED*TestUAF*" 0 + verify_log_message 0 "*DING (type 254) RECEIVED*TestMAX*" 0 + } + + test "VM_RegisterClusterMessageReceiver - dangling callback after MODULE UNLOAD" { + set loglines [count_log_lines 0] + + # Register the receivers on node1 + assert_equal OK [$node1 test.register_receiver] + + # Unload the module on node1 + assert_equal OK [$node1 MODULE UNLOAD cluster] + + # Another node sends a packet; node1 receives it and, on the buggy code, + # would invoke the dangling callback and crash. After the fix the entry + # is gone, so the packet is simply ignored. + R 0 CONFIG RESETSTAT + assert_equal OK [$node2 test.send_msg_uaf] + + # Verify node1 is still alive (the receiving node must not have crashed). + wait_for_condition 50 100 { + [CI 0 cluster_stats_messages_module_received] >= 2 + } else { + fail "node1 didn't receive cluster module message" + } + verify_no_log_message 0 "*DING (type 3) RECEIVED*TestUAF*" $loglines + verify_no_log_message 0 "*DING (type 254) RECEIVED*TestMAX*" $loglines + assert_equal PONG [$node1 PING] } } From 1e7a8f3105eefa85bf43c7361dbe9bc87b5327bd Mon Sep 17 00:00:00 2001 From: Quanye Yang Date: Fri, 14 Aug 2026 04:36:00 +0800 Subject: [PATCH 24/27] Skip IO-thread read-done followup unless update_state sync-invokes handlers (#4401) Follow-up of #3611. Fixes a regression introduced by that PR. The May 27 On-Demand run on this PR (SET/GET, 96B, io-threads 2/10, pipeline 1/10) showed no significant RPS change. After `17ec23f` removed `post_read_done_postpone_mask`, `processClientIOReadsDone()` started postponing READ and returning `needs_post_read_update = 1` for every non-ACCEPTING completed read. That second phase (`lookupClientByID` + `processPendingCommandAndInputBuffer` + `connUpdateState`) is only required when `update_state` may synchronously invoke handlers. For other transports it is per-completion overhead. This matches the post-merge dashboard drop: small payloads, io-threads, **P1 worse than P10**. This PR restores the original gate without bringing `struct client` into the connection driver (the review concern that led to `17ec23f`): - `ConnectionType.sync_handlers_in_update_state` (0 by default) - Set only where `update_state` can sync-call handlers - Mask is still computed in `networking.c` from IO state - `connUpdateState()` still runs immediately, including ACCEPTING --------- Signed-off-by: quanyeyang --- src/connection.h | 8 ++++++++ src/networking.c | 6 ++++-- src/rdma.c | 2 ++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/connection.h b/src/connection.h index dc85db96639..5527ee2a769 100644 --- a/src/connection.h +++ b/src/connection.h @@ -147,6 +147,10 @@ typedef struct ConnectionType { void (*postpone_update_state)(struct connection *conn, int postpone_mask); /* Called by the main-thread */ void (*update_state)(struct connection *conn); + /* 1 if update_state may synchronously invoke read/write handlers. + * When set, processClientIOReadsDone defers clearing postpone until after + * command batching; leave 0 for transports that do not need that. */ + int sync_handlers_in_update_state; /* TLS specified methods */ sds (*get_peer_cert)(struct connection *conn); @@ -526,6 +530,10 @@ static inline void connSetPostponeUpdateState(connection *conn, int postpone_mas } } +static inline int connUpdateStateMayInvokeHandlers(connection *conn) { + return conn && conn->type && conn->type->sync_handlers_in_update_state; +} + static inline int connIsIntegrityChecked(connection *conn) { return conn->type->connIntegrityChecked && conn->type->connIntegrityChecked(); } diff --git a/src/networking.c b/src/networking.c index faa3f4e04d2..abf29985a43 100644 --- a/src/networking.c +++ b/src/networking.c @@ -6568,10 +6568,12 @@ int processClientIOReadsDone(client *c) { int in_accept_state = (connGetState(c->conn) == CONN_STATE_ACCEPTING); int needs_post_read_update = 0; + /* Defer the post-batch update only when update_state may sync-invoke + * handlers. Always call update_state, including ACCEPTING. */ if (c->conn) { int mask = 0; - if (!in_accept_state) { - mask |= CONN_POSTPONE_READ; + if (!in_accept_state && connUpdateStateMayInvokeHandlers(c->conn)) { + mask = CONN_POSTPONE_READ; if (c->io_write_state != CLIENT_IDLE) mask |= CONN_POSTPONE_WRITE; needs_post_read_update = 1; } diff --git a/src/rdma.c b/src/rdma.c index c0def54d5cd..198721021a3 100644 --- a/src/rdma.c +++ b/src/rdma.c @@ -1860,6 +1860,8 @@ static ConnectionType CT_RDMA = { .process_pending_data = rdmaProcessPendingData, .postpone_update_state = postPoneUpdateRdmaState, .update_state = updateRdmaState, + /* updateRdmaState → connRdmaEventHandler may sync-call read/write handlers. */ + .sync_handlers_in_update_state = 1, /* Miscellaneous */ .connIntegrityChecked = NULL, From 917de6ac19a63d7c9aa5d57b8151e4d07c424862 Mon Sep 17 00:00:00 2001 From: michellee-10 Date: Thu, 13 Aug 2026 14:41:01 -0700 Subject: [PATCH 25/27] Log EXEC in commandlog (#4267) Log EXEC in commandlog. Catches cases where there's no individually slow command. --------- Signed-off-by: Michelle Lee --- src/commands.def | 2 +- src/commands/exec.json | 3 +-- tests/unit/commandlog.tcl | 29 +++++++++++++++++++++++++++-- tests/unit/slowlog.tcl | 29 +++++++++++++++++++++++++++-- 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/commands.def b/src/commands.def index ad401849799..942c7e8a5bb 100644 --- a/src/commands.def +++ b/src/commands.def @@ -12158,7 +12158,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("substr","Returns a substring from a string value.","O(N) where N is the length of the returned string. The complexity is ultimately determined by the returned length, but because creating a substring from an existing string is very cheap, it can be considered O(1) for small strings.","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SUBSTR_History,0,SUBSTR_Tips,0,getrangeCommand,4,CMD_READONLY,ACL_CATEGORY_READ|ACL_CATEGORY_SLOW|ACL_CATEGORY_STRING,NULL,SUBSTR_Keyspecs,1,NULL,3),.args=SUBSTR_Args}, /* transactions */ {MAKE_CMD("discard","Discards a transaction.","O(N), when N is the number of queued commands","2.0.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,DISCARD_History,0,DISCARD_Tips,0,discardCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_FAST|ACL_CATEGORY_TRANSACTION,NULL,DISCARD_Keyspecs,0,NULL,0)}, -{MAKE_CMD("exec","Executes all commands in a transaction.","Depends on commands in the transaction","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,EXEC_History,0,EXEC_Tips,0,execCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SKIP_COMMANDLOG,ACL_CATEGORY_SLOW|ACL_CATEGORY_TRANSACTION,NULL,EXEC_Keyspecs,0,NULL,0)}, +{MAKE_CMD("exec","Executes all commands in a transaction.","Depends on commands in the transaction","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,EXEC_History,0,EXEC_Tips,0,execCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_SLOW|ACL_CATEGORY_TRANSACTION,NULL,EXEC_Keyspecs,0,NULL,0)}, {MAKE_CMD("multi","Starts a transaction.","O(1)","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,MULTI_History,0,MULTI_Tips,0,multiCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_NO_MULTI|CMD_ALLOW_BUSY,ACL_CATEGORY_FAST|ACL_CATEGORY_TRANSACTION,NULL,MULTI_Keyspecs,0,NULL,0)}, {MAKE_CMD("unwatch","Forgets about watched keys of a transaction.","O(1)","2.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,UNWATCH_History,0,UNWATCH_Tips,0,unwatchCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_FAST|ACL_CATEGORY_TRANSACTION,NULL,UNWATCH_Keyspecs,0,NULL,0)}, {MAKE_CMD("watch","Monitors changes to keys to determine the execution of a transaction.","O(1) for every key.","2.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,WATCH_History,0,WATCH_Tips,0,watchCommand,-2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_NO_MULTI|CMD_ALLOW_BUSY,ACL_CATEGORY_FAST|ACL_CATEGORY_TRANSACTION,NULL,WATCH_Keyspecs,1,NULL,1),.args=WATCH_Args}, diff --git a/src/commands/exec.json b/src/commands/exec.json index 14f050fbac6..8c4aba48a4a 100644 --- a/src/commands/exec.json +++ b/src/commands/exec.json @@ -9,8 +9,7 @@ "command_flags": [ "NOSCRIPT", "LOADING", - "STALE", - "SKIP_COMMANDLOG" + "STALE" ], "acl_categories": [ "SLOW", diff --git a/tests/unit/commandlog.tcl b/tests/unit/commandlog.tcl index 1e8a5efceba..3c405656451 100644 --- a/tests/unit/commandlog.tcl +++ b/tests/unit/commandlog.tcl @@ -291,18 +291,43 @@ start_server {tags {"commandlog"} overrides {commandlog-execution-slower-than 10 lindex $e 3 } {sadd set foo {AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA... (1 more bytes)}} - test {COMMANDLOG slow - EXEC is not logged, just executed commands} { + test {COMMANDLOG slow - EXEC is logged alongside slow inner commands} { r config set commandlog-execution-slower-than 100000 r commandlog reset slow assert_equal [r commandlog len slow] 0 r multi r debug sleep 0.2 r exec + assert_equal [r commandlog len slow] 2 + set entries [r commandlog get -1 slow] + assert_equal [lindex [lindex $entries 0] 3] {exec} + assert_equal [lindex [lindex $entries 1] 3] {debug sleep 0.2} + } {} {needs:debug} + + test {COMMANDLOG slow - EXEC records total transaction time when inner commands are individually fast} { + r config set commandlog-execution-slower-than 100000 + r commandlog reset slow + r multi + for {set i 0} {$i < 10} {incr i} { + r debug sleep 0.03 + } + r exec assert_equal [r commandlog len slow] 1 set e [lindex [r commandlog get -1 slow] 0] - assert_equal [lindex $e 3] {debug sleep 0.2} + assert_equal [lindex $e 3] {exec} + assert {[lindex $e 2] >= 100000} } {} {needs:debug} + test {COMMANDLOG slow - EXEC is not logged when transaction is below threshold} { + r config set commandlog-execution-slower-than 100000 + r commandlog reset slow + r multi + r set foo bar + r get foo + r exec + assert_equal [r commandlog len slow] 0 + } + test {COMMANDLOG slow - can clean older entries} { r client setname lastentry_client r config set commandlog-slow-execution-max-len 1 diff --git a/tests/unit/slowlog.tcl b/tests/unit/slowlog.tcl index 1be530d37fa..1e0dbe5f0f2 100644 --- a/tests/unit/slowlog.tcl +++ b/tests/unit/slowlog.tcl @@ -172,18 +172,43 @@ start_server {tags {"slowlog"} overrides {slowlog-log-slower-than 1000000}} { lindex $e 3 } {sadd set foo {AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA... (1 more bytes)}} - test {SLOWLOG - EXEC is not logged, just executed commands} { + test {SLOWLOG - EXEC is logged alongside slow inner commands} { r config set slowlog-log-slower-than 100000 r slowlog reset assert_equal [r slowlog len] 0 r multi r debug sleep 0.2 r exec + assert_equal [r slowlog len] 2 + set entries [r slowlog get] + assert_equal [lindex [lindex $entries 0] 3] {exec} + assert_equal [lindex [lindex $entries 1] 3] {debug sleep 0.2} + } {} {needs:debug} + + test {SLOWLOG - EXEC records total transaction time when inner commands are individually fast} { + r config set slowlog-log-slower-than 100000 + r slowlog reset + r multi + for {set i 0} {$i < 10} {incr i} { + r debug sleep 0.03 + } + r exec assert_equal [r slowlog len] 1 set e [lindex [r slowlog get] 0] - assert_equal [lindex $e 3] {debug sleep 0.2} + assert_equal [lindex $e 3] {exec} + assert {[lindex $e 2] >= 100000} } {} {needs:debug} + test {SLOWLOG - EXEC is not logged when transaction is below threshold} { + r config set slowlog-log-slower-than 100000 + r slowlog reset + r multi + r set foo bar + r get foo + r exec + assert_equal [r slowlog len] 0 + } + test {SLOWLOG - can clean older entries} { r client setname lastentry_client r config set slowlog-max-len 1 From 2d69b73063e206b7dae75bdb80a33e6bc6ff789e Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Wed, 12 Aug 2026 16:29:20 -0700 Subject: [PATCH 26/27] Comment addressing Signed-off-by: harrylin98 --- src/networking.c | 5 ++++- src/server.c | 6 +++--- src/throttle_repl.c | 24 +++++++++--------------- src/unit/test_throttle_repl.cpp | 2 +- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/networking.c b/src/networking.c index 2bd59a6cf48..73a74c347ed 100644 --- a/src/networking.c +++ b/src/networking.c @@ -6284,7 +6284,10 @@ int checkClientOutputBufferLimits(client *c) { } else { c->obuf_soft_limit_reached_time = 0; } - if ((soft || hard) && throttleRepl_isClientExemptFromCobLimits(c)) return 0; + /* The steady-state throttle may exempt a replica from the soft limit to give throttling + * time to converge; the hard limit is never suppressed, so a replica that reaches it is + * always disconnected. */ + if (soft && !hard && throttleRepl_isClientExemptFromCobLimits(c)) return 0; return soft || hard; } diff --git a/src/server.c b/src/server.c index 3c80cbe7e6d..cd561baf1d4 100644 --- a/src/server.c +++ b/src/server.c @@ -6871,10 +6871,10 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { } } - /* Throttle */ - if (all_sections || (dictFind(section_dict, "throttle") != NULL)) { + /* Throttling */ + if (all_sections || (dictFind(section_dict, "throttling") != NULL)) { if (sections++) info = sdscat(info, "\r\n"); - info = sdscat(info, "# Throttle\r\n"); + info = sdscat(info, "# Throttling\r\n"); info = throttle_sdscatInfoMetrics(info); info = throttleRepl_sdscatInfoMetrics(info); } diff --git a/src/throttle_repl.c b/src/throttle_repl.c index 363fa2e5147..7e6ed460f00 100644 --- a/src/throttle_repl.c +++ b/src/throttle_repl.c @@ -118,9 +118,10 @@ static bool evaluateSteadyStateThrottle(client *c, int64_t cob_size) { /* --- Public API --- */ -/* In some cases, we want to protect replicas from being killed by the COB limits. When - * throttling hasn't had time to adjust and there is no severe memory condition, it makes - * sense to allow the replica to live until throttling can stabilize the situation. */ +/* Determines whether a replica should be temporarily exempted from the soft client output + * buffer limit. While the steady-state throttle is converging, exempting the soft limit + * prevents a premature disconnect and allows the throttler to reduce the replica's buffer + * back below target. */ bool throttleRepl_isClientExemptFromCobLimits(client *c) { if (!throttle_repl_config.repl_throttle_steady_state_enabled || !isThrottlerActive()) return false; if (!iAmPrimary()) return false; @@ -132,15 +133,14 @@ bool throttleRepl_isClientExemptFromCobLimits(client *c) { /* There's no need to protect the replica if it's already using less than the target size. */ if (client_cob_size < getReplicaSteadyStateCobTargetSize()) return false; - /* Don't exempt if server is over maxmemory. - * When eviction is already running, we can't afford to let - * replica output buffers grow further. */ + /* Don't exempt if the server is over maxmemory. + * When eviction is already running, we can't afford to let replica output buffers grow further. */ if (server.maxmemory && getMaxmemoryState(NULL, NULL, NULL, NULL) == C_ERR) return false; /* Don't protect if throttle has been working too long without success. */ time_t elapsed = server.unixtime - c->obuf_soft_limit_reached_time; if (elapsed > 4 * STEADY_STATE_CONVERGENCE_SECS) return false; - /* Otherwise, allow the replica to exceed the configured limits, giving the throttler time to correct. */ + /* Otherwise, allow the replica to exceed the soft limit, giving the throttler time to correct. */ return true; } @@ -189,14 +189,8 @@ void throttleRepl_adjustThrottling(void) { sds throttleRepl_sdscatInfoMetrics(sds info) { info = sdscatprintf(info, - "repl_throttle_active:%d\r\n", - metrics.is_throttler_active ? 1 : 0); - - if (metrics.is_throttler_active) { - info = sdscatprintf(info, - "repl_throttle_rate:%.2f\r\n", - metrics.current_throttle_rate); - } + "repl_throttle_rate:%.2f\r\n", + metrics.is_throttler_active ? metrics.current_throttle_rate : 0); throttleMetrics throttle_metrics; throttle_getMetrics(METRICS_NAME, &throttle_metrics); diff --git a/src/unit/test_throttle_repl.cpp b/src/unit/test_throttle_repl.cpp index 392bf8cf757..4083ec53ec5 100644 --- a/src/unit/test_throttle_repl.cpp +++ b/src/unit/test_throttle_repl.cpp @@ -84,7 +84,7 @@ class ThrottleReplTest : public ::testing::Test { } bool isReplThrottlerActive() { - return (long)readMetric("repl_throttle_active") == 1; + return readMetric("repl_throttle_rate") > 0.0; } double getThrottlerRate() { From 8cd535ddb3dc80aceb154e6ef9dbe11d256e2209 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Thu, 13 Aug 2026 16:53:50 -0700 Subject: [PATCH 27/27] Tests: EXEC is now entry 0 of the MULTI redaction commandlog test (#4404) `unit/commandlog` fails on unstable at 917de6ac1: ``` *** [err]: COMMANDLOG slow - Redaction does not leak to later commands in a MULTI in tests/unit/commandlog.tcl Expected 'set foo bar' to be equal to 'exec' (context: type eval line 12 cmd {assert_equal {set foo bar} [lindex [lindex $slowlog_resp 0] 3]} proc ::test) ``` #4267 dropped `SKIP_COMMANDLOG` from EXEC (`src/commands/exec.json:12`), so EXEC is now logged after the commands it ran, which puts it at entry 0 of the newest-first `COMMANDLOG GET`. The redaction test at `tests/unit/commandlog.tcl:212`, added by #4323, still reads entry 0 and gets the EXEC instead of the SET. Neither PR was rebased on the other, so this only broke on merge and not in either PR's CI. The log after `MULTI; ACL SETUSER commandlog-test-user +get; SET foo bar; EXEC`: | Entry | Command | |---|---| | 0 | `exec` | | 1 | `set foo bar` | | 2 | `acl setuser (redacted) (redacted)` | Read the SET from entry 1 instead. The other tests #4267 touched already assert entry 0 is `exec` and entry 1 is the inner command, so this matches. Also assert entries 0 and 2 rather than only the SET. The point of the test is that the ACL SETUSER redaction stops at the ACL SETUSER, and asserting entry 2 is redacted is what keeps it from passing if redaction breaks entirely. Asserting entry 0 is `exec` makes the index arithmetic fail loudly next time the entry order changes, instead of silently comparing against the wrong entry the way it just did. ## Testing `unit/commandlog` goes from 29 passed, 1 failed to 30 passed. `unit/slowlog` and `unit/multi` are unchanged at 100 passed. Reverting the `src/` hunks of 917de6ac1 confirms that commit is the trigger. The new entry-0 assert fails, along with the two EXEC tests 917de6ac1 added: ``` *** [err]: COMMANDLOG slow - Redaction does not leak to later commands in a MULTI in tests/unit/commandlog.tcl Expected 'exec' to be equal to 'set foo bar' (context: type eval line 13 cmd {assert_equal {exec} [lindex [lindex $slowlog_resp 0] 3]} proc ::test) *** [err]: COMMANDLOG slow - EXEC is logged alongside slow inner commands in tests/unit/commandlog.tcl Expected '1' to be equal to '2' (context: type eval line 8 cmd {assert_equal [r commandlog len slow] 2} proc ::test) *** [err]: COMMANDLOG slow - EXEC records total transaction time when inner commands are individually fast in tests/unit/commandlog.tcl Expected '0' to be equal to '1' (context: type eval line 9 cmd {assert_equal [r commandlog len slow] 1} proc ::test) ``` *This was generated by AI but verified, with love, by a human.* Signed-off-by: Madelyn Olson --- tests/unit/commandlog.tcl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/commandlog.tcl b/tests/unit/commandlog.tcl index 3c405656451..3128b2ce960 100644 --- a/tests/unit/commandlog.tcl +++ b/tests/unit/commandlog.tcl @@ -208,8 +208,11 @@ start_server {tags {"commandlog"} overrides {commandlog-execution-slower-than 10 r config set commandlog-execution-slower-than -1 set slowlog_resp [r commandlog get -1 slow] - # The ACL SETUSER redaction must not carry over to the following SET - assert_equal {set foo bar} [lindex [lindex $slowlog_resp 0] 3] + # Entry 0 is the EXEC itself, entry 1 is the SET and entry 2 is the ACL SETUSER. + # The ACL SETUSER redaction must not carry over to the following SET. + assert_equal {exec} [lindex [lindex $slowlog_resp 0] 3] + assert_equal {set foo bar} [lindex [lindex $slowlog_resp 1] 3] + assert_equal {acl setuser (redacted) (redacted)} [lindex [lindex $slowlog_resp 2] 3] r acl deluser commandlog-test-user }