From 34f10426e25987276c16287d804d9bf082db0dba Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Tue, 14 Jul 2026 21:45:48 +1000 Subject: [PATCH 01/21] clusterer_controller: zero-config HA clusterer control module (devel) Port of the clusterer_controller module and its clusterer-module integration to the devel branch: zero-config HA clusterer control over encrypted UDP multicast, per-cluster node identity, hybrid native+controller topologies, cl_ctr_* MI/variables/functions, join flood-DoS hardening, and input validation. Devel adaptations: - cc_discover_bin_sockets() iterates protos[PROTO_BIN].listeners as struct socket_info_full (next/prev), reading the embedded socket_info. - The clusterer core edits are re-seated onto devel's clusterer.c / clusterer_mod.c / node_info.h, merging cleanly with the new bridges feature and the clusterer_enable_rerouting guard. Robustness / log hygiene: - cc_decrypt_pkt distinguishes bootstrap-key failures (real wrong-password / foreign-cluster / tampering -> WARN) from transient session-key mismatches during a (re)key or split-brain heal (-> DBG). - the split-brain defer budget resets on a fresh higher-IP JOIN_REQ so a lower-IP node waits for a live higher-IP peer instead of self-promoting (bounded by CC_JOIN_DEFER_HARDMAX); JOIN_REQ sends are rate-limited. - clusterer: the seed sync-fallback on a fresh start is downgraded from ERROR to DBG (the SYNC_IN_PROGRESS partial-sync failure path is unchanged). - cc_elect_master enforces "MASTER_ALIVE keepalive armed <=> I am the elected master": a node demoted purely by election now stops broadcasting MASTER_ALIVE, fixing an oscillation where a lower-IP node flapped between two masters. Validated end-to-end on a 3-node cluster: controller / native / hybrid modes, master failover and election stability (staggered + simultaneous starts converge to a single stable master), multiple controller clusters sharing one BIN socket (distinct multicast per cluster), MI outputs and error paths, buffer overrun/underrun fuzzing, wrong-password rejection, and config-mismatch. Docs and generated README included (admin guide + HA test appendix). --- modules/clusterer/api.h | 1 - modules/clusterer/clusterer.c | 153 +- modules/clusterer/clusterer_ctrl.c | 391 ++ modules/clusterer/clusterer_ctrl.h | 172 + modules/clusterer/clusterer_mod.c | 124 +- modules/clusterer/node_info.c | 36 +- modules/clusterer/node_info.h | 37 +- modules/clusterer/sharing_tags.c | 125 +- modules/clusterer/sharing_tags.h | 2 + modules/clusterer/sync.c | 32 +- modules/clusterer/topology.c | 42 +- modules/clusterer_controller/Makefile | 29 + modules/clusterer_controller/README | 1611 +++++ .../clusterer_controller.c | 5902 +++++++++++++++++ .../doc/clusterer_controller.xml | 22 + .../doc/clusterer_controller_admin.xml | 1813 +++++ .../doc/clusterer_controller_tests.xml | 266 + .../clusterer_controller/doc/contributors.xml | 25 + .../test/cc_join_reject_test.py | 135 + 19 files changed, 10809 insertions(+), 109 deletions(-) create mode 100644 modules/clusterer/clusterer_ctrl.c create mode 100644 modules/clusterer/clusterer_ctrl.h create mode 100644 modules/clusterer_controller/Makefile create mode 100644 modules/clusterer_controller/README create mode 100644 modules/clusterer_controller/clusterer_controller.c create mode 100644 modules/clusterer_controller/doc/clusterer_controller.xml create mode 100644 modules/clusterer_controller/doc/clusterer_controller_admin.xml create mode 100644 modules/clusterer_controller/doc/clusterer_controller_tests.xml create mode 100644 modules/clusterer_controller/doc/contributors.xml create mode 100755 modules/clusterer_controller/test/cc_join_reject_test.py diff --git a/modules/clusterer/api.h b/modules/clusterer/api.h index b6b8d3143c4..ad7ba8cd727 100644 --- a/modules/clusterer/api.h +++ b/modules/clusterer/api.h @@ -345,6 +345,5 @@ static inline module_dependency_t *get_deps_clusterer(const param_export_t *para return alloc_module_dep(MOD_TYPE_DEFAULT, "clusterer", DEP_ABORT); } - #endif /* CLUSTERER_API_H */ diff --git a/modules/clusterer/clusterer.c b/modules/clusterer/clusterer.c index ae6c16cbc70..b8b36c2885f 100644 --- a/modules/clusterer/clusterer.c +++ b/modules/clusterer/clusterer.c @@ -87,6 +87,7 @@ void sync_check_timer(utime_t ticks, void *param) lock_start_read(cl_list_lock); for (cl = *cluster_list; cl; cl = cl->next) { + if (!cl->current_node) continue; lock_get(cl->current_node->lock); if (!(cl->current_node->flags & NODE_STATE_ENABLED)) { lock_release(cl->current_node->lock); @@ -109,11 +110,27 @@ void sync_check_timer(utime_t ticks, void *param) cap->flags &= ~(CAP_SYNC_PENDING|CAP_SYNC_STARTUP); sr_set_status(cl_srg, STR2CI(cap->reg.sr_id), CAP_SR_SYNCED, STR2CI(CAP_SR_STATUS_STR(CAP_SR_SYNCED)), 0); - sr_add_report_fmt(cl_srg, STR2CI(cap->reg.sr_id), 0, - "ERROR: Sync request aborted! (no donor found in due time)" - " => fallback to synced state"); - LM_ERR("Sync request aborted! (no donor found in due time)" - ", falling back to synced state\n"); + /* Seed-fallback for a still-PENDING sync (the transfer never + * started - no donor responded to our request within + * seed_fb_interval). This is exactly what the seed + * fallback is *for*: on a fresh or simultaneous cold start + * no peer has any data to donate, so the seed proceeds as + * authoritative. It is expected, not an error - the state + * is still recorded in the status report for visibility. A + * genuine partial-sync failure is a *different* branch (the + * SYNC_IN_PROGRESS timeout below) and is logged there. */ + if (cl->node_list == NULL) + sr_add_report_fmt(cl_srg, STR2CI(cap->reg.sr_id), 0, + "No peers present — self-synced as first node in cluster"); + else + sr_add_report_fmt(cl_srg, STR2CI(cap->reg.sr_id), 0, + "Seed fallback — no donor sent data in due time, " + "self-synced as seed"); + LM_DBG("Seed fallback in cluster %d: capability '%.*s' " + "self-marked as synced (%s)\n", cl->cluster_id, + cap->reg.name.len, cap->reg.name.s, + cl->node_list == NULL ? "first/lone node" + : "no donor sent data in due time"); /* send update about the state of this capability */ send_single_cap_update(cl, cap, 1); @@ -155,7 +172,7 @@ int cl_set_state(int cluster_id, int node_id, enum cl_node_state state) return -1; } - if (node_id != current_id) { + if (node_id != cluster_self_id(cluster)) { node = get_node_by_id(cluster, node_id); if (!node) { lock_stop_read(cl_list_lock); @@ -190,13 +207,16 @@ int cl_set_state(int cluster_id, int node_id, enum cl_node_state state) LM_INFO("Set state: %s for node: %d in cluster: %d\n", state ? "enabled" : "disabled", node_id, cluster_id); - if (db_mode && update_db_state(cluster_id, node_id, state) < 0) + if (cl_db_mode(cluster) && update_db_state(cluster_id, node_id, state) < 0) LM_ERR("Failed to update state in clusterer DB for node [%d] cluster [%d]\n", node_id, cluster_id); return 0; } + if (!cluster->current_node) + return -1; + lock_get(cluster->current_node->lock); if (state == STATE_DISABLED && cluster->current_node->flags & NODE_STATE_ENABLED) @@ -226,7 +246,7 @@ int cl_set_state(int cluster_id, int node_id, enum cl_node_state state) LM_INFO("Set state: %s for local node in cluster: %d\n", state ? "enabled" : "disabled", cluster_id); - if (db_mode && update_db_state(cluster_id, current_id, state) < 0) + if (cl_db_mode(cluster) && update_db_state(cluster_id, current_id, state) < 0) LM_ERR("Failed to update state in clusterer DB for cluster [%d]\n", cluster->cluster_id); return 0; @@ -628,11 +648,27 @@ clusterer_bridges_bcast_msg(bin_packet_t *packet, int src_cid) } +/* This node's id in @cluster_id, resolved without taking cl_list_lock: clusters + * persist for the module's lifetime, and callers may already hold cl_list_lock + * (read) or a different lock (shtags) whose order is cl_list_lock -> shtags_lock, + * so taking cl_list_lock here would risk a reverse-order deadlock. Matches the + * lock-free nature of the GET_CURRENT_ID this replaces. Returns -1 if the + * cluster is unknown or this node's identity in it is not yet established. */ +static int trailer_self_id(int cluster_id) +{ + cluster_info_t *cl; + + for (cl = *cluster_list; cl; cl = cl->next) + if (cl->cluster_id == cluster_id) + return cluster_self_id(cl); + return -1; +} + int msg_add_trailer(bin_packet_t *packet, int cluster_id, int dst_id) { if (bin_push_int(packet, cluster_id) < 0) return -1; - if (bin_push_int(packet, current_id) < 0) + if (bin_push_int(packet, trailer_self_id(cluster_id)) < 0) return -1; if (bin_push_int(packet, dst_id) < 0) return -1; @@ -947,7 +983,7 @@ static void handle_cap_update(bin_packet_t *packet, node_info_t *source) for (i = 0; i < nr_nodes; i++) { bin_pop_int(packet, &node_id); - if (node_id == current_id) { + if (node_id == cluster_self_id(source->cluster)) { bin_pop_int(packet, &nr_cap); for (j = 0; j < nr_cap; j++) { bin_pop_str(packet, &cap); @@ -1128,12 +1164,15 @@ static void handle_remove_node(bin_packet_t *packet, cluster_info_t *cl) bin_pop_int(packet, &target_node); LM_DBG("Received remove node command for node id: [%d]\n", target_node); - if (db_mode) { + if (cl_db_mode(cl)) { LM_DBG("We are in DB mode, ignoring received remove node command\n"); return; } - if (target_node == current_id) { + if (target_node == cluster_self_id(cl)) { + if (!cl->current_node) + return; + lock_get(cl->current_node->lock); if (cl->current_node->flags & NODE_STATE_ENABLED) { @@ -1180,14 +1219,9 @@ void bin_rcv_cl_extra_packets(bin_packet_t *packet, int packet_type, LM_DBG("received clusterer message from: %s:%hu with source id: %d and" " cluster id: %d\n", ip, port, source_id, cluster_id); - if (source_id == current_id) { - LM_ERR("Received message with bad source - same node id as this instance\n"); - return; - } - gettimeofday(&now, NULL); - if (!db_mode && packet_type == CLUSTERER_REMOVE_NODE) + if ((!db_mode || use_controller) && packet_type == CLUSTERER_REMOVE_NODE) lock_start_write(cl_list_lock); else lock_start_read(cl_list_lock); @@ -1199,6 +1233,11 @@ void bin_rcv_cl_extra_packets(bin_packet_t *packet, int packet_type, goto exit; } + if (source_id == cluster_self_id(cl)) { + LM_ERR("Received message with bad source - same node id as this instance\n"); + goto exit; + } + lock_get(cl->current_node->lock); if (!(cl->current_node->flags & NODE_STATE_ENABLED)) { lock_release(cl->current_node->lock); @@ -1236,7 +1275,7 @@ void bin_rcv_cl_extra_packets(bin_packet_t *packet, int packet_type, } else lock_release(node->lock); - if (dest_id != current_id) { + if (dest_id != cluster_self_id(cl)) { if (clusterer_enable_rerouting == 0) { LM_WARN("Received message for destination id [%d] but rerouting disabled\n", dest_id); goto exit; @@ -1292,7 +1331,7 @@ void bin_rcv_cl_extra_packets(bin_packet_t *packet, int packet_type, } exit: - if (!db_mode && packet_type == CLUSTERER_REMOVE_NODE) + if ((!db_mode || use_controller) && packet_type == CLUSTERER_REMOVE_NODE) lock_stop_write(cl_list_lock); else lock_stop_read(cl_list_lock); @@ -1318,12 +1357,7 @@ void bin_rcv_cl_packets(bin_packet_t *packet, int packet_type, LM_DBG("received clusterer message from: %s:%hu with source id: %d and " "cluster id: %d\n", ip, port, source_id, cl_id); - if (source_id == current_id) { - LM_ERR("Received message with bad source - same node id as this instance\n"); - return; - } - - if (!db_mode && (packet_type == CLUSTERER_NODE_DESCRIPTION || + if ((!db_mode || use_controller) && (packet_type == CLUSTERER_NODE_DESCRIPTION || packet_type == CLUSTERER_FULL_TOP_UPDATE)) lock_start_write(cl_list_lock); else @@ -1335,6 +1369,22 @@ void bin_rcv_cl_packets(bin_packet_t *packet, int packet_type, goto exit; } + /* current_node is legitimately NULL while this node's identity is being + * (re)established for a dynamically constructed cluster (clusterer_ctrl + * update_identity: the cluster can already exist and receive BIN packets + * before current_node is assigned). Drop the packet instead of + * dereferencing NULL. */ + if (!cl->current_node) { + LM_INFO("Received message for cluster [%d] before local identity is " + "established, ignoring\n", cl_id); + goto exit; + } + + if (source_id == cluster_self_id(cl)) { + LM_ERR("Received message with bad source - same node id as this instance\n"); + goto exit; + } + lock_get(cl->current_node->lock); if (!(cl->current_node->flags & NODE_STATE_ENABLED)) { lock_release(cl->current_node->lock); @@ -1347,7 +1397,7 @@ void bin_rcv_cl_packets(bin_packet_t *packet, int packet_type, if (!node) { LM_INFO("Received message with unknown source id [%d]\n", source_id); - if (!db_mode) + if (!cl_db_mode(cl)) handle_internal_msg_unknown(packet, cl, packet_type, &ri->src_su, ri->proto, source_id); } else { @@ -1374,7 +1424,7 @@ void bin_rcv_cl_packets(bin_packet_t *packet, int packet_type, } exit: - if (!db_mode && (packet_type == CLUSTERER_NODE_DESCRIPTION || + if ((!db_mode || use_controller) && (packet_type == CLUSTERER_NODE_DESCRIPTION || packet_type == CLUSTERER_FULL_TOP_UPDATE)) lock_stop_write(cl_list_lock); else @@ -1472,11 +1522,6 @@ static void bin_rcv_mod_packets(bin_packet_t *packet, int packet_type, "cluster ids: %d->%d\n", ip, port, source_id, src_cluster_id, cluster_id); } - if (source_id == current_id) { - LM_ERR("Received message with bad source - same node id as this instance\n"); - return; - } - cap = (struct capability_reg *)ptr; if (!cap) { LM_ERR("Failed to get bin callback parameter\n"); @@ -1493,6 +1538,11 @@ static void bin_rcv_mod_packets(bin_packet_t *packet, int packet_type, goto exit; } + if (source_id == cluster_self_id(cl)) { + LM_ERR("Received message with bad source - same node id as this instance\n"); + goto exit; + } + lock_get(cl->current_node->lock); if (!(cl->current_node->flags & NODE_STATE_ENABLED)) { lock_release(cl->current_node->lock); @@ -1543,7 +1593,7 @@ static void bin_rcv_mod_packets(bin_packet_t *packet, int packet_type, } else lock_release(node->lock); - if (dest_id != current_id) { + if (dest_id != cluster_self_id(cl)) { /* route the message */ bin_push_int(packet, cluster_id); bin_push_int(packet, source_id); @@ -1619,6 +1669,7 @@ int send_single_cap_update(cluster_info_t *cluster, struct local_cap *cap, timestamp = (int)(unsigned long)time(NULL); + if (!cluster->current_node) return -1; lock_get(cluster->current_node->lock); for (neigh = cluster->current_node->neighbour_list; neigh; @@ -1637,7 +1688,7 @@ int send_single_cap_update(cluster_info_t *cluster, struct local_cap *cap, return -1; } bin_push_int(&packet, cluster->cluster_id); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(cluster)); bin_push_int(&packet, ++cluster->current_node->cap_seq_no); bin_push_int(&packet, timestamp); @@ -1646,7 +1697,7 @@ int send_single_cap_update(cluster_info_t *cluster, struct local_cap *cap, /* only the current node */ bin_push_int(&packet, 1); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(cluster)); /* only a single capability */ bin_push_int(&packet, 1); @@ -1656,7 +1707,7 @@ int send_single_cap_update(cluster_info_t *cluster, struct local_cap *cap, bin_push_int(&packet, 0); /* don't require reply */ bin_push_int(&packet, 1); /* path length is 1, only current node at this point */ - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(cluster)); bin_get_buffer(&packet, &bin_buffer); for (i = 0; i < no_dests; i++) @@ -1704,7 +1755,7 @@ int send_cap_update(node_info_t *dest_node, int require_reply) return -1; } bin_push_int(&packet, dest_node->cluster->cluster_id); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(dest_node->cluster)); lock_get(dest_node->cluster->current_node->lock); @@ -1719,7 +1770,7 @@ int send_cap_update(node_info_t *dest_node, int require_reply) for (cl_cap = dest_node->cluster->capabilities, nr_cap = 0; cl_cap; cl_cap = cl_cap->next, nr_cap++) ; if (nr_cap) { - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(dest_node->cluster)); bin_push_int(&packet, nr_cap); for (cl_cap=dest_node->cluster->capabilities;cl_cap;cl_cap=cl_cap->next) { bin_push_str(&packet, &cl_cap->reg.name); @@ -1750,7 +1801,7 @@ int send_cap_update(node_info_t *dest_node, int require_reply) bin_push_int(&packet, require_reply); bin_push_int(&packet, 1); /* path length is 1, only current node at this point */ - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(dest_node->cluster)); bin_get_buffer(&packet, &bin_buffer); if (msg_send(dest_node->cluster->send_sock, dest_node->proto, &dest_node->addr, @@ -1903,9 +1954,25 @@ int cl_register_cap(str *cap, cl_packet_cb_f packet_cb, cl_event_cb_f event_cb, cluster = get_cluster_by_id(cluster_id); if (!cluster) { - LM_ERR("cluster id %d is not defined in the %s\n", cluster_id, - db_mode ? "DB" : "script"); - return -1; + if (use_controller) { + cluster = shm_malloc(sizeof *cluster); + if (!cluster) { LM_ERR("no shm\n"); return -1; } + memset(cluster, 0, sizeof *cluster); + cluster->cluster_id = cluster_id; + cluster->controller_managed = 1; /* on-demand controller stub */ + if ((cluster->lock = lock_alloc()) == NULL || !lock_init(cluster->lock)) { + shm_free(cluster); return -1; + } + if (cl_list_lock) lock_start_write(cl_list_lock); + cluster->next = *cluster_list; + *cluster_list = cluster; + if (cl_list_lock) lock_stop_write(cl_list_lock); + LM_INFO("clusterer: auto-created stub for cluster %d\n", cluster_id); + } else { + LM_ERR("cluster id %d is not defined in the %s\n", cluster_id, + db_mode ? "DB" : "script"); + return -1; + } } new_cl_cap = shm_malloc(sizeof *new_cl_cap + cap->len + CAP_SR_ID_PREFIX_LEN); diff --git a/modules/clusterer/clusterer_ctrl.c b/modules/clusterer/clusterer_ctrl.c new file mode 100644 index 00000000000..e27ef114aa3 --- /dev/null +++ b/modules/clusterer/clusterer_ctrl.c @@ -0,0 +1,391 @@ +/* + * clusterer_ctrl.c — Controller API implementation for clusterer + * + * Copyright (C) 2026 Yury Kirsanov + * VoIPLine Telecom + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include "../../dprint.h" +#include "../../rw_locking.h" +#include "../../mem/shm_mem.h" +#include "../../locking.h" + +#include "node_info.h" /* add_node_info, remove_node_list, + get_cluster_by_id, get_node_by_id, + cluster_list, cl_list_lock, current_id */ +#include "clusterer.h" /* LS_DOWN, do_actions_node_ev, MAX_NO_CLUSTERS */ +#include "sharing_tags.h" +#include "topology.h" /* delete_neighbour */ /* shtag_event_handler */ +#include "clusterer_ctrl.h" + +/* declared in clusterer.c — raises E_CLUSTERER_NODE_STATE_CHANGED */ +int report_node_state(enum clusterer_event event, int cluster_id, int node_id); + +/* Free a current_node entry that is NOT in node_list. + * remove_node_list() walks node_list looking for the pointer — if current_node + * was never added there (our set_my_identity path) it crashes. */ +static void free_current_node(node_info_t *node) +{ + if (!node) return; + if (node->lock) { + lock_destroy(node->lock); + lock_dealloc(node->lock); + } + if (node->sp_info) shm_free(node->sp_info); + if (node->description.s) shm_free(node->description.s); + if (node->sip_addr.s) shm_free(node->sip_addr.s); + if (node->url.s) shm_free(node->url.s); + shm_free(node); +} + +/** + * clusterer_ctrl_set_identity() - register this node's own identity. + * + * CRITICAL: current_id MUST be set before calling add_node_info(). + * add_node_info() checks (node_id == current_id) to decide whether to + * place the entry in cluster->current_node (self, not pinged) or + * cluster->node_list (peer, pinged). Setting it after causes the local + * node to land in node_list and get pinged — "same node id" errors. + */ +int clusterer_ctrl_set_identity(int cluster_id, int node_id, str *bin_url) +{ + node_info_t *new_node = NULL; + cluster_info_t *cl; + int int_vals[NO_DB_INT_VALS]; + str str_vals[NO_DB_STR_VALS]; + static str desc = str_init("controller"); + static str seed = str_init("seed"); + + int_vals[INT_VALS_ID_COL] = 0; + int_vals[INT_VALS_CLUSTER_ID_COL] = cluster_id; + int_vals[INT_VALS_NODE_ID_COL] = node_id; + int_vals[INT_VALS_STATE_COL] = 1; + int_vals[INT_VALS_NO_PING_RETRIES_COL] = DEFAULT_NO_PING_RETRIES; + int_vals[INT_VALS_PRIORITY_COL] = DEFAULT_PRIORITY; + + memset(str_vals, 0, sizeof str_vals); + str_vals[STR_VALS_URL_COL] = *bin_url; + str_vals[STR_VALS_FLAGS_COL] = seed; + str_vals[STR_VALS_DESCRIPTION_COL] = desc; + + /* Set current_id BEFORE add_node_info */ + current_id = node_id; + if (_current_id_shm) *_current_id_shm = node_id; + + lock_start_write(cl_list_lock); + + if (add_node_info(&new_node, cluster_list, int_vals, str_vals, node_id) < 0) { + lock_stop_write(cl_list_lock); + LM_ERR("clusterer: set_my_identity: add_node_info failed for " + "cluster %d node %d\n", cluster_id, node_id); + return -1; + } + + cl = get_cluster_by_id(cluster_id); + if (cl && new_node && !cl->current_node) + cl->current_node = new_node; + + lock_stop_write(cl_list_lock); + + LM_INFO("clusterer: [cluster %d] identity set: node_id=%d url=%.*s\n", + cluster_id, node_id, bin_url->len, bin_url->s); + return 0; +} + +/** + * clusterer_ctrl_add_node() - add a discovered peer at runtime. + */ +int clusterer_ctrl_add_node(int cluster_id, int node_id, str *bin_url) +{ + node_info_t *new_node = NULL; + cluster_info_t *cl; + int int_vals[NO_DB_INT_VALS]; + str str_vals[NO_DB_STR_VALS]; + static str desc = str_init("controller"); + static str seed = str_init("seed"); + + lock_start_write(cl_list_lock); + + cl = get_cluster_by_id(cluster_id); + if (cl && get_node_by_id(cl, node_id)) { + lock_stop_write(cl_list_lock); + LM_DBG("clusterer: [cluster %d] node %d already present\n", + cluster_id, node_id); + return 0; + } + + int_vals[INT_VALS_ID_COL] = 0; + int_vals[INT_VALS_CLUSTER_ID_COL] = cluster_id; + int_vals[INT_VALS_NODE_ID_COL] = node_id; + int_vals[INT_VALS_STATE_COL] = 1; + int_vals[INT_VALS_NO_PING_RETRIES_COL] = DEFAULT_NO_PING_RETRIES; + int_vals[INT_VALS_PRIORITY_COL] = DEFAULT_PRIORITY; + + memset(str_vals, 0, sizeof str_vals); + str_vals[STR_VALS_URL_COL] = *bin_url; + str_vals[STR_VALS_FLAGS_COL] = seed; + str_vals[STR_VALS_DESCRIPTION_COL] = desc; + + if (add_node_info(&new_node, cluster_list, int_vals, str_vals, cluster_self_id(cl)) < 0) { + lock_stop_write(cl_list_lock); + LM_ERR("clusterer: add_node: add_node_info failed for " + "cluster %d node %d\n", cluster_id, node_id); + return -1; + } + + lock_stop_write(cl_list_lock); + + LM_INFO("clusterer: [cluster %d] added peer node_id=%d url=%.*s\n", + cluster_id, node_id, bin_url->len, bin_url->s); + return 0; +} + +/** + * clusterer_ctrl_remove_node() - remove a departed peer at runtime. + */ +int clusterer_ctrl_remove_node(int cluster_id, int node_id) +{ + cluster_info_t *cl; + node_info_t *node; + + lock_start_write(cl_list_lock); + + cl = get_cluster_by_id(cluster_id); + if (!cl) { + lock_stop_write(cl_list_lock); + LM_WARN("clusterer: remove_node: cluster %d not found\n", cluster_id); + return -1; + } + + node = get_node_by_id(cl, node_id); + if (!node) { + lock_stop_write(cl_list_lock); + LM_WARN("clusterer: remove_node: node %d not found in cluster %d\n", + node_id, cluster_id); + return -1; + } + + /* Purge all topology references to the departing node BEFORE + * freeing it: neighbour lists of current_node and every peer, + * plus next_hop pointers. Freed-node reuse (same node_id + * reassigned later) otherwise leaves dangling pointers that + * crash with bogus proto/node values. */ + { + node_info_t *it; + if (cl->current_node) + delete_neighbour(cl->current_node, node); + for (it = cl->node_list; it; it = it->next) { + if (it == node) continue; + lock_get(it->lock); + delete_neighbour(it, node); + if (it->next_hop && it->next_hop->node_id == node_id) + it->next_hop = NULL; + lock_release(it->lock); + } + } + + /* Remove node from list, then fire callbacks outside the lock. + * Callbacks (dialog rcv_cluster_event) call back into clusterer + * to send BIN packets and need cl_list_lock for read. */ + remove_node_list(cl, node); + + { + struct local_cap *cap_it; + struct local_cap *caps = cl->capabilities; + lock_stop_write(cl_list_lock); + for (cap_it = caps; cap_it; cap_it = cap_it->next) + if (cap_it->reg.event_cb) + cap_it->reg.event_cb(CLUSTER_NODE_DOWN, node_id); + report_node_state(CLUSTER_NODE_DOWN, cluster_id, node_id); + } + + LM_INFO("clusterer: [cluster %d] removed node_id=%d\n", + cluster_id, node_id); + return 0; +} + +/** + * clusterer_ctrl_update_identity() - correct this node's node_id. + * + * Replaces the optimistic node_id=1 with the real master-assigned id. + * No-op if id unchanged. + * + * CRITICAL: current_id must be set BEFORE free+add so add_node_info + * routes the new entry to current_node (self) not node_list (peer). + * current_node is NOT in node_list so we free it directly — calling + * remove_node_list() on it would crash walking the list for a pointer + * that isn't there. + */ +int clusterer_ctrl_update_identity(int cluster_id, int new_node_id, str *bin_url) +{ + cluster_info_t *cl; + node_info_t *new_node = NULL; + node_info_t *old_node; + int int_vals[NO_DB_INT_VALS]; + str str_vals[NO_DB_STR_VALS]; + static str desc = str_init("controller"); + static str seed = str_init("seed"); + + lock_start_write(cl_list_lock); + + cl = get_cluster_by_id(cluster_id); + if (!cl) { + lock_stop_write(cl_list_lock); + LM_ERR("clusterer: update_identity: cluster %d not found\n", cluster_id); + return -1; + } + + if (cl->current_node && cl->current_node->node_id == new_node_id) { + lock_stop_write(cl_list_lock); + return 0; /* no-op */ + } + + /* Set current_id BEFORE free+add */ + current_id = new_node_id; + if (_current_id_shm) *_current_id_shm = new_node_id; + + old_node = cl->current_node; + cl->current_node = NULL; + + /* Purge all peer neighbour references to the old current_node before + * freeing it — same as clusterer_ctrl_remove_node does for peers. */ + if (old_node) { + node_info_t *it; + for (it = cl->node_list; it; it = it->next) { + lock_get(it->lock); + delete_neighbour(it, old_node); + if (it->next_hop && it->next_hop->node_id == old_node->node_id) + it->next_hop = NULL; + lock_release(it->lock); + } + } + + int_vals[INT_VALS_ID_COL] = 0; + int_vals[INT_VALS_CLUSTER_ID_COL] = cluster_id; + int_vals[INT_VALS_NODE_ID_COL] = new_node_id; + int_vals[INT_VALS_STATE_COL] = 1; + int_vals[INT_VALS_NO_PING_RETRIES_COL] = DEFAULT_NO_PING_RETRIES; + int_vals[INT_VALS_PRIORITY_COL] = DEFAULT_PRIORITY; + + memset(str_vals, 0, sizeof str_vals); + str_vals[STR_VALS_URL_COL] = *bin_url; + str_vals[STR_VALS_FLAGS_COL] = seed; + str_vals[STR_VALS_DESCRIPTION_COL] = desc; + + if (add_node_info(&new_node, cluster_list, int_vals, str_vals, new_node_id) < 0) { + lock_stop_write(cl_list_lock); + LM_ERR("clusterer: update_identity: add_node_info failed for " + "cluster %d node %d\n", cluster_id, new_node_id); + free_current_node(old_node); + return -1; + } + + cl->current_node = new_node; + + lock_stop_write(cl_list_lock); + + /* Free old entry outside the lock */ + free_current_node(old_node); + + LM_INFO("clusterer: [cluster %d] identity updated to node_id=%d url=%.*s\n", + cluster_id, new_node_id, bin_url->len, bin_url->s); + return 0; +} + +/** + * load_clusterer_ctrl_binds() - fill the API struct for use by controller. + */ +int clusterer_ctrl_sync_current_id(void) +{ + cluster_info_t *cl; + + if (!cl_list_lock || !cluster_list || !*cluster_list) + return 0; + + lock_start_read(cl_list_lock); + for (cl = *cluster_list; cl; cl = cl->next) { + if (cl->current_node) { + current_id = cl->current_node->node_id; + break; + } + } + lock_stop_read(cl_list_lock); + return 0; +} + +int clusterer_ctrl_activate_backup_shtags(int cluster_id) +{ + return shtag_activate_all_backup(cluster_id); +} + +int clusterer_ctrl_force_backup_shtags(int cluster_id) +{ + return shtag_force_all_backup(cluster_id); +} + +int clusterer_ctrl_set_shtag_managed(int cluster_id) +{ + cluster_info_t *cl; + + lock_start_write(cl_list_lock); + cl = get_cluster_by_id(cluster_id); + if (!cl) { + lock_stop_write(cl_list_lock); + LM_ERR("clusterer: set_shtag_managed: cluster %d not found\n", + cluster_id); + return -1; + } + cl->shtag_managed = 1; + lock_stop_write(cl_list_lock); + + LM_INFO("clusterer: [cluster %d] sharing tags are now " + "controller-managed (MI and script changes blocked)\n", + cluster_id); + return 0; +} + +int clusterer_ctrl_unset_shtag_managed(int cluster_id) +{ + cluster_info_t *cl; + + lock_start_write(cl_list_lock); + cl = get_cluster_by_id(cluster_id); + if (!cl) { + lock_stop_write(cl_list_lock); + LM_ERR("clusterer: unset_shtag_managed: cluster %d not found\n", + cluster_id); + return -1; + } + cl->shtag_managed = 0; + lock_stop_write(cl_list_lock); + + LM_INFO("clusterer: [cluster %d] sharing tags are no longer " + "controller-managed (MI and script changes allowed again)\n", + cluster_id); + return 0; +} + +int load_clusterer_ctrl_binds(clusterer_ctrl_binds_t *binds) +{ + if (!binds) { + LM_ERR("clusterer: load_clusterer_ctrl_binds: NULL binds\n"); + return -1; + } + binds->set_my_identity = clusterer_ctrl_set_identity; + binds->add_node = clusterer_ctrl_add_node; + binds->remove_node = clusterer_ctrl_remove_node; + binds->update_identity = clusterer_ctrl_update_identity; + binds->sync_current_id = clusterer_ctrl_sync_current_id; + binds->activate_backup_shtags = clusterer_ctrl_activate_backup_shtags; + binds->set_shtag_managed = clusterer_ctrl_set_shtag_managed; + binds->unset_shtag_managed = clusterer_ctrl_unset_shtag_managed; + binds->force_backup_shtags = clusterer_ctrl_force_backup_shtags; + return 0; +} diff --git a/modules/clusterer/clusterer_ctrl.h b/modules/clusterer/clusterer_ctrl.h new file mode 100644 index 00000000000..af4f8b9f47b --- /dev/null +++ b/modules/clusterer/clusterer_ctrl.h @@ -0,0 +1,172 @@ +/* + * clusterer_ctrl.h — Controller API for the clusterer module + * + * Allows an external module (clusterer_controller) to drive the clusterer + * topology at runtime without any DB or static configuration. + * + * Copyright (C) 2026 Yury Kirsanov + * VoIPLine Telecom + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#ifndef CLUSTERER_CTRL_H +#define CLUSTERER_CTRL_H + +#include "../../str.h" + +/** + * clusterer_ctrl_binds - API struct loaded by clusterer_controller. + * + * Usage in clusterer_controller mod_init(): + * + * #include "../clusterer/clusterer_ctrl.h" + * static clusterer_ctrl_binds_t clctl; + * if (load_clusterer_ctrl_binds(&clctl) < 0) { ... } + * + * Then from the worker process / callbacks: + * + * str url = str_init("bin:10.22.23.191:5566"); + * clctl.set_my_identity(1, my_node_id, &url); + * + * str peer_url = str_init("bin:10.22.23.192:5566"); + * clctl.add_node(1, peer_node_id, &peer_url); + * + * clctl.remove_node(1, departed_node_id); + */ +typedef struct clusterer_ctrl_binds { + /** + * set_my_identity() — register this node's identity in a cluster. + * + * Creates the cluster_info_t if it does not yet exist. + * Sets global current_id and marks cluster->current_node. + * Must be called before add_node() for the same cluster_id. + * + * Called by controller after node_id is allocated (either from + * existing master via NODE_ASSIGN or from join deadline expiry). + * + * @cluster_id integer cluster identifier (matches controller cluster) + * @node_id integer allocated by the controller master (>= 1) + * @bin_url str pointing to "bin:IP:PORT" + * @return 0 on success, -1 on error + */ + int (*set_my_identity)(int cluster_id, int node_id, str *bin_url); + + /** + * add_node() — add a peer node to a cluster at runtime. + * + * Creates the node_info_t, adds it to the cluster's node_list. + * The clusterer ping timer picks it up within one ping interval + * and establishes the BIN link automatically. + * + * Called on every CC_PKT_NODE_ASSIGN received for a peer. + * Safe to call if the node already exists — returns 0 (no-op). + * + * @cluster_id must match a cluster initialised by set_my_identity() + * @node_id peer's allocated node_id + * @bin_url peer's "bin:IP:PORT" string + * @return 0 on success, -1 on error + */ + int (*add_node)(int cluster_id, int node_id, str *bin_url); + + /** + * remove_node() — remove a peer node from a cluster at runtime. + * + * Removes from the node_list and cleans up routing state. + * The BIN connection is closed by the clusterer's own cleanup path. + * + * Called on CC_PKT_GOODBYE or when election-window expiry removes + * the peer from the controller's own peer table. + * + * @cluster_id cluster the node belongs to + * @node_id node_id to remove + * @return 0 on success, -1 if cluster or node not found + */ + int (*remove_node)(int cluster_id, int node_id); + /** + * update_identity() — correct this node's node_id after master assignment. + * + * Called when the real node_id arrives via NODE_ASSIGN and differs from + * the optimistic value set at startup. Removes the old current_node + * entry and adds a new one with the correct node_id, updating both + * global current_id and cluster->current_node atomically. + * + * Safe to call with the same node_id as already set — returns 0 (no-op). + * + * @cluster_id cluster to update + * @new_node_id the master-assigned node_id + * @bin_url this node's "bin:IP:PORT" (may be identical to current) + * @return 0 on success, -1 on error + */ + int (*update_identity)(int cluster_id, int new_node_id, str *bin_url); + + /** + * sync_current_id() - sync local current_id from shared memory. + * + * Must be called from child_init() in every child process after fork. + * current_id is a process-local global — after fork each child inherits + * the pre-fork value. This re-reads the correct id from the shared + * cluster->current_node so BIN packets carry the right source node_id. + * + * @return 0 always + */ + int (*sync_current_id)(void); + + /** + * activate_backup_shtags() - activate all BACKUP sharing tags + * for the given cluster. Called only by the controller master + * when a peer departs or when this node becomes new master. + */ + int (*activate_backup_shtags)(int cluster_id); + + /** + * force_backup_shtags() - force all local sharing tags to BACKUP + * regardless of =active config. Called by the controller at + * startup when it manages tags itself. + */ + int (*force_backup_shtags)(int cluster_id); + + /** + * set_shtag_managed() - mark a cluster's sharing tags as controller-managed. + * + * Once set, the MI command clusterer_set_tag_active and the $shtag() + * script variable setter are blocked for this cluster, returning an + * error to the caller. This prevents manual or event-route-driven + * shtag changes from conflicting with controller-managed failover. + * + * Called from clusterer_controller mod_init() for every cluster that + * has manage_shtags=1. + * + * @cluster_id cluster to lock + * @return 0 on success, -1 if cluster not found + */ + int (*set_shtag_managed)(int cluster_id); + + /** + * unset_shtag_managed() - stop treating a cluster's sharing tags as + * controller-managed, re-allowing MI/script changes. Used when a node + * adopts a running cluster's manage_shtags=0 setting at runtime + * (on_config_mismatch=adopt). + * + * @cluster_id cluster to unlock + * @return 0 on success, -1 if cluster not found + */ + int (*unset_shtag_managed)(int cluster_id); +} clusterer_ctrl_binds_t; + +/** + * load_clusterer_ctrl_binds() — fill a clusterer_ctrl_binds_t struct. + * + * Called from clusterer_controller's mod_init(). Returns -1 if clusterer + * is not loaded or was not built with use_controller support. + */ +typedef int (*load_clusterer_ctrl_binds_f)(clusterer_ctrl_binds_t *binds); + +int load_clusterer_ctrl_binds(clusterer_ctrl_binds_t *binds); + +#endif /* CLUSTERER_CTRL_H */ diff --git a/modules/clusterer/clusterer_mod.c b/modules/clusterer/clusterer_mod.c index 92a6540b792..a837da4192b 100644 --- a/modules/clusterer/clusterer_mod.c +++ b/modules/clusterer/clusterer_mod.c @@ -39,6 +39,7 @@ #include "clusterer.h" #include "sync.h" #include "sharing_tags.h" +#include "clusterer_ctrl.h" #include "clusterer_evi.h" int ping_interval = DEFAULT_PING_INTERVAL; @@ -47,8 +48,24 @@ int ping_timeout = DEFAULT_PING_TIMEOUT; int seed_fb_interval = DEFAULT_SEED_FB_INTERVAL; int sync_timeout = DEFAULT_SYNC_TIMEOUT; int current_id = -1; +int *_current_id_shm = NULL; int db_mode = 1; int clusterer_enable_rerouting = 1; +int use_controller = 0; + +/* cluster_ids to pre-create when use_controller=1 */ +static int cc_stub_ids[64]; +static int cc_stub_count = 0; + +static int cc_add_cluster_id(modparam_t type, void *val) +{ + if (cc_stub_count >= 64) { + LM_ERR("clusterer: too many cluster_id entries\n"); + return -1; + } + cc_stub_ids[cc_stub_count++] = (int)(long)val; + return 0; +} str clusterer_db_url = {NULL, 0}; @@ -106,6 +123,7 @@ int cmd_check_addr(struct sip_msg *msg, int *cluster_id, str *ip_str, */ static const cmd_export_t cmds[] = { + {"load_clusterer_ctrl_binds", (cmd_function)load_clusterer_ctrl_binds, {{0,0,0}}, 0}, {"load_clusterer", (cmd_function)load_clusterer, {{0,0,0}}, 0}, {"cluster_broadcast_req", (cmd_function)cmd_broadcast_req, { {CMD_PARAM_INT,0,0}, @@ -158,6 +176,8 @@ static const param_export_t params[] = { {"flags_col", STR_PARAM, &flags_col.s }, {"description_col", STR_PARAM, &description_col.s }, {"db_mode", INT_PARAM, &db_mode }, + {"use_controller", INT_PARAM, &use_controller }, + {"cluster_id", INT_PARAM|USE_FUNC_PARAM, (void*)cc_add_cluster_id}, {"neighbor_node_info", STR_PARAM|USE_FUNC_PARAM, (void*)&provision_neighbor}, {"my_node_info", STR_PARAM|USE_FUNC_PARAM, @@ -389,11 +409,23 @@ static int mod_init(void) flags_col.len = strlen(flags_col.s); description_col.len = strlen(description_col.s); - /* only allow the DB URL to be skipped in "P2P discovery" mode */ + /* db_mode defaults to 1 (DB-backed). If no DB URL is configured there are + * no DB-backed native clusters, so drop to non-DB mode - this keeps + * controller-only and purely static (my_node_info) setups working without + * an explicit db_mode=0. A DB-native or hybrid setup provides a db_url and + * keeps db_mode. */ + if (!clusterer_db_url.s && !db_default_url) + db_mode = 0; + + /* The DB URL is required whenever native clusters are DB-backed + * (db_mode != 0), even alongside the controller. */ init_db_url(clusterer_db_url, db_mode == 0); - if (current_id < 1) { - LM_CRIT("Invalid 'my_node_id' parameter\n"); + /* my_node_id identifies this node within its *native* clusters (DB or + * static my_node_info). Controller clusters get their id assigned at + * runtime, so my_node_id is only required when native clusters exist. */ + if (current_id < 1 && (db_mode != 0 || (cluster_list && *cluster_list != NULL))) { + LM_CRIT("Invalid 'my_node_id' parameter (required for DB or static native clusters)\n"); return -1; } if (ping_interval <= 0) { @@ -421,6 +453,21 @@ static int mod_init(void) LM_CRIT("Failed to init lock\n"); return -1; } + /* Move GET_CURRENT_ID to shared memory so all forked processes + * see the same value when it is updated by the controller. */ + { + int *_shm_id = shm_malloc(sizeof(int)); + if (!_shm_id) { + LM_CRIT("No shm memory for GET_CURRENT_ID\n"); + return -1; + } + /* Seed with the static my_node_id so native (DB/static) clusters see + * the right identity via GET_CURRENT_ID at load time; -1 (a sentinel + * that matches no node) is only used when there is no static id, i.e. + * a controller-only node whose id is assigned at runtime. */ + *_shm_id = (current_id >= 1) ? current_id : -1; + _current_id_shm = _shm_id; + } /* if statistics are disabled, prevent their registration to core */ if (clusterer_enable_stats==0) @@ -443,16 +490,21 @@ static int mod_init(void) goto error; } - if (cl->current_node->node_id != current_id) { + if (cl->current_node && cl->current_node->node_id != current_id) { LM_ERR("Bad 'my_node_id' parameter, value: %d different than" - " the node_id property in the 'my_node_info' parameter\n", current_id); + " the node_id property in the 'my_node_info' parameter\n", GET_CURRENT_ID); goto error; } } } + /* Native clusters backed by the DB (db_mode != 0): bind and load them. In a + * hybrid this runs even when the controller is also active - DB-native and + * controller clusters are no longer mutually exclusive. Controller clusters + * are never stored in the DB, so load_db_info only ever brings in native + * ones. Static my_node_info/neighbor provisioning (db_mode==0) has already + * populated cluster_list at config-parse time. */ if (db_mode) { - /* bind to the mysql module */ if (db_bind_mod(&clusterer_db_url, &dr_dbf)) { LM_CRIT("Cannot bind to database module! " "Did you forget to load a database module ?\n"); @@ -462,7 +514,6 @@ static int mod_init(void) LM_CRIT("Given SQL DB does not provide query types needed by this module!\n"); goto error; } - /* init DB connection */ if ((db_hdl = dr_dbf.init(&clusterer_db_url)) == 0) { LM_ERR("cannot initialize database connection\n"); goto error; @@ -471,11 +522,57 @@ static int mod_init(void) LM_ERR("Failed to load info from DB\n"); goto error; } - dr_dbf.close(db_hdl); db_hdl = NULL; } + /* Controller-managed clusters: pre-create a stub for each cluster_id the + * controller registered (via the 'cluster_id' modparam), so cl_register_cap() + * succeeds for tm/dialog before clusterer_controller injects the real + * identity. Each stub is flagged controller_managed: it never touches the DB + * and behaves as db_mode=0, so it coexists with DB-native clusters. A + * cluster_id must be exclusively controller-managed OR native, never both - + * the native clusters (DB loaded just above, or static from parse time) are + * already in cluster_list, so an overlap is caught here. */ + if (use_controller) { + int _ci; + for (_ci = 0; _ci < cc_stub_count; _ci++) { + cluster_info_t *_ex; + for (_ex = *cluster_list; _ex; _ex = _ex->next) + if (_ex->cluster_id == cc_stub_ids[_ci]) { + LM_ERR("clusterer: cluster_id %d is registered as " + "controller-managed (cluster_id modparam) but is " + "already defined via native config " + "(my_node_info/neighbor_node_info/DB) or listed " + "twice - a cluster_id must be unique and either " + "controller-managed or native, not both\n", + cc_stub_ids[_ci]); + return -1; + } + + cluster_info_t *_cl = shm_malloc(sizeof *_cl); + if (!_cl) { + LM_ERR("clusterer: no shm for cluster %d stub\n", + cc_stub_ids[_ci]); + return -1; + } + memset(_cl, 0, sizeof *_cl); + _cl->cluster_id = cc_stub_ids[_ci]; + _cl->controller_managed = 1; + _cl->lock = lock_alloc(); + if (!_cl->lock || lock_init(_cl->lock) == NULL) { + LM_ERR("clusterer: lock_alloc failed for cluster %d\n", + cc_stub_ids[_ci]); + shm_free(_cl); + return -1; + } + _cl->next = *cluster_list; + *cluster_list = _cl; + LM_INFO("clusterer: pre-created controller-managed cluster %d stub\n", + cc_stub_ids[_ci]); + } + } + /* register timer */ heartbeats_timer_interval = gcd(ping_interval*1000, ping_timeout); heartbeats_timer_interval = gcd(heartbeats_timer_interval, node_timeout*1000); @@ -536,7 +633,8 @@ static int mod_init(void) /* check if the cluster IDs in the the sharing tag list are valid */ shtag_init_list(); shtag_init_reporting(); - shtag_validate_list(); + if (!use_controller) + shtag_validate_list(); return 0; error: @@ -893,7 +991,7 @@ static mi_response_t *clusterer_list_topology(const mi_params_t *params, if (!node_item) goto error; - if (add_mi_number(node_item, MI_SSTR("node_id"), current_id) < 0) + if (add_mi_number(node_item, MI_SSTR("node_id"), GET_CURRENT_ID) < 0) goto error; neigh_arr = add_mi_array(node_item, MI_SSTR("Neighbours")); @@ -925,7 +1023,7 @@ static mi_response_t *clusterer_list_topology(const mi_params_t *params, } if (n_info->link_state == LS_UP) - if (add_mi_number(neigh_arr, 0,0, current_id) < 0) { + if (add_mi_number(neigh_arr, 0,0, GET_CURRENT_ID) < 0) { lock_release(n_info->lock); goto error; } @@ -1054,7 +1152,7 @@ static mi_response_t *cluster_send_mi(const mi_params_t *params, return init_mi_param_error(); if (node_id < 1) return init_mi_error(400, MI_SSTR("Bad value for 'destination'")); - if (node_id == current_id) + if (node_id == GET_CURRENT_ID) return init_mi_error(400, MI_SSTR("Local node specified as destination")); if (get_mi_string_param(params, "cmd_name", &cmd_name.s, &cmd_name.len) < 0) @@ -1250,7 +1348,7 @@ static inline void generate_msg_tag(pv_value_t *tag_val, int cluster_id) memcpy(tag_val->rs.s, tmp, len); tag_val->rs.s[len] = '-'; tag_val->rs.len = len + 1; - tmp = int2str(current_id, &len); + tmp = int2str(GET_CURRENT_ID, &len); memcpy(tag_val->rs.s + tag_val->rs.len, tmp, len); tag_val->rs.s[tag_val->rs.len + len] = '-'; tag_val->rs.len += len + 1; diff --git a/modules/clusterer/node_info.c b/modules/clusterer/node_info.c index fd09726dd67..6025734a062 100644 --- a/modules/clusterer/node_info.c +++ b/modules/clusterer/node_info.c @@ -77,7 +77,7 @@ cluster_info_t **cluster_list; int load_cluster_bridges(cluster_info_t *cl_list); int add_node_info(node_info_t **new_info, cluster_info_t **cl_list, int *int_vals, - str *str_vals) + str *str_vals, int self_id) { char *host; int hlen, port; @@ -140,7 +140,7 @@ int add_node_info(node_info_t **new_info, cluster_info_t **cl_list, int *int_val else (*new_info)->flags &= ~NODE_STATE_ENABLED; - if (int_vals[INT_VALS_NODE_ID_COL] != current_id) + if (int_vals[INT_VALS_NODE_ID_COL] != self_id) (*new_info)->link_state = LS_RESTART_PINGING; else (*new_info)->link_state = LS_UP; @@ -213,7 +213,7 @@ int add_node_info(node_info_t **new_info, cluster_info_t **cl_list, int *int_val (*new_info)->proto = proto; - if (int_vals[INT_VALS_NODE_ID_COL] != current_id) { + if (int_vals[INT_VALS_NODE_ID_COL] != self_id) { he = sip_resolvehost(&st, (unsigned short *) &port, (unsigned short *)&proto, 0, 0); if (!he) { @@ -259,7 +259,7 @@ int add_node_info(node_info_t **new_info, cluster_info_t **cl_list, int *int_val } (*new_info)->sp_info->node = *new_info; - if (int_vals[INT_VALS_NODE_ID_COL] != current_id) { + if (int_vals[INT_VALS_NODE_ID_COL] != self_id) { (*new_info)->next = cluster->node_list; cluster->node_list = *new_info; cluster->no_nodes++; @@ -382,7 +382,7 @@ int load_db_info(db_func_t *dr_dbf, db_con_t* db_hdl, cluster_info_t **cl_list) LM_DBG("DB query - retrieve the list of clusters" " in which the local node runs\n"); - VAL_INT(&clusterer_node_id_value) = current_id; + VAL_INT(&clusterer_node_id_value) = GET_CURRENT_ID; /* first we see in which clusters the local node runs*/ if (dr_dbf->query(db_hdl, &clusterer_node_id_key, &op_eq, @@ -499,12 +499,12 @@ int load_db_info(db_func_t *dr_dbf, db_con_t* db_hdl, cluster_info_t **cl_list) strlen(str_vals[STR_VALS_DESCRIPTION_COL].s) : 0; /* add info to backing list */ - if ((rc = add_node_info(&_, cl_list, int_vals, str_vals)) != 0) { + if ((rc = add_node_info(&_, cl_list, int_vals, str_vals, GET_CURRENT_ID)) != 0) { LM_ERR("Unable to add node info to backing list\n"); if (rc < 0) { /* serious error happened, better give up */ goto error; - } else if (int_vals[INT_VALS_NODE_ID_COL] == current_id) { + } else if (int_vals[INT_VALS_NODE_ID_COL] == GET_CURRENT_ID) { LM_ERR("Invalid info for local node\n"); /* the node info is bogus, but cannot be skipped * as it is the current node) */ @@ -770,7 +770,7 @@ int provision_neighbor(modparam_t type, void *val) *cluster_list = NULL; } - if (add_node_info(&new_info, cluster_list, int_vals, str_vals) < 0) { + if (add_node_info(&new_info, cluster_list, int_vals, str_vals, GET_CURRENT_ID) < 0) { LM_ERR("Unable to add node info to backing list\n"); return -1; } @@ -803,13 +803,13 @@ int provision_current(modparam_t type, void *val) return -1; } - if (int_vals[INT_VALS_NODE_ID_COL] == -1 && current_id == -1) { + if (int_vals[INT_VALS_NODE_ID_COL] == -1 && GET_CURRENT_ID == -1) { LM_ERR("Node ID not defined. Set either the value of the 'node_id' proprety" " of 'my_node_info' or set 'my_node_id' parameter before 'my_node_info'!\n"); return -1; } - if (current_id != -1 && int_vals[INT_VALS_NODE_ID_COL] != -1 && - int_vals[INT_VALS_NODE_ID_COL] != current_id) { + if (GET_CURRENT_ID != -1 && int_vals[INT_VALS_NODE_ID_COL] != -1 && + int_vals[INT_VALS_NODE_ID_COL] != GET_CURRENT_ID) { LM_ERR("Bad value in 'my_node_info' parameter, node_id: %d different" " than 'my_node_id' parameter\n", int_vals[INT_VALS_NODE_ID_COL]); return -1; @@ -817,7 +817,7 @@ int provision_current(modparam_t type, void *val) if (int_vals[INT_VALS_NODE_ID_COL] != -1) current_id = int_vals[INT_VALS_NODE_ID_COL]; else - int_vals[INT_VALS_NODE_ID_COL] = current_id; + int_vals[INT_VALS_NODE_ID_COL] = GET_CURRENT_ID; int_vals[INT_VALS_STATE_COL] = 1; if (int_vals[INT_VALS_NO_PING_RETRIES_COL] == -1) @@ -837,7 +837,7 @@ int provision_current(modparam_t type, void *val) *cluster_list = NULL; } - if (add_node_info(&new_info, cluster_list, int_vals, str_vals) != 0) { + if (add_node_info(&new_info, cluster_list, int_vals, str_vals, GET_CURRENT_ID) != 0) { LM_ERR("Unable to add node info to backing list\n"); return -1; } @@ -864,10 +864,10 @@ int update_db_state(int cluster_id, int node_id, int state) { VAL_NULL(&update_val) = 0; VAL_INT(&update_val) = state; - if (node_id == current_id) { + if (node_id == GET_CURRENT_ID) { VAL_TYPE(&node_id_val) = DB_INT; VAL_NULL(&node_id_val) = 0; - VAL_INT(&node_id_val) = current_id; + VAL_INT(&node_id_val) = GET_CURRENT_ID; if (dr_dbf.update(db_hdl, &node_id_key, 0, &node_id_val, &update_key, &update_val, 1, 1) < 0) @@ -1121,7 +1121,7 @@ void api_free_next_hop(clusterer_node_t *next_hop) int cl_get_my_id(void) { - return current_id; + return GET_CURRENT_ID; } int cl_get_my_sip_addr(int cluster_id, str *out_addr) @@ -1149,7 +1149,7 @@ int cl_get_my_sip_addr(int cluster_id, str *out_addr) memset(out_addr, 0, sizeof *out_addr); rc = 0; } else { - if (pkg_str_dup(out_addr, &cl->current_node->sip_addr) != 0) { + if (cl->current_node && pkg_str_dup(out_addr, &cl->current_node->sip_addr) != 0) { LM_ERR("oom\n"); memset(out_addr, 0, sizeof *out_addr); rc = -1; @@ -1203,7 +1203,7 @@ int cl_get_my_index(int cluster_id, str *capability, int *nr_nodes) sorted[j+1] = tmp; } - for (i = 0; i < *nr_nodes && sorted[i] < current_id; i++) ; + for (i = 0; i < *nr_nodes && sorted[i] < cluster_self_id(cl); i++) ; (*nr_nodes)++; return i; diff --git a/modules/clusterer/node_info.h b/modules/clusterer/node_info.h index 649a7f7d829..a95661d7ded 100644 --- a/modules/clusterer/node_info.h +++ b/modules/clusterer/node_info.h @@ -137,6 +137,17 @@ struct cluster_info { cluster_bridge_t *bridges; /* replication links to other clusters */ + /* Set by clusterer_controller when manage_shtags=1 for this cluster. + * Blocks MI and script-variable shtag activation to prevent conflicts + * with controller-managed failover. */ + int shtag_managed; + + /* 1 = this cluster's topology and identity are driven at runtime by + * clusterer_controller (registered via the 'cluster_id' modparam); it never + * touches the DB and behaves as db_mode=0 regardless of the global db_mode. + * 0 = a native cluster defined via DB or static my_node_info/neighbor. */ + int controller_managed; + struct cluster_info *next; }; @@ -167,16 +178,39 @@ extern str clnk_shtag_col; extern str clnk_dst_node_col; extern int current_id; +extern int *_current_id_shm; +/* Read current_id from shm if available (cross-process after fork) */ +#define GET_CURRENT_ID (_current_id_shm ? *_current_id_shm : current_id) + +/* This node's node_id *within a specific cluster*. With the controller a node + * can hold a different node_id in each cluster, so the per-cluster identity in + * shared memory (cl->current_node) is authoritative. Returns -1 (a node_id + * that matches nothing) when this cluster's identity is not yet established, so + * a not-yet-joined cluster can never accidentally match or stamp a real id + * (in particular it never borrows another cluster's id via the legacy global). */ +static inline int cluster_self_id(const struct cluster_info *cl) +{ + return (cl && cl->current_node) ? cl->current_node->node_id : -1; +} extern int db_mode; +extern int use_controller; extern rw_lock_t *cl_list_lock; extern cluster_info_t **cluster_list; +/* Effective db_mode *for one cluster*. Controller-managed clusters never use + * the DB (their topology is injected at runtime), so they always behave as + * db_mode=0 even in a hybrid where native clusters are DB-backed (db_mode!=0). */ +static inline int cl_db_mode(const struct cluster_info *cl) +{ + return (cl && cl->controller_managed) ? 0 : db_mode; +} + int update_db_state(int cluster_id, int node_id, int state); int load_db_info(db_func_t *dr_dbf, db_con_t* db_hdl, cluster_info_t **cl_list); void free_info(cluster_info_t *cl_list); int add_node_info(node_info_t **new_info, cluster_info_t **cl_list, int *int_vals, - str *str_vals); + str *str_vals, int self_id); void remove_node_list(cluster_info_t *cl, node_info_t *node); int provision_neighbor(modparam_t type, void* val); @@ -196,6 +230,7 @@ static inline cluster_info_t *get_cluster_by_id(int cluster_id) { cluster_info_t *cl; + if (!cluster_list || !*cluster_list) return NULL; for (cl = *cluster_list; cl; cl = cl->next) if (cl->cluster_id == cluster_id) return cl; diff --git a/modules/clusterer/sharing_tags.c b/modules/clusterer/sharing_tags.c index d486d813a62..04ce5f6c1ca 100644 --- a/modules/clusterer/sharing_tags.c +++ b/modules/clusterer/sharing_tags.c @@ -387,12 +387,29 @@ int shtag_modparam_func(modparam_t type, void *val_s) tag_name.len, tag_name.s); return -1; } - /* force the given state */ - tag->state = init_state; - - if (init_state == SHTAG_STATE_ACTIVE) - /* broadcast (later) in cluster that this tag is active */ - tag->send_active_msg = 1; + /* Force the given state. Only a *controller-managed* cluster's active tag + * is forced to backup here (its controller master activates it when + * appropriate); a native cluster keeps its configured =active state - the + * old global use_controller check wrongly demoted native tags too in a + * hybrid. A controller cluster's stub is created later, in mod_init, so at + * this parse-time point it is normally not yet in cluster_list; the + * controller then forces its tags to backup at runtime via + * set_shtag_managed()/shtag_force_all_backup() (which also clears any pending + * active broadcast set below). */ + { + cluster_info_t *_tcl = get_cluster_by_id(c_id); + if (_tcl && _tcl->controller_managed && init_state == SHTAG_STATE_ACTIVE) { + tag->state = SHTAG_STATE_BACKUP; + LM_INFO("clusterer: [cluster %d] sharing tag [%.*s] " + "forced to backup (controller-managed)\n", + tag->cluster_id, tag->name.len, tag->name.s); + } else { + tag->state = init_state; + if (init_state == SHTAG_STATE_ACTIVE) + /* broadcast (later) in cluster that this tag is active */ + tag->send_active_msg = 1; + } + } return 0; } @@ -876,6 +893,71 @@ int handle_shtag_active(bin_packet_t *packet, int cluster_id, int source_id) } +/** + * shtag_force_all_backup() - force every sharing tag for the given + * cluster to BACKUP state, ignoring the =active config value. + * Called by clusterer_controller at startup when manage_shtags=1 so + * that tag activation is decided solely by the controller master. + * Runs pre-cluster-join: no BIN broadcast needed. + */ +int shtag_force_all_backup(int cluster_id) +{ + struct sharing_tag *tag; + int lock_old_flag; + + if (!shtags_list || !*shtags_list) + return 0; + + lock_start_sw_read(shtags_lock); + for (tag = *shtags_list; tag; tag = tag->next) { + if (tag->cluster_id != cluster_id || + tag->state != SHTAG_STATE_ACTIVE) + continue; + lock_switch_write(shtags_lock, lock_old_flag); + tag->state = SHTAG_STATE_BACKUP; + tag->send_active_msg = 0; /* suppress pending active broadcast */ + lock_switch_read(shtags_lock, lock_old_flag); + LM_INFO("clusterer: [cluster %d] sharing tag [%.*s] forced to " + "backup (controller-managed)\n", + cluster_id, tag->name.len, tag->name.s); + } + lock_stop_sw_read(shtags_lock); + return 0; +} + +/** + * shtag_activate_all_backup() - activate every BACKUP sharing tag for + * the given cluster. Called by clusterer_controller master on node departure. + */ +int shtag_activate_all_backup(int cluster_id) +{ + struct sharing_tag *tag; + /* collect names under lock to avoid O(n²) restart-from-head loop */ +#define SHTAG_MAX_ACTIVATE 64 + str to_activate[SHTAG_MAX_ACTIVATE]; + int n = 0, i; + + if (!shtags_list || !*shtags_list) + return 0; + + lock_start_read(shtags_lock); + for (tag = *shtags_list; tag && n < SHTAG_MAX_ACTIVATE; tag = tag->next) { + if (tag->cluster_id != cluster_id || + tag->state != SHTAG_STATE_BACKUP) + continue; + to_activate[n++] = tag->name; + } + lock_stop_read(shtags_lock); + + for (i = 0; i < n; i++) { + LM_INFO("clusterer: [cluster %d] promoting sharing tag " + "[%.*s] from backup to active (controller master)\n", + cluster_id, to_activate[i].len, to_activate[i].s); + shtag_activate(&to_activate[i], cluster_id, MI_SSTR("controller master")); + } + return 0; +} + void shtag_event_handler(int cluster_id, enum clusterer_event ev, int node_id) { if (ev == CLUSTER_NODE_UP) @@ -960,9 +1042,19 @@ mi_response_t *shtag_mi_set_active(const mi_params_t *params, tag.len, tag.s, c_id); lock_start_read(cl_list_lock); - if (!get_cluster_by_id(c_id)) { - lock_stop_read(cl_list_lock); - return init_mi_error(404, MI_SSTR("Cluster ID not found")); + { + cluster_info_t *_cl = get_cluster_by_id(c_id); + if (!_cl) { + lock_stop_read(cl_list_lock); + return init_mi_error(404, MI_SSTR("Cluster ID not found")); + } + if (_cl->shtag_managed) { + lock_stop_read(cl_list_lock); + LM_WARN("clusterer: MI shtag activation blocked for cluster %d " + "— sharing tags are controller-managed\n", c_id); + return init_mi_error(403, MI_SSTR("Sharing tag is " + "controller-managed; manual activation not allowed")); + } } lock_stop_read(cl_list_lock); @@ -1057,7 +1149,20 @@ int var_set_sh_tag(struct sip_msg* msg, pv_param_t *param, int op, return 0; } - if (shtag_activate( &v_name->shtag, v_name->cluster_id, + lock_start_read(cl_list_lock); + { + cluster_info_t *_cl = get_cluster_by_id(v_name->cluster_id); + if (_cl && _cl->shtag_managed) { + lock_stop_read(cl_list_lock); + LM_WARN("clusterer: script shtag activation blocked for " + "tag <%.*s/%d> — sharing tags are controller-managed\n", + v_name->shtag.len, v_name->shtag.s, v_name->cluster_id); + return -1; + } + } + lock_stop_read(cl_list_lock); + + if (shtag_activate( &v_name->shtag, v_name->cluster_id, MI_SSTR("script variable"))==-1) { LM_ERR("failed to set sharing tag <%.*s/%d> to new state %d\n", v_name->shtag.len, v_name->shtag.s, v_name->cluster_id, state); diff --git a/modules/clusterer/sharing_tags.h b/modules/clusterer/sharing_tags.h index ae7a26e411c..64da0717fd3 100644 --- a/modules/clusterer/sharing_tags.h +++ b/modules/clusterer/sharing_tags.h @@ -43,6 +43,8 @@ int send_shtag_active_info(int c_id, str *tag_name, int node_id); void shtag_flush_state(int c_id, int node_id); void shtag_event_handler(int cluster_id, enum clusterer_event ev, int node_id); +int shtag_activate_all_backup(int cluster_id); +int shtag_force_all_backup(int cluster_id); mi_response_t *shtag_mi_list(const mi_params_t *params, struct mi_handler *async_hdl); diff --git a/modules/clusterer/sync.c b/modules/clusterer/sync.c index 187e825c0cf..4a47e09f3ff 100644 --- a/modules/clusterer/sync.c +++ b/modules/clusterer/sync.c @@ -70,7 +70,7 @@ static int get_sync_source(cluster_info_t *cluster, str *capability, if (get_next_hop(node) == 0) continue; - if (!match_node(cluster->current_node, node, match_cond)) + if (!cluster->current_node || !match_node(cluster->current_node, node, match_cond)) continue; lock_get(node->lock); @@ -94,14 +94,40 @@ static int get_sync_source(cluster_info_t *cluster, str *capability, int queue_sync_request(cluster_info_t *cluster, struct local_cap *lcap) { lock_get(cluster->lock); + + /* If we are a seed node with no peers yet, skip the pending queue and + * self-mark as synced immediately. There is nobody to sync from, and + * the seed-fallback timer would just fire after seed_fb_interval and + * log a spurious ERROR. When peers join later the normal event-driven + * sync path (CLUSTER_NODE_UP callback) will re-trigger if needed. */ + if (cluster->current_node && + (cluster->current_node->flags & NODE_IS_SEED) && + cluster->node_list == NULL) { + lcap->flags |= CAP_STATE_OK; + lcap->flags &= ~(CAP_SYNC_PENDING | CAP_SYNC_STARTUP); + lock_release(cluster->lock); + LM_DBG("No peers in cluster %d — capability '%.*s' self-marked as " + "synced (first/lone seed node)\n", + cluster->cluster_id, lcap->reg.name.len, lcap->reg.name.s); + sr_set_status(cl_srg, STR2CI(lcap->reg.sr_id), CAP_SR_SYNCED, + STR2CI(CAP_SR_STATUS_STR(CAP_SR_SYNCED)), 0); + sr_add_report_fmt(cl_srg, STR2CI(lcap->reg.sr_id), 0, + "No peers present — self-synced as first node in cluster"); + send_single_cap_update(cluster, lcap, 1); + return 0; + } + lcap->flags |= CAP_SYNC_PENDING; if (sr_get_core_status() == STATE_INITIALIZING) lcap->flags |= CAP_SYNC_STARTUP; else lcap->flags &= ~CAP_SYNC_STARTUP; - if (cluster->current_node->flags & NODE_IS_SEED) - gettimeofday(&lcap->sync_req_time, NULL); + /* Always record when we started waiting — if current_node is not yet set + * (controller mode, identity assigned post-fork), sync_req_time would + * stay at zero (epoch) and TIME_DIFF would be huge, causing the seed + * fallback timer to fire immediately once identity is assigned. */ + gettimeofday(&lcap->sync_req_time, NULL); lock_release(cluster->lock); diff --git a/modules/clusterer/topology.c b/modules/clusterer/topology.c index 07603958637..e85936c0972 100644 --- a/modules/clusterer/topology.c +++ b/modules/clusterer/topology.c @@ -50,7 +50,7 @@ static int send_ping(node_info_t *node, int req_node_list) return -1; } bin_push_int(&packet, node->cluster->cluster_id); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(node->cluster)); bin_push_int(&packet, req_node_list); /* request list of known nodes ? */ bin_get_buffer(&packet, &send_buffer); @@ -195,6 +195,8 @@ void heartbeats_timer(void) lock_start_read(cl_list_lock); for (clusters_it = *cluster_list; clusters_it; clusters_it = clusters_it->next) { + if (!clusters_it->current_node) + continue; /* identity not yet set by controller */ lock_get(clusters_it->current_node->lock); if (!(clusters_it->current_node->flags & NODE_STATE_ENABLED)) { lock_release(clusters_it->current_node->lock); @@ -500,7 +502,7 @@ int flood_message(bin_packet_t *packet, cluster_info_t *cluster, bin_push_int(packet, path_len + 1); /* go to end of the buffer and include current node in path */ bin_skip_int_packet_end(packet, path_len); - bin_push_int(packet, current_id); + bin_push_int(packet, cluster_self_id(cluster)); bin_get_buffer(packet, &bin_buffer); msg_altered = 1; } @@ -562,7 +564,7 @@ static int send_full_top_update(node_info_t *dest_node, int nr_nodes, int *node_ return -1; } bin_push_int(&packet, dest_node->cluster->cluster_id); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(dest_node->cluster)); bin_push_int(&packet, ++dest_node->cluster->current_node->top_seq_no); bin_push_int(&packet, timestamp); @@ -574,7 +576,7 @@ static int send_full_top_update(node_info_t *dest_node, int nr_nodes, int *node_ bin_push_int(&packet, dest_node->cluster->no_nodes); /* the first adjacency list in the message is for the current node */ - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(dest_node->cluster)); bin_push_int(&packet, 0); /* no description for current node */ bin_push_int(&packet, dest_node->cluster->current_node->ls_seq_no); bin_push_int(&packet, dest_node->cluster->current_node->ls_timestamp); @@ -622,7 +624,7 @@ static int send_full_top_update(node_info_t *dest_node, int nr_nodes, int *node_ } bin_push_int(&packet, 1); /* path length is 1, only current node at this point */ - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(dest_node->cluster)); bin_get_buffer(&packet, &bin_buffer); if (msg_send(dest_node->cluster->send_sock, dest_node->proto, &dest_node->addr, @@ -669,7 +671,7 @@ static int send_ls_update(node_info_t *node, clusterer_link_state new_ls) return -1; } bin_push_int(&packet, node->cluster->cluster_id); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(node->cluster)); bin_push_int(&packet, ++node->cluster->current_node->ls_seq_no); bin_push_int(&packet, timestamp); @@ -680,7 +682,7 @@ static int send_ls_update(node_info_t *node, clusterer_link_state new_ls) /* path length is 1, only current node at this point */ bin_push_int(&packet, 1); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(node->cluster)); lock_release(node->cluster->current_node->lock); @@ -714,7 +716,7 @@ static node_info_t *add_node(bin_packet_t *received, cluster_info_t *cl, int_vals[INT_VALS_NODE_ID_COL] = src_node_id; int_vals[INT_VALS_STATE_COL] = 1; /* enabled */ - if (add_node_info(&new_node, &cl, int_vals, str_vals) != 0) { + if (add_node_info(&new_node, &cl, int_vals, str_vals, cluster_self_id(cl)) != 0) { LM_ERR("Unable to add node info to backing list\n"); return NULL; } @@ -1044,13 +1046,13 @@ void handle_full_top_update(bin_packet_t *packet, node_info_t *source, for (i = 0; i < no_nodes; i++) { skip = 0; - if (top_node_id[i] == current_id) + if (top_node_id[i] == cluster_self_id(source->cluster)) skip = 1; top_node = get_node_by_id(source->cluster, top_node_id[i]); if (!skip && !top_node) { - if (db_mode) { + if (cl_db_mode(source->cluster)) { skip = 1; } else if (!top_node_info[i][0]) { LM_WARN("Unknown node id [%d] in topology update with " @@ -1092,8 +1094,8 @@ void handle_full_top_update(bin_packet_t *packet, node_info_t *source, no_present_nodes = 0; for (j = 0; j < top_node_info[i][3]; j++) { top_neigh = get_node_by_id(source->cluster, top_node_info[i][j+4]); - if (!top_neigh && top_node_info[i][j+4] != current_id) { - if (db_mode) + if (!top_neigh && top_node_info[i][j+4] != cluster_self_id(source->cluster)) { + if (cl_db_mode(source->cluster)) continue; for (n_idx = 0; n_idx < no_nodes && top_node_info[i][j+4] != top_node_id[n_idx]; @@ -1117,7 +1119,7 @@ void handle_full_top_update(bin_packet_t *packet, node_info_t *source, } } - if (top_node_info[i][j+4] == current_id) { + if (top_node_info[i][j+4] == cluster_self_id(source->cluster)) { lock_get(top_node->lock); if (top_node->link_state == LS_DOWN && top_node->flags & NODE_STATE_ENABLED) { @@ -1180,7 +1182,7 @@ void handle_internal_msg_unknown(bin_packet_t *received, cluster_info_t *cl, return; } bin_push_int(&packet, cl->cluster_id); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(cl)); bin_get_buffer(&packet, &bin_buffer); if (msg_send(cl->send_sock, proto, src_su, 0, bin_buffer.s, @@ -1234,8 +1236,8 @@ void handle_ls_update(bin_packet_t *received, node_info_t *src_node, bin_pop_int(received, &neigh_id); bin_pop_int(received, &new_ls); ls_neigh = get_node_by_id(src_node->cluster, neigh_id); - if (!ls_neigh && neigh_id != current_id) { - if (!db_mode) + if (!ls_neigh && neigh_id != cluster_self_id(src_node->cluster)) { + if (!cl_db_mode(src_node->cluster)) LM_WARN("Received link state update about unknown node id [%d]\n", neigh_id); lock_release(src_node->lock); return; @@ -1244,7 +1246,7 @@ void handle_ls_update(bin_packet_t *received, node_info_t *src_node, LM_DBG("Received link state update with source [%d] about node [%d], new state=%s\n", src_node->node_id, neigh_id, new_ls ? "DOWN" : "UP"); - if (neigh_id == current_id) { + if (neigh_id == cluster_self_id(src_node->cluster)) { if ((new_ls == LS_UP && src_node->link_state == LS_DOWN) || (new_ls == LS_DOWN && src_node->link_state == LS_UP)) { lock_release(src_node->lock); @@ -1276,14 +1278,14 @@ void handle_unknown_id(node_info_t *src_node) return; } bin_push_int(&packet, src_node->cluster->cluster_id); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(src_node->cluster)); /* include info about current node */ bin_push_node_info(&packet, src_node->cluster->current_node); /* path length is 1, only current node at this point */ bin_push_int(&packet, 1); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(src_node->cluster)); bin_get_buffer(&packet, &bin_buffer); if (msg_send(src_node->cluster->send_sock, src_node->proto, &src_node->addr, @@ -1316,7 +1318,7 @@ void handle_ping(bin_packet_t *received, node_info_t *src_node, return; } bin_push_int(&packet, src_node->cluster->cluster_id); - bin_push_int(&packet, current_id); + bin_push_int(&packet, cluster_self_id(src_node->cluster)); if (req_list) { /* include a list of known nodes */ diff --git a/modules/clusterer_controller/Makefile b/modules/clusterer_controller/Makefile new file mode 100644 index 00000000000..5bf0dd4fe8b --- /dev/null +++ b/modules/clusterer_controller/Makefile @@ -0,0 +1,29 @@ +# WARNING: do not run this directly, it should be run by the master Makefile + +include ../../Makefile.defs +auto_gen= +NAME= clusterer_controller.so + +LIBS += -L../tls_wolfssl/lib/lib/ -lwolfssl -lm +DEFS += -I../tls_wolfssl/lib/include/ +DEPS += ../tls_wolfssl/lib/lib/libwolfssl.a + +# Optional stronger crypto suite: if libsodium (dev) is detected on the build +# host, compile with XChaCha20-Poly1305 + Argon2id instead of AES-256-GCM + +# scrypt. Distro static libs are usually non-PIC (can't link into a .so), so +# we link libsodium dynamically — target hosts then need the libsodium runtime +# package (e.g. `apt install libsodium23`). All nodes in a cluster must be +# built the same way (the wire formats are not interoperable). +SODIUM_EXISTS := $(shell pkg-config --exists libsodium 2>/dev/null && echo yes) +ifeq ($(SODIUM_EXISTS),yes) + DEFS += -DCC_HAVE_SODIUM $(shell pkg-config --cflags libsodium) + LIBS += $(shell pkg-config --libs libsodium) + $(info clusterer_controller: libsodium found -> XChaCha20-Poly1305 + Argon2id) +else + $(info clusterer_controller: libsodium not found -> AES-256-GCM + scrypt) +endif + +include ../../Makefile.modules + +../tls_wolfssl/lib/lib/libwolfssl.a: + $(MAKE) -C ../tls_wolfssl lib/lib/libwolfssl.a diff --git a/modules/clusterer_controller/README b/modules/clusterer_controller/README new file mode 100644 index 00000000000..5cda8f9f1e5 --- /dev/null +++ b/modules/clusterer_controller/README @@ -0,0 +1,1611 @@ +CLUSTERER_CONTROLLER Module + __________________________________________________________ + + Table of Contents + + 1. Admin Guide + + 1.1. Overview + 1.2. Discovery Protocol + 1.3. Master Election + 1.4. Security Architecture + 1.5. Dependencies + + 1.5.1. OpenSIPS Modules + 1.5.2. External Libraries or Applications + + 1.6. Exported Parameters + + 1.6.1. cluster (string) + 1.6.2. my_ip (string) + 1.6.3. interface (string) + 1.6.4. query_time (integer) + 1.6.5. password (string) + 1.6.6. manage_shtags (integer) + 1.6.7. master_stickiness (integer) + 1.6.8. on_config_mismatch (string) + + 1.7. Exported MI Functions + + 1.7.1. cl_ctr_list_members + 1.7.2. cl_ctr_node_info + 1.7.3. cl_ctr_list_config + 1.7.4. cl_ctr_shtag_force + 1.7.5. cl_ctr_shtag_auto + + 1.8. Exported Pseudo-Variables + 1.9. Exported Functions + 1.10. Multiple Clusters + 1.11. Hybrid Topologies (native + controller clusters) + 1.12. Configuration Example + 1.13. Limitations + 1.14. Planned Features + + A. HA Behaviour Tests + + A.1. Baseline + A.2. Test 1 — Stop the active node + A.3. Test 2 — Stop a backup node + A.4. Test 3 — Stop both backup nodes + A.5. Test 4 — Stop active node and one backup + A.6. Test 5 — Full cluster restart + A.7. Test 6 — Multiple clusters over one BIN socket + A.8. Summary + + 2. Contributors + + 2.1. Contributors + 2.2. Documentation Contributors + + List of Examples + + 1.1. Set cluster parameter — single cluster + 1.2. Set cluster parameter — multiple clusters on separate + networks (one BIN socket per network) + + 1.3. Set cluster parameter — multiple clusters sharing a single + BIN socket (recommended default) + + 1.4. Set my_ip parameter + 1.5. Set interface parameter + 1.6. Set query_time parameter + 1.7. Set password parameter + 1.8. Global manage_shtags — applies to all clusters + 1.9. Per-cluster override — opt one cluster out of automatic + failover + + 1.10. Global opt-out with one cluster opting in + 1.11. Typical full configuration with manage_shtags=1 (default) + 1.12. Set master_stickiness parameter + 1.13. Set on_config_mismatch parameter + 1.14. cl_ctr_list_members usage + 1.15. cl_ctr_node_info usage + 1.16. cl_ctr_list_config usage + 1.17. cl_ctr_shtag_force usage + 1.18. cl_ctr_shtag_auto usage + 1.19. Using $cl_ctr_* in the script + 1.20. Per-peer lookup functions + 1.21. Multiple clusters — dialog on cluster 1, usrloc on + cluster 2 + + 1.22. Multiple clusters — same multicast IP, different ports + 1.23. Hybrid — DB-native cluster 10 + controller cluster 1 + 1.24. Minimal HA cluster configuration + +Chapter 1. Admin Guide + +1.1. Overview + + The clusterer_controller module provides automatic peer + discovery and topology management for the clusterer module via + authenticated, encrypted UDP multicast. It eliminates the need + for static node configuration or a database — nodes discover + each other automatically at startup and the cluster topology is + maintained dynamically at runtime. + + When use_controller=1 is set in the clusterer module, the + clusterer_controller module takes over all topology management: + it allocates unique node IDs, discovers peer BIN socket + addresses, and calls the clusterer internal API to add or + remove nodes as they join or leave the cluster. + + By default (manage_shtags=1), the module also provides fully + automatic sharing tag failover. The controller master node is + the single decision point for which node holds the active tag — + no event routes, MI commands, or seed_fallback_interval + configuration is needed. Sharing tags are forced to backup at + startup regardless of the =active config value, and the active + tag is claimed by the controller master automatically when the + cluster forms or when the active node departs. An operator can + override this automatic allocation and pin the active tag to a + chosen node with the cl_ctr_shtag_force MI command, reverting + to automatic allocation with cl_ctr_shtag_auto. + + The minimal configuration per node is a single multicast group + address. No IP addresses, node IDs, BIN URLs, or sharing tag + management scripts need to be hardcoded or maintained. Any + number of nodes can join or leave without any configuration + change on the remaining nodes. + +1.2. Discovery Protocol + + All traffic uses UDP multicast to the configured multicast + address and port. Clusters are kept apart in two independent + ways: by the multicast endpoint (two clusters may use different + IP addresses, or the same address with different UDP ports), + and by the id (cluster_id) carried in the cleartext of every + packet. A node silently ignores any packet whose cluster_id + differs from its own, so several clusters can safely share one + multicast group and port. Two clusters merge only if they share + all of the multicast address, the UDP port, the cluster_id and + the password — i.e. they are configured identically, which is + the operator's responsibility to avoid. + + Every packet is encrypted and authenticated with an AEAD — + AES-256-GCM by default, or XChaCha20-Poly1305 when the module + is built against libsodium (see the Security Architecture + section). Two distinct encryption keys are used depending on + the communication phase: + * Bootstrap key — derived from the configured password with a + memory-hard KDF (scrypt N=2^16, r=8, p=1, or Argon2id in + the libsodium build, with a per-cluster salt), so a + password captured from a bootstrap packet cannot be + brute-forced cheaply offline. Derived once at startup. Used + for the admission handshake (JOIN_REQ, KEY_GRANT, + JOIN_REJECT) and the split-brain MASTER_BEACON — traffic + that must be readable before a session key exists, or by + masters holding different session keys. + * Session key — derived via HKDF-SHA256 from the password and + a 32-byte master salt generated once when the cluster first + bootstraps. All normal cluster traffic uses this key. It is + preserved across master changes (a new master reuses the + key every member already holds), so failover needs no + re-keying. + + Each packet wire format begins with a 2-byte magic value + identifying the key type and a 2-byte cluster_id, both in + cleartext (the magic and cluster_id must be readable before + decryption to select the key and to filter foreign clusters), + followed by a random AEAD nonce (12 bytes for AES-256-GCM, 24 + for XChaCha20-Poly1305) and the ciphertext. The magic and + cluster_id are additionally bound into the authentication tag + as AAD, so they cannot be altered undetected. Only the payload + is encrypted; the authenticated plaintext begins with a 1-byte + packet type and a 4-byte monotonic sequence number used for + replay protection. A 16-byte authentication tag follows the + ciphertext. Packets for a different cluster_id are dropped + before decryption; packets encrypted with a different password + fail authentication and are silently discarded. + + The following packet types are defined: + * ALIVE — periodic heartbeat sent by every active node every + query_time seconds. Carries the sender IP, its X25519 + public key (so peers can prepare for key agreement) and a + small descriptor of the sender's consistency-critical + settings (manage_shtags, master_stickiness, query_time) + used for configuration-drift detection (see Security + Architecture). Encrypted with the session key. + * JOIN_REQ — sent at startup by a new node, carrying its IP, + BIN socket list, X25519 ephemeral public key, and a 16-byte + random join nonce. Encrypted with the bootstrap key so it + can be sent before a session key exists. + * MEMBER_LIST — sent by the master in response to a JOIN_REQ, + carrying the member count, the operator-forced sharing-tag + holder node_id (0 = automatic), and the full peer IP list + so the joining node can participate in elections. Only + accepted from the current master (except during initial + join when no master is yet known). + * NODE_ASSIGN — sent by the master to multicast, allocating a + node_id and BIN socket record for a joining node. All + cluster members receive and apply it. + * GOODBYE — sent on graceful shutdown so peers can remove the + node immediately without waiting for timeout. Uses the + sender's monotonic sequence counter to prevent forgery. + * MASTER_ALIVE — keepalive sent by the master every 1 second + (independent of query_time). Used by all peers to detect + master failure quickly (3-second timeout). Encrypted with + the session key. + * KEY_GRANT — unicast response to a JOIN_REQ, sent by the + master directly to the joining node. Contains the master's + X25519 public key, the original join nonce, and the master + salt wrapped with a per-exchange key derived from + ECDH(master_priv, joiner_pub) + password + join_nonce via + HKDF-SHA256. Encrypted with the bootstrap key. + * KEY_HANDOFF — unicast packet sent by the outgoing master on + graceful shutdown to the next-highest-IP peer, delivering + the master salt so that peer can become the new master + without a full re-join cycle. Encrypted with the session + key. + * JOIN_REJECT — sent by the master to a joining node whose + JOIN_REQ repeatedly fails authentication (wrong password). + After CC_JOIN_FAIL_LIMIT (3) consecutive bootstrap-key + decryption failures from the same source IP the master + sends a JOIN_REJECT to that IP. Encrypted with the + bootstrap key so it cannot be forged by a node that does + not know the cluster password. The joining node logs a + critical error and shuts down OpenSIPS on receipt. + * MASTER_BEACON — a master-only announcement multicast every + few MASTER_ALIVE ticks, carrying this partition's member + count. Unlike MASTER_ALIVE it is encrypted with the + bootstrap key, so it is readable even by a master that + holds a different session key. This is how a split brain + between two independently bootstrapped partitions is + detected and merged (see Master Election). + +1.3. Master Election + + Each cluster has three roles: master (the active coordinator), + backup (the standby promoted when the master fails, always the + highest-IP non-master) and member. The election uses a + quantized time window so that all nodes evaluate the same + eligible peer set and reach the same result deterministically. + No NTP synchronisation between nodes is required for correct + election results. + + The master_stickiness parameter (default 1) controls whether a + live master is kept when a higher-IP node joins. With + stickiness enabled, the master stays put and the higher-IP + joiner becomes the backup, minimising handovers; with + stickiness disabled the highest-IP node always becomes master. + In either mode two live masters are reconciled + deterministically (see Split-brain handling below). See the + master_stickiness parameter for details. + + Only the master handles JOIN_REQ packets, allocates node_ids, + and sends NODE_ASSIGN and MEMBER_LIST packets. Non-master nodes + are passive during join events. A joining node receives the + current session key from the master (via KEY_GRANT) and joins + as a member or backup; it never seizes mastership during the + join handshake. + + Preserved session key: the session key is generated once, when + the first node bootstraps the cluster, and is then preserved + across every master change. A new master does not re-key; + because every member already holds the key (obtained when it + joined), master transitions require no re-keying and no re-JOIN + cycle. + + Fast master failure detection: the master sends MASTER_ALIVE + packets every 1 second. All non-master peers maintain a + 3-second watchdog timer that fires if no MASTER_ALIVE is + received. On expiry the silent master is aged out of the + election window and each peer immediately re-elects, promoting + the backup (highest-IP survivor) — which already holds the + session key, so it starts serving within one keepalive + interval. + + Graceful master handoff: when the current master shuts down + cleanly, it sends a KEY_HANDOFF packet directly to the + next-highest-IP peer before sending GOODBYE to multicast. This + confirms the master salt to the incoming master so it can + assume control immediately. + + Split-brain handling. A split brain (more than one node + believing it is master) is prevented and, if it still occurs, + healed by three cooperating mechanisms: + * Prevention at join time. When several nodes start + simultaneously they all exchange (bootstrap-decryptable) + JOIN_REQs and thus learn about each other. At the join + deadline, a node that has seen a higher-IP node also still + joining defers its own self-promotion (for a few bounded + rounds) and joins that node instead, so only the highest-IP + starter becomes master and no independent-key lone masters + are created. + * Same-key yield. Two masters that share a session key (for + example after a network partition heals) can read each + other's MASTER_ALIVE; the lower-IP master immediately + yields to the higher-IP one. + * Divergent-key merge. Two masters that were bootstrapped + independently hold different session keys and so cannot + read each other's MASTER_ALIVE. Each therefore emits a + MASTER_BEACON encrypted with the shared bootstrap key. On + hearing a beacon from a superior partition — larger member + count, ties broken by higher IP — a node abandons its + partition, re-joins the superior master and adopts its + session key, converging the whole cluster onto a single + master and key. + +1.4. Security Architecture + + The module uses a two-phase key agreement to provide forward + secrecy and replay protection for all cluster traffic. + + Payload encryption and header binding: every packet's payload + is sealed with an AEAD. The 2-byte magic (a key selector that + must be readable before decryption) and the 2-byte cluster_id + that precede the nonce are cleartext framing, but they are + bound into the AEAD tag as additional authenticated data (AAD): + a captured packet cannot be re-stamped with a different + cluster_id and still authenticate, which matters when two + clusters share one multicast group and password. A node also + drops any packet whose cluster_id does not match its own before + attempting decryption, so foreign-cluster traffic on the group + never counts as an authentication failure. + + Crypto suite (selected at build time): by default the module + uses AES-256-GCM (12-byte nonce) for the payload AEAD and + scrypt (N=2^16, r=8, p=1) for the bootstrap-key derivation, + through WolfSSL. If libsodium is detected on the build host, + the module is compiled instead with XChaCha20-Poly1305 (24-byte + nonce, whose 192-bit nonce space removes any random-nonce + collision concern) and Argon2id. The two wire formats are not + interoperable, so every node in a cluster must be built with + the same suite; the active suite is reported in the startup log + (crypto=...). + + Phase 1 — bootstrap (JOIN_REQ / KEY_GRANT): each worker process + generates an ephemeral X25519 keypair on startup. When joining, + the node sends its public key and a 16-byte random join nonce + in the JOIN_REQ, encrypted with the bootstrap key (a + memory-hard KDF of the password — scrypt, or Argon2id in the + libsodium build). The master responds with a KEY_GRANT + encrypted with the same bootstrap key, containing its own + public key, the echoed join nonce, and the master salt wrapped + with a per-exchange key derived via HKDF-SHA256 from: +IKM = ECDH(master_priv, joiner_pub) || password || join_nonce + + This ensures that even if the password is compromised, + previously recorded exchanges cannot be decrypted without both + the ephemeral private key and the per-exchange join nonce. + + Phase 2 — session (all other packets): once the master salt is + known, all nodes derive the session key as: +session_key = HKDF-SHA256(IKM=password, salt=master_salt, info="cc-sessi +on-key") + + The session key is generated once, when the first node + bootstraps the cluster, and preserved across every master + change: a new master reuses the key that every member already + holds, so master transitions require no re-keying. All normal + cluster traffic (ALIVE, MEMBER_LIST, NODE_ASSIGN, GOODBYE, + MASTER_ALIVE, KEY_HANDOFF) is encrypted with this key. + + Replay protection: each sender maintains a monotonically + increasing 32-bit sequence number embedded in the authenticated + plaintext of every session-key packet. Each receiver tracks the + last accepted sequence number per source IP and rejects any + packet whose sequence is not strictly greater than the last + accepted value. Sequence counters are reset to zero whenever + the session key is (re)derived — at cluster bootstrap and when + a joiner adopts the key via KEY_GRANT/KEY_HANDOFF — and on peer + restart detection (JOIN_REQ received from a known IP resets + that peer's counter; MEMBER_LIST upsert resets all listed + peers). This protection does not depend on clock + synchronisation. + + Rate limiting: a per-source rate limiter (256 slots, 20 + packets/second limit) is applied before any decryption attempt. + This prevents CPU exhaustion from packet floods directed at the + multicast group. + + Join authentication and rejection: the master tracks + consecutive bootstrap-key decryption failures per source IP in + a small worker-local table (CC_JOIN_FAIL_TABLE_SZ = 8 slots). + When any source IP accumulates CC_JOIN_FAIL_LIMIT (3) + consecutive failures — indicating a node attempting to join + with the wrong password — the master sends an encrypted + JOIN_REJECT packet and stops responding to further JOIN_REQs + from that IP. + + On the joining side, a received JOIN_REJECT is only acted on + while the node is still in the initial join phase (CC_NODE_NEW + state) and is addressed to this node; an already-active cluster + member ignores any JOIN_REJECT unconditionally, so a node with + the correct password can never be evicted by a peer. + + A node joining with the wrong password cannot decrypt the + JOIN_REJECT (it is encrypted with the master's bootstrap key), + so it relies on a self-contained signal instead: while joining + it counts packets received from other peers that it cannot + decrypt. If, at the join deadline, the node is still unjoined + and has seen CC_JOIN_FAIL_LIMIT or more such undecryptable + packets, it concludes that a cluster it cannot authenticate to + exists on the group and shuts down OpenSIPS with a critical log + message — rather than promoting itself into a lone, split-brain + master (which, with managed sharing tags, would create a + duplicate active tag). This counter is reset the moment a + KEY_GRANT is successfully processed, so a legitimate joiner + that briefly saw an undecryptable packet before receiving its + key is never affected. + + Rogue traffic isolation: a node requests a re-key in response + to an undecryptable session-key packet only when that packet + came from its current master (a legitimate key rotation). + Undecryptable session packets from any other source — for + example a wrong-password or malicious node broadcasting on the + multicast group — are ignored, so such traffic cannot drive the + cluster into a re-JOIN churn. + + Peer table exhaustion defence: the peer table is bounded at + CC_MAX_PEERS (256) entries. When the table is full, the master + rejects JOIN_REQ packets from unknown IPs with a JOIN_REJECT + response. Known peers that are reconnecting after a restart + continue to be admitted regardless of the table count, since + they already own a slot. This prevents an attacker with the + cluster password from exhausting the peer table by flooding + JOIN_REQs from spoofed source addresses. + + Configuration-consistency enforcement: all nodes of a cluster + must use identical consistency-critical settings + (manage_shtags, master_stickiness and query_time); a per-node + mismatch would otherwise cause silent, inconsistent failover + and sharing-tag behaviour (for example, a master with + manage_shtags=0 would leave no node holding the active tag). + Each node advertises these effective settings in its ALIVE + heartbeat and in its JOIN_REQ, so mismatches are detected. What + happens then is controlled by the on_config_mismatch modparam: + * reject (default) - when a node tries to join an established + cluster (a master is alive) with different settings, the + master logs the attempt and returns a JOIN_REJECT; the + joining node logs the offending settings and shuts down, so + a misconfigured node never joins. + * warn - the node is allowed to join, but any peer that + observes a different value logs a single loud CONFIG + MISMATCH warning (repeated only if the peer's advertised + configuration changes, cleared once it matches). + * adopt - the joining node adopts the running cluster's + (master's) settings at runtime and continues; the adopted + values are what cl_ctr_list_config reports. + + This turns an easy-to-miss misconfiguration into an obvious log + line, a refused join, or a self-correction rather than a + hard-to-diagnose HA failure. + + Node identity and node_id allocation: node_id values are + allocated exclusively by the current master, serialised under + the peer-table lock, and a joining node never picks its own id. + The master hands out the lowest unused id by scanning the live + peer table, so a node that has failed but is not yet timed out + still occupies its slot and its id is never handed to a + different joiner — new nodes always receive a distinct id even + during the failure-detection window. A node that restarts and + rejoins from the same address reuses its previous id (and has + its replay counter reset), so ids stay stable across restarts. + Because peers are keyed by source IP address, every node in a + cluster must present a stable, unique source IP: two distinct + nodes that appear behind the same address (for example through + NAT) would share a single peer slot and node_id. Deploy the + cluster on a network where each member has its own routable + address on the BIN/multicast interface. + + Trust model — shared secret, not per-node identity: the cluster + is a single shared-secret trust domain. Authentication proves + only that a peer holds the cluster password; it does not bind a + cryptographic identity to an individual node, and there is no + per-node authorisation or revocation. Consequently any party in + possession of the password is a fully trusted member and can + legitimately win the highest-IP master election and assume the + master role — there is no distinction between "may be a member" + and "may be master". An attacker without the password cannot + affect the election at all: forged or replayed MASTER_ALIVE and + beacon packets fail AEAD authentication (or the strict + per-source sequence check) and are dropped before any election + logic runs. The residual exposure is therefore a malicious or + compromised insider that already holds the shared key. Protect + the password accordingly, and rotate it if a node is + decommissioned or suspected compromised. Removing this + limitation — per-node keypairs with enrolment and revocation, + so a single node can be distrusted without re-keying the whole + cluster — is planned future work. + +1.5. Dependencies + +1.5.1. OpenSIPS Modules + + The following modules are required by this module: + * proto_bin — required so that BIN listeners are registered + and available for discovery when clusterer_controller + initialises and scans the proto_bin listener list. + * clusterer — required with use_controller=1 set, so that the + clusterer internal API (load_clusterer_ctrl_binds) is + exported and available at initialisation time. + + Both dependencies are declared in the module's dep_export_t. + OpenSIPS will refuse to start if either dependency is not + satisfied. This does not imply that the modules must appear in + a particular order in the configuration file — OpenSIPS + resolves the dependency at runtime and will initialize the + required modules first regardless of loadmodule order. + + All other modules that use the clusterer interface (tm, dialog, + dispatcher, usrloc etc.) may be loaded in any order relative to + clusterer_controller. The clusterer module automatically + creates a cluster stub when use_controller=1 is set and a + module attempts to register a capability for an unknown + cluster. + +1.5.2. External Libraries or Applications + + The following libraries must be installed: + * tls_wolfssl — the clusterer_controller module links + statically against the WolfSSL library built by the + tls_wolfssl module + (modules/tls_wolfssl/lib/lib/libwolfssl.a). WolfSSL + provides AES-256-GCM authenticated encryption, X25519 ECDH + key agreement, HKDF-SHA256 key derivation, the scrypt + password KDF, and SHA-256. The tls_wolfssl module must be + present in the source tree; its static library is built + automatically as a dependency if not already present. + * libsodium (optional) — if the build host has libsodium + development files (detected via pkg-config), the module is + compiled with the stronger crypto suite: XChaCha20-Poly1305 + for the payload AEAD and Argon2id for the bootstrap-key + derivation, in place of AES-256-GCM and scrypt. X25519 ECDH + and HKDF continue to use WolfSSL. libsodium is linked + dynamically, so each target host then also needs the + libsodium runtime package (for example libsodium23). + Because the two suites produce incompatible wire formats, + every node in a cluster must be built the same way; the + active suite is printed in the startup log. + +1.6. Exported Parameters + +1.6.1. cluster (string) + + Define a cluster to participate in. The value is a + comma-separated key=value string with the following fields: + * id (required) — positive integer cluster identifier, must + match the cluster_id used by clusterer consumers (dialog, + usrloc, dispatcher, etc.). + * multicast (required) — IPv4 multicast address and UDP port + in the form A.B.C.D:PORT. The address must be in the + 224.0.0.0/4 range. + * password (optional) — AES-256 encryption key material. All + nodes in the same cluster must use the same password. Falls + back to the global password modparam if not set. + * bin_socket (optional) — BIN socket to advertise for this + cluster, in the form bin:IP:PORT. Required when multiple + clusters are defined (so the controller knows which + listener to advertise), but it need not be distinct — + several clusters may share the same BIN socket. When only + one cluster is defined and only one BIN socket exists, the + socket is auto-detected from the proto_bin listeners. + * manage_shtags (optional) — per-cluster override for the + global manage_shtags modparam. Set to 1 to enable automatic + sharing tag failover for this cluster, or 0 to disable it. + When omitted, the global manage_shtags value applies, + regardless of the order in which cluster and manage_shtags + modparams appear in the config file. + + This parameter may be set multiple times to participate in + multiple clusters simultaneously. Each cluster runs its own + independent worker process. + + No default value. At least one cluster must be defined. + + Example 1.1. Set cluster parameter — single cluster +... +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333") +... + + Example 1.2. Set cluster parameter — multiple clusters on + separate networks (one BIN socket per network) +... +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:10.0.1.10:5566") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.2:3333,bin_socket=bin:10.0.2.10:5566") +... + + Example 1.3. Set cluster parameter — multiple clusters sharing + a single BIN socket (recommended default) +... +# one proto_bin listener serves both clusters; the cluster_id in each +# BIN packet keeps their replication traffic separate +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:10.0.0.10:5566") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.2:3333,bin_socket=bin:10.0.0.10:5566") +... + +Note + + One BIN socket can serve any number of clusters. Every + clusterer BIN packet carries its cluster_id, so a single + proto_bin listener demultiplexes traffic for all clusters. + Unless you specifically want different clusters on different + interfaces or networks, set the same bin_socket value on every + cluster entry (it is mandatory once more than one cluster is + defined, but need not be distinct). Using multiple BIN sockets + is for network/interface segregation, not throughput: the + replication data plane runs over BIN (TCP) and scales with the + shared TCP worker pool (tcp_children) and IPC dispatch, + independent of the number of BIN sockets. What does scale with + the number of clusters is the controller's own worker set — it + spawns one lightweight control worker (UDP multicast: + discovery, election, keepalives) per cluster — so it is the + cluster count, not the BIN-socket count, that adds processes. + +Note + + Each cluster needs a distinct multicast endpoint, but clusters + may share one BIN socket. The two sockets play opposite roles. + The controller's multicast endpoint is its control plane: every + cluster defined in one instance must use a different multicast + address:port — two cluster entries with the same multicast + value are rejected at startup (duplicate multicast). Use a + different port (e.g. :3333 and :3334) or a different group per + cluster. The bin_socket is the replication data plane and is + the opposite: it may be freely shared across clusters, since + every BIN packet carries its cluster_id. (The cluster_id filter + on the multicast wire header is for a different purpose: + letting separate deployments coexist on a shared multicast + group, not two clusters inside one instance.) + +1.6.2. my_ip (string) + + Explicitly set the local IPv4 address used by the controller + for its own node identity and master election. This is the IP + that the controller advertises to peers in JOIN_REQ and + NODE_ASSIGN packets and uses for the highest-IP master election + algorithm. + + Note: this parameter controls the controller's identity only. + The BIN socket address advertised to clusterer peers is + discovered separately from the proto_bin listener list and is + independent of this setting. Do not confuse my_ip with the BIN + socket IP defined by the socket=bin:IP:PORT core parameter. + + When set, the module walks the interface list to find which + local interface owns this address and uses that interface for + multicast traffic. Startup fails if no local interface owns the + given address. + + The module supports three identity resolution modes depending + on which modparams are provided: + + Mode 1 — my_ip set: The given IP is used directly. The owning + interface is resolved automatically from the system interface + list. Use this mode on multi-homed hosts where you want to pin + the controller identity to a specific IP. + + Mode 2 — interface set, my_ip not set: The first IPv4 address + on the named interface is used as the controller identity IP. A + warning is logged if the interface has multiple IPv4 addresses. + + Mode 3 — neither set (default): A throw-away UDP socket is + connected to the multicast group and getsockname() is called to + determine which source IP the kernel would select. The + interface name is resolved from the returned IP. Suitable for + single-homed hosts. + + Default: auto-detected (Mode 3). + + Example 1.4. Set my_ip parameter +... +modparam("clusterer_controller", "my_ip", "10.22.23.191") +... + +1.6.3. interface (string) + + Explicitly set the network interface name to use for multicast + traffic (e.g. eth0, enp6s18). The module takes the first IPv4 + address assigned to this interface as the controller's identity + IP. This corresponds to Mode 2 described in the my_ip parameter + documentation above. + + Like my_ip, this parameter affects the controller's own + identity only and has no effect on the BIN socket addresses + advertised to clusterer peers. + + If the interface has more than one IPv4 address, a warning is + logged and the first address (in the order returned by the + kernel) is used. Set my_ip explicitly to avoid ambiguity on + multi-address interfaces. + + Ignored if my_ip is also set — my_ip takes precedence. + + Default: auto-detected (Mode 3 — see my_ip). + + Example 1.5. Set interface parameter +... +modparam("clusterer_controller", "interface", "eth0") +... + +1.6.4. query_time (integer) + + How often (in seconds) each active node sends an ALIVE + heartbeat to the multicast group. This value also controls the + election window (3 × query_time) and the peer purge window (6 × + query_time). + + Smaller values mean faster failure detection but higher + multicast traffic. Valid range: 1–60. + + Default value is “5”. + + Example 1.6. Set query_time parameter +... +modparam("clusterer_controller", "query_time", 5) +... + +1.6.5. password (string) + + Global default encryption password for all clusters. All nodes + in a cluster must use the same password. The password serves + two purposes: + * Bootstrap key — the password is stretched with scrypt + (memory-hard, per-cluster salt) to encrypt JOIN_REQ and + KEY_GRANT packets during the initial key exchange phase + before a session key is established. + * Session key material — the password is fed into HKDF-SHA256 + together with the master salt to derive the session key + used for all normal cluster traffic. + + Can be overridden per cluster using the password= key in the + cluster parameter. + + Default value is “3eCrEt*5629”. Change this in production. Use + a long, high-entropy secret rather than a memorable phrase — + scrypt raises the cost of an offline guess, but only a strong + secret removes the risk. A generated key is ideal, e.g. openssl + rand -base64 32. The module logs a startup warning if the + configured password is the default or has an estimated entropy + below 80 bits. + + Example 1.7. Set password parameter +... +modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") +... + +1.6.6. manage_shtags (integer) + + When set to 1 (the default), the controller master node + automatically manages sharing tag failover for all clusters. + The controller becomes the single decision point for which node + holds the active tag, eliminating races between nodes and + requiring no script-level event routes or MI commands to handle + failover. While active, the clusterer_set_tag_active MI command + and the $shtag() script variable setter are blocked for + controller-managed clusters, returning an error to the caller. + + Behaviour when manage_shtags=1: + + Startup: all local sharing tags are forced to backup state + during module initialisation, regardless of the =active value + in the clusterer sharing_tag modparam. The deferred BIN + broadcast flag is also cleared so no SHTAG_ACTIVE packet is + ever sent at startup. This ensures that no node can steal the + active tag from an existing cluster member simply by + restarting. + + Bootstrap: when the first node starts alone and no existing + master responds within query_time seconds (join deadline), it + elects itself master and activates all local backup tags + exactly once. Nodes that join an existing cluster are never + eligible for this bootstrap path and never self-activate. + + Failover: when any node departs (graceful shutdown via GOODBYE + packet, or timeout-based removal), the controller master + activates its own backup tags for that cluster. This covers all + departure scenarios: last node standing, master still present, + and post re-election. + + Rejoin: a node rejoining an existing cluster always starts in + backup state and never reclaims the active tag from the current + holder, even if =active appears in its config. + + When set to 0, the controller does not touch sharing tag state + at all. The =active config value, seed_fallback_interval, and + external MI/event-route scripts behave exactly as in stock + clusterer without the controller. Use this when you have + existing tag management scripts and want to opt out of + automatic failover. + + Default value is “1”. + + Global vs per-cluster scope: This modparam sets a global + default that applies to every cluster defined via the cluster + modparam. Individual clusters can override it by including + manage_shtags=0 or manage_shtags=1 directly in the cluster + string. The global default is resolved at startup after all + modparams are processed, so the order of manage_shtags and + cluster lines in the config file does not matter. + + Example 1.8. Global manage_shtags — applies to all clusters +... +# Enable automatic failover for every cluster (this is also the default) +modparam("clusterer_controller", "manage_shtags", 1) +modparam("clusterer_controller", "cluster", "id=1,multicast=239.0.90.1:3 +333") +modparam("clusterer_controller", "cluster", "id=2,multicast=239.0.90.2:3 +333") +... + + Example 1.9. Per-cluster override — opt one cluster out of + automatic failover +... +# manage_shtags=1 globally, but cluster 2 uses its own MI/event-route sc +ripts +modparam("clusterer_controller", "manage_shtags", 1) +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.2:3333,manage_shtags=0") +... + + Example 1.10. Global opt-out with one cluster opting in +... +# Disable automatic failover globally; enable it only for cluster 1 +modparam("clusterer_controller", "manage_shtags", 0) +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,manage_shtags=1") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.2:3333") +... + + Example 1.11. Typical full configuration with manage_shtags=1 + (default) +# All nodes use identical config — only the BIN socket IP differs per no +de. +# The =active tag value in sharing_tag is ignored by the controller; +# it is kept in the config only for compatibility with manage_shtags=0. + +socket=bin:10.22.23.191:3857 + +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "sharing_tag", "vip1/1=active") +modparam("clusterer", "ping_interval", 4) +modparam("clusterer", "ping_timeout", 1500) + +loadmodule "clusterer_controller.so" +modparam("clusterer_controller", "cluster", "id=1,multicast=239.0.90.1: +3333") +modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") +# manage_shtags defaults to 1 — no need to set it explicitly + +1.6.7. master_stickiness (integer) + + Controls whether a live master keeps its role when a higher-IP + node joins. Default 1 (sticky). + + The module recognises three roles per cluster: master (the + active coordinator), backup (the standby that takes over when + the master fails), and member (all other nodes). The backup is + always the highest-IP node that is not the master. + * master_stickiness=1 (default): the master is sticky — a + live master keeps the role and is not displaced when a + higher-IP node joins. The newly joined node becomes the + backup (replacing the previous backup if it has a higher + IP); the master only changes when the current master + actually fails, at which point the backup is promoted. This + minimises the number of master handovers. + * master_stickiness=0: pure highest-IP election — a higher-IP + node takes over as master as soon as it appears. This + produces more handovers but always keeps the highest-IP + node as master. + + In both modes a split-brain (two nodes each believing they are + master, e.g. after a network partition heals) is resolved + deterministically: the lower-IP master yields to the higher-IP + one. + + Global vs per-cluster scope: like manage_shtags, this sets a + global default that individual clusters can override with + master_stickiness=0 or master_stickiness=1 in the cluster + string. Resolution happens at startup regardless of modparam + order. + + Example 1.12. Set master_stickiness parameter +... +# Global default (sticky) — omit entirely for the same effect +modparam("clusterer_controller", "master_stickiness", 1) + +# Per-cluster override: cluster 2 always promotes the highest-IP node +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.2:3333,master_stickiness=0") +... + +1.6.8. on_config_mismatch (string) + + Policy applied when a node's consistency-critical settings + (manage_shtags, master_stickiness, query_time) differ from + those of the running cluster. All nodes of a cluster are + expected to use identical values; this parameter decides what + happens when they do not. One of: + * reject (default) - the master refuses the join with a + JOIN_REJECT and the joining node shuts down after logging + which settings differ. + * warn - the node is allowed to join; a single CONFIG + MISMATCH warning is logged per mismatching peer. + * adopt - the joining node adopts the running cluster's + settings at runtime and continues. + + Global only (not per-cluster). Default: reject. + + Example 1.13. Set on_config_mismatch parameter +... +modparam("clusterer_controller", "on_config_mismatch", "reject") +... + +1.7. Exported MI Functions + +1.7.1. cl_ctr_list_members + + List all current cluster members with their node_id, status and + BIN socket addresses. Status is one of master, backup (the + standby that will be promoted if the master fails) or member. + Only peers within the current election window are shown. + + Parameters: none + + Example 1.14. cl_ctr_list_members usage +opensips-cli -x mi cl_ctr_list_members +[ + { + "cluster_id": 1, + "members": [ + { + "ip": "10.22.23.191", + "node_id": 1, + "status": "master", + "bin_sockets": [ "bin:10.22.23.191:3857" ] + }, + { + "ip": "10.22.23.193", + "node_id": 2, + "status": "backup", + "bin_sockets": [ "bin:10.22.23.193:3857" ] + }, + { + "ip": "10.22.23.192", + "node_id": 3, + "status": "member", + "bin_sockets": [ "bin:10.22.23.192:3857" ] + } + ] + } +] + +1.7.2. cl_ctr_node_info + + Return full information for a specific node identified by its + allocated node_id. + + Parameters: + * node_id — the integer node_id to look up. + + Example 1.15. cl_ctr_node_info usage +opensips-cli -x mi cl_ctr_node_info node_id=2 +{ + "node_id": 2, + "ip": "10.22.23.192", + "cluster_id": 1, + "status": "backup", + "bin_sockets": [ "bin:10.22.23.192:3857" ] +} + +1.7.3. cl_ctr_list_config + + List all configured clusters and their resolved settings — the + effective values actually in use after global defaults and + per-cluster overrides have been applied. Useful for confirming + that a per-cluster master_stickiness or manage_shtags override + took effect. The cluster password is never exposed. + + The shtag_mode field reports the current sharing-tag allocation + policy: auto when the active tag follows the master + automatically, or override: when an operator has + pinned a fixed holder with cl_ctr_shtag_force. + + Parameters: none + + Example 1.16. cl_ctr_list_config usage +opensips-cli -x mi cl_ctr_list_config +[ + { + "cluster_id": 1, + "multicast": "239.0.90.1:3333", + "my_ip": "10.22.23.191", + "bin_socket": "bin:10.22.23.191:3857", + "query_time": 5, + "master_stickiness": 1, + "manage_shtags": 1, + "shtag_mode": "auto", + "member_count": 3 + } +] + +1.7.4. cl_ctr_shtag_force + + Force a specific node to hold the active sharing tag, + overriding the normal master-driven allocation. This is useful + for planned maintenance or manual traffic steering: the chosen + node becomes the sole active shtag holder cluster-wide while + every other node — including the master — is put into backup + for that tag. + + The command must be issued on the current master (it returns an + error otherwise). The override is propagated to all members in + the MEMBER_LIST and survives master fail-over: a newly elected + master keeps honouring it rather than reclaiming the tag. + Automatic allocation stays suspended until cl_ctr_shtag_auto is + called. If the forced node leaves the cluster or times out, the + override is cleared automatically and automatic allocation + resumes. + + Parameters: + * cluster_id — the target cluster. + * node_id — the node that must hold the active tag; it must + be a current member of the cluster. + + Example 1.17. cl_ctr_shtag_force usage +opensips-cli -x mi cl_ctr_shtag_force cluster_id=1 node_id=3 + +1.7.5. cl_ctr_shtag_auto + + Clear any override set by cl_ctr_shtag_force and resume + automatic, master-driven sharing-tag allocation — the active + tag follows the master again. Must be issued on the current + master. + + Parameters: + * cluster_id — the target cluster. + + Example 1.18. cl_ctr_shtag_auto usage +opensips-cli -x mi cl_ctr_shtag_auto cluster_id=1 + +1.8. Exported Pseudo-Variables + + These read-only pseudo-variables expose live cluster state to + the routing script, so a decision such as “only the master runs + this job” can be made without an MI call. They read directly + from shared memory and are therefore available in every process + (SIP workers included). + + Each variable optionally takes a cluster id as its argument, + e.g. $cl_ctr_role(2). The bare form ($cl_ctr_role) resolves to + the only configured cluster; when several clusters are defined + the bare form returns NULL and logs a one-time warning, so the + cluster must be named explicitly. An unknown cluster id, or a + value that is not currently known (e.g. no master yet), returns + NULL. All of them are read-only - assigning to them fails. + * $cl_ctr_role — this node's role in the cluster: master, + backup, member, or joining (still authenticating / before + the first election). + * $cl_ctr_is_master — 1 if this node is the cluster master, 0 + otherwise. A fast path for the most common check. + * $cl_ctr_master_ip — IP of the current master (NULL if none + is elected yet). + * $cl_ctr_backup_ip — IP of the current backup / standby + master (NULL if none). + * $cl_ctr_node_id — this node's node_id within that cluster + (NULL until assigned). A node may hold different ids in + different clusters. + * $cl_ctr_my_ip — the controller identity IP of this node. + * $cl_ctr_members — number of live members currently in the + cluster. + * $cl_ctr_shtag_mode — auto (tags follow the elected master) + or forced (an operator pinned them with + cl_ctr_shtag_force). + * $cl_ctr_forced_node — the node_id the active sharing tag is + pinned to (NULL when in auto mode). + + Example 1.19. Using $cl_ctr_* in the script +# single cluster: run a periodic job only on the master +if ($cl_ctr_is_master) + route(do_master_only_work); + +# several clusters: name the one you mean +xlog("cluster 2 master is $cl_ctr_master_ip(2), I am $cl_ctr_role(2)\n") +; + + +1.9. Exported Functions + + Per-peer lookups take two arguments (cluster_id, node_id) and + are therefore script functions, not pseudo-variables (a comma + inside a variable's parentheses is ambiguous when the variable + is used as a function argument). Boolean checks return + true/false for use directly in an if; value lookups write into + an output variable. All are usable from any route. + * cl_ctr_node_is_master(cluster_id, node_id) — true if that + node is the cluster master. + * cl_ctr_node_present(cluster_id, node_id) — true if that + node_id is a live member. + * cl_ctr_get_node_role(cluster_id, node_id, out_var) — write + that node's role (master/backup/member) into out_var; + returns false if it is not a member. + * cl_ctr_get_node_ip(cluster_id, node_id, out_var) — write + that node's IP into out_var. + + Example 1.20. Per-peer lookup functions +if (cl_ctr_node_present(1, 3) && cl_ctr_node_is_master(1, 3)) { + cl_ctr_get_node_ip(1, 3, $var(ip)); + xlog("node 3 ($var(ip)) leads cluster 1\n"); +} + +1.10. Multiple Clusters + + A single OpenSIPS instance can participate in multiple clusters + simultaneously by repeating the cluster modparam. Each cluster + runs an independent worker process with its own multicast + socket, peer table, master election, and node_id space. + + Clusters are distinguished by the combination of their + multicast IP address and UDP port. Two useful topologies are + possible: + * Different ports, same multicast IP — convenient when all + clusters share the same L2 segment. The port number alone + separates traffic for each cluster. Each cluster's password + should also differ to provide an additional encryption + barrier. + * Different multicast IPs — useful when clusters span + different network segments or when multicast routing is + scoped differently per cluster. + + When multiple clusters are defined and the node has more than + one BIN socket, the bin_socket= key must be specified in each + cluster string to indicate which BIN socket to advertise for + that cluster. If only one BIN socket exists, it is used for all + clusters automatically. + + Each cluster has its own independent clusterer cluster_id, + allowing different OpenSIPS subsystems to replicate on + different clusters: + + Example 1.21. Multiple clusters — dialog on cluster 1, usrloc + on cluster 2 +# Two BIN sockets, one per cluster +socket=bin:10.0.1.10:5566 +socket=bin:10.0.2.10:5566 + +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +modparam("clusterer", "use_controller", 1) + +loadmodule "clusterer_controller.so" +# Cluster 1 — dialog replication group, LAN segment 10.0.1.0/24 +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:10.0.1.10:5566") +# Cluster 2 — usrloc replication group, LAN segment 10.0.2.0/24 +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.1:3334,bin_socket=bin:10.0.2.10:5566") + +loadmodule "dialog.so" +modparam("dialog", "dialog_replication_cluster", 1) + +loadmodule "usrloc.so" +modparam("usrloc", "cluster_id", 2) + + Example 1.22. Multiple clusters — same multicast IP, different + ports +socket=bin:10.22.23.191:3857 + +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +modparam("clusterer", "use_controller", 1) + +loadmodule "clusterer_controller.so" +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,password=ClusterOneSecret") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.1:3334,password=ClusterTwoSecret") + +loadmodule "dialog.so" +modparam("dialog", "dialog_replication_cluster", 1) + +loadmodule "dispatcher.so" +modparam("dispatcher", "cluster_id", 2) + +1.11. Hybrid Topologies (native + controller clusters) + + A single OpenSIPS instance can run native clusterer clusters + (defined in the database or statically via + my_node_info/neighbor_node_info) and controller-managed + clusters side by side. This allows, for example, a fixed, + DB-provisioned replication cluster to coexist with a + zero-config controller-driven HA cluster on the same node. + + Which kind a cluster is follows from how it is declared: + * Controller-managed — every cluster_id registered with the + clusterer module via modparam("clusterer", "cluster_id", N) + (and matched by a cluster entry in this module). Its + topology and this node's node_id are driven at runtime by + the controller; it never touches the database and always + behaves as db_mode=0, regardless of the global db_mode. + * Native — every cluster loaded from the clusterer database + (db_mode!=0) or provisioned statically. It behaves exactly + as classic clusterer: fixed my_node_id, DB persistence + (when DB-backed), script/MI-managed sharing tags. + + The rules that keep the two kinds apart: + * A cluster_id is exclusively controller-managed or native — + declaring the same id both ways is rejected at startup. + * my_node_id identifies this node in its native clusters only + and is required only when native clusters exist; controller + clusters get their node id assigned at runtime, and this + node may well hold different node ids in different + clusters. + * db_url/db_mode apply to native clusters only. A + controller-only deployment needs neither. + * Sharing tags: only tags of controller-managed clusters are + forced to backup at startup and driven by the controller + master; tags of native clusters keep their configured state + (=active included) and stay script/MI-managed. + * Native and controller clusters may share the same BIN + socket - the cluster_id carried in every BIN packet keeps + their traffic apart. + + Example 1.23. Hybrid — DB-native cluster 10 + controller + cluster 1 +socket=bin:10.0.0.10:5566 + +loadmodule "db_mysql.so" +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +# native side: cluster 10 lives in the DB (nodes provisioned there) +modparam("clusterer", "db_mode", 1) +modparam("clusterer", "db_url", "mysql://opensips:pass@localhost/opensip +s") +modparam("clusterer", "my_node_id", 5) # this node's id in cluster +10 +# controller side: cluster 1 is dynamic +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) + +loadmodule "clusterer_controller.so" +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:10.0.0.10:5566,passwo +rd=S3cret") + +loadmodule "dialog.so" +modparam("dialog", "dialog_replication_cluster", 1) # controller-manag +ed + +loadmodule "dispatcher.so" +modparam("dispatcher", "cluster_id", 10) # DB-native + +1.12. Configuration Example + + The following example shows a minimal two-module configuration + for zero-config HA clustering with dialog replication. The + loadmodule order does not matter — the dependency system + enforces correct initialization order automatically. + + Example 1.24. Minimal HA cluster configuration +# Each node needs an explicit BIN socket (no wildcard) +socket=bin:10.22.23.191:3857 + +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "sharing_tag", "vip1/1=active") +modparam("clusterer", "ping_interval", 4) +modparam("clusterer", "ping_timeout", 1500) + +loadmodule "clusterer_controller.so" +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333") +modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") + +loadmodule "tm.so" +modparam("tm", "tm_replication_cluster", 1) + +loadmodule "dialog.so" +modparam("dialog", "dialog_replication_cluster", 1) +modparam("dialog", "cluster_auto_sync", 1) + +loadmodule "dispatcher.so" +modparam("dispatcher", "cluster_id", 1) +modparam("dispatcher", "cluster_probing_mode", "distributed") + + The same configuration file (with the node-specific + socket=bin:IP:PORT line changed per node) is used on every + node. No other per-node customization is required. + +1.13. Limitations + + * IPv4 only. IPv6 multicast is not currently supported. + * The module pre-allocates approximately 152 KB of shared + memory (-m) per cluster and 6 KB of private memory (-M) per + worker process at startup. If either allocation fails, + OpenSIPS will refuse to start with an error in the log. + * Wildcard BIN sockets (bin:*:PORT) are rejected at startup. + An explicit IP address must be used in the socket= line. + * Node IDs are not persistent across a full cluster restart + (all nodes down simultaneously). IDs are reallocated + starting from 1 when the cluster reforms. This has no + operational impact as long as at least one node remains up + during rolling restarts. + * The multicast network must support IP multicast routing + between all cluster nodes. Nodes on different L3 segments + require PIM or similar multicast routing. + * PIM-DM networks: PIM Dense Mode periodically re-floods + multicast traffic (prune state typically expires every 3 + minutes). On slow or congested networks this brief + re-flood/prune cycle could cause a gap in MASTER_ALIVE + delivery and trigger a spurious master re-election. If this + occurs, increase the effective timeout by raising + CC_MASTER_KA_MISSED in the source from 3 to 5 or higher. + PIM Sparse Mode (PIM-SM) or networks with IGMP snooping do + not have this issue. + * L2 overlay tunnels (Geneve, VXLAN, GRE): if the overlay + presents a flat L2 segment with multicast support, the + controller works transparently. However, some VXLAN + deployments disable multicast entirely (no underlay + multicast group, no BUM replication) — in that case the + controller will not function, as it has no unicast + fallback. Encapsulation overhead also adds latency and + jitter; on high-latency overlays consider raising + CC_MASTER_KA_MISSED to avoid spurious re-elections. + * IPsec-protected links: native IPsec multicast requires + GDOI/GET VPN (RFC 6407), which is rarely deployed. The + recommended approach is to run multicast inside an inner + tunnel (GRE-over-IPsec, Geneve-over-IPsec) that presents a + multicast-capable interface. Running the controller over + such a setup results in double encryption + (application-layer AES-256-GCM plus IPsec ESP), which is + harmless but adds minor CPU overhead. IPsec ESP tunnel mode + also reduces the effective MTU by approximately 50 bytes, + which compounds the MEMBER_LIST fragmentation issue + described below. + * MEMBER_LIST fragmentation: the MEMBER_LIST packet grows + with cluster size and reaches approximately 4395 bytes at + the maximum of 256 nodes. This exceeds the 1472-byte UDP + payload budget of a standard 1500-byte MTU Ethernet link + and requires IP fragmentation: + + Standard Ethernet (1500 MTU): 3 fragments + + IPsec ESP tunnel (~1400 MTU): 4 fragments + + GRE-over-IPsec (~1350 MTU): 4–5 fragments + All other packet types (ALIVE, JOIN_REQ, KEY_GRANT, + GOODBYE, etc.) fit comfortably within a single datagram on + any of these links. The DF bit is not set, so IP + fragmentation occurs transparently where the network allows + it. However, firewalls or stateless middleboxes that + silently drop fragmented UDP will prevent new nodes from + joining, since MEMBER_LIST is required to complete the join + sequence. Verify that fragmented UDP is permitted on all + paths between cluster nodes, particularly over VPN tunnels + and across datacenter firewalls. + +1.14. Planned Features + + The following features are planned for future releases: + * Node maintenance mode — take a node out of duty for a + rolling upgrade while it stays in the cluster. A node in + maintenance keeps replicating and answering pings and stays + visible in cl_ctr_list_members, but is excluded from + election (never master or backup; if it is master it hands + over gracefully first) and sheds its sharing tags (a + cl_ctr_shtag_force pin on it auto-clears). Two levels are + planned: + + evicted — out of election and tags; the routing script + refuses new work while established dialogs finish. + + full — additionally marks the node down for clusterer + consumers so peers stop routing replication work to + it. + The state is cluster-wide (carried in the ALIVE/MEMBER_LIST + control plane, so every node agrees and it survives master + failover) and runtime-only — a restart brings the node back + in service. + Interfaces (following the read-only variables and MI + commands above): + + MI cl_ctr_maintenance (any node, the master propagates + it) sets a target node's state evicted/full/off; + cl_ctr_list_members gains a maintenance column. + + Script function cl_ctr_set_maintenance() (a verb - an + action) for a node to put itself in or out of + maintenance from the routing logic. + + Read-only variables $cl_ctr_maintenance (this node: + none/evicted/full) and $cl_ctr_node_maint(cluster_id, + node_id) (any peer); $cl_ctr_role gains a maintenance + value. + + Event E_CL_CTR_MAINTENANCE raised on every node when + any member's maintenance state changes, so an + event_route can react (e.g. shift dispatcher weights). + * Statistics — module statistics (current role, member count, + master changes, nodes joined/left, JOIN_REJECTs, decrypt + failures, config mismatches, split-brain merges) exposed + via get_statistics and monitoring exporters. + * Events — events raised on state transitions (became master, + demoted, node joined/left, split-brain merged, config + mismatch, authentication reject), named E_CL_CTR_* and + consumable from an event_route or any event subscriber + transport. + * IPv6 multicast — the control plane currently uses IPv4 + multicast (groups in 224.0.0.0/4). Add IPv6 multicast + support (ff00::/8 groups, AF_INET6 sockets and membership) + so the controller can run on IPv6-only or dual-stack + deployments. + + Read-only script variables for cluster state ($cl_ctr_role, + $cl_ctr_is_master, and the rest) are already available - see + Exported Pseudo-Variables. + +Appendix A. HA Behaviour Tests + + The following tests were performed on a three-node cluster + (nodes A=10.22.23.191, B=10.22.23.192, C=10.22.23.193) to + verify correct failover, tag stability, and no-steal-on-join + behaviour. The sharing tag under test is vip1 in cluster 1. All + nodes run with manage_shtags=1. + + Tag state was queried after each operation via: +opensips-cli -x mi clusterer_list_shtags + +A.1. Baseline + + All three nodes running. B holds the active tag; A and C are + backup. +A (10.22.23.191) svc=active tag=backup +B (10.22.23.192) svc=active tag=active +C (10.22.23.193) svc=active tag=backup + +A.2. Test 1 — Stop the active node + + B (active) is stopped. The remaining nodes must elect a new + active holder. B must rejoin as backup and must not steal the + tag from whichever node became active. +# Stop B +A svc=active tag=backup +B svc=inactive tag=(down) +C svc=active tag=active <-- C promoted + +# Start B +A svc=active tag=backup +B svc=active tag=backup <-- rejoined as backup +C svc=active tag=active <-- C retains active + + Result: PASS. Failover within the dead-node detection window; + rejoining node did not steal the active tag. + +A.3. Test 2 — Stop a backup node + + A (backup) is stopped. The active tag must remain on C without + any transition. A must rejoin as backup. +# Stop A +A svc=inactive tag=(down) +B svc=active tag=backup +C svc=active tag=active <-- unchanged + +# Start A +A svc=active tag=backup <-- rejoined as backup +B svc=active tag=backup +C svc=active tag=active <-- still active + + Result: PASS. Removing a backup node causes no tag movement; + rejoining node started in backup state. + +A.4. Test 3 — Stop both backup nodes + + A and B (both backup) are stopped simultaneously. The lone + remaining node C must retain the active tag. A and B must + rejoin as backup. +# Stop A and B +A svc=inactive tag=(down) +B svc=inactive tag=(down) +C svc=active tag=active <-- unchanged, lone node + +# Start A, then B +A svc=active tag=backup <-- rejoined as backup +B svc=active tag=backup <-- rejoined as backup +C svc=active tag=active <-- still active + + Result: PASS. Active node remained stable while running alone; + both rejoining nodes came up in backup state. + +A.5. Test 4 — Stop active node and one backup + + B (active) and C (backup) are stopped. The sole remaining node + A must become active. B and C must rejoin as backup. +# Stop B and C +A svc=active tag=active <-- A promoted, now lone node +B svc=inactive tag=(down) +C svc=inactive tag=(down) + +# Start B +A svc=active tag=active <-- retains active +B svc=active tag=backup <-- rejoined as backup +C svc=inactive tag=(down) + +# Start C +A svc=active tag=active <-- retains active +B svc=active tag=backup +C svc=active tag=backup <-- rejoined as backup + + Result: PASS. The surviving node correctly claimed the active + tag; each rejoining node started in backup state without + challenging the active holder. + +A.6. Test 5 — Full cluster restart + + All three nodes are stopped (full outage). Nodes are then + started one at a time. The first node up must self-elect as + active (no peers available to sync from). Subsequent nodes must + join as backup and must not steal the active tag. +# All stopped +A svc=inactive tag=(down) +B svc=inactive tag=(down) +C svc=inactive tag=(down) + +# Start A first +A svc=active tag=active <-- first/lone seed, self-synced +B svc=inactive tag=(down) +C svc=inactive tag=(down) + +# Start B +A svc=active tag=active <-- retains active +B svc=active tag=backup <-- joined as backup, did not steal +C svc=inactive tag=(down) + +# Start C +A svc=active tag=active <-- retains active +B svc=active tag=backup +C svc=active tag=backup <-- joined as backup, did not steal + + Result: PASS. The first node to start elected itself active and + no spurious sync errors were logged. Each subsequent node + joined as backup without challenging the active holder. + +A.7. Test 6 — Multiple clusters over one BIN socket + + Two controller clusters (id=1 and id=2) are configured on all + three nodes. Each cluster has its own multicast endpoint (a + distinct port is required - two clusters on the same multicast + address:port are rejected at startup with duplicate multicast), + but both advertise the same BIN socket (bin:IP:3857). Each + cluster must form independently and its replication must stay + isolated over the shared socket. +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "cluster_id", 2) +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:IP:3857") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.1:3334,bin_socket=bin:IP:3857") + +# both clusters converge, each with its own master/backup/member roles; +# clusterer_list shows cluster 1 and cluster 2 both using bin:IP:3857 + + Result: PASS. Both clusters formed independently (each + member_count=3 with consistent roles), the BIN links of both + were Up over the single shared socket, and there were zero + decrypt / cross-talk / foreign-cluster errors on any node - the + cluster_id in each BIN packet keeps the two clusters' + replication traffic separate. A control test confirmed that + configuring both clusters on the same multicast address:port is + refused at startup. + +A.8. Summary + + Test Scenario Result + 1 Stop active node; rejoin PASS + 2 Stop backup node; rejoin PASS + 3 Stop both backups simultaneously; rejoin PASS + 4 Stop active + one backup; rejoin PASS + 5 Full cluster restart; sequential startup PASS + 6 Two clusters sharing one BIN socket (distinct multicast) PASS + + In all five failover tests (1–5): exactly one node held the + active sharing tag at all times (including during the failure + window), and no rejoining node stole the active tag from the + current holder. Test 6 additionally confirmed that two clusters + can share a single BIN socket with fully isolated replication. + +Chapter 2. Contributors + +2.1. Contributors + + Definition, design and implementation of this module was made + by: + * Yury Kirsanov — VoIPLine Telecom + +2.2. Documentation Contributors + + Documentation was written by: + * Yury Kirsanov — VoIPLine Telecom + + Documentation Copyrights: + + Copyright © 2026 VoIPLine Telecom diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c new file mode 100644 index 00000000000..6247b1c4862 --- /dev/null +++ b/modules/clusterer_controller/clusterer_controller.c @@ -0,0 +1,5902 @@ +/* + * clusterer_controller - multicast extension for the clusterer module + * + * Copyright (C) 2026 Yury Kirsanov + * VoIPLine Telecom + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * ========================================================================= + * MODULE: clusterer_controller + * ========================================================================= + * + * PROTOCOL OVERVIEW + * ----------------- + * All traffic is UDP multicast to multicast_address:multicast_port. Every + * packet's payload is encrypted and authenticated with an AEAD. The cleartext + * framing that precedes it - a 2-byte magic (key selector, must be readable + * before decryption) and a 2-byte cluster_id - is bound into the AEAD tag as + * additional authenticated data (AAD), so it cannot be altered undetected (this + * blocks re-stamping a captured packet onto a different cluster_id when two + * clusters share a multicast group and password). + * + * CRYPTO SUITE (build-time choice; every node in a cluster must match): + * - Default (WolfSSL): AES-256-GCM payload AEAD, 12-byte nonce; + * scrypt bootstrap-key KDF (N=2^16, r=8, p=1). + * - If libsodium is detected on the build host (-DCC_HAVE_SODIUM, see the + * Makefile): XChaCha20-Poly1305 payload AEAD, 24-byte nonce (its 192-bit + * nonce removes any random-nonce collision worry); Argon2id bootstrap KDF. + * The two wire formats are NOT interoperable (nonce size and primitives + * differ). The active suite is logged at startup ("crypto=..."). + * + * Two key types are used, selected by the 2-byte magic: + * + * Bootstrap key - KDF(password, salt="...v1:"+multicast) [scrypt or Argon2id] + * Used for the admission handshake (JOIN_REQ, KEY_GRANT, JOIN_REJECT) and + * the split-brain MASTER_BEACON - i.e. traffic that must be readable before + * a session key exists, or by masters holding different session keys. + * Memory-hard KDF (derived once at startup) so a password captured from a + * bootstrap packet cannot be brute-forced cheaply offline. + * + * Session key - HKDF-SHA256 over an X25519-ECDH-agreed random master_salt. + * Used for all normal traffic. Generated once when the first node + * bootstraps the cluster and then preserved across every master change: + * a new master reuses the key that every member already holds, so master + * transitions require no re-keying. + * + * Wire format (all packets): + * [2B magic] [2B cluster_id] [12B|24B nonce] [ciphertext] [16B tag] + * AAD = magic || cluster_id. A node drops packets whose cluster_id does not + * match its own BEFORE decryption, so foreign-cluster traffic sharing the + * group never counts as an authentication failure. + * + * Authenticated plaintext layout: + * [1B: packet type] [4B: seq BE] [payload] + * + * The 32-bit monotonic sequence number is per-sender and validated per source + * IP for session-key packets. Prevents replay without any dependency on clock + * synchronisation. + * + * PACKET TYPES + * ------------ + * ALIVE - session key, multicast + * Every active node every query_time seconds. + * Payload: IP(16B) + pubkey(32B). Peers learn X25519 pubkeys here. + * + * JOIN_REQ - bootstrap key, multicast + * Sent by a new node on startup. + * Payload: IP(16B) + bin_info + pubkey(32B) + join_nonce(16B) + * join_nonce is random per-exchange; folded into the KEY_GRANT wrap key. + * + * MEMBER_LIST - session key, multicast + * Master -> all: member count, the operator-forced sharing-tag holder + * node_id (0 = automatic), and the full peer IP list, so all nodes elect + * identically. Only accepted from the current master (CC_NODE_NEW aside). + * + * GOODBYE - session key, multicast + * Graceful shutdown. Peers remove sender immediately without timeout. + * + * NODE_ASSIGN - session key, multicast + * Master -> all: allocate node_id + BIN socket for a joining node. + * + * MASTER_ALIVE - session key, multicast + * Master-only keepalive every CC_MASTER_KA_INTERVAL seconds. Peers declare + * the master dead after CC_MASTER_KA_TIMEOUT seconds of silence and trigger + * re-election. Two masters that share a session key resolve split-brain + * here: the lower-IP one yields. + * + * KEY_GRANT - bootstrap key, unicast to joiner + * Master reply to JOIN_REQ. + * Payload: IP(16B) + master_pubkey(32B) + join_nonce(16B) + wrapped_salt(32B) + * wrapped_salt = master_salt XOR HKDF(ECDH(shared) || password || join_nonce) + * + * KEY_HANDOFF - session key, unicast to next master + * Outgoing master on graceful shutdown -> next-highest-IP peer. Transfers + * master_salt so the new master avoids a full re-join cycle. + * Payload: IP(16B) + sender_pubkey(32B) + wrapped_salt(32B) + * + * JOIN_REJECT - bootstrap key, multicast + * Master -> a source whose bootstrap packets repeatedly fail to decrypt + * (wrong password). Encrypted, so only a correctly-configured node can + * read it; a wrong-password joiner also self-terminates at its deadline. + * + * MASTER_BEACON - bootstrap key, multicast + * Master-only, every CC_MASTER_BEACON_EVERY keepalive ticks. Because it + * uses the bootstrap key it is readable even by a master holding a + * DIFFERENT session key, which is how a split brain between independently + * bootstrapped partitions is detected and merged. Payload: member count. + * + * NODE STATE MACHINE + * ------------------ + * CC_NODE_NEW --> (MEMBER_LIST or KEY_GRANT received) --> CC_NODE_ACTIVE + * --> (join_deadline expired) --> CC_NODE_ACTIVE + * + * CC_NODE_NEW: receive only; do NOT send ALIVE or MASTER_ALIVE. + * CC_NODE_ACTIVE: send ALIVE every query_time seconds. + * Master also sends MASTER_ALIVE every CC_MASTER_KA_INTERVAL s. + * + * MASTER ELECTION + * --------------- + * Three roles per cluster: MASTER (active coordinator), BACKUP (standby, always + * the highest-IP non-master) and MEMBER. Election uses a quantized window so + * all nodes evaluate identical peer sets and reach the same result + * deterministically. No NTP synchronisation required. + * + * master_stickiness (modparam, default 1): a live master keeps the role - a + * higher-IP node that joins becomes the BACKUP instead of preempting the + * master, so handovers are minimised. With master_stickiness=0 the highest-IP + * node always becomes master. + * + * Fast failure detection: MASTER_ALIVE at 1 s; 3 s timeout. On master failure + * the silent master is aged out of the election window and the BACKUP + * (highest-IP survivor) is promoted immediately - it already holds the + * preserved session key, so there is no re-keying and no re-JOIN cycle. + * Graceful handoff: KEY_HANDOFF + GOODBYE before shutdown. + * + * SPLIT-BRAIN HANDLING (three layers) + * ----------------------------------- + * 1. Prevention at join time: simultaneously-starting nodes see each other's + * JOIN_REQs, so at the join deadline a node that has seen a higher-IP + * starter DEFERS self-promotion (bounded) and joins that node instead of + * everyone becoming an independent-key lone master. + * 2. Same-key yield: two masters that share a session key see each other's + * MASTER_ALIVE; the lower-IP one yields (see MASTER_ALIVE above). + * 3. Divergent-key merge: masters that DO NOT share a session key cannot read + * each other's MASTER_ALIVE, so each emits a bootstrap-key MASTER_BEACON. + * On hearing a superior beacon (larger member count, ties broken by higher + * IP) a node re-joins that master and adopts its key. + * + * SHARING TAGS (manage_shtags, default 1) + * --------------------------------------- + * The controller drives clusterer sharing tags: normally the MASTER is the sole + * active holder and every other node is backup. An operator can override this + * with the cl_ctr_shtag_force MI command (pin the active tag to a chosen node) and + * revert with cl_ctr_shtag_auto; the override is carried in MEMBER_LIST, survives + * master fail-over, and auto-clears if the forced node departs. cl_ctr_list_config + * reports the current mode (auto / override:). + * + * ========================================================================= + */ + +#include +#include +#include /* strcasecmp() */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include /* O_NONBLOCK, fcntl() */ +#include /* getifaddrs(), freeifaddrs() */ +#include /* IF_NAMESIZE, struct ifreq, SO_BINDTODEVICE */ + +#include "../../sr_module.h" /* module_exports, MODULE_VERSION, proc_export_t, + PROC_FLAG_*, dep_export_t, DEP_ABORT, + param_export_t, STR_PARAM, INT_PARAM */ +#include "../../dprint.h" /* LM_ERR / LM_WARN / LM_INFO / LM_DBG */ +#include "../../mem/shm_mem.h" /* shm_malloc / shm_free */ +#include "../../locking.h" /* gen_lock_t - base spinlock primitive */ +#include "../../rw_locking.h" /* rw_lock_t - reader-writer lock built on top */ +#include "../../mi/mi.h" /* mi_export_t, mi_response_t, MI helpers */ +#include "../../timer.h" /* get_uticks(), utime_t - us since start */ +#include "../../socket_info.h" /* struct socket_info, PROTO_BIN */ +#include "../../net/api_proto.h" /* protos[] array */ +#include "../../globals.h" /* process_no - this process's index */ +#include "../../ipc.h" /* ipc_send_rpc() - cross-process job dispatch */ +#include "../../pvar.h" /* pv_export_t - read-only $cl_ctr_* variables */ + +#include "../clusterer/clusterer_ctrl.h" /* set_my_identity, add_node, remove_node */ + +#include /* must be first: build-time feature flags */ +#include /* Aes, wc_AesGcmSetKey/Encrypt/Decrypt */ +#include /* WC_RNG, wc_RNG_GenerateBlock */ +#include /* wc_HKDF */ +#include /* curve25519_key, wc_curve25519_* */ +#include /* wc_Sha256Hash */ +#include /* wc_scrypt (bootstrap key hardening) */ +#include /* timerfd_create(), timerfd_settime() */ +#include "../../reactor_proc.h" /* reactor_proc_init/add_fd/loop */ + +/* Optional stronger crypto suite. Selected at BUILD time: if libsodium is + * detected on the build host (see the module Makefile -> -DCC_HAVE_SODIUM), the + * payload AEAD becomes XChaCha20-Poly1305 (192-bit nonce -> no random-nonce + * collision worry, even for the static bootstrap key) and the bootstrap KDF + * becomes Argon2id. Otherwise we fall back to WolfSSL AES-256-GCM + scrypt. + * NOTE: the wire formats are NOT interoperable (nonce size and primitives + * differ), so every node in a cluster must be built with the same suite. */ +#ifdef CC_HAVE_SODIUM +#include +#define CC_CRYPTO_SUITE "XChaCha20-Poly1305 + Argon2id" +#else +#define CC_CRYPTO_SUITE "AES-256-GCM + scrypt" +#endif + +/* ========================================================================= + * Wire-format constants + * ========================================================================= */ + +/* 2-byte wire magic - a cleartext key-selector at the start of every packet + * (it must be readable before decryption to choose bootstrap vs session key). + * Both share the 0xCC prefix; the second byte distinguishes the key tier. + * Only a sanity/routing tag on a dedicated multicast group:port - the real + * confidentiality and integrity come from the AES-256-GCM payload. */ +#define CC_MAGIC_SZ 2 +static const unsigned char CC_PACKET_MAGIC[CC_MAGIC_SZ] = { 0xCC, 0x00 }; +static const unsigned char CC_BOOTSTRAP_MAGIC[CC_MAGIC_SZ] = { 0xCC, 0x01 }; + +/* Packet type bytes */ +#define CC_PKT_ALIVE 0x01 +#define CC_PKT_JOIN_REQ 0x02 +#define CC_PKT_MEMBER_LIST 0x03 /* master -> joining node: here is the cluster */ +#define CC_PKT_GOODBYE 0x04 /* graceful shutdown notification */ +#define CC_PKT_NODE_ASSIGN 0x05 /* master -> multicast: here is your node_id */ +#define CC_PKT_MASTER_ALIVE 0x06 /* master-only keepalive; ~1 s interval */ +#define CC_PKT_KEY_GRANT 0x07 /* master -> joiner: ECDH-wrapped master_salt */ +#define CC_PKT_KEY_HANDOFF 0x08 /* outgoing master -> next master: salt handoff */ +#define CC_PKT_JOIN_REJECT 0x09 /* master -> joiner: authentication rejected */ +#define CC_PKT_MASTER_BEACON 0x0A /* master-only announce (BOOTSTRAP key) so + * masters with divergent session keys can + * still discover each other and merge a + * split brain; payload = member count 2B BE */ + +/* Number of consecutive bootstrap-decrypt failures before a JOIN_REJECT is sent */ +#define CC_JOIN_FAIL_LIMIT 3 +#define CC_JOIN_FAIL_TABLE_SZ 8 /* max simultaneous rejected IPs tracked by master */ + +#define CC_MAX_IP_LEN 15 /* "255.255.255.255" without NUL */ +#define CC_PUBKEY_SZ 32 /* X25519 public key */ +#define CC_JOIN_NONCE_SZ 16 /* per-exchange nonce in JOIN_REQ/KEY_GRANT */ +#define CC_MASTER_SALT_SZ 32 /* random salt generated by each new master */ +/* MEMBER_LIST entry: IP (16B null-padded) + is_master (1B) = 17B. + * Pubkeys are NOT carried here - nodes learn them from ALIVE packets, + * keeping MEMBER_LIST small enough to avoid excessive IP fragmentation. */ +#define CC_IP_ENTRY_SZ 17 +#define CC_LIST_COUNT_SZ 2 /* MEMBER_LIST count field: uint16_t BE */ +#define CC_NODE_ID_SZ 2 /* uint16_t node_id, big-endian */ +#define CC_MAX_BIN_SOCKETS 8 /* max BIN listeners per node */ +#define CC_MAX_BIN_SOCK_LEN 64 /* "bin:255.255.255.255:65535" = 26 chars */ +/* BIN info block: [bin_count 1B][sock1 NUL-term]...[sockN NUL-term] */ +#define CC_BIN_INFO_MAX_SZ (1 + CC_MAX_BIN_SOCKETS * CC_MAX_BIN_SOCK_LEN) + +/* AES-256-GCM encryption constants + * wire: [magic 2B][cluster_id 2B BE][nonce 12B][ciphertext][GCM tag 16B] + * plaintext: [type 1B][seq 4B][payload] + * The cluster_id is cleartext (like the magic) so a node can drop packets that + * belong to a different cluster sharing the same multicast group WITHOUT its + * key - before decryption, so foreign traffic never counts as an auth failure. */ +#ifdef CC_HAVE_SODIUM +#define CC_NONCE_SZ 24 /* XChaCha20-Poly1305 nonce (192-bit) */ +#else +#define CC_NONCE_SZ 12 /* AES-GCM nonce, random per packet */ +#endif +#define CC_TAG_SZ 16 /* AEAD tag (16 for both AES-GCM & XChaCha) */ +#define CC_SEQ_SZ 4 /* uint32_t monotonic sequence in plaintext */ +#define CC_CLUSTER_ID_SZ 2 /* cleartext uint16 cluster_id (BE) selector */ +#define CC_NONCE_OFF (CC_MAGIC_SZ + CC_CLUSTER_ID_SZ) /* nonce starts here */ +#define CC_WIRE_HDR_SZ (CC_MAGIC_SZ + CC_CLUSTER_ID_SZ + CC_NONCE_SZ) /* 16 (GCM) / 28 (XChaCha) */ +#define CC_PLAIN_HDR_SZ (1 + CC_SEQ_SZ) /* type + seq = 5 */ + +/* Bootstrap-key hardening: the join/admission key is derived from the shared + * password with scrypt (memory-hard) instead of a single SHA-256, so a + * password captured from a JOIN_REQ cannot be brute-forced cheaply offline. + * Derived ONCE in mod_init (main process, before fork), so the cost is a + * transient startup cost only - cost=16/r=8 is ~64 MiB for ~0.3 s, freed + * immediately; workers inherit the 32-byte key and never run scrypt. (Argon2id + * would be preferable but this WolfSSL build does not provide it.) */ +#define CC_SCRYPT_COST 16 /* log2(N): N = 65536 (2x offline cost) */ +#define CC_SCRYPT_BLOCKSIZE 8 /* r */ +#define CC_SCRYPT_PARALLEL 1 /* p */ +#ifdef CC_HAVE_SODIUM +/* Argon2id parameters (libsodium crypto_pwhash). Fixed so every node derives + * the same key; ~64 MiB to match the scrypt fallback's memory hardness. */ +#define CC_ARGON2_OPSLIMIT 3UL +#define CC_ARGON2_MEMLIMIT (64UL * 1024 * 1024) +#endif +/* Minimum estimated password entropy (bits) before a startup warning fires. */ +#define CC_MIN_PASSWORD_BITS 80 +#define CC_DEFAULT_PASSWORD "3eCrEt*5629" /* insecure placeholder; warn if used */ + +/* Master keepalive: master sends CC_PKT_MASTER_ALIVE every CC_MASTER_KA_INTERVAL + * seconds. Peers declare master dead after CC_MASTER_KA_MISSED missed packets. */ +#define CC_MASTER_KA_INTERVAL 1 /* seconds between MASTER_ALIVE sends */ +#define CC_MASTER_KA_MISSED 3 /* missed keepalives before re-election */ +#define CC_MASTER_KA_TIMEOUT (CC_MASTER_KA_INTERVAL * CC_MASTER_KA_MISSED) + +/* Split-brain merge: a master emits a CC_PKT_MASTER_BEACON (bootstrap key) once + * every CC_MASTER_BEACON_EVERY MASTER_ALIVE ticks. This is the only traffic two + * masters with divergent session keys can both read, so it bounds split-brain + * convergence to ~CC_MASTER_BEACON_EVERY seconds while keeping bootstrap-key use + * (and thus exposure) rare compared with the 1 s session keepalive. */ +#define CC_MASTER_BEACON_EVERY 5 /* MASTER_ALIVE ticks between beacons (~5 s) */ + +/* Split-brain PREVENTION at join time. When several nodes cold-start together + * they all exchange (bootstrap-decryptable) JOIN_REQs, so each learns the other + * starters. At the join deadline a node that has seen a higher-IP starter does + * NOT self-promote; it defers (re-sending JOIN_REQ) so the highest-IP starter + * becomes the single master and everyone joins it - no divergent keys ever form. + * Bounded so a higher-IP node that heard-then-died cannot stall us forever. */ +#define CC_JOIN_DEFER_SECS 1 /* seconds per deferral round */ +#define CC_JOIN_DEFER_MAX 4 /* consecutive deferrals for a *silent* */ + /* higher-IP peer before we promote anyway */ +#define CC_JOIN_DEFER_HARDMAX 20 /* absolute cap on deferrals incl. resets - */ + /* a peer stuck joining can't stall forever */ +#define CC_JOIN_REQ_MIN_US 500000 /* min microseconds between JOIN_REQ sends */ + +/* Per-source-IP rate limiter: checked before decryption to shed floods cheaply. + * Tracks up to CC_RATE_TBL_SZ source IPs with a 1-second sliding window. */ +#define CC_RATE_TBL_SZ 256 /* one slot per peer; matches max cluster size */ +#define CC_RATE_LIMIT 20 /* max packets per second per source IP */ + +typedef struct { + uint32_t ip; /* network byte order; 0 = empty slot */ + time_t window_start; + int count; +} cc_rate_entry_t; + +/* Max packet sizes: wire(20) = magic(8) + nonce(12); plain(5) = type(1) + seq(4) + * MASTER_ALIVE : wire(20) + plain(5) + tag(16) = 41 bytes + * ALIVE : wire(20) + plain(5) + IP(16) + pubkey(32) + tag(16) = 89 bytes + * GOODBYE : wire(20) + plain(5) + IP(16) + tag(16) = 57 bytes + * KEY_GRANT : wire(20) + plain(5) + IP(16) + master_pubkey(32) + * + join_nonce(16) + wrapped_salt(32) + tag(16) = 137 bytes + * KEY_HANDOFF : wire(20) + plain(5) + IP(16) + sender_pubkey(32) + * + wrapped_salt(32) + tag(16) = 121 bytes + * JOIN_REQ max : wire(20) + plain(5) + IP(16) + bin_info(513) + pubkey(32) + * + join_nonce(16) + tag(16) = 618 bytes + * NODE_ASSIGN max : wire(20) + plain(5) + node_id(2) + IP(16) + bin_info(513) + * + tag(16) = 572 bytes + * MEMBER_LIST max : wire(20) + plain(5) + count(2) + 256x17 + tag(16) = 4395 bytes + * + * Pubkeys are distributed via ALIVE (89 bytes) rather than MEMBER_LIST so that + * MEMBER_LIST payload stays bounded to 256x17=4352 bytes max. + * + * Fragmentation note: all packets except MEMBER_LIST fit in a single datagram + * on any standard link (Ethernet 1472B payload budget). MEMBER_LIST at 4395B + * requires IP fragmentation: + * - Standard Ethernet (1500 MTU): 3 fragments + * - IPsec ESP tunnel (~1400 MTU): 4 fragments + * - GRE-over-IPsec (~1350 MTU): 4-5 fragments + * Firewalls that block fragmented UDP packets will silently drop MEMBER_LIST, + * preventing new nodes from joining. The DF bit is not set so fragmentation + * occurs transparently where the network allows it. */ +#define CC_SMALL_PKT_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 + CC_TAG_SZ) +/* Consistency-critical settings advertised in ALIVE so peers can detect + * accidental per-node config drift for the same cluster: + * manage_shtags(1B) + master_stickiness(1B) + query_time(2B BE). */ +#define CC_CONFIG_SZ 4 +/* JOIN_REQ: [ip NUL][bin_count 1B][sockets...][pubkey 32B][join_nonce 16B] */ +#define CC_JOIN_PKT_MAX_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 \ + + CC_BIN_INFO_MAX_SZ + CC_PUBKEY_SZ + CC_JOIN_NONCE_SZ \ + + CC_CONFIG_SZ + CC_TAG_SZ) +/* KEY_GRANT: [target_ip NUL][master_pubkey 32B][join_nonce 16B][wrapped_salt 32B] */ +#define CC_KEY_GRANT_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 \ + + CC_PUBKEY_SZ + CC_JOIN_NONCE_SZ + CC_MASTER_SALT_SZ + CC_TAG_SZ) +/* KEY_HANDOFF: [target_ip NUL][sender_pubkey 32B][wrapped_salt 32B] */ +#define CC_KEY_HANDOFF_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 \ + + CC_PUBKEY_SZ + CC_MASTER_SALT_SZ + CC_TAG_SZ) +/* NODE_ASSIGN: [node_id 2B][ip NUL][bin_count 1B][sockets...] */ +#define CC_NODE_ASSIGN_MAX_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_NODE_ID_SZ \ + + CC_MAX_IP_LEN + 1 + CC_BIN_INFO_MAX_SZ + CC_TAG_SZ) +#define CC_LIST_PKT_MAX_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_LIST_COUNT_SZ \ + + CC_NODE_ID_SZ /* forced-shtag node_id */ \ + + CC_MAX_PEERS * CC_IP_ENTRY_SZ + CC_TAG_SZ) +/* Large enough to receive a fully reassembled UDP datagram (max 65507 bytes) */ +#define CC_RECV_BUF_SZ 65536 + +/* ========================================================================= + * Peer-table constants + * ========================================================================= */ + +#define CC_MAX_PEERS 256 + +/* + * CC_ELECT_FACTOR - election window = query_time x CC_ELECT_FACTOR. + * QUANTIZED: all nodes evaluate the same cutoff simultaneously. + * + * CC_PURGE_FACTOR - memory-cleanup window = query_time x CC_PURGE_FACTOR. + * Not quantized; only affects when entries are freed, not who is elected. + */ +#define CC_ELECT_FACTOR 3 +#define CC_PURGE_FACTOR 6 + +/* ========================================================================= + * Node-join state machine + * ========================================================================= */ + +typedef enum { + CC_NODE_NEW = 0, /* sent JOIN_REQ, awaiting MEMBER_LIST or timeout */ + CC_NODE_ACTIVE = 1 /* fully participating, sends ALIVE */ +} cc_node_state_t; + +/* TODO: maintenance mode + * + * Add CC_NODE_MAINTENANCE = 2 to cc_node_state_t. A node in maintenance + * must never become master, must not participate in elections (treated as + * absent from cc_elect_master), and must not become active shtag holder for + * any cluster even when manage_shtags=1. + * + * Entry / exit: new MI command cc_maintenance {on|off} sets the flag on the + * local node. Advertise the state in the ALIVE payload so all peers know to + * exclude this node from elections without waiting for a MEMBER_LIST refresh. + * The maintenance flag should survive MEMBER_LIST resets (it is local policy, + * not part of the cluster-wide peer table - store it in cc_cluster_t, not + * cc_peer_t). + * + * In cc_elect_master: skip any peer whose ALIVE-advertised maintenance flag + * is set (treat it as not in the election window even if last_seen is fresh). + * + * In cc_transition_to_active and shtag activation paths: gate all + * activate_backup_shtags / set_active_shtag calls behind + * if (cl->peers->node_state != CC_NODE_MAINTENANCE) + * + * TODO: shtag override mode + * + * Add MI command cc_set_shtag_holder to force a + * specific non-maintenance node to be the active shtag holder for a cluster, + * overriding the normal master-drives-shtag logic. Persist the override in + * a new field cc_peers_t.shtag_override_ip[CC_MAX_IP_LEN+1] (in shm so + * it is visible across processes). + * + * When shtag_override_ip is set for a cluster: + * - the designated node calls set_active_shtag regardless of mastership + * - all other nodes stay in backup shtag state + * - the override is NOT cleared on MEMBER_LIST or key rotation - it is + * explicit operator intent and must be cleared only by + * cc_clear_shtag_override (see below) or automatically when the + * overridden node enters maintenance mode + * - if the overridden node is put into maintenance mode, the override is + * automatically cleared and normal master-driven shtag logic resumes + * + * TODO: shtag override clear + * + * Add MI command cc_clear_shtag_override to cancel a + * previously set shtag override and return the cluster to normal mode where + * the elected master drives shtag activation. + * + * Implementation: + * - zero cc_peers_t.shtag_override_ip for the cluster + * - the current master immediately calls set_active_shtag on itself and + * activate_backup_shtags on all other nodes, restoring normal state + * - non-master nodes that were held in backup shtag state due to the + * override need no explicit action - the next election cycle reapplies + * correct shtag assignments automatically + * - log at INFO: "shtag override cleared for cluster , resuming + * normal master-driven shtag assignment (master: )" + * - return error if no override is active for the given cluster_id + * + * TODO: MI status commands + * + * cc_list_shtags - list all clusters with their active shtag holder, + * whether the holder was elected normally or overridden, and which nodes + * are in maintenance mode. Output columns: cluster_id, shtag, holder_ip, + * status in {elected | overridden | maintenance | no_holder}. + * + * cl_ctr_list_members should be extended to include a 'mode' field per node: + * active | maintenance, and an 'shtag_status' field: holder | backup | + * overridden | n/a (when manage_shtags=0 for that cluster). + */ + +/* ========================================================================= + * Cluster table + * ========================================================================= */ + +#define CC_MAX_CLUSTERS 16 /* max cluster= entries */ + +/* Forward-declared so cc_cluster_t can embed a pointer */ +typedef struct cc_peers_ cc_peers_t; + +/** + * cc_cluster_t - per-cluster runtime state. + * One instance per "cluster" modparam; one worker process per instance. + */ +typedef struct cc_cluster_ { + int cluster_id; + char multicast_address[INET_ADDRSTRLEN]; + int multicast_port; + char password[1025]; + unsigned char key[32]; /* bootstrap key = SHA256(password); JOIN only */ + unsigned char session_key[32]; /* group key = HKDF(password, master_salt) */ + int manage_shtags; /* per-cluster override; defaults to global manage_shtags */ + int master_stickiness; /* per-cluster override; -1 = inherit global */ + cc_peers_t *peers; /* per-cluster peer table in shm */ + /* BIN socket resolved at mod_init - advertised in JOIN_REQ/NODE_ASSIGN */ + char bin_socket[CC_MAX_BIN_SOCK_LEN]; /* "bin:IP:PORT" */ + /* Worker-process fds and state - valid only inside cc_worker after fork */ + int sock; /* multicast UDP socket */ + int alive_tfd; /* periodic ALIVE timer */ + int join_tfd; /* one-shot join deadline */ + int rejoin_tfd; /* 1-second JOIN_REQ retry */ + int master_alive_tfd; /* master sends MASTER_ALIVE 1/s */ + int master_dead_tfd; /* non-master: fires on ka miss */ + int identity_registered; /* 1 once update_identity called */ + int shtag_bootstrapped; /* -1 = eligible, 1 = done */ + /* ECDH keypair - generated in cc_worker after fork, never leaves process */ + unsigned char my_privkey[CC_PUBKEY_SZ]; + unsigned char my_pubkey[CC_PUBKEY_SZ]; + /* Per-exchange nonce sent in our JOIN_REQ; used to verify/unwrap KEY_GRANT */ + unsigned char my_join_nonce[CC_JOIN_NONCE_SZ]; + /* Set while a re-key JOIN_REQ is in flight; cleared on KEY_GRANT success + * or master transition to prevent nonce stomping under packet flood. */ + int join_pending; + /* 1 once a valid session_key has been established - either generated at + * cluster bootstrap (cc_on_became_master) or adopted from the current master + * master via KEY_GRANT / KEY_HANDOFF. A node must NOT act as master + * (broadcast MASTER_ALIVE) while this is 0, or it would encrypt with an + * underived key that no member can decrypt. */ + int have_session_key; + /* Master-side per-IP table tracking bootstrap-decrypt failures. + * Worker-local (no shm, no lock needed). After CC_JOIN_FAIL_LIMIT + * failures from the same source IP the master sends JOIN_REJECT. */ + struct { + uint32_t ip_num; + char ip[CC_MAX_IP_LEN + 1]; + int count; + int rejected; /* 1 = JOIN_REJECT already sent; suppress repeats */ + } join_fail_tbl[CC_JOIN_FAIL_TABLE_SZ]; + /* Joiner-side auth-failure detection - no lock needed (worker-local). */ + int bootstrap_auth_fails; /* consecutive bootstrap decrypt failures + during CC_NODE_NEW; reset on KEY_GRANT */ + int join_attempt_count; /* rejoin_tfd fires since last KEY_GRANT */ + /* Count of packets from OTHER peers during CC_NODE_NEW that we could not + * decrypt (any magic). Non-zero means a cluster (or rogue) whose key we do + * not share exists on the group - evidence we may have the wrong password. + * This is only *evidence*: cc_on_join_tfd never self-terminates on it + * immediately (that would let start-up noise or a flood kill a healthy + * node); it defers and re-joins, and a KEY_GRANT resets this counter. Only + * a correct-password joiner ever receives a KEY_GRANT, so persistence of + * this counter across the whole defer budget is what marks a real + * wrong-password / foreign-cluster condition. */ + int auth_fail_pkts; + /* Number of join rounds we have already deferred because we still could not + * authenticate. We only give up (shut down) after CC_JOIN_DEFER_MAX such + * rounds, so a KEY_GRANT that is merely slow, or a brief burst of start-up + * noise or crafted garbage, never self-terminates a correctly-configured + * node. Reset on successful authentication. */ + int auth_defer_count; + /* master_salt lives in cl->peers->master_salt (shm) so mod_destroy can + * read it. session_key is the worker-local derived key cache. */ + /* Per-source-IP rate limiter table - pkg_malloc'd in cc_worker after fork */ + cc_rate_entry_t *rate_tbl; + /* Last shtag decision this worker applied, so cc_apply_shtags_decision() + * logs the *reason* only when the decision (or its cause) actually + * changes - not on every idempotent re-apply. Worker-local. + * shtag_last_active: -1 unknown, 0 backup, 1 active. + * shtag_last_forced: the forced node_id in effect at that time. */ + int shtag_last_active; + uint16_t shtag_last_forced; + /* Counts MASTER_ALIVE ticks so a beacon is emitted every + * CC_MASTER_BEACON_EVERY of them. Worker-local (master path only). */ + unsigned int beacon_tick; + /* How many times we have deferred self-promotion at the join deadline + * because a higher-IP node was also still joining (split-brain + * prevention). Reset to 0 whenever a fresh JOIN_REQ from a higher-IP peer + * arrives (it is demonstrably still alive and joining, so we keep waiting + * for it rather than self-promoting into a divergent-key split brain). + * Worker-local; reset once we leave the NEW state. */ + int join_defer_count; + /* Total deferrals across resets - an absolute cap (CC_JOIN_DEFER_HARDMAX) + * so a peer that keeps sending JOIN_REQ yet never becomes master cannot + * defer us forever. Worker-local; reset once we leave the NEW state. */ + int join_defer_total; + /* utime (us since start) of the last JOIN_REQ we transmitted, for a + * minimum-interval throttle so a key-mismatch/split-brain burst cannot + * flood the group with JOIN_REQs. 0 = never sent. Worker-local. */ + utime_t last_join_req_utime; + /* 1 while our MASTER_ALIVE keepalive timer is armed (i.e. we are acting as + * master and broadcasting). Set by cc_arm_master_timers(). Lets + * cc_elect_master() enforce the invariant "keepalive armed <=> I am the + * elected master": an election that demotes us (clears is_master) without + * going through a yield/member-list path must still stop the keepalive, + * otherwise a demoted node keeps broadcasting MASTER_ALIVE and lower-IP + * peers oscillate between two masters. Worker-local. */ + int master_ka_armed; +} cc_cluster_t; + +static cc_cluster_t cc_clusters[CC_MAX_CLUSTERS]; +static int cc_cluster_count = 0; + +/* Raw "cluster" strings collected during modparam parsing */ +static char *cc_cluster_strs[CC_MAX_CLUSTERS]; +static int cc_cluster_str_count = 0; + +/* ========================================================================= + * Module parameters + * ========================================================================= */ + +/* Global modparams - apply to all clusters unless overridden per-cluster */ +static char *my_ip = NULL; /* explicit IP, or NULL for auto-detect */ +static char *my_interface = NULL; /* explicit interface name, or NULL */ +static int query_time = 5; +static char *password = CC_DEFAULT_PASSWORD; /* default; falls back per cluster */ + +/* Policy when a node's consistency-critical settings (manage_shtags/ + * master_stickiness/query_time) differ from the running cluster (a master is + * alive). Set via the on_config_mismatch modparam string: + * "warn" - admit/keep the node but log a CONFIG MISMATCH warning; + * "reject" - the master refuses the join (JOIN_REJECT) and the node shuts + * down with a clear message (default); + * "adopt" - the node adopts the master's (authoritative) settings at + * runtime and continues. */ +#define CC_CFGMISMATCH_WARN 0 +#define CC_CFGMISMATCH_REJECT 1 +#define CC_CFGMISMATCH_ADOPT 2 +/* JOIN_REJECT reason codes (1 byte after the target IP in the payload). */ +#define CC_REJECT_GENERIC 0 /* wrong password / unauthorized / table full */ +#define CC_REJECT_CONFIG 1 /* different cluster settings (reject policy) */ +static char *on_config_mismatch_s = NULL; /* raw modparam string */ +static int on_config_mismatch = CC_CFGMISMATCH_REJECT; /* resolved; default reject */ + +/* Resolved at mod_init time - always valid after cc_resolve_local_identity() */ +static char my_ip_buf[INET_ADDRSTRLEN]; +static char my_interface_buf[IF_NAMESIZE]; + +static WC_RNG cc_rng; + +/* Local node identity - populated at mod_init by scanning the config file */ +static uint16_t my_node_id = 0; + +/* clusterer integration - loaded at mod_init if clusterer use_controller=1 */ +static clusterer_ctrl_binds_t clctl; +static int clctl_loaded = 0; +static int manage_shtags = 1; +/* master_stickiness (global default; per-cluster override via "cluster" string): + * 1 (default) = the master is "sticky": a live master keeps the role and is + * NOT displaced when a higher-IP node joins. The highest-IP + * non-master is designated BACKUP and takes over only when the + * master fails. A higher-IP joiner just replaces the backup. + * Result: fewer master handovers. + * 0 = not sticky - pure highest-IP election, so a higher-IP node + * takes over as master as soon as it appears (more handovers). */ +static int master_stickiness = 1; +static char my_bin_sockets[CC_MAX_BIN_SOCKETS][CC_MAX_BIN_SOCK_LEN]; +static int my_bin_count = 0; + + +/** + * cc_add_cluster_param() - collect "cluster" modparam strings. + * Actual parsing happens in mod_init() after all params are set. + */ +static int cc_add_cluster_param(modparam_t type, void *val) +{ + if (cc_cluster_str_count >= CC_MAX_CLUSTERS) { + LM_ERR("clusterer_controller: too many clusters (max %d)\n", + CC_MAX_CLUSTERS); + return -1; + } + { + size_t _len = strlen((char *)val) + 1; + cc_cluster_strs[cc_cluster_str_count] = pkg_malloc(_len); + if (!cc_cluster_strs[cc_cluster_str_count]) { + LM_ERR("clusterer_controller: pkg_malloc failed\n"); + return -1; + } + memcpy(cc_cluster_strs[cc_cluster_str_count], (char *)val, _len); + } + cc_cluster_str_count++; + return 0; +} + +static const param_export_t params[] = { + {"cluster", STR_PARAM | USE_FUNC_PARAM, (void *)cc_add_cluster_param}, + {"my_ip", STR_PARAM, &my_ip}, + {"interface", STR_PARAM, &my_interface}, + {"query_time", INT_PARAM, &query_time}, + {"password", STR_PARAM, &password}, + {"manage_shtags", INT_PARAM, &manage_shtags}, + {"master_stickiness", INT_PARAM, &master_stickiness}, + {"on_config_mismatch", STR_PARAM, &on_config_mismatch_s}, + {0, 0, 0} +}; + +/* ========================================================================= + * Peer table (shared memory) + * ========================================================================= */ + +typedef struct cc_peer_ { + char ip[CC_MAX_IP_LEN + 1]; + unsigned int ip_num; + time_t last_seen; + int is_master; + int is_backup; /* 1 = standby master (highest-IP non-master) */ + int in_election; /* 1 = currently inside the election window */ + uint16_t node_id; /* allocated by master; 0 = not yet assigned */ + uint8_t bin_count; /* number of BIN listeners reported */ + char bin_sockets[CC_MAX_BIN_SOCKETS][CC_MAX_BIN_SOCK_LEN]; + unsigned char pubkey[CC_PUBKEY_SZ]; /* X25519 public key; zero if unknown */ + unsigned char join_nonce[CC_JOIN_NONCE_SZ]; /* per-exchange nonce from JOIN_REQ */ + uint32_t last_seq; /* highest seq accepted from this peer */ + /* Peer's advertised consistency-critical config (from ALIVE), used to warn + * on accidental per-node config drift. cfg_known=0 until first advertised; + * cfg_warned deduplicates the mismatch warning. */ + int cfg_known; + int cfg_manage_shtags; + int cfg_master_stickiness; + int cfg_query_time; + int cfg_warned; +} cc_peer_t; + +struct cc_peers_ { + cc_peer_t entries[CC_MAX_PEERS]; + int count; + /* rw_lock_t allows concurrent readers (MI, future script functions) + * while still serialising the single writer (cc_worker). */ + rw_lock_t *lock; + cc_node_state_t node_state; + time_t join_deadline; + /* last elected master IP - used to detect and log master changes */ + char last_master[CC_MAX_IP_LEN + 1]; + /* master_salt: generated by each new master, shared here so mod_destroy + * (running in main process) can derive session_key for GOODBYE. */ + unsigned char master_salt[CC_MASTER_SALT_SZ]; + /* my_seq: monotonic send counter; in shm so mod_destroy can use it for + * GOODBYE without needing the worker's private state. Reset to 0 on + * every session key rotation so last_seq counters reset cleanly. */ + uint32_t my_seq; + /* Sharing-tag override: 0 = automatic (master-driven) allocation; nonzero = + * an operator has forced this node_id to be the active shtag holder for the + * cluster (cl_ctr_shtag_force MI), suspending automatic allocation until + * cl_ctr_shtag_auto clears it. Propagated to all nodes in the MEMBER_LIST. */ + uint16_t shtag_forced_node_id; + /* worker_proc_no: OpenSIPS process index of this cluster's cc_worker, + * published here (shm) after fork so MI handlers running in a different + * process can target the worker with ipc_send_rpc(). -1 until set. */ + int worker_proc_no; + /* Effective (possibly adopted) consistency-critical settings, mirrored in + * shm so MI handlers in another process (cl_ctr_list_config) report the value + * actually in force after an on_config_mismatch=adopt. The worker keeps + * these in sync with its own cl->manage_shtags / master_stickiness / + * query_time. Initialised from the resolved config at mod_init. */ + int eff_manage_shtags; + int eff_master_stickiness; + int eff_query_time; +}; + + +/* ========================================================================= + * timerfd helpers + * ========================================================================= */ + +/* Drain the expiration counter so the fd stops being readable. */ +static void cc_drain_tfd(int tfd) +{ + uint64_t exp; + if (read(tfd, &exp, sizeof(exp)) < 0 && errno != EAGAIN) + LM_WARN("clusterer_controller: timerfd read: %s\n", strerror(errno)); +} + +/* Arm a timerfd. Pass sec_value=0 to disarm. */ +static void cc_arm_tfd(int tfd, time_t sec_value, time_t sec_interval) +{ + struct itimerspec its; + memset(&its, 0, sizeof(its)); + its.it_value.tv_sec = sec_value; + its.it_interval.tv_sec = sec_interval; + if (timerfd_settime(tfd, 0, &its, NULL) < 0) + LM_WARN("clusterer_controller: timerfd_settime: %s\n", strerror(errno)); +} + +/* ========================================================================= + * Forward declarations + * ========================================================================= */ + +static int mod_init(void); +static int cc_child_init(int rank); +static void mod_destroy(void); +static void cc_worker(int rank); +static int cc_on_sock(int fd, void *param, int was_timeout); +static int cc_on_alive_tfd(int fd, void *param, int was_timeout); +static void cc_arm_master_timers(cc_cluster_t *cl, int i_am_master); +static int cc_on_join_tfd(int fd, void *param, int was_timeout); +static int cc_on_rejoin_tfd(int fd, void *param, int was_timeout); +static int cc_on_master_alive_tfd(int fd, void *param, int was_timeout); +static int cc_on_master_dead_tfd(int fd, void *param, int was_timeout); +static mi_response_t *mi_cl_ctr_members(const mi_params_t *params, + struct mi_handler *hdl); +static void cc_handle_member_list(const char *payload, int payload_len, + const char *sender_ip, cc_cluster_t *cl); +static void cc_handle_join_req(int sock, const char *payload, int payload_len, + cc_cluster_t *cl); +static void cc_handle_node_assign(const char *payload, int payload_len, + const char *sender_ip, cc_cluster_t *cl); +static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl); +static void cc_handle_master_alive(const char *sender_ip, cc_cluster_t *cl); +static void cc_handle_key_grant(const char *payload, int payload_len, + const char *sender_ip, cc_cluster_t *cl); +static void cc_handle_key_handoff(const char *payload, int payload_len, + const char *sender_ip, cc_cluster_t *cl); +static void cc_handle_join_reject(const char *payload, int payload_len, + const char *sender_ip, cc_cluster_t *cl); +static void cc_send_join_reject(int sock, const char *target_ip, cc_cluster_t *cl, + int reason); +static mi_response_t *mi_cl_ctr_node_info(const mi_params_t *params, + struct mi_handler *hdl); +static mi_response_t *mi_cl_ctr_config(const mi_params_t *params, + struct mi_handler *hdl); +static mi_response_t *mi_cl_ctr_shtag_force(const mi_params_t *params, + struct mi_handler *hdl); +static mi_response_t *mi_cl_ctr_shtag_auto(const mi_params_t *params, + struct mi_handler *hdl); + +/* ========================================================================= + * Extra-process export (layout from mi_fifo.c) + * ========================================================================= */ + +static proc_export_t procs[] = { + {"clusterer_controller worker", 0, 0, cc_worker, 1, + PROC_FLAG_INITCHILD | PROC_FLAG_HAS_IPC}, + {0, 0, 0, 0, 0, 0} +}; + +/* ========================================================================= + * Module dependency + * ========================================================================= */ + +static const dep_export_t deps = { + { + /* proto_bin must load before us so its listeners are registered */ + { MOD_TYPE_DEFAULT, "proto_bin", DEP_ABORT }, + { MOD_TYPE_DEFAULT, "clusterer", DEP_ABORT }, + { MOD_TYPE_NULL, NULL, 0 }, + }, + { { NULL, NULL } }, +}; + +/* ========================================================================= + * MI command export table + * ========================================================================= */ + +static const mi_export_t mi_cmds[] = { + { + "cl_ctr_list_members", + "List all current cluster members with node_id, status and BIN sockets", + 0, 0, + { + {mi_cl_ctr_members, {0}}, + {EMPTY_MI_RECIPE} + } + }, + { + "cl_ctr_node_info", + "Return full info for a node_id across all clusters", + 0, 0, + { + {mi_cl_ctr_node_info, {"node_id", 0}}, + {EMPTY_MI_RECIPE} + } + }, + { + "cl_ctr_list_config", + "List all configured clusters and their resolved settings", + 0, 0, + { + {mi_cl_ctr_config, {0}}, + {EMPTY_MI_RECIPE} + } + }, + { + "cl_ctr_shtag_force", + "Force a node to hold the active sharing tag (master only); " + "suspends automatic allocation until cl_ctr_shtag_auto", + 0, 0, + { + {mi_cl_ctr_shtag_force, {"cluster_id", "node_id", 0}}, + {EMPTY_MI_RECIPE} + } + }, + { + "cl_ctr_shtag_auto", + "Resume automatic master-driven sharing-tag allocation (master only)", + 0, 0, + { + {mi_cl_ctr_shtag_auto, {"cluster_id", 0}}, + {EMPTY_MI_RECIPE} + } + }, + {EMPTY_MI_EXPORT} +}; + +/* ========================================================================= + * Read-only script variables ($cl_ctr_*) + * + * Nouns, never verbs (actions are script functions / MI). Each variable + * optionally takes a cluster id: $cl_ctr_role(2). The bare form + * ($cl_ctr_role) resolves to the sole configured cluster; with several + * clusters it returns NULL and warns once, so nobody silently reads the + * wrong cluster. All values are read from the shm peer table under a read + * lock, so every process (SIP workers included) sees live state. Read-only: + * no setter is exported. + * ========================================================================= */ + +enum cc_pv_field { + CC_PV_ROLE, /* master | backup | member | joining */ + CC_PV_IS_MASTER, /* 1 / 0 */ + CC_PV_MASTER_IP, /* current master's IP, NULL if none */ + CC_PV_BACKUP_IP, /* current backup's IP, NULL if none */ + CC_PV_NODE_ID, /* this node's id in the cluster, NULL if unassigned */ + CC_PV_MY_IP, /* controller identity IP */ + CC_PV_MEMBERS, /* live member count */ + CC_PV_SHTAG_MODE, /* auto | forced */ + CC_PV_FORCED_NODE, /* node pinned by cl_ctr_shtag_force, NULL if auto */ +}; + +/* Optional (cluster_id) argument; bare form leaves the spec zeroed (cid 0). */ +static int cc_pv_parse_cluster(pv_spec_p sp, const str *in) +{ + unsigned int cid; + str s; + + if (!sp) + return -1; + if (!in || !in->s || in->len == 0) { + sp->pvp.pvn.u.isname.name.n = 0; /* bare form -> sole cluster */ + return 0; + } + s = *in; + trim(&s); + if (s.len == 0) { + sp->pvp.pvn.u.isname.name.n = 0; + return 0; + } + /* str2int rejects any non-digit (so '-5', 'abc', '1.2' all fail); the + * length cap keeps a huge value from silently overflowing/wrapping into a + * valid id (999999999 < INT_MAX). */ + if (s.len > 9 || str2int(&s, &cid) < 0 || cid == 0) { + LM_ERR("clusterer_controller: invalid cluster id '%.*s' in $cl_ctr_* " + "variable (expected a positive integer 1..999999999)\n", + in->len, in->s); + return -1; + } + sp->pvp.pvn.u.isname.name.n = (int)cid; + return 0; +} + +static int cc_pv_get(struct sip_msg *msg, pv_param_t *param, pv_value_t *res, + enum cc_pv_field field) +{ + static char cc_pv_ipbuf[INET_ADDRSTRLEN]; + static int cc_pv_warned_ambiguous; + cc_cluster_t *cl = NULL; + cc_peer_t *me = NULL, *master = NULL, *backup = NULL; + const char *sval = NULL; + int cid, i, have_int = 0, ival = 0, joining; + + cid = param->pvn.u.isname.name.n; + if (cid == 0) { + if (cc_cluster_count == 1) { + cl = &cc_clusters[0]; + } else { + if (!cc_pv_warned_ambiguous) { + LM_WARN("clusterer_controller: bare $cl_ctr_* used with %d " + "clusters configured - specify the cluster id, e.g. " + "$cl_ctr_role(%d)\n", cc_cluster_count, + cc_cluster_count ? cc_clusters[0].cluster_id : 1); + cc_pv_warned_ambiguous = 1; + } + return pv_get_null(msg, param, res); + } + } else { + for (i = 0; i < cc_cluster_count; i++) + if (cc_clusters[i].cluster_id == cid) { + cl = &cc_clusters[i]; + break; + } + } + if (!cl || !cl->peers) + return pv_get_null(msg, param, res); + + lock_start_read(cl->peers->lock); + + joining = (cl->peers->node_state == CC_NODE_NEW); + for (i = 0; i < cl->peers->count; i++) { + cc_peer_t *e = &cl->peers->entries[i]; + if (e->is_master) + master = e; + if (e->is_backup) + backup = e; + if (my_ip && strcmp(e->ip, my_ip) == 0) + me = e; + } + + switch (field) { + case CC_PV_ROLE: + if (joining) + sval = "joining"; + else if (me && me->is_master) + sval = "master"; + else if (me && me->is_backup) + sval = "backup"; + else + sval = "member"; + break; + case CC_PV_IS_MASTER: + have_int = 1; + ival = (!joining && me && me->is_master) ? 1 : 0; + break; + case CC_PV_MASTER_IP: + if (master) { + strncpy(cc_pv_ipbuf, master->ip, sizeof(cc_pv_ipbuf) - 1); + cc_pv_ipbuf[sizeof(cc_pv_ipbuf) - 1] = '\0'; + sval = cc_pv_ipbuf; + } + break; + case CC_PV_BACKUP_IP: + if (backup) { + strncpy(cc_pv_ipbuf, backup->ip, sizeof(cc_pv_ipbuf) - 1); + cc_pv_ipbuf[sizeof(cc_pv_ipbuf) - 1] = '\0'; + sval = cc_pv_ipbuf; + } + break; + case CC_PV_NODE_ID: + if (me && me->node_id > 0) { + have_int = 1; + ival = me->node_id; + } + break; + case CC_PV_MY_IP: + sval = my_ip; /* resolved in mod_init, constant afterwards */ + break; + case CC_PV_MEMBERS: + have_int = 1; + ival = cl->peers->count; + break; + case CC_PV_SHTAG_MODE: + sval = cl->peers->shtag_forced_node_id ? "forced" : "auto"; + break; + case CC_PV_FORCED_NODE: + if (cl->peers->shtag_forced_node_id) { + have_int = 1; + ival = cl->peers->shtag_forced_node_id; + } + break; + } + + lock_stop_read(cl->peers->lock); + + if (have_int) + return pv_get_uintval(msg, param, res, (unsigned int)ival); + if (sval) { + str s = {(char *)sval, (int)strlen(sval)}; + return pv_get_strval(msg, param, res, &s); + } + return pv_get_null(msg, param, res); +} + +#define CC_PV_WRAP(_fn, _field) \ +static int _fn(struct sip_msg *msg, pv_param_t *param, pv_value_t *res) \ +{ return cc_pv_get(msg, param, res, _field); } + +/* ------------------------------------------------------------------------- + * Per-peer lookups: query a *specific* node in a cluster. These are script + * FUNCTIONS, not pseudo-variables, because they take two arguments + * (cluster_id, node_id) - a comma inside a pvar's parentheses is ambiguous to + * the config parser when the pvar is used as a function argument. Boolean + * checks return true/false for use in if(); value lookups write an output var. + * ------------------------------------------------------------------------- */ + +/* Find peer (cluster_id, node_id); cid<=0 => the sole configured cluster. + * Returns 0 and fills the requested out params on success, -1 if the cluster + * or node is unknown. Caller must not hold the peers lock. */ +static int cc_find_peer(int cid, int nid, int *is_master, int *is_backup, + char *ipbuf, int ipbuf_sz) +{ + cc_cluster_t *cl = NULL; + int i, rc = -1; + + if (cid <= 0) { + if (cc_cluster_count == 1) + cl = &cc_clusters[0]; + } else { + for (i = 0; i < cc_cluster_count; i++) + if (cc_clusters[i].cluster_id == cid) { + cl = &cc_clusters[i]; + break; + } + } + if (!cl || !cl->peers) + return -1; + + lock_start_read(cl->peers->lock); + for (i = 0; i < cl->peers->count; i++) { + cc_peer_t *e = &cl->peers->entries[i]; + if (e->node_id != nid) + continue; + if (is_master) *is_master = e->is_master; + if (is_backup) *is_backup = e->is_backup; + if (ipbuf) { + strncpy(ipbuf, e->ip, ipbuf_sz - 1); + ipbuf[ipbuf_sz - 1] = '\0'; + } + rc = 0; + break; + } + lock_stop_read(cl->peers->lock); + return rc; +} + +static int cc_out_str(struct sip_msg *msg, pv_spec_t *out, const char *str_s) +{ + pv_value_t val; + memset(&val, 0, sizeof val); + val.flags = PV_VAL_STR; + val.rs.s = (char *)str_s; + val.rs.len = strlen(str_s); + return pv_set_value(msg, out, 0, &val); +} + +/* cl_ctr_node_is_master(cluster_id, node_id) -> true if that node is master */ +static int w_cl_ctr_node_is_master(struct sip_msg *msg, int *cid, int *nid) +{ + int im = 0; + if (cc_find_peer(cid ? *cid : 0, nid ? *nid : 0, &im, NULL, NULL, 0) < 0) + return -1; + return im ? 1 : -1; +} + +/* cl_ctr_node_present(cluster_id, node_id) -> true if node_id is a live member */ +static int w_cl_ctr_node_present(struct sip_msg *msg, int *cid, int *nid) +{ + return cc_find_peer(cid ? *cid : 0, nid ? *nid : 0, NULL, NULL, NULL, 0) == 0 + ? 1 : -1; +} + +/* cl_ctr_get_node_role(cluster_id, node_id, out) -> out=master|backup|member */ +static int w_cl_ctr_get_node_role(struct sip_msg *msg, int *cid, int *nid, + pv_spec_t *out) +{ + int im = 0, ib = 0; + if (cc_find_peer(cid ? *cid : 0, nid ? *nid : 0, &im, &ib, NULL, 0) < 0) + return -1; + return cc_out_str(msg, out, im ? "master" : (ib ? "backup" : "member")) == 0 + ? 1 : -1; +} + +/* cl_ctr_get_node_ip(cluster_id, node_id, out) -> out=that node's IP */ +static int w_cl_ctr_get_node_ip(struct sip_msg *msg, int *cid, int *nid, + pv_spec_t *out) +{ + char ipbuf[INET_ADDRSTRLEN]; + if (cc_find_peer(cid ? *cid : 0, nid ? *nid : 0, NULL, NULL, + ipbuf, sizeof ipbuf) < 0) + return -1; + return cc_out_str(msg, out, ipbuf) == 0 ? 1 : -1; +} + +CC_PV_WRAP(cc_pv_role, CC_PV_ROLE) +CC_PV_WRAP(cc_pv_is_master, CC_PV_IS_MASTER) +CC_PV_WRAP(cc_pv_master_ip, CC_PV_MASTER_IP) +CC_PV_WRAP(cc_pv_backup_ip, CC_PV_BACKUP_IP) +CC_PV_WRAP(cc_pv_node_id, CC_PV_NODE_ID) +CC_PV_WRAP(cc_pv_my_ip, CC_PV_MY_IP) +CC_PV_WRAP(cc_pv_members, CC_PV_MEMBERS) +CC_PV_WRAP(cc_pv_shtag_mode, CC_PV_SHTAG_MODE) +CC_PV_WRAP(cc_pv_forced_node, CC_PV_FORCED_NODE) + +static const pv_export_t cc_mod_vars[] = { + { {"cl_ctr_role", sizeof("cl_ctr_role")-1}, 1100, + cc_pv_role, 0, cc_pv_parse_cluster, 0, 0, 0 }, + { {"cl_ctr_is_master", sizeof("cl_ctr_is_master")-1}, 1101, + cc_pv_is_master, 0, cc_pv_parse_cluster, 0, 0, 0 }, + { {"cl_ctr_master_ip", sizeof("cl_ctr_master_ip")-1}, 1102, + cc_pv_master_ip, 0, cc_pv_parse_cluster, 0, 0, 0 }, + { {"cl_ctr_backup_ip", sizeof("cl_ctr_backup_ip")-1}, 1103, + cc_pv_backup_ip, 0, cc_pv_parse_cluster, 0, 0, 0 }, + { {"cl_ctr_node_id", sizeof("cl_ctr_node_id")-1}, 1104, + cc_pv_node_id, 0, cc_pv_parse_cluster, 0, 0, 0 }, + { {"cl_ctr_my_ip", sizeof("cl_ctr_my_ip")-1}, 1105, + cc_pv_my_ip, 0, cc_pv_parse_cluster, 0, 0, 0 }, + { {"cl_ctr_members", sizeof("cl_ctr_members")-1}, 1106, + cc_pv_members, 0, cc_pv_parse_cluster, 0, 0, 0 }, + { {"cl_ctr_shtag_mode", sizeof("cl_ctr_shtag_mode")-1}, 1107, + cc_pv_shtag_mode, 0, cc_pv_parse_cluster, 0, 0, 0 }, + { {"cl_ctr_forced_node", sizeof("cl_ctr_forced_node")-1}, 1108, + cc_pv_forced_node, 0, cc_pv_parse_cluster, 0, 0, 0 }, + { {0, 0}, 0, 0, 0, 0, 0, 0, 0 } +}; + +/* Script functions: per-peer lookups (two args -> functions, not variables). */ +static const cmd_export_t cc_cmds[] = { + {"cl_ctr_node_is_master", (cmd_function)w_cl_ctr_node_is_master, { + {CMD_PARAM_INT, 0, 0}, {CMD_PARAM_INT, 0, 0}, {0, 0, 0}}, ALL_ROUTES}, + {"cl_ctr_node_present", (cmd_function)w_cl_ctr_node_present, { + {CMD_PARAM_INT, 0, 0}, {CMD_PARAM_INT, 0, 0}, {0, 0, 0}}, ALL_ROUTES}, + {"cl_ctr_get_node_role", (cmd_function)w_cl_ctr_get_node_role, { + {CMD_PARAM_INT, 0, 0}, {CMD_PARAM_INT, 0, 0}, {CMD_PARAM_VAR, 0, 0}, {0, 0, 0}}, ALL_ROUTES}, + {"cl_ctr_get_node_ip", (cmd_function)w_cl_ctr_get_node_ip, { + {CMD_PARAM_INT, 0, 0}, {CMD_PARAM_INT, 0, 0}, {CMD_PARAM_VAR, 0, 0}, {0, 0, 0}}, ALL_ROUTES}, + {0, 0, {{0, 0, 0}}, 0} +}; + +/* ========================================================================= + * module_exports + * ========================================================================= */ + +struct module_exports exports = { + "clusterer_controller", + MOD_TYPE_DEFAULT, + MODULE_VERSION, + DEFAULT_DLFLAGS, + 0, + &deps, + cc_cmds, /* cmds - per-peer lookup functions */ + 0, /* acmds */ + params, + 0, /* stats */ + mi_cmds, + cc_mod_vars, /* pvs - read-only $cl_ctr_* variables */ + 0, /* transforms */ + procs, + 0, /* pre_init_f */ + mod_init, + 0, /* response_f */ + mod_destroy, + cc_child_init, /* child_init_f */ + 0 /* reload_confirm_f */ +}; + +/* ========================================================================= + * Internal helpers + * ========================================================================= */ + +static unsigned int ip_to_num(const char *ip) +{ + struct in_addr addr; + if (inet_aton(ip, &addr) == 0) + return 0; + return ntohl(addr.s_addr); +} + +/** + * cc_election_cutoff() - quantized stale cutoff for master election. + * + * All NTP-synchronized nodes compute the same value at the same second, + * so they always evaluate the identical eligible-peer set and elect the + * same master. + */ +static time_t cc_election_cutoff(void) +{ + time_t now = time(NULL); + return (now / (time_t)query_time) * (time_t)query_time + - (time_t)(query_time * CC_ELECT_FACTOR); +} + +/** + * cc_elect_master(cl) - mark the peer with the highest IP as master. + * + * Uses the quantized election window so all NTP-synchronized nodes evaluate + * the same eligible set and elect the same master. + * + * Also tracks two state transitions and logs them at INFO: + * + * in_election 1->0 A peer's last_seen fell outside the election window - + * the node is considered down. Logged immediately so the + * operator sees the event without waiting for cc_prune_stale(cl) + * (which only fires at CC_PURGE_FACTOR x query_time). + * + * last_master The elected master IP changed - either because the + * previous master went down, or a higher-IP node joined. + * + * Must be called with cl->peers->lock held. + */ +static void cc_elect_master(cc_cluster_t *cl) +{ + time_t cutoff = cc_election_cutoff(); + unsigned int top_num = 0; + int i, n_in = 0, top_idx = -1, cur_master_idx = -1, master_idx = -1; + int i_am_elected = 0; + char prev_master[CC_MAX_IP_LEN + 1]; + char prev_backup[CC_MAX_IP_LEN + 1]; + + prev_backup[0] = '\0'; + /* Snapshot the master we had before this election, for change reporting. */ + { + size_t _l = strnlen(cl->peers->last_master, CC_MAX_IP_LEN); + memcpy(prev_master, cl->peers->last_master, _l); + prev_master[_l] = '\0'; + } + + for (i = 0; i < cl->peers->count; i++) { + cc_peer_t *e = &cl->peers->entries[i]; + int now_in = (e->last_seen >= cutoff); + + /* Detect peer dropping out of the election window */ + if (e->in_election && !now_in) + LM_INFO("clusterer_controller: peer %s went down " + "(last seen %lds ago)\n", + e->ip, (long)(time(NULL) - e->last_seen)); + + e->in_election = now_in; + + /* Remember the live current master and the previous backup before + * clearing the flags - used for sticky election and change logging. */ + if (e->is_master && now_in) + cur_master_idx = i; + if (e->is_backup) { + size_t _l = strnlen(e->ip, CC_MAX_IP_LEN); + memcpy(prev_backup, e->ip, _l); + prev_backup[_l] = '\0'; + } + + e->is_master = 0; + e->is_backup = 0; + + if (now_in) { + n_in++; + if (e->ip_num > top_num) { + top_num = e->ip_num; + top_idx = i; + } + } + } + + /* Choose the master: + * master_stickiness == 1 (default, sticky): a live current master keeps the + * role - a higher-IP peer does NOT preempt it (fewer handovers). Only + * when there is no live master do we elect the highest-IP peer. + * master_stickiness == 0: pure highest-IP election - the highest-IP peer + * always wins, preempting any lower-IP current master. + * Split-brain (two live masters, e.g. after a partition heal) is resolved + * separately in cc_handle_master_alive by yielding to the highest IP. */ + if (cl->master_stickiness == 1 && cur_master_idx >= 0) + master_idx = cur_master_idx; + else + master_idx = top_idx; + + if (master_idx >= 0) { + const char *m_ip, *b_ip, *why; + int b_idx = -1, master_changed, backup_changed; + unsigned int b_num = 0; + int j; + + cl->peers->entries[master_idx].is_master = 1; + m_ip = cl->peers->entries[master_idx].ip; + i_am_elected = (strcmp(m_ip, my_ip) == 0); + + /* Designate the BACKUP: the highest-IP in-window peer that is not the + * master. Deterministic across all nodes, so everyone agrees who takes + * over next; on master failure cc_elect_master (no live current master) + * promotes exactly this node. */ + for (j = 0; j < cl->peers->count; j++) { + cc_peer_t *e = &cl->peers->entries[j]; + if (j == master_idx || !e->in_election) + continue; + if (e->ip_num > b_num) { + b_num = e->ip_num; + b_idx = j; + } + } + if (b_idx >= 0) + cl->peers->entries[b_idx].is_backup = 1; + b_ip = (b_idx >= 0) ? cl->peers->entries[b_idx].ip : NULL; + + master_changed = (strcmp(prev_master, m_ip) != 0); + backup_changed = (strcmp(prev_backup, b_ip ? b_ip : "") != 0); + + /* Persist the elected master for the next round / other handlers. */ + { + size_t _l = strnlen(m_ip, CC_MAX_IP_LEN); + memcpy(cl->peers->last_master, m_ip, _l); + cl->peers->last_master[_l] = '\0'; + } + + /* One clear line whenever the master or backup role changes, stating + * who holds each role, which is this node, and WHY the master was + * chosen (highest IP, sticky current master kept over a higher-IP peer, or + * sole surviving node). */ + if (master_changed || backup_changed) { + if (n_in <= 1) + why = "sole node in window"; + else if (cl->master_stickiness == 1 && master_idx == cur_master_idx && + top_idx >= 0 && top_idx != master_idx) + why = "sticky: current master kept over higher-IP node"; + else + why = "highest IP in window"; + + LM_INFO("clusterer_controller: [cluster %d] roles: MASTER=%s%s (%s); " + "BACKUP=%s%s (highest-IP non-master); %d node(s) in window\n", + cl->cluster_id, + m_ip, + strcmp(m_ip, my_ip) == 0 ? " [me]" : "", + why, + b_ip ? b_ip : "(none)", + (b_ip && strcmp(b_ip, my_ip) == 0) ? " [me]" : "", + n_in); + } + } else { + /* No eligible peer - cluster has no master */ + if (cl->peers->last_master[0] != '\0') { + LM_INFO("clusterer_controller: [cluster %d] master lost (%s), " + "no eligible peers in election window\n", + cl->cluster_id, cl->peers->last_master); + cl->peers->last_master[0] = '\0'; + } + } + + /* Invariant: MASTER_ALIVE keepalive armed <=> I am the elected master. + * If this election demoted us (someone else won, or no master at all) but + * our keepalive is still running, stop it now - otherwise we keep + * broadcasting MASTER_ALIVE as a phantom master and lower-IP peers + * oscillate between two masters. Only disarm here: promoting a new master + * (arming) is done by the became-master paths, which also establish the + * session key. cc_arm_master_timers() only issues timerfd syscalls, so it + * is safe under cl->peers->lock. */ + if (!i_am_elected && cl->master_ka_armed) + cc_arm_master_timers(cl, 0); +} + +/** + * cc_i_am_master_locked(cl) - return 1 if my_ip is currently elected master. + * Must be called with cl->peers->lock held. + */ +static int cc_i_am_master_locked(cc_cluster_t *cl) +{ + time_t cutoff = cc_election_cutoff(); + int i; + + for (i = 0; i < cl->peers->count; i++) { + if (cl->peers->entries[i].is_master && + cl->peers->entries[i].last_seen >= cutoff && + strcmp(cl->peers->entries[i].ip, my_ip) == 0) + return 1; + } + return 0; +} + +/** + * cc_ip_beats_master_locked() - check whether a candidate IP would displace + * the current master in an election. + * + * Returns 1 (re-election is worth running) when: + * - ip_num is strictly greater than the current master's ip_num, OR + * - there is no current master in the election window (no-one to defend). + * + * Returns 0 when the current master has a higher or equal IP - it would + * win the election anyway, so running one is pointless. + * + * Must be called with cl->peers->lock held. + */ +static int cc_ip_beats_master_locked(unsigned int ip_num, cc_cluster_t *cl) +{ + time_t cutoff = cc_election_cutoff(); + int i; + + for (i = 0; i < cl->peers->count; i++) { + if (cl->peers->entries[i].is_master && + cl->peers->entries[i].last_seen >= cutoff) + return (ip_num > cl->peers->entries[i].ip_num); + } + return 1; /* no master in the election window - election is needed */ +} + +/** + * cc_apply_shtags_decision() - (de)activate this node's sharing tags per policy. + * + * Decides whether THIS node should be the active sharing-tag holder for the + * cluster and calls clusterer accordingly. Idempotent (activate/force-backup + * are no-ops when already in that state), so it is safe to call on any relevant + * event. Takes NO lock - the caller passes the state it already read, so this + * is safe both inside and outside cl->peers->lock (clctl uses its own locks). + * + * forced != 0 : the operator pinned node_id 'forced' as the active holder + * (cl_ctr_shtag_force). Only that node activates; all others go + * to backup. Automatic allocation is suspended. + * forced == 0 : automatic mode - the current master is the active holder. + */ +static void cc_apply_shtags_decision(cc_cluster_t *cl, int i_am_master, + uint16_t forced) +{ + int activate; + + if (!cl->manage_shtags || !clctl_loaded || !clctl.activate_backup_shtags) + return; + + if (forced != 0) + activate = (my_node_id != 0 && (uint16_t)my_node_id == forced); + else + activate = i_am_master; + + /* Log the reason on this node, but only when the decision or its cause + * changes - cc_apply_shtags_decision() is called on every relevant event + * and clusterer's own activate/force calls are idempotent, so logging + * unconditionally would flood. This makes the "why" visible on EVERY + * node (master, forced holder, and passive backups alike). */ + if (activate != cl->shtag_last_active || forced != cl->shtag_last_forced) { + if (activate && forced != 0) + LM_INFO("clusterer_controller: [cluster %d] activating sharing tags " + "- operator forced this node (node_id %u) as active holder\n", + cl->cluster_id, forced); + else if (activate) + LM_INFO("clusterer_controller: [cluster %d] activating sharing tags " + "- this node is the cluster master\n", cl->cluster_id); + else if (forced != 0) + LM_INFO("clusterer_controller: [cluster %d] keeping sharing tags in " + "backup - operator forced node_id %u as active holder\n", + cl->cluster_id, forced); + else + LM_INFO("clusterer_controller: [cluster %d] keeping sharing tags in " + "backup - active holder is the cluster master\n", + cl->cluster_id); + cl->shtag_last_active = activate; + cl->shtag_last_forced = forced; + } + + if (activate) + clctl.activate_backup_shtags(cl->cluster_id); + else if (clctl.force_backup_shtags) + clctl.force_backup_shtags(cl->cluster_id); +} + +/** + * cc_apply_shtags() - convenience wrapper that reads the current state under a + * read lock and applies the shtag decision. Call WITHOUT cl->peers->lock held. + */ +static void cc_apply_shtags(cc_cluster_t *cl) +{ + int i_am_master; + uint16_t forced; + + lock_start_read(cl->peers->lock); + i_am_master = cc_i_am_master_locked(cl); + forced = cl->peers->shtag_forced_node_id; + lock_stop_read(cl->peers->lock); + + cc_apply_shtags_decision(cl, i_am_master, forced); +} + +/** + * cc_prune_stale(cl) - free entries far outside the election window. + * Memory management only - does not affect election outcomes. + * Must be called with cl->peers->lock held. + */ +static void cc_prune_stale(cc_cluster_t *cl) +{ + time_t cutoff = time(NULL) - (time_t)(query_time * CC_PURGE_FACTOR); + int i; + + for (i = 0; i < cl->peers->count; i++) { + if (cl->peers->entries[i].last_seen < cutoff) { + uint16_t pruned_id = cl->peers->entries[i].node_id; + LM_INFO("clusterer_controller: purging timed-out peer %s\n", + cl->peers->entries[i].ip); + cl->peers->count--; + if (i < cl->peers->count) + cl->peers->entries[i] = cl->peers->entries[cl->peers->count]; + memset(&cl->peers->entries[cl->peers->count], 0, sizeof(cc_peer_t)); + i--; + /* cl_list_lock and cl->peers->lock are independent - no deadlock */ + if (clctl_loaded && pruned_id > 0) + clctl.remove_node(cl->cluster_id, pruned_id); + /* If the operator-forced shtag holder timed out, drop the override + * so automatic allocation resumes instead of leaving no active tag. */ + if (pruned_id != 0 && cl->peers->shtag_forced_node_id == pruned_id) { + LM_WARN("clusterer_controller: [cluster %d] forced shtag node " + "%u timed out - resuming automatic allocation\n", + cl->cluster_id, pruned_id); + cl->peers->shtag_forced_node_id = 0; + } + /* Re-apply shtag policy (override-aware); inside the lock, so pass + * the state directly rather than calling the locking wrapper. */ + cc_apply_shtags_decision(cl, cc_i_am_master_locked(cl), + cl->peers->shtag_forced_node_id); + } + } +} + +/** + * cc_apply_master_from_list_locked() - apply master designation from a + * received MEMBER_LIST/MEMBER_LIST packet. + * + * Zeros all is_master flags in the peer table, then sets is_master=1 for + * the entry matching master_ip. Updates last_master accordingly. + * Must be called with cl->peers->lock held. + */ +static void cc_apply_master_from_list_locked(const char *master_ip, cc_cluster_t *cl) +{ + int i; + + for (i = 0; i < cl->peers->count; i++) { + if (strcmp(cl->peers->entries[i].ip, master_ip) == 0) { + cl->peers->entries[i].is_master = 1; + } else { + cl->peers->entries[i].is_master = 0; + } + } + + memcpy(cl->peers->last_master, master_ip, + strnlen(master_ip, CC_MAX_IP_LEN)); + cl->peers->last_master[strnlen(master_ip, CC_MAX_IP_LEN)] = '\0'; +} + +/** + * cc_upsert_peer_locked() - insert or refresh a peer entry. + * Does NOT call cc_elect_master(cl); callers do that explicitly. + * Must be called with cl->peers->lock held. + */ +static void cc_upsert_peer_locked(const char *src_ip, cc_cluster_t *cl) +{ + int i; + unsigned int src_num = ip_to_num(src_ip); + time_t now = time(NULL); + + if (src_num == 0) { + LM_WARN("clusterer_controller: ignoring invalid IP '%s'\n", src_ip); + return; + } + + for (i = 0; i < cl->peers->count; i++) { + if (strcmp(cl->peers->entries[i].ip, src_ip) == 0) { + cl->peers->entries[i].last_seen = now; + return; /* updated */ + } + } + + /* New entry */ + if (cl->peers->count >= CC_MAX_PEERS) { + LM_WARN("clusterer_controller: peer table full, ignoring %s\n", + src_ip); + return; + } + { + cc_peer_t *e = &cl->peers->entries[cl->peers->count]; + memcpy(e->ip, src_ip, strnlen(src_ip, CC_MAX_IP_LEN)); + e->ip[strnlen(src_ip, CC_MAX_IP_LEN)] = '\0'; + e->ip_num = src_num; + e->last_seen = now; + e->is_master = 0; + cl->peers->count++; + LM_INFO("clusterer_controller: new peer %s (total=%d)\n", + src_ip, cl->peers->count); + } +} + +/** + * cc_alloc_node_id_locked() - find the lowest unused node_id >= 1. + * Must be called with cl->peers->lock held for write. + */ +static uint16_t cc_alloc_node_id_locked(cc_cluster_t *cl) +{ + uint16_t id; + int i, used; + + for (id = 1; id < 65535; id++) { + used = 0; + for (i = 0; i < cl->peers->count; i++) { + if (cl->peers->entries[i].node_id == id) { + used = 1; + break; + } + } + if (!used) + return id; + } + return 0; /* table full (shouldn't happen with CC_MAX_PEERS=256) */ +} + +/** + * cc_update_peer_bin_locked() - store node_id and BIN sockets for a peer. + * Must be called with cl->peers->lock held. + */ +static void cc_update_peer_bin_locked(const char *ip, uint16_t node_id, + uint8_t bin_count, + const char (*bin_sockets)[CC_MAX_BIN_SOCK_LEN], + cc_cluster_t *cl) +{ + int i; + for (i = 0; i < cl->peers->count; i++) { + if (strcmp(cl->peers->entries[i].ip, ip) == 0) { + cl->peers->entries[i].node_id = node_id; + cl->peers->entries[i].bin_count = bin_count; + if (bin_count > 0) + memcpy(cl->peers->entries[i].bin_sockets, bin_sockets, + bin_count * CC_MAX_BIN_SOCK_LEN); + return; + } + } +} + +/* ========================================================================= + * AES-256-GCM packet encryption / decryption + * + * Every packet is fully encrypted and authenticated with AES-256-GCM. + * A 12-byte random nonce (generated fresh for each packet via WolfSSL RNG) + * ensures that even identical payloads produce different ciphertext. + * + * A 4-byte monotonic sequence number is included inside the plaintext. + * Receivers track the highest sequence seen from each peer and reject any + * packet whose sequence is not strictly greater. This stops replays + * without requiring NTP-synchronised clocks or a finite nonce cache. + * The counter resets to 0 on every session key rotation; old packets + * encrypted with the previous key fail AES-GCM authentication anyway. + * Bootstrap-key packets (CC_BOOTSTRAP_MAGIC) skip the sequence check - + * their per-exchange join_nonce provides equivalent replay protection. + * + * Wire layout: + * [magic 2B][cluster_id 2B][nonce 12B][ciphertext][GCM tag 16B] + * Plaintext: + * [type 1B][seq 4B][payload] + * ========================================================================= */ + +static int cc_hkdf_sha256(const unsigned char *ikm, size_t ikm_len, + const unsigned char *salt, size_t salt_len, + const char *info, + unsigned char out[32]) +{ + size_t info_len = info ? strlen(info) : 0; + return wc_HKDF(WC_SHA256, + ikm, (word32)ikm_len, + salt, (word32)salt_len, + (const byte *)info, (word32)info_len, + out, 32) == 0 ? 0 : -1; +} + +static int cc_gen_ecdh_keypair(unsigned char *privkey, unsigned char *pubkey) +{ + curve25519_key k; + word32 len = CC_PUBKEY_SZ; + int rc = -1; + + if (wc_curve25519_init(&k) != 0) goto done; + if (wc_curve25519_make_key(&cc_rng, 32, &k) != 0) goto free; + if (wc_curve25519_export_private_raw(&k, privkey, &len) != 0) goto free; + len = CC_PUBKEY_SZ; + if (wc_curve25519_export_public(&k, pubkey, &len) != 0) goto free; + rc = 0; +free: + wc_curve25519_free(&k); +done: + if (rc < 0) + LM_ERR("clusterer_controller: X25519 keygen failed\n"); + return rc; +} + +static int cc_ecdh_shared(const unsigned char *my_priv, + const unsigned char *peer_pub, + unsigned char out[CC_PUBKEY_SZ]) +{ + curve25519_key priv_k, pub_k; + word32 len = CC_PUBKEY_SZ; + int rc = -1; + + if (wc_curve25519_init(&priv_k) != 0) return -1; + if (wc_curve25519_init(&pub_k) != 0) { wc_curve25519_free(&priv_k); return -1; } + + if (wc_curve25519_import_private(my_priv, CC_PUBKEY_SZ, &priv_k) != 0) goto done; + if (wc_curve25519_import_public(peer_pub, CC_PUBKEY_SZ, &pub_k) != 0) goto done; + if (wc_curve25519_shared_secret(&priv_k, &pub_k, out, &len) != 0) goto done; + rc = 0; +done: + wc_curve25519_free(&priv_k); + wc_curve25519_free(&pub_k); + if (rc < 0) + LM_ERR("clusterer_controller: X25519 derive failed\n"); + return rc; +} + +/** + * cc_wrap_salt() - XOR-encrypt master_salt for a specific peer using ECDH. + * wrap_key = HKDF(ECDH(my_priv, peer_pub) || password [|| nonce], info) + * wrapped = master_salt XOR wrap_key + * + * nonce/nonce_len are optional (pass NULL/0 for KEY_HANDOFF). + * For KEY_GRANT, nonce is the per-exchange join_nonce from the JOIN_REQ, + * making every wrap_key unique even if the password is later compromised. + */ +static int cc_wrap_salt(const unsigned char *my_priv, + const unsigned char *peer_pub, + const char *password, + const unsigned char *master_salt, + const char *info, + const unsigned char *nonce, + size_t nonce_len, + unsigned char wrapped[CC_MASTER_SALT_SZ]) +{ + unsigned char ss[CC_PUBKEY_SZ]; + unsigned char ikm[CC_PUBKEY_SZ + 1024 + CC_JOIN_NONCE_SZ]; + size_t pass_len = strlen(password); + size_t ikm_len; + unsigned char wrap_key[32]; + int i; + + if (cc_ecdh_shared(my_priv, peer_pub, ss) < 0) + return -1; + + /* IKM = ss || password [|| nonce] */ + memcpy(ikm, ss, CC_PUBKEY_SZ); + if (pass_len > 1024) pass_len = 1024; + memcpy(ikm + CC_PUBKEY_SZ, password, pass_len); + ikm_len = CC_PUBKEY_SZ + pass_len; + if (nonce && nonce_len > 0) { + if (nonce_len > CC_JOIN_NONCE_SZ) nonce_len = CC_JOIN_NONCE_SZ; + memcpy(ikm + ikm_len, nonce, nonce_len); + ikm_len += nonce_len; + } + + if (cc_hkdf_sha256(ikm, ikm_len, + ss, CC_PUBKEY_SZ, /* use ss as salt too */ + info, wrap_key) < 0) + return -1; + + for (i = 0; i < CC_MASTER_SALT_SZ; i++) + wrapped[i] = master_salt[i] ^ wrap_key[i]; + return 0; +} + +/** + * cc_derive_session_key() - derive group key from password + master_salt. + * Reads master_salt from cl->peers->master_salt (shm, caller holds write lock). + * Stores result in cl->session_key (worker-local cache). + * Must be called with cl->peers->lock held for WRITE. + */ +static int cc_derive_session_key(cc_cluster_t *cl) +{ + int i; + size_t pass_len = strlen(cl->password); + if (cc_hkdf_sha256((unsigned char *)cl->password, pass_len, + cl->peers->master_salt, CC_MASTER_SALT_SZ, + "cc_session", cl->session_key) < 0) { + LM_ERR("clusterer_controller: [cluster %d] session key derivation failed\n", + cl->cluster_id); + return -1; + } + /* Reset sequence counters: old packets encrypted with the previous key + * fail AES-GCM authentication, so starting from 0 is safe. */ + cl->peers->my_seq = 0; + for (i = 0; i < cl->peers->count; i++) + cl->peers->entries[i].last_seq = 0; + cl->have_session_key = 1; /* a valid group key now exists */ + return 0; +} + + +/** + * cc_password_entropy_bits() - conservative estimate of a password's entropy. + * bits ~= length * floor(log2(charset)), where charset is the union of the + * character classes present. floor() makes it a slight under-estimate, so the + * weak-password warning errs toward firing. No libm dependency. + */ +static int cc_password_entropy_bits(const char *p) +{ + int have_lower = 0, have_upper = 0, have_digit = 0, have_sym = 0; + int charset, bits_per_char = 0, tmp; + size_t i, len = strlen(p); + + for (i = 0; i < len; i++) { + unsigned char c = (unsigned char)p[i]; + if (c >= 'a' && c <= 'z') have_lower = 1; + else if (c >= 'A' && c <= 'Z') have_upper = 1; + else if (c >= '0' && c <= '9') have_digit = 1; + else have_sym = 1; + } + charset = have_lower * 26 + have_upper * 26 + have_digit * 10 + have_sym * 33; + if (charset < 2) + charset = 2; + for (tmp = charset; tmp > 1; tmp >>= 1) + bits_per_char++; + return (int)(len * (size_t)bits_per_char); +} + +/** + * cc_derive_key() - derive the 32-byte bootstrap (admission) key from the + * shared password. Used only for the join handshake (JOIN_REQ / KEY_GRANT / + * JOIN_REJECT); normal traffic uses the ECDH-agreed session key. + * + * Uses scrypt (memory-hard) rather than a single SHA-256 so that a password + * captured from a JOIN packet cannot be brute-forced cheaply offline. Called + * once per cluster from mod_init() in the main process before fork, so the + * ~64 MiB scrypt working set is a transient startup cost only; workers inherit + * the derived key and never run scrypt. + */ +static int cc_derive_key(cc_cluster_t *cl) +{ + char salt[64]; + int saltlen, bits; + + /* Warn on a weak or default admission password. scrypt raises the cost of + * each offline guess, but only a high-entropy secret removes the risk. */ + if (strcmp(cl->password, CC_DEFAULT_PASSWORD) == 0) { + LM_WARN("clusterer_controller: [cluster %d] using the built-in default " + "password - set a strong 'password' (e.g. `openssl rand -base64 32`)\n", + cl->cluster_id); + } else { + bits = cc_password_entropy_bits(cl->password); + if (bits < CC_MIN_PASSWORD_BITS) + LM_WARN("clusterer_controller: [cluster %d] weak password (~%d bits " + "of entropy) - an attacker who captures a JOIN packet can " + "brute-force it offline; use a long random string, e.g. " + "`openssl rand -base64 32`\n", cl->cluster_id, bits); + } + + /* Per-cluster salt: a fixed domain-separation label plus the multicast + * address, so the same password on different clusters yields different + * bootstrap keys. The salt is public by design - scrypt's work factor, + * not salt secrecy, is what defeats brute force. */ + saltlen = snprintf(salt, sizeof(salt), "opensips-cc-bootstrap-v1:%s", + cl->multicast_address); + if (saltlen < 0 || saltlen >= (int)sizeof(salt)) + saltlen = (int)strlen(salt); + +#ifdef CC_HAVE_SODIUM + /* Argon2id. crypto_pwhash needs a fixed-length 16-byte salt, so fold our + * variable-length domain-separation salt into one with BLAKE2b. */ + { + unsigned char salt16[crypto_pwhash_SALTBYTES]; + crypto_generichash(salt16, sizeof(salt16), + (const unsigned char *)salt, (size_t)saltlen, NULL, 0); + if (crypto_pwhash(cl->key, 32, + cl->password, strlen(cl->password), + salt16, CC_ARGON2_OPSLIMIT, CC_ARGON2_MEMLIMIT, + crypto_pwhash_ALG_ARGON2ID13) != 0) { + LM_ERR("clusterer_controller: key derivation (Argon2id) failed for " + "cluster %d (out of memory?)\n", cl->cluster_id); + return -1; + } + } +#else + if (wc_scrypt(cl->key, (const byte *)cl->password, (int)strlen(cl->password), + (const byte *)salt, saltlen, + CC_SCRYPT_COST, CC_SCRYPT_BLOCKSIZE, CC_SCRYPT_PARALLEL, + 32) != 0) { + LM_ERR("clusterer_controller: key derivation (scrypt) failed for " + "cluster %d\n", cl->cluster_id); + return -1; + } +#endif + return 0; +} + +/** + * cc_encrypt_pkt() - encrypt plaintext in-place and append the GCM tag. + * + * On entry: buf[0..CC_MAGIC_SZ-1] = magic (set by caller) + * buf[plain_off..] = plaintext to encrypt + * On return: buf[CC_MAGIC_SZ..] = cleartext cluster_id (BE) + * buf[CC_NONCE_OFF..] = random nonce + * buf[plain_off..] = ciphertext (same length) + * buf[plain_off+plain_len..+CC_TAG_SZ-1] = GCM tag + * + * @return total packet length, or -1 on error + */ +static int cc_encrypt_pkt(char *buf, int plain_off, int plain_len, + const unsigned char *key, int cluster_id) +{ + uint16_t cid_be = htons((uint16_t)cluster_id); + + memcpy(buf + CC_MAGIC_SZ, &cid_be, CC_CLUSTER_ID_SZ); /* cleartext selector */ + + /* AAD = cleartext header (magic + cluster_id): binding it to the tag stops + * a captured packet being re-stamped with another cluster_id on a shared + * multicast+password group (cross-cluster injection). The nonce is the + * AEAD IV, already bound. Random nonces are safe here: at this volume the + * AES-GCM 96-bit collision bound is unreachable, and XChaCha20's 192-bit + * nonce removes the concern outright. */ +#ifdef CC_HAVE_SODIUM + { + unsigned char nonce[CC_NONCE_SZ]; + unsigned long long clen = 0; + randombytes_buf(nonce, CC_NONCE_SZ); + memcpy(buf + CC_NONCE_OFF, nonce, CC_NONCE_SZ); + if (crypto_aead_xchacha20poly1305_ietf_encrypt( + (unsigned char *)buf + plain_off, &clen, + (const unsigned char *)buf + plain_off, (unsigned long long)plain_len, + (const unsigned char *)buf, CC_MAGIC_SZ + CC_CLUSTER_ID_SZ, + NULL, nonce, key) != 0) { + LM_ERR("clusterer_controller: XChaCha20-Poly1305 encrypt failed\n"); + return -1; + } + return plain_off + (int)clen; /* clen = plain_len + CC_TAG_SZ */ + } +#else + { + Aes aes; + unsigned char nonce[CC_NONCE_SZ]; + unsigned char tag[CC_TAG_SZ]; + if (wc_RNG_GenerateBlock(&cc_rng, nonce, CC_NONCE_SZ) != 0) { + LM_ERR("clusterer_controller: RNG failed\n"); + return -1; + } + memcpy(buf + CC_NONCE_OFF, nonce, CC_NONCE_SZ); + if (wc_AesGcmSetKey(&aes, key, 32) != 0) { + LM_ERR("clusterer_controller: AesGcmSetKey failed\n"); + return -1; + } + if (wc_AesGcmEncrypt(&aes, + (byte *)buf + plain_off, + (const byte *)buf + plain_off, (word32)plain_len, + nonce, CC_NONCE_SZ, + tag, CC_TAG_SZ, + (const byte *)buf, CC_MAGIC_SZ + CC_CLUSTER_ID_SZ) != 0) { + LM_ERR("clusterer_controller: AesGcmEncrypt failed\n"); + wc_AesFree(&aes); + return -1; + } + wc_AesFree(&aes); + memcpy(buf + plain_off + plain_len, tag, CC_TAG_SZ); + return plain_off + plain_len + CC_TAG_SZ; + } +#endif +} + +/** + * cc_decrypt_pkt() - authenticate and decrypt a received packet in-place. + * + * On entry: buf = [magic 2B][cluster_id 2B][nonce][ciphertext][tag 16B] + * On return: buf[CC_WIRE_HDR_SZ..] = plaintext (type + seq + payload) + * + * AAD must match cc_encrypt_pkt(): the cleartext header (magic+cluster_id), so + * a packet re-stamped with another cluster_id fails authentication. + * + * @return 0 on success, -1 to drop (wrong key, tampered, or too short) + */ +static int cc_decrypt_pkt(char *buf, ssize_t n, const char *sender_ip, + const unsigned char *key, int is_bootstrap) +{ + ssize_t cipher_len = n - CC_WIRE_HDR_SZ - CC_TAG_SZ; /* plaintext length */ + + if (cipher_len <= 0) { + LM_INFO("clusterer_controller: packet from %s too short to decrypt " + "(%zd bytes)\n", sender_ip, n); + return -1; + } + +#ifdef CC_HAVE_SODIUM + { + unsigned char *nonce = (unsigned char *)buf + CC_NONCE_OFF; + unsigned long long mlen = 0; + if (crypto_aead_xchacha20poly1305_ietf_decrypt( + (unsigned char *)buf + CC_WIRE_HDR_SZ, &mlen, NULL, + (const unsigned char *)buf + CC_WIRE_HDR_SZ, + (unsigned long long)(cipher_len + CC_TAG_SZ), + (const unsigned char *)buf, CC_MAGIC_SZ + CC_CLUSTER_ID_SZ, + nonce, key) != 0) { + if (is_bootstrap) + LM_WARN("clusterer_controller: bootstrap decryption failed " + "from %s - wrong password, foreign cluster, or " + "tampered packet\n", sender_ip); + else + LM_DBG("clusterer_controller: session packet from %s did not " + "decrypt - transient key mismatch during (re)key or " + "split-brain heal\n", sender_ip); + return -1; + } + } +#else + { + Aes aes; + unsigned char *nonce = (unsigned char *)buf + CC_NONCE_OFF; + unsigned char *tag = (unsigned char *)buf + n - CC_TAG_SZ; + int ret; + if (wc_AesGcmSetKey(&aes, key, 32) != 0) { + LM_ERR("clusterer_controller: AesGcmSetKey failed\n"); + return -1; + } + ret = wc_AesGcmDecrypt(&aes, + (byte *)buf + CC_WIRE_HDR_SZ, + (const byte *)buf + CC_WIRE_HDR_SZ, (word32)cipher_len, + nonce, CC_NONCE_SZ, + tag, CC_TAG_SZ, + (const byte *)buf, CC_MAGIC_SZ + CC_CLUSTER_ID_SZ); + wc_AesFree(&aes); + if (ret != 0) { + if (is_bootstrap) + LM_WARN("clusterer_controller: bootstrap decryption failed " + "from %s - wrong password, foreign cluster, or " + "tampered packet\n", sender_ip); + else + LM_DBG("clusterer_controller: session packet from %s did not " + "decrypt - transient key mismatch during (re)key or " + "split-brain heal\n", sender_ip); + return -1; + } + } +#endif + return 0; +} + +/** + * cc_check_and_update_seq() - reject replayed or reordered packets. + * Looks up sender_ip in the peer table; requires pkt_seq > last_seq. + * Updates last_seq on accept. Unknown senders (new nodes not yet in + * the peer table) are accepted so their first packet (ALIVE/JOIN_REQ) + * can populate the table. + * Only called for CC_PACKET_MAGIC packets; bootstrap packets use join_nonce. + * Single-threaded caller (cc_worker reactor); no lock needed for the check. + * @return 0 to accept, -1 to drop. + */ +static int cc_check_and_update_seq(const char *sender_ip, uint32_t pkt_seq, + cc_cluster_t *cl) +{ + int i; + for (i = 0; i < cl->peers->count; i++) { + if (strcmp(cl->peers->entries[i].ip, sender_ip) == 0) { + if (pkt_seq <= cl->peers->entries[i].last_seq) { + LM_WARN("clusterer_controller: replay from %s seq=%u last=%u, dropping\n", + sender_ip, pkt_seq, cl->peers->entries[i].last_seq); + return -1; + } + cl->peers->entries[i].last_seq = pkt_seq; + return 0; + } + } + return 0; /* unknown sender: accept, handler will upsert into peer table */ +} + +/* ========================================================================= + * Socket setup + * ========================================================================= */ + +static int cc_setup_socket(cc_cluster_t *cl) +{ + int sock; + int yes = 1; + unsigned char loop = 1, ttl = 32; + struct sockaddr_in local; + struct ip_mreq mreq; + + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock < 0) { + LM_ERR("clusterer_controller: socket(): %s\n", strerror(errno)); + return -1; + } + + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0) { + LM_ERR("clusterer_controller: SO_REUSEADDR: %s\n", strerror(errno)); + close(sock); + return -1; + } + + /* Expand the kernel receive buffer so it can hold a fully reassembled + * MEMBER_LIST datagram (up to ~4 KB with 256 peers) even when IP + * fragmentation is in play on a low-MTU link such as PPP at 128 bytes. */ + { + int rcvbuf = 1 << 20; /* request 1 MB; kernel may cap lower */ + if (setsockopt(sock, SOL_SOCKET, SO_RCVBUF, + &rcvbuf, sizeof(rcvbuf)) < 0) + LM_WARN("clusterer_controller: SO_RCVBUF: %s\n", strerror(errno)); + } + + memset(&local, 0, sizeof(local)); + local.sin_family = AF_INET; + local.sin_port = htons((uint16_t)cl->multicast_port); + local.sin_addr.s_addr = htonl(INADDR_ANY); + + if (bind(sock, (struct sockaddr *)&local, sizeof(local)) < 0) { + LM_ERR("clusterer_controller: bind() port %d: %s\n", + cl->multicast_port, strerror(errno)); + close(sock); + return -1; + } + + memset(&mreq, 0, sizeof(mreq)); + mreq.imr_multiaddr.s_addr = inet_addr(cl->multicast_address); + mreq.imr_interface.s_addr = htonl(INADDR_ANY); + + if (setsockopt(sock, IPPROTO_IP, IP_ADD_MEMBERSHIP, + &mreq, sizeof(mreq)) < 0) { + LM_ERR("clusterer_controller: IP_ADD_MEMBERSHIP (%s): %s\n", + cl->multicast_address, strerror(errno)); + close(sock); + return -1; + } + + if (setsockopt(sock, IPPROTO_IP, IP_MULTICAST_LOOP, + &loop, sizeof(loop)) < 0) + LM_WARN("clusterer_controller: IP_MULTICAST_LOOP: %s\n", + strerror(errno)); + + if (setsockopt(sock, IPPROTO_IP, IP_MULTICAST_TTL, + &ttl, sizeof(ttl)) < 0) + LM_WARN("clusterer_controller: IP_MULTICAST_TTL: %s\n", + strerror(errno)); + + /* Pin the sending interface to my_ip so that loopback packets carry + * my_ip as source address - this makes self-loopback detection in + * cc_handle_member_list() reliable on multi-homed hosts. */ + { + struct in_addr local_if; + local_if.s_addr = inet_addr(my_ip); + if (setsockopt(sock, IPPROTO_IP, IP_MULTICAST_IF, + &local_if, sizeof(local_if)) < 0) + LM_WARN("clusterer_controller: [cluster %d] IP_MULTICAST_IF: %s\n", + cl->cluster_id, + strerror(errno)); + } + + /* Bind socket to the resolved interface by name for stricter routing. + * This is more reliable than IP_MULTICAST_IF alone on multi-homed hosts + * because it works at the socket level regardless of routing tables. */ + if (my_interface_buf[0] != '\0') { + struct ifreq ifr; + memset(&ifr, 0, sizeof(ifr)); + memcpy(ifr.ifr_name, my_interface_buf, + strnlen(my_interface_buf, IF_NAMESIZE - 1)); + if (setsockopt(sock, SOL_SOCKET, SO_BINDTODEVICE, + &ifr, sizeof(ifr)) < 0) + /* Requires CAP_NET_RAW - not available after privilege drop. + * IP_MULTICAST_IF (set above) already pins the interface by + * IP, so this is belt-and-suspenders only; failure is safe. */ + LM_DBG("clusterer_controller: SO_BINDTODEVICE (%s): %s " + "(non-fatal, IP_MULTICAST_IF covers this)\n", + my_interface_buf, strerror(errno)); + } + + LM_INFO("clusterer_controller: [cluster %d] socket ready, joined %s:%d\n", + cl->cluster_id, cl->multicast_address, cl->multicast_port); + + /* Set non-blocking so sendto() never hangs the worker if the kernel + * UDP send buffer fills up. recvfrom() already relies on select() + * for readiness, so O_NONBLOCK is safe and consistent for both. */ + if (fcntl(sock, F_SETFL, fcntl(sock, F_GETFL, 0) | O_NONBLOCK) < 0) { + LM_WARN("clusterer_controller: fcntl O_NONBLOCK: %s\n", + strerror(errno)); + /* non-fatal - we continue; send paths handle EAGAIN explicitly */ + } + + return sock; +} + +/* ========================================================================= + * Packet senders + * ========================================================================= */ + +/** + * cc_send_pkt_with_ip() - build and multicast a small (ALIVE/GOODBYE) packet. + * JOIN_REQ is handled by cc_send_join_req_pkt() which carries BIN socket info. + * + * ALIVE: [type 1B][seq 4B][ip NUL][pubkey 32B] - peers learn our pubkey here + * GOODBYE: [type 1B][seq 4B][ip NUL] - no pubkey needed + */ +static void cc_send_pkt_with_ip(int sock, unsigned char type, cc_cluster_t *cl) +{ + /* Sized for ALIVE which carries an extra pubkey + config descriptor */ + char pkt[CC_SMALL_PKT_SZ + CC_PUBKEY_SZ + CC_CONFIG_SZ]; + uint32_t seq = htonl(++cl->peers->my_seq); + int ip_len = (int)strlen(my_ip); + int plain_len, total_len; + struct sockaddr_in dest; + + if (ip_len > CC_MAX_IP_LEN) + ip_len = CC_MAX_IP_LEN; + + memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); + + pkt[CC_WIRE_HDR_SZ] = (char)type; + memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + memcpy(pkt + CC_WIRE_HDR_SZ + 1 + CC_SEQ_SZ, my_ip, ip_len); + pkt[CC_WIRE_HDR_SZ + 1 + CC_SEQ_SZ + ip_len] = '\0'; + plain_len = 1 + CC_SEQ_SZ + ip_len + 1; + + /* ALIVE appends our X25519 public key so peers accumulate pubkeys + * without bloating MEMBER_LIST (avoids excessive IP fragmentation). */ + if (type == CC_PKT_ALIVE) { + memcpy(pkt + CC_WIRE_HDR_SZ + plain_len, cl->my_pubkey, CC_PUBKEY_SZ); + plain_len += CC_PUBKEY_SZ; + /* Advertise our effective consistency-critical config so peers can + * detect accidental per-node config drift for the same cluster. */ + { + char *c = pkt + CC_WIRE_HDR_SZ + plain_len; + uint16_t qt = htons((uint16_t)(query_time & 0xFFFF)); + c[0] = (char)(cl->manage_shtags ? 1 : 0); + c[1] = (char)(cl->master_stickiness ? 1 : 0); + memcpy(c + 2, &qt, 2); + plain_len += CC_CONFIG_SZ; + } + } + + total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->session_key, cl->cluster_id); + if (total_len < 0) + return; + + memset(&dest, 0, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons((uint16_t)cl->multicast_port); + dest.sin_addr.s_addr = inet_addr(cl->multicast_address); + + if (sendto(sock, pkt, total_len, 0, + (struct sockaddr *)&dest, sizeof(dest)) < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) + LM_DBG("clusterer_controller: [cluster %d] sendto (type=0x%02x) would block\n", + cl->cluster_id, type); + else + LM_ERR("clusterer_controller: [cluster %d] sendto (type=0x%02x): %s\n", + cl->cluster_id, type, strerror(errno)); + } else { + LM_DBG("clusterer_controller: [cluster %d] sent 0x%02x\n", cl->cluster_id, type); + } +} + +#define cc_send_alive(sock, cl) cc_send_pkt_with_ip((sock), CC_PKT_ALIVE, (cl)) +#define cc_send_join_req(sock, cl) cc_send_join_req_pkt((sock), (cl)) + +/** + * cc_send_list_pkt() - encrypt and multicast the active peer table. + * + * Wire: [magic 2B][cluster_id 2B][nonce 12B][AES-256-GCM([type 1B][seq 4B][count 2B][entries...])][tag 16B] + */ +static void cc_send_list_pkt(int sock, unsigned char type, cc_cluster_t *cl) +{ + char pkt[CC_LIST_PKT_MAX_SZ]; + uint32_t seq = htonl(++cl->peers->my_seq); + uint16_t count = 0; + uint16_t count_be; + char *p; + time_t cutoff; + struct sockaddr_in dest; + int i, plain_len, total_len; + + memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); + /* nonce at [8..19] written by cc_encrypt_pkt */ + + /* Plaintext: [type][seq][count BE][forced_shtag_node_id BE][entries...] */ + pkt[CC_WIRE_HDR_SZ] = (char)type; + memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + /* count filled after iteration; forced_shtag_node_id filled below */ + p = pkt + CC_WIRE_HDR_SZ + 1 + CC_SEQ_SZ + CC_LIST_COUNT_SZ + CC_NODE_ID_SZ; + + cutoff = time(NULL) - (time_t)(query_time * CC_ELECT_FACTOR); + + lock_start_read(cl->peers->lock); + { + uint16_t forced_be = htons(cl->peers->shtag_forced_node_id); + memcpy(pkt + CC_WIRE_HDR_SZ + 1 + CC_SEQ_SZ + CC_LIST_COUNT_SZ, + &forced_be, CC_NODE_ID_SZ); + } + + for (i = 0; i < cl->peers->count && count < CC_MAX_PEERS; i++) { + cc_peer_t *e = &cl->peers->entries[i]; + if (e->last_seen < cutoff) + continue; + /* Entry layout: [ip 16B null-padded][is_master 1B] = 17B */ + memset(p, 0, CC_IP_ENTRY_SZ); + memcpy(p, e->ip, strnlen(e->ip, CC_MAX_IP_LEN)); + p[CC_IP_ENTRY_SZ - 1] = (char)(e->is_master ? 1 : 0); + p += CC_IP_ENTRY_SZ; + count++; + } + + lock_stop_read(cl->peers->lock); + + count_be = htons(count); + memcpy(pkt + CC_WIRE_HDR_SZ + 1 + CC_SEQ_SZ, &count_be, CC_LIST_COUNT_SZ); + + plain_len = 1 + CC_SEQ_SZ + CC_LIST_COUNT_SZ + CC_NODE_ID_SZ + + count * CC_IP_ENTRY_SZ; + total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->session_key, cl->cluster_id); + if (total_len < 0) + return; + + memset(&dest, 0, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons((uint16_t)cl->multicast_port); + dest.sin_addr.s_addr = inet_addr(cl->multicast_address); + + if (sendto(sock, pkt, total_len, 0, + (struct sockaddr *)&dest, sizeof(dest)) < 0) + LM_ERR("clusterer_controller: [cluster %d] sendto MEMBER_LIST: %s\n", + cl->cluster_id, strerror(errno)); + else + LM_INFO("clusterer_controller: [cluster %d] sent MEMBER_LIST (%d members)\n", + cl->cluster_id, count); +} + +#define cc_send_member_list(sock, cl) cc_send_list_pkt((sock), CC_PKT_MEMBER_LIST, (cl)) + +/** + * cc_send_join_req_pkt() - send CC_PKT_JOIN_REQ with BIN socket info. + * + * Payload: [ip NUL][bin_count 1B][sock1 NUL]...[sockN NUL] + */ +static void cc_send_join_req_pkt(int sock, cc_cluster_t *cl) +{ + char pkt[CC_JOIN_PKT_MAX_SZ]; + uint32_t seq; + + /* Rate-limit JOIN_REQ transmissions. During a key-mismatch / split-brain + * heal several code paths (defer timer, session-mismatch re-key, rejoin + * timer) can each ask to (re)send a JOIN_REQ within the same second. A + * JOIN_REQ is idempotent - the master answers whichever one arrives - so + * drop any that lands within CC_JOIN_REQ_MIN_US of the previous send; the + * next timer tick resends if the join is still pending. The very first + * send (last_join_req_utime == 0) is never throttled. */ + { + utime_t now_us = get_uticks(); + if (cl->last_join_req_utime != 0 && + (utime_t)(now_us - cl->last_join_req_utime) < CC_JOIN_REQ_MIN_US) + return; + cl->last_join_req_utime = now_us; + } + + seq = htonl(++cl->peers->my_seq); + int ip_len = (int)strlen(my_ip); + char *p; + int plain_len, total_len; + struct sockaddr_in dest; + + if (ip_len > CC_MAX_IP_LEN) + ip_len = CC_MAX_IP_LEN; + + memcpy(pkt, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ); /* JOIN_REQ uses bootstrap key */ + + /* Plaintext: [type][seq][ip NUL][bin_count][sock1 NUL]...[sockN NUL][pubkey 32B] */ + pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_JOIN_REQ; + memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + p = pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + + memcpy(p, my_ip, ip_len); + p[ip_len] = '\0'; + p += ip_len + 1; + + /* Advertise only the BIN socket resolved for this specific cluster */ + { + int slen = (int)strnlen(cl->bin_socket, CC_MAX_BIN_SOCK_LEN - 1); + *p++ = 1; /* bin_count */ + memcpy(p, cl->bin_socket, slen); + p[slen] = '\0'; + p += slen + 1; + } + + /* Append our X25519 public key so master can wrap the session salt for us */ + memcpy(p, cl->my_pubkey, CC_PUBKEY_SZ); + p += CC_PUBKEY_SZ; + + /* Append per-exchange nonce; master echoes it in KEY_GRANT to bind the + * ECDH wrap to this specific exchange even if password is later compromised */ + if (wc_RNG_GenerateBlock(&cc_rng, cl->my_join_nonce, CC_JOIN_NONCE_SZ) != 0) { + LM_ERR("clusterer_controller: RNG for join_nonce failed\n"); + return; + } + memcpy(p, cl->my_join_nonce, CC_JOIN_NONCE_SZ); + p += CC_JOIN_NONCE_SZ; + + /* Advertise our consistency-critical config so the master can reject (or + * warn about) a join with settings that differ from the running cluster. */ + { + uint16_t qt = htons((uint16_t)(query_time & 0xFFFF)); + *p++ = (char)(cl->manage_shtags ? 1 : 0); + *p++ = (char)(cl->master_stickiness ? 1 : 0); + memcpy(p, &qt, 2); + p += 2; + } + + plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); + total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->key, cl->cluster_id); + if (total_len < 0) + return; + + memset(&dest, 0, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons((uint16_t)cl->multicast_port); + dest.sin_addr.s_addr = inet_addr(cl->multicast_address); + + if (sendto(sock, pkt, total_len, 0, + (struct sockaddr *)&dest, sizeof(dest)) < 0) + LM_ERR("clusterer_controller: [cluster %d] sendto JOIN_REQ: %s\n", + cl->cluster_id, strerror(errno)); + else + LM_DBG("clusterer_controller: [cluster %d] sent JOIN_REQ bin=%s\n", + cl->cluster_id, cl->bin_socket); +} + +/** + * cc_send_node_assign() - send CC_PKT_NODE_ASSIGN to multicast. + * + * Payload: [node_id 2B BE][ip NUL][bin_count 1B][sock1 NUL]...[sockN NUL] + * + * Sent by master after allocating a node_id. All cluster members receive + * it and update their peer tables accordingly. + */ +static void cc_send_node_assign(int sock, const char *ip, uint16_t node_id, + uint8_t bin_count, + const char (*bin_sockets)[CC_MAX_BIN_SOCK_LEN], + cc_cluster_t *cl) +{ + char pkt[CC_NODE_ASSIGN_MAX_SZ]; + uint32_t seq = htonl(++cl->peers->my_seq); + uint16_t nid_be = htons(node_id); + int ip_len = (int)strnlen(ip, CC_MAX_IP_LEN); + char *p; + int i, plain_len, total_len; + struct sockaddr_in dest; + + memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); + + pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_NODE_ASSIGN; + memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + p = pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + + /* node_id (2B BE) */ + memcpy(p, &nid_be, CC_NODE_ID_SZ); + p += CC_NODE_ID_SZ; + + /* IP NUL */ + memcpy(p, ip, ip_len); + p[ip_len] = '\0'; + p += ip_len + 1; + + /* BIN sockets */ + *p++ = (char)bin_count; + for (i = 0; i < bin_count; i++) { + int slen = (int)strnlen(bin_sockets[i], CC_MAX_BIN_SOCK_LEN - 1); + memcpy(p, bin_sockets[i], slen); + p[slen] = '\0'; + p += slen + 1; + } + + plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); + /* NODE_ASSIGN carries the CC_PACKET_MAGIC session magic, so the receiver + * decrypts it with session_key. It MUST therefore be encrypted with + * session_key, not the bootstrap key - otherwise every NODE_ASSIGN fails + * GCM auth on receipt ("session key mismatch"), driving a JOIN_REQ storm. + * KEY_GRANT is sent before NODE_ASSIGN so the joiner already holds the + * session key by the time this arrives. */ + total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->session_key, cl->cluster_id); + if (total_len < 0) + return; + + memset(&dest, 0, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons((uint16_t)cl->multicast_port); + dest.sin_addr.s_addr = inet_addr(cl->multicast_address); + + if (sendto(sock, pkt, total_len, 0, + (struct sockaddr *)&dest, sizeof(dest)) < 0) + LM_ERR("clusterer_controller: [cluster %d] sendto NODE_ASSIGN: %s\n", + cl->cluster_id, strerror(errno)); + else + LM_INFO("clusterer_controller: [cluster %d] NODE_ASSIGN node_id=%u ip=%s\n", + cl->cluster_id, node_id, ip); +} + + +/** + * cc_send_master_alive() - master-only keepalive, no payload beyond the header. + * Encrypted with session_key (CC_PACKET_MAGIC). + */ +static void cc_send_master_alive(int sock, cc_cluster_t *cl) +{ + char pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_TAG_SZ]; + uint32_t seq = htonl(++cl->peers->my_seq); + int total_len; + struct sockaddr_in dest; + + memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); + pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_MASTER_ALIVE; + memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + /* no payload beyond type+seq */ + + total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, CC_PLAIN_HDR_SZ, + cl->session_key, cl->cluster_id); + if (total_len < 0) return; + + memset(&dest, 0, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons((uint16_t)cl->multicast_port); + dest.sin_addr.s_addr = inet_addr(cl->multicast_address); + + if (sendto(sock, pkt, total_len, 0, + (struct sockaddr *)&dest, sizeof(dest)) < 0) + LM_ERR("clusterer_controller: [cluster %d] sendto MASTER_ALIVE: %s\n", + cl->cluster_id, strerror(errno)); +} + +/** + * cc_send_master_beacon() - master-only split-brain merge announcement. + * + * Encrypted with the BOOTSTRAP key (CC_BOOTSTRAP_MAGIC) - unlike MASTER_ALIVE, + * which uses the per-cluster session key. Every correctly-configured node + * shares the bootstrap key, so two masters that hold *different* session keys + * (e.g. after an all-node simultaneous cold start) can still read each other's + * beacon and reconcile. Payload is this partition's member count (2B BE); the + * sender IP comes from the datagram source. See cc_handle_master_beacon(). + */ +static void cc_send_master_beacon(int sock, cc_cluster_t *cl) +{ + char pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + 2 + CC_TAG_SZ]; + uint32_t seq = htonl(++cl->peers->my_seq); + uint16_t cnt_be; + int total_len; + struct sockaddr_in dest; + + lock_start_read(cl->peers->lock); + cnt_be = htons((uint16_t)cl->peers->count); + lock_stop_read(cl->peers->lock); + + memcpy(pkt, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ); + pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_MASTER_BEACON; + memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + memcpy(pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ, &cnt_be, 2); + + total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, CC_PLAIN_HDR_SZ + 2, + cl->key, cl->cluster_id); + if (total_len < 0) return; + + memset(&dest, 0, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons((uint16_t)cl->multicast_port); + dest.sin_addr.s_addr = inet_addr(cl->multicast_address); + + if (sendto(sock, pkt, total_len, 0, + (struct sockaddr *)&dest, sizeof(dest)) < 0) + LM_ERR("clusterer_controller: [cluster %d] sendto MASTER_BEACON: %s\n", + cl->cluster_id, strerror(errno)); +} + +/** + * cc_send_key_grant() - send ECDH-wrapped master_salt to a joining node. + * Encrypted with bootstrap key (CC_BOOTSTRAP_MAGIC) so the joining node + * can read it before having the session key. + * + * Payload: [target_ip NUL][my_pubkey 32B][join_nonce 16B][wrapped_salt 32B] + * + * join_nonce is the per-exchange nonce from the JOIN_REQ. It is echoed back + * here (inside the authenticated GCM envelope) and folded into cc_wrap_salt's + * IKM so the wrap_key is unique per exchange even if the password leaks. + */ +static void cc_send_key_grant(int sock, const char *target_ip, cc_cluster_t *cl, + const unsigned char *joiner_pubkey, + const unsigned char *joiner_nonce) +{ + char pkt[CC_KEY_GRANT_SZ]; + uint32_t seq = htonl(++cl->peers->my_seq); + unsigned char wrapped[CC_MASTER_SALT_SZ]; + char *p; + int ip_len, plain_len, total_len; + struct sockaddr_in dest; + + if (cc_wrap_salt(cl->my_privkey, joiner_pubkey, cl->password, + cl->peers->master_salt, "cc_key_grant", + joiner_nonce, CC_JOIN_NONCE_SZ, wrapped) < 0) + return; + + ip_len = (int)strnlen(target_ip, CC_MAX_IP_LEN); + + memcpy(pkt, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ); + pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_KEY_GRANT; + memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + p = pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + + memcpy(p, target_ip, ip_len); p[ip_len] = '\0'; p += ip_len + 1; + memcpy(p, cl->my_pubkey, CC_PUBKEY_SZ); p += CC_PUBKEY_SZ; + memcpy(p, joiner_nonce, CC_JOIN_NONCE_SZ); p += CC_JOIN_NONCE_SZ; + memcpy(p, wrapped, CC_MASTER_SALT_SZ); p += CC_MASTER_SALT_SZ; + + plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); + total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->key, cl->cluster_id); + if (total_len < 0) return; + + memset(&dest, 0, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons((uint16_t)cl->multicast_port); + dest.sin_addr.s_addr = inet_addr(cl->multicast_address); + + if (sendto(sock, pkt, total_len, 0, + (struct sockaddr *)&dest, sizeof(dest)) < 0) + LM_ERR("clusterer_controller: [cluster %d] sendto KEY_GRANT: %s\n", + cl->cluster_id, strerror(errno)); + else + LM_INFO("clusterer_controller: [cluster %d] sent KEY_GRANT to %s\n", + cl->cluster_id, target_ip); +} + +/** + * cc_send_key_handoff() - send master_salt to the next master before departing. + * Encrypted with session_key (CC_PACKET_MAGIC); only next master can unwrap. + * + * Payload: [next_master_ip NUL][my_pubkey 32B][wrapped_salt 32B] + */ +static void cc_send_key_handoff(int sock, const char *next_master_ip, + const unsigned char *next_master_pubkey, + cc_cluster_t *cl) +{ + char pkt[CC_KEY_HANDOFF_SZ]; + uint32_t seq = htonl(++cl->peers->my_seq); + unsigned char wrapped[CC_MASTER_SALT_SZ]; + char *p; + int ip_len, plain_len, total_len; + struct sockaddr_in dest; + + if (cc_wrap_salt(cl->my_privkey, next_master_pubkey, cl->password, + cl->peers->master_salt, "cc_key_handoff", + NULL, 0, wrapped) < 0) + return; + + ip_len = (int)strnlen(next_master_ip, CC_MAX_IP_LEN); + + memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); + pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_KEY_HANDOFF; + memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + p = pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + + memcpy(p, next_master_ip, ip_len); p[ip_len] = '\0'; p += ip_len + 1; + memcpy(p, cl->my_pubkey, CC_PUBKEY_SZ); p += CC_PUBKEY_SZ; + memcpy(p, wrapped, CC_MASTER_SALT_SZ); p += CC_MASTER_SALT_SZ; + + plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); + total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->session_key, cl->cluster_id); + if (total_len < 0) return; + + memset(&dest, 0, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons((uint16_t)cl->multicast_port); + dest.sin_addr.s_addr = inet_addr(cl->multicast_address); + + if (sendto(sock, pkt, total_len, 0, + (struct sockaddr *)&dest, sizeof(dest)) < 0) + LM_ERR("clusterer_controller: [cluster %d] sendto KEY_HANDOFF: %s\n", + cl->cluster_id, strerror(errno)); + else + LM_INFO("clusterer_controller: [cluster %d] sent KEY_HANDOFF to %s\n", + cl->cluster_id, next_master_ip); +} + +/* ========================================================================= + * State transition helper + * ========================================================================= */ + +/** + * cc_on_became_master() - side effects when this node wins an election. + * Generates fresh master_salt, derives session_key, arms MASTER_ALIVE timer, + * disarms master_dead watchdog. Must be called with cl->peers->lock held WRITE. + */ +static void cc_on_became_master(cc_cluster_t *cl) +{ + cl->join_pending = 0; /* any pending re-key join is now moot */ + if (wc_RNG_GenerateBlock(&cc_rng, cl->peers->master_salt, CC_MASTER_SALT_SZ) != 0) { + LM_ERR("clusterer_controller: [cluster %d] RNG for master_salt failed\n", + cl->cluster_id); + return; + } + cc_derive_session_key(cl); /* reads cl->peers->master_salt */ + LM_INFO("clusterer_controller: [cluster %d] became master - " + "new master_salt generated, session key rotated\n", cl->cluster_id); + /* Timer ops outside lock (timerfd is worker-local, no shm concern) */ +} + +/** + * cc_arm_master_timers() - arm/disarm keepalive timers after election. + * Call WITHOUT cl->peers->lock held. + * i_am_master: 1 = we won, arm MASTER_ALIVE, disarm dead-watchdog. + * 0 = we lost, arm dead-watchdog, disarm MASTER_ALIVE. + */ +static void cc_arm_master_timers(cc_cluster_t *cl, int i_am_master) +{ + cl->master_ka_armed = i_am_master ? 1 : 0; + if (i_am_master) { + cc_arm_tfd(cl->master_alive_tfd, CC_MASTER_KA_INTERVAL, CC_MASTER_KA_INTERVAL); + cc_arm_tfd(cl->master_dead_tfd, 0, 0); /* disarm watchdog */ + } else { + cc_arm_tfd(cl->master_alive_tfd, 0, 0); /* disarm sender */ + cc_arm_tfd(cl->master_dead_tfd, CC_MASTER_KA_TIMEOUT, 0); + } +} + +/** + * cc_request_rekey() - ask the current key-holder for the session key. + * Sends a single JOIN_REQ (bootstrap key), guarded by join_pending so a fresh + * nonce is not stomped while one exchange is in flight. Used when a node is + * elected master but has not yet adopted the cluster key (KEY_GRANT lost). + * Call WITHOUT cl->peers->lock held. + */ +static void cc_request_rekey(cc_cluster_t *cl) +{ + if (cl->join_pending) + return; + cl->join_pending = 1; + cc_send_join_req(cl->sock, cl); +} + +/** + * cc_transition_to_active() - switch from CC_NODE_NEW to CC_NODE_ACTIVE. + * + * Disarms the join-phase timers, sends the first ALIVE immediately, then + * arms the periodic ALIVE timer. Called from both: + * - the join_tfd handler (deadline expired, fresh cluster), and + * - cc_handle_member_list (master responded before deadline). + * Must be called with cl->peers->lock NOT held. + */ +static void cc_transition_to_active(cc_cluster_t *cl) +{ + cc_arm_tfd(cl->join_tfd, 0, 0); /* disarm one-shot deadline */ + cc_arm_tfd(cl->rejoin_tfd, 0, 0); /* disarm JOIN_REQ retry */ + cc_send_alive(cl->sock, cl); /* first heartbeat now */ + cc_arm_tfd(cl->alive_tfd, query_time, query_time); /* periodic from here */ +} + +/* ========================================================================= + * Packet handlers + * ========================================================================= */ + +/** + * cc_fmt_cfg_diff() - render only the consistency-critical settings that + * actually DIFFER into 'out' (e.g. "manage_shtags cluster=1 node=0"), so a + * mismatch log names just the offending setting(s) rather than all three. + * 'la'/'lb' are the labels for the local/peer sides ("cluster"/"node" or + * "local"/"peer"). + */ +static void cc_fmt_cfg_diff(char *out, int outsz, const char *la, const char *lb, + int a_manage, int b_manage, int a_stick, int b_stick, + int a_qt, int b_qt) +{ + int n = 0; + out[0] = '\0'; +#define CC_DIFF_APPEND(cond, name, av, bv) \ + do { \ + if (cond) { \ + int _w = snprintf(out + n, (n < outsz) ? (outsz - n) : 0, \ + "%s" name " %s=%d %s=%d", \ + n ? ", " : "", la, (av), lb, (bv)); \ + if (_w > 0) { n += _w; if (n > outsz) n = outsz; } \ + } \ + } while (0) + CC_DIFF_APPEND(a_manage != b_manage, "manage_shtags", a_manage, b_manage); + CC_DIFF_APPEND(a_stick != b_stick, "master_stickiness", a_stick, b_stick); + CC_DIFF_APPEND(a_qt != b_qt, "query_time", a_qt, b_qt); +#undef CC_DIFF_APPEND +} + +/** + * cc_adopt_config() - adopt the running cluster's consistency-critical settings + * (on_config_mismatch=adopt). Called on a non-master node when the master's + * advertised config differs from ours. Call WITHOUT cl->peers->lock held. + */ +static void cc_adopt_config(cc_cluster_t *cl, int new_manage, int new_stick, + int new_qt, int is_active) +{ + int old_manage = cl->manage_shtags ? 1 : 0; + + LM_INFO("clusterer_controller: [cluster %d] adopting cluster settings from " + "master (manage_shtags %d->%d, master_stickiness %d->%d, " + "query_time %d->%d)\n", + cl->cluster_id, old_manage, new_manage ? 1 : 0, + cl->master_stickiness ? 1 : 0, new_stick ? 1 : 0, query_time, new_qt); + + cl->master_stickiness = new_stick ? 1 : 0; + + /* query_time is declared global but is a per-process copy after fork, and + * there is one worker process per cluster, so updating it here affects only + * THIS cluster's worker - never another cluster on a multi-cluster node. + * Re-arm the periodic ALIVE timer if we are already active; the election + * window and purge derive from query_time dynamically and need no re-arm. */ + if (new_qt >= 1 && new_qt != query_time) { + query_time = new_qt; + if (is_active) + cc_arm_tfd(cl->alive_tfd, query_time, query_time); + } + + /* Mirror the effective values into shm so cl_ctr_list_config (MI process) + * reports what is actually in force after adoption. */ + cl->peers->eff_manage_shtags = new_manage ? 1 : 0; + cl->peers->eff_master_stickiness = cl->master_stickiness; + cl->peers->eff_query_time = query_time; + + if ((new_manage ? 1 : 0) != old_manage) { + cl->manage_shtags = new_manage ? 1 : 0; + if (clctl_loaded) { + if (cl->manage_shtags) { + if (clctl.set_shtag_managed) + clctl.set_shtag_managed(cl->cluster_id); + } else if (clctl.unset_shtag_managed) { + clctl.unset_shtag_managed(cl->cluster_id); + } + } + /* Reconcile tag state under the new policy (no-op when now unmanaged). */ + cc_apply_shtags(cl); + } +} + +/** + * cc_handle_alive() - process a CC_PKT_ALIVE packet. + * + * Regular heartbeat path: upsert the sender, re-elect. + * Only called in CC_NODE_ACTIVE state; ignored while joining (cc_recv_one + * still dispatches them so the peer table builds up before the timeout). + */ +static void cc_handle_alive(const char *src_ip, + const unsigned char *pubkey, /* may be NULL */ + int cfg_present, int peer_manage, + int peer_stick, int peer_qt, + cc_cluster_t *cl) +{ + int prev_master, now_master; + int warn = 0, adopt = 0, is_active = 0, ent = -1; + char warn_ip[CC_MAX_IP_LEN + 1] = ""; + int loc_manage = cl->manage_shtags ? 1 : 0; + int loc_stick = cl->master_stickiness ? 1 : 0; + int loc_qt = query_time; + int mism = 0, changed = 0; + + lock_start_write(cl->peers->lock); + prev_master = cc_i_am_master_locked(cl); + cc_upsert_peer_locked(src_ip, cl); + { + int _i; + for (_i = 0; _i < cl->peers->count; _i++) { + cc_peer_t *e = &cl->peers->entries[_i]; + if (strcmp(e->ip, src_ip) != 0) + continue; + ent = _i; + /* Store pubkey so master can use it for KEY_HANDOFF / KEY_GRANT */ + if (pubkey) + memcpy(e->pubkey, pubkey, CC_PUBKEY_SZ); + if (cfg_present) { + changed = (!e->cfg_known + || e->cfg_manage_shtags != peer_manage + || e->cfg_master_stickiness != peer_stick + || e->cfg_query_time != peer_qt); + mism = (peer_manage != loc_manage + || peer_stick != loc_stick + || peer_qt != loc_qt); + e->cfg_known = 1; + e->cfg_manage_shtags = peer_manage; + e->cfg_master_stickiness = peer_stick; + e->cfg_query_time = peer_qt; + } + break; + } + } + cc_elect_master(cl); + now_master = cc_i_am_master_locked(cl); + is_active = (cl->peers->node_state == CC_NODE_ACTIVE); + /* Config-consistency handling, decided once the master is known. In + * 'adopt' mode a non-master node takes the master's settings; otherwise a + * mismatch is logged once per peer (re-logged only if the peer's advertised + * config changes, cleared when it matches). cc_elect_master does not + * reorder entries, so the captured index stays valid. */ + if (cfg_present && ent >= 0) { + cc_peer_t *e = &cl->peers->entries[ent]; + int sender_is_master = (cl->peers->last_master[0] != '\0' + && strcmp(src_ip, cl->peers->last_master) == 0); + if (!mism) { + e->cfg_warned = 0; + } else if (on_config_mismatch == CC_CFGMISMATCH_ADOPT + && sender_is_master && !now_master) { + adopt = 1; + e->cfg_warned = 0; + } else if (changed || !e->cfg_warned) { + warn = 1; + e->cfg_warned = 1; + memcpy(warn_ip, e->ip, sizeof(warn_ip)); + } + } + lock_stop_write(cl->peers->lock); + + if (warn) { + char diff[160]; + cc_fmt_cfg_diff(diff, sizeof(diff), "local", "peer", + loc_manage, peer_manage, loc_stick, peer_stick, + loc_qt, peer_qt); + LM_WARN("clusterer_controller: [cluster %d] CONFIG MISMATCH with peer %s " + "- all nodes of a cluster MUST use identical settings; mismatched " + "values cause inconsistent failover/sharing-tag behaviour (%s)\n", + cl->cluster_id, warn_ip, diff); + } + if (adopt) + cc_adopt_config(cl, peer_manage, peer_stick, peer_qt, is_active); + /* Defer acting as master until we hold the cluster key. In normal + * operation a higher-IP joiner has already adopted the key via KEY_GRANT + * before winning here, so this only guards the pathological case where the + * KEY_GRANT was lost during the join. Request a re-key instead of + * broadcasting an undecryptable MASTER_ALIVE. */ + if (now_master && !cl->have_session_key) { + cc_request_rekey(cl); + return; + } + if (prev_master != now_master) + cc_arm_master_timers(cl, now_master); +} + +/** + * cc_handle_join_req() - process a CC_PKT_JOIN_REQ packet. + * + * Payload: [ip NUL][bin_count 1B][sock1 NUL]...[sockN NUL] + * + * Non-masters ignore JOIN_REQ - the master handles discovery exclusively. + * + * Master behaviour: + * 1. Parse joining node's IP and BIN sockets from payload. + * 2. Upsert peer; store BIN info and allocate a node_id. + * 3. Send NODE_ASSIGN (joining node + all existing peers) so every + * node in the cluster learns the full updated picture. + * 4. If joining IP > own IP: run election, send MEMBER_LIST. + * 5. If joining IP <= own IP: send MEMBER_LIST (self still master). + */ +static void cc_handle_join_req(int sock, const char *payload, int payload_len, + cc_cluster_t *cl) +{ + const char *p = payload; + const char *end = payload + payload_len; + char src_ip[CC_MAX_IP_LEN + 1]; + char bin_socks[CC_MAX_BIN_SOCKETS][CC_MAX_BIN_SOCK_LEN]; + unsigned char joiner_pubkey[CC_PUBKEY_SZ]; + unsigned char joiner_nonce[CC_JOIN_NONCE_SZ]; + uint8_t bin_cnt = 0; + int ip_len, was_master, i; + int j_cfg_present = 0, j_manage = 0, j_stick = 0, j_qt = 0; + uint16_t new_id; + + /* --- Parse IP --- */ + ip_len = (int)strnlen(p, CC_MAX_IP_LEN); + if (p + ip_len >= end) { + LM_WARN("clusterer_controller: JOIN_REQ payload truncated\n"); + return; + } + memcpy(src_ip, p, ip_len); + src_ip[ip_len] = '\0'; + p += ip_len + 1; + + /* Ignore our own JOIN_REQ via loopback */ + if (strcmp(src_ip, my_ip) == 0) + return; + + /* --- Parse BIN sockets --- */ + memset(bin_socks, 0, sizeof(bin_socks)); + if (p < end) { + bin_cnt = (uint8_t)*p++; + if (bin_cnt > CC_MAX_BIN_SOCKETS) + bin_cnt = CC_MAX_BIN_SOCKETS; + for (i = 0; i < (int)bin_cnt && p < end; i++) { + int slen = (int)strnlen(p, CC_MAX_BIN_SOCK_LEN - 1); + memcpy(bin_socks[i], p, slen); + bin_socks[i][slen] = '\0'; + p += slen + 1; + } + } + + /* --- Parse X25519 public key (appended after BIN info) --- */ + memset(joiner_pubkey, 0, CC_PUBKEY_SZ); + if (p + CC_PUBKEY_SZ <= end) { + memcpy(joiner_pubkey, p, CC_PUBKEY_SZ); + p += CC_PUBKEY_SZ; + } + + /* --- Parse per-exchange join_nonce (appended after pubkey) --- */ + memset(joiner_nonce, 0, CC_JOIN_NONCE_SZ); + if (p + CC_JOIN_NONCE_SZ <= end) { + memcpy(joiner_nonce, p, CC_JOIN_NONCE_SZ); + p += CC_JOIN_NONCE_SZ; + } + + /* --- Parse the joiner's consistency-critical config (after join_nonce) --- */ + if (p + CC_CONFIG_SZ <= end) { + uint16_t qt_be; + j_manage = (unsigned char)p[0]; + j_stick = (unsigned char)p[1]; + memcpy(&qt_be, p + 2, 2); + j_qt = ntohs(qt_be); + j_cfg_present = 1; + p += CC_CONFIG_SZ; + } + + LM_INFO("clusterer_controller: [cluster %d] JOIN_REQ from %s " + "(%d BIN socket(s))\n", cl->cluster_id, src_ip, bin_cnt); + + lock_start_write(cl->peers->lock); + + was_master = (cl->peers->node_state == CC_NODE_ACTIVE) && + cc_i_am_master_locked(cl); + + if (!was_master) { + /* Split-brain prevention: while we are ourselves still joining, record + * the other joiner so the join-deadline election can defer to the + * highest-IP starter instead of every node self-promoting into a + * divergent-key lone master. (An active non-master simply ignores it; + * the master drives discovery.) */ + if (cl->peers->node_state == CC_NODE_NEW) { + cc_upsert_peer_locked(src_ip, cl); + /* A fresh JOIN_REQ from a *higher-IP* peer proves it is still alive + * and joining, so reset our split-brain defer budget: keep waiting + * for it to become master instead of prematurely self-promoting into + * a divergent-key split brain. join_defer_total still grows (hard + * cap) so a peer that is stuck forever cannot stall us indefinitely. */ + if (ip_to_num(src_ip) > ip_to_num(my_ip)) + cl->join_defer_count = 0; + } + lock_stop_write(cl->peers->lock); + LM_DBG("clusterer_controller: non-master ignoring JOIN_REQ from %s\n", + src_ip); + return; + } + + /* Config-consistency gate (master side): if the joiner advertises + * consistency-critical settings that differ from the running cluster and + * the policy is 'reject', refuse the join so the node shuts down rather + * than joining and behaving inconsistently. (warn/adopt admit the node; + * the joiner then warns or adopts on the master's ALIVE.) */ + if (j_cfg_present && on_config_mismatch == CC_CFGMISMATCH_REJECT) { + int loc_manage = cl->manage_shtags ? 1 : 0; + int loc_stick = cl->master_stickiness ? 1 : 0; + if (j_manage != loc_manage || j_stick != loc_stick || j_qt != query_time) { + char diff[160]; + lock_stop_write(cl->peers->lock); + cc_fmt_cfg_diff(diff, sizeof(diff), "cluster", "node", + loc_manage, j_manage, loc_stick, j_stick, + query_time, j_qt); + LM_WARN("clusterer_controller: [cluster %d] rejecting JOIN_REQ from %s: " + "different settings than the running cluster (%s)\n", + cl->cluster_id, src_ip, diff); + cc_send_join_reject(sock, src_ip, cl, CC_REJECT_CONFIG); + return; + } + } + + /* Reject JOIN_REQ from an unknown IP when the peer table is full. + * Known peers (reconnecting after restart) are still allowed through + * since they already occupy a slot. */ + if (cl->peers->count >= CC_MAX_PEERS) { + int _found = 0, _fi; + for (_fi = 0; _fi < cl->peers->count; _fi++) { + if (strcmp(cl->peers->entries[_fi].ip, src_ip) == 0) { + _found = 1; + break; + } + } + if (!_found) { + lock_stop_write(cl->peers->lock); + LM_WARN("clusterer_controller: [cluster %d] peer table full " + "(%d/%d), rejecting JOIN_REQ from %s\n", + cl->cluster_id, cl->peers->count, CC_MAX_PEERS, src_ip); + cc_send_join_reject(sock, src_ip, cl, CC_REJECT_GENERIC); + return; + } + } + + /* Upsert peer and store BIN info + node_id. + * If this IP already has a node_id (rejoining after crash/restart), + * reuse it so the id stays stable and clusterer stays in sync. */ + cc_upsert_peer_locked(src_ip, cl); + { + int _i; + new_id = 0; + for (_i = 0; _i < cl->peers->count; _i++) { + if (strcmp(cl->peers->entries[_i].ip, src_ip) == 0) { + new_id = cl->peers->entries[_i].node_id; + break; + } + } + if (new_id == 0) + new_id = cc_alloc_node_id_locked(cl); + } + cc_update_peer_bin_locked(src_ip, new_id, + bin_cnt, + (const char (*)[CC_MAX_BIN_SOCK_LEN])bin_socks, + cl); + + /* Store joiner's public key, join_nonce, and reset last_seq in peer table. + * Resetting last_seq is essential: a restarted node begins its seq counter + * from 0, and without this reset peers would permanently reject its new + * packets (old last_seq > new seq) until the next key rotation. */ + { + int _k; + for (_k = 0; _k < cl->peers->count; _k++) { + if (strcmp(cl->peers->entries[_k].ip, src_ip) == 0) { + memcpy(cl->peers->entries[_k].pubkey, joiner_pubkey, CC_PUBKEY_SZ); + memcpy(cl->peers->entries[_k].join_nonce, joiner_nonce, CC_JOIN_NONCE_SZ); + cl->peers->entries[_k].last_seq = 0; + break; + } + } + } + + lock_stop_write(cl->peers->lock); + + /* JOIN_REQ decrypted successfully - clear any failure record for this IP + * so a node that fixes its password isn't immediately rejected again. */ + { + uint32_t _ip_num = ip_to_num(src_ip); + int _fi; + for (_fi = 0; _fi < CC_JOIN_FAIL_TABLE_SZ; _fi++) { + if (cl->join_fail_tbl[_fi].ip_num == _ip_num) { + memset(&cl->join_fail_tbl[_fi], 0, sizeof(cl->join_fail_tbl[_fi])); + break; + } + } + } + + /* Send KEY_GRANT first (bootstrap key) so joiner can derive session_key, + * then NODE_ASSIGN + MEMBER_LIST (session key). */ + if (joiner_pubkey[0] || joiner_pubkey[1]) /* non-zero pubkey present */ + cc_send_key_grant(sock, src_ip, cl, joiner_pubkey, joiner_nonce); + + lock_start_write(cl->peers->lock); + + /* Send NODE_ASSIGN for joining node */ + cc_send_node_assign(sock, src_ip, new_id, bin_cnt, + (const char (*)[CC_MAX_BIN_SOCK_LEN])bin_socks, cl); + + /* Send NODE_ASSIGN for each existing peer so joining node learns + * all current node_ids and BIN sockets */ + for (i = 0; i < cl->peers->count; i++) { + cc_peer_t *e = &cl->peers->entries[i]; + if (strcmp(e->ip, src_ip) == 0 || e->node_id == 0) + continue; + cc_send_node_assign(sock, e->ip, e->node_id, e->bin_count, + (const char (*)[CC_MAX_BIN_SOCK_LEN])e->bin_sockets, + cl); + } + + /* A current master NEVER hands mastership to a joining node during the + * join handshake - doing so forced the joiner to broadcast MASTER_ALIVE + * before it had adopted the cluster key, producing a key-mismatch loop. + * + * Instead we stay master and designate ourselves in the MEMBER_LIST. The + * joiner adopts our session key via the KEY_GRANT sent above and joins as + * a member. If it has a higher IP it will win the very next ALIVE-driven + * election (cc_handle_alive) - but by then it holds the key, so when it + * takes over it broadcasts with a key every member already has. This + * keeps the deterministic "highest IP is master" outcome while deferring + * the actual takeover until after the key has been transferred. */ + lock_stop_write(cl->peers->lock); + LM_INFO("clusterer_controller: [cluster %d] I am master, " + "new node %s assigned node_id=%u\n", + cl->cluster_id, src_ip, new_id); + cc_send_member_list(sock, cl); +} + +/** + * cc_handle_member_list() - process a CC_PKT_MEMBER_LIST from the master. + * + * This is THE authoritative packet for cluster state. All nodes - both + * the joining node and existing active members - update their peer tables + * and master designation directly from this list. No independent election + * is run; the master's word is final. + * + * Joining node (CC_NODE_NEW): + * - Replaces its empty peer table with the master's list. + * - Transitions to CC_NODE_ACTIVE. + * - If the list designates us as master (our IP has is_master=1): we + * accept mastership immediately and log accordingly. + * + * Active member / old master (CC_NODE_ACTIVE): + * - Upserts any new peers from the list. + * - Applies master designation from the list (may demote old master). + * - Logs who is now master. + */ +static void cc_handle_member_list(const char *payload, int payload_len, + const char *sender_ip, cc_cluster_t *cl) +{ + uint16_t count; + uint16_t forced_node_id; + int i; + const char *p; + char designated_master[CC_MAX_IP_LEN + 1]; + + /* MEMBER_LIST is authoritative cluster state and must only come from the + * current master. Any cluster member holding the session key could forge + * one; accepting it would let an insider demote the real master, inject + * fake peers, or trigger spurious re-elections. + * Own loopback (sender == my_ip) is allowed when we are the master. */ + if (strcmp(sender_ip, my_ip) == 0) { + LM_DBG("clusterer_controller: ignoring own MEMBER_LIST loopback\n"); + return; + } + { + int _from_master; + lock_start_read(cl->peers->lock); + _from_master = (cl->peers->last_master[0] != '\0' && + strcmp(sender_ip, cl->peers->last_master) == 0); + lock_stop_read(cl->peers->lock); + if (!_from_master) { + /* Also allow during CC_NODE_NEW: we have no master yet, so any + * MEMBER_LIST is our first authoritative view of the cluster. */ + int _is_new; + lock_start_read(cl->peers->lock); + _is_new = (cl->peers->node_state == CC_NODE_NEW); + lock_stop_read(cl->peers->lock); + if (!_is_new) { + LM_WARN("clusterer_controller: MEMBER_LIST from non-master %s " + "(master is %s), dropping\n", + sender_ip, + cl->peers->last_master[0] ? cl->peers->last_master : "(none)"); + return; + } + } + } + + designated_master[0] = '\0'; + + if (payload_len < CC_LIST_COUNT_SZ + CC_NODE_ID_SZ) { + LM_WARN("clusterer_controller: MEMBER_LIST too short\n"); + return; + } + + { + uint16_t count_be, forced_be; + memcpy(&count_be, payload, CC_LIST_COUNT_SZ); + count = ntohs(count_be); + /* forced-shtag node_id follows the count (0 = automatic allocation) */ + memcpy(&forced_be, payload + CC_LIST_COUNT_SZ, CC_NODE_ID_SZ); + forced_node_id = ntohs(forced_be); + } + + if (count > CC_MAX_PEERS) { + LM_WARN("clusterer_controller: MEMBER_LIST count %u exceeds " + "max peers %d, dropping\n", count, CC_MAX_PEERS); + return; + } + + if (payload_len < CC_LIST_COUNT_SZ + CC_NODE_ID_SZ + + (int)count * CC_IP_ENTRY_SZ) { + LM_WARN("clusterer_controller: MEMBER_LIST truncated " + "(count=%u, got %d bytes)\n", count, payload_len); + return; + } + + p = payload + CC_LIST_COUNT_SZ + CC_NODE_ID_SZ; + + /* First pass: collect designated master IP */ + for (i = 0; i < (int)count; i++) { + const char *entry = p + i * CC_IP_ENTRY_SZ; + unsigned char is_master = (unsigned char)entry[CC_IP_ENTRY_SZ - 1]; + if (is_master) { + memcpy(designated_master, entry, CC_MAX_IP_LEN); + designated_master[CC_MAX_IP_LEN] = '\0'; + break; + } + } + + lock_start_write(cl->peers->lock); + + /* Second pass: upsert all peers and reset their last_seq. + * Resetting last_seq here covers the case where a peer restarted and + * sent JOIN_REQ: the MEMBER_LIST is the broadcast announcement that a + * join event occurred. Without the reset, non-master peers would reject + * the restarted node's new packets (old last_seq > new low seq). */ + for (i = 0; i < (int)count; i++, p += CC_IP_ENTRY_SZ) { + char ip_buf[CC_MAX_IP_LEN + 1]; + int _j; + memcpy(ip_buf, p, CC_MAX_IP_LEN); + ip_buf[CC_MAX_IP_LEN] = '\0'; + if (ip_buf[0] == '\0') + continue; + cc_upsert_peer_locked(ip_buf, cl); + for (_j = 0; _j < cl->peers->count; _j++) { + if (strcmp(cl->peers->entries[_j].ip, ip_buf) == 0) { + cl->peers->entries[_j].last_seq = 0; + break; + } + } + } + + /* Snapshot master status BEFORE cc_apply_master_from_list_locked clears it */ + int was_master_before = cc_i_am_master_locked(cl); + + /* Apply the master designation from the list - no local election */ + if (designated_master[0] != '\0') + cc_apply_master_from_list_locked(designated_master, cl); + + /* The master is authoritative for the shtag override too. */ + cl->peers->shtag_forced_node_id = forced_node_id; + + int was_new = (cl->peers->node_state == CC_NODE_NEW); + if (was_new) { + /* Only act as master if the list designates us AND we already hold the + * cluster key. Incumbent masters no longer hand over during a join, so + * in normal operation a MEMBER_LIST never designates a NEW node - this + * guard is defensive: without a key we would broadcast MASTER_ALIVE that + * no member can decrypt. Without the key we join as a plain member and + * let a later election promote us once KEY_GRANT has landed. */ + int _taking_over = (designated_master[0] != '\0' + && strcmp(designated_master, my_ip) == 0 + && cl->have_session_key); + cl->peers->node_state = CC_NODE_ACTIVE; + if (_taking_over) { + lock_stop_write(cl->peers->lock); + LM_INFO("clusterer_controller: received MEMBER_LIST (%u members) " + "from existing master %s - taking over mastership " + "(my IP %s is higher)\n", + count, sender_ip, my_ip); + } else { + lock_stop_write(cl->peers->lock); + LM_INFO("clusterer_controller: received MEMBER_LIST (%u members) " + "from %s - joined cluster as member, master is %s\n", + count, sender_ip, + designated_master[0] ? designated_master : "(none)"); + } + cc_transition_to_active(cl); + if (_taking_over) + cc_arm_master_timers(cl, 1); /* start MASTER_ALIVE, disarm dead watchdog */ + } else { + /* Active node - log the master update (may be self-demotion) */ + int i_am_master = (designated_master[0] != '\0' && + strcmp(designated_master, my_ip) == 0); + lock_stop_write(cl->peers->lock); + if (i_am_master) { + LM_INFO("clusterer_controller: MEMBER_LIST received - " + "I am master (%d members)\n", count); + } else if (was_master_before) { + /* A genuine demotion: we held mastership until this MEMBER_LIST. */ + LM_INFO("clusterer_controller: demoted to member - new master is %s " + "(%d members in cluster)\n", + designated_master[0] ? designated_master : "(none)", + count); + /* Fix up keepalive timers: stop sending MASTER_ALIVE, arm watchdog. */ + cc_arm_master_timers(cl, 0); + } else { + /* Already a member - this is just a routine MEMBER_LIST refresh (e.g. + * a periodic re-broadcast or a shtag-override update). No role change, + * so don't cry "demoted"; log quietly at debug level. */ + LM_DBG("clusterer_controller: MEMBER_LIST refreshed - master is %s " + "(%d members)\n", + designated_master[0] ? designated_master : "(none)", count); + } + } + + /* (Re)apply the shtag decision now that the forced-node override and the + * master designation from this MEMBER_LIST have both been stored. */ + cc_apply_shtags(cl); +} + +/** + * cc_handle_goodbye() - process a CC_PKT_GOODBYE packet. + * + * Remove the departing node from the peer table immediately. + * + * Re-election is triggered ONLY when: + * 1. Only one node remains - we are alone and must assume mastership. + * 2. Our IP is higher than the current master's IP, or the master entry + * no longer exists because the departing node was the master. + * cc_ip_beats_master_locked() covers both cases: it returns 1 when + * no is_master entry is present (departed master) or when our IP + * numerically exceeds the current master's. + * + * All other departures (a member leaves while a higher-IP master is alive) + * require no immediate action - the next periodic ALIVE cycle runs + * cc_elect_master(cl) within query_time seconds and self-corrects if needed. + */ +static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) +{ + int i, i_am_master, master_unchanged, remaining; + char prev_master[CC_MAX_IP_LEN + 1]; + char new_master[CC_MAX_IP_LEN + 1]; + uint16_t departed_node_id = 0; + + LM_INFO("clusterer_controller: GOODBYE from %s\n", src_ip); + + lock_start_write(cl->peers->lock); + + for (i = 0; i < cl->peers->count; i++) { + if (strcmp(cl->peers->entries[i].ip, src_ip) == 0) { + departed_node_id = cl->peers->entries[i].node_id; + cl->peers->count--; + if (i < cl->peers->count) + cl->peers->entries[i] = cl->peers->entries[cl->peers->count]; + memset(&cl->peers->entries[cl->peers->count], 0, sizeof(cc_peer_t)); + break; + } + } + + /* If the operator-forced shtag holder departed, drop the override so + * automatic allocation resumes rather than leaving no active holder. */ + if (departed_node_id != 0 && + cl->peers->shtag_forced_node_id == departed_node_id) { + LM_WARN("clusterer_controller: [cluster %d] forced shtag node %u " + "departed - resuming automatic allocation\n", + cl->cluster_id, departed_node_id); + cl->peers->shtag_forced_node_id = 0; + } + + remaining = cl->peers->count; + + /* --- Decide whether re-election is warranted --- */ + if (remaining <= 1) { + /* We are the only node left - no election needed, promote directly. */ + int was_master = cc_i_am_master_locked(cl); + cc_apply_master_from_list_locked(my_ip, cl); + lock_stop_write(cl->peers->lock); + if (was_master) { + LM_INFO("clusterer_controller: %s departed - I am the only " + "node remaining, I remain master\n", src_ip); + } else { + LM_INFO("clusterer_controller: %s departed - I am the only " + "node remaining, promoted myself to master\n", src_ip); + } + if (clctl_loaded && departed_node_id > 0) + clctl.remove_node(cl->cluster_id, departed_node_id); + cc_apply_shtags(cl); + return; + } + + if (cc_ip_beats_master_locked(ip_to_num(my_ip), cl)) { + /* Two sub-cases both return 1 from cc_ip_beats_master_locked: + * a) departing node was the master (no master entry remains) + * b) our IP is genuinely higher than the current master (anomaly) */ + if (strcmp(src_ip, cl->peers->last_master) == 0) { + LM_INFO("clusterer_controller: %s departed - it was the master, " + "triggering re-election\n", src_ip); + } else { + LM_INFO("clusterer_controller: %s departed - our IP %s is higher " + "than current master %s, triggering re-election\n", + src_ip, my_ip, + cl->peers->last_master[0] ? cl->peers->last_master : "(none)"); + } + } else { + /* Snapshot before releasing - must not read last_master after unlock */ + { + size_t _l = strnlen(cl->peers->last_master, CC_MAX_IP_LEN); + memcpy(prev_master, cl->peers->last_master, _l); + prev_master[_l] = '\0'; + } + lock_stop_write(cl->peers->lock); + LM_INFO("clusterer_controller: %s departed - master %s still " + "active, no re-election needed (%d node(s) remaining)\n", + src_ip, + prev_master[0] ? prev_master : "(none)", + remaining); + if (clctl_loaded && departed_node_id > 0) + clctl.remove_node(cl->cluster_id, departed_node_id); + /* Re-apply shtag policy: the still-active master takes over the + * departed node's tags, unless an operator override is in effect. */ + cc_apply_shtags(cl); + return; + } + + memcpy(prev_master, cl->peers->last_master, + strnlen(cl->peers->last_master, CC_MAX_IP_LEN)); + prev_master[strnlen(cl->peers->last_master, CC_MAX_IP_LEN)] = '\0'; + + { + cc_elect_master(cl); + i_am_master = cc_i_am_master_locked(cl); + } + + master_unchanged = (strcmp(prev_master, cl->peers->last_master) == 0); + i_am_master = cc_i_am_master_locked(cl); + /* Snapshot post-election master before releasing - used in member log */ + { + size_t _l = strnlen(cl->peers->last_master, CC_MAX_IP_LEN); + memcpy(new_master, cl->peers->last_master, _l); + new_master[_l] = '\0'; + } + + lock_stop_write(cl->peers->lock); + + if (i_am_master) { + if (master_unchanged) { + LM_INFO("clusterer_controller: re-election complete - " + "I remain master (%d node(s) in cluster)\n", remaining); + } else { + LM_INFO("clusterer_controller: re-election complete - " + "I reclaimed mastership after %s departed " + "(%d node(s) remaining) - sending MEMBER_LIST\n", + src_ip, remaining); + cc_arm_master_timers(cl, 1); + cc_send_member_list(sock, cl); + } + } else { + LM_INFO("clusterer_controller: re-election complete - " + "master is %s, my role is member (%d node(s) remaining)\n", + new_master[0] ? new_master : "(none)", + remaining); + } + if (clctl_loaded && departed_node_id > 0) + clctl.remove_node(cl->cluster_id, departed_node_id); + cc_apply_shtags(cl); +} + +/** + * cc_handle_node_assign() - process a CC_PKT_NODE_ASSIGN from the master. + * + * Payload: [node_id 2B BE][ip NUL][bin_count 1B][sock1 NUL]...[sockN NUL] + * + * All nodes (including master via loopback) apply the assignment: + * - Upsert the peer entry if not already present. + * - Store node_id and BIN sockets. + * - If ip == my_ip: record my_node_id. + */ +static void cc_handle_node_assign(const char *payload, int payload_len, + const char *sender_ip, cc_cluster_t *cl) +{ + const char *p = payload; + const char *end = payload + payload_len; + uint16_t node_id; + char ip[CC_MAX_IP_LEN + 1]; + char bin_socks[CC_MAX_BIN_SOCKETS][CC_MAX_BIN_SOCK_LEN]; + uint8_t bin_cnt = 0; + int ip_len, i; + + if (payload_len < (int)(CC_NODE_ID_SZ + 2)) { + LM_WARN("clusterer_controller: NODE_ASSIGN payload too short\n"); + return; + } + + /* node_id (2B BE) */ + memcpy(&node_id, p, CC_NODE_ID_SZ); + node_id = ntohs(node_id); + p += CC_NODE_ID_SZ; + + /* IP */ + ip_len = (int)strnlen(p, CC_MAX_IP_LEN); + memcpy(ip, p, ip_len); + ip[ip_len] = '\0'; + p += ip_len + 1; + + /* BIN sockets */ + memset(bin_socks, 0, sizeof(bin_socks)); + if (p < end) { + bin_cnt = (uint8_t)*p++; + if (bin_cnt > CC_MAX_BIN_SOCKETS) + bin_cnt = CC_MAX_BIN_SOCKETS; + for (i = 0; i < (int)bin_cnt && p < end; i++) { + int slen = (int)strnlen(p, CC_MAX_BIN_SOCK_LEN - 1); + memcpy(bin_socks[i], p, slen); + bin_socks[i][slen] = '\0'; + p += slen + 1; + } + } + + lock_start_write(cl->peers->lock); + cc_upsert_peer_locked(ip, cl); + cc_update_peer_bin_locked(ip, node_id, bin_cnt, + (const char (*)[CC_MAX_BIN_SOCK_LEN])bin_socks, + cl); + lock_stop_write(cl->peers->lock); + + /* Record our own node_id - also the first signal that a master exists */ + if (strcmp(ip, my_ip) == 0) { + /* Always our own entry - update identity regardless of my_node_id */ + if (my_node_id == 0) { + int is_joining; + lock_start_read(cl->peers->lock); + is_joining = (cl->peers->node_state == CC_NODE_NEW); + lock_stop_read(cl->peers->lock); + if (is_joining) + LM_INFO("clusterer_controller: [cluster %d] found existing master " + "at %s - receiving cluster state\n", + cl->cluster_id, sender_ip); + } + my_node_id = node_id; + LM_INFO("clusterer_controller: [cluster %d] master %s assigned us " + "node_id=%u\n", cl->cluster_id, sender_ip, node_id); + /* Correct the optimistic node_id=1 set at startup if needed */ + if (clctl_loaded) { + str url = {cl->bin_socket, (int)strlen(cl->bin_socket)}; + clctl.update_identity(cl->cluster_id, node_id, &url); + } + } else { + LM_INFO("clusterer_controller: [cluster %d] master %s assigned " + "node_id=%u to %s\n", cl->cluster_id, sender_ip, node_id, ip); + /* Add peer to clusterer */ + if (clctl_loaded && bin_cnt > 0) { + str url = {bin_socks[0], (int)strlen(bin_socks[0])}; + clctl.add_node(cl->cluster_id, node_id, &url); + } + } +} + +/** + * cc_handle_master_alive() - process a master keepalive. + * + * Beyond rearming the master-dead watchdog, this keeps every node agreeing on + * the master's identity (the 1s keepalive is more frequent than MEMBER_LIST, + * so it is the authoritative "who is master" signal) and resolves split-brain: + * if this node also believes it is master and the announcing node has a higher + * IP, it yields. Highest-IP-wins is the deterministic tiebreak in BOTH + * preemption modes, so two masters (e.g. after a network partition heals) can + * never both stick. + */ +static void cc_handle_master_alive(const char *sender_ip, cc_cluster_t *cl) +{ + int i_am_master, yielded = 0; + int from_self = (strcmp(sender_ip, my_ip) == 0); + + lock_start_write(cl->peers->lock); + i_am_master = cc_i_am_master_locked(cl); + + if (i_am_master) { + if (!from_self && ip_to_num(sender_ip) > ip_to_num(my_ip)) { + /* Split-brain: a higher-IP node also claims mastership - yield. */ + cc_upsert_peer_locked(sender_ip, cl); + cc_apply_master_from_list_locked(sender_ip, cl); + yielded = 1; + } + /* else: my own loopback, or a lower-IP claimant that will yield to us */ + } else if (!from_self) { + /* Track the announcing node as the current master so all nodes agree + * even if a MEMBER_LIST was missed, and so sticky cc_elect_master + * preserves the correct current master. */ + cc_upsert_peer_locked(sender_ip, cl); + cc_apply_master_from_list_locked(sender_ip, cl); + } + lock_stop_write(cl->peers->lock); + + if (yielded) { + LM_INFO("clusterer_controller: [cluster %d] yielding mastership to " + "higher-IP master %s (split-brain resolution)\n", + cl->cluster_id, sender_ip); + cc_arm_master_timers(cl, 0); /* stop MASTER_ALIVE, arm dead watchdog */ + } + + /* Non-masters (including a node that just yielded) watch the keepalive. */ + if (!i_am_master || yielded) + cc_arm_tfd(cl->master_dead_tfd, CC_MASTER_KA_TIMEOUT, 0); +} + +/** + * cc_rejoin_superior_master() - abandon our current allegiance and merge into + * the partition led by 'superior_ip', adopting its session key. + * + * Used by the split-brain merge (cc_handle_master_beacon): our node - whether a + * lone/independent master or a member of a smaller partition - records + * superior_ip as master, stops asserting mastership, and issues a JOIN_REQ so + * the superior master answers with NODE_ASSIGN + KEY_GRANT. Once the KEY_GRANT + * lands we can decrypt the superior partition's session traffic and are fully + * merged. Call WITHOUT cl->peers->lock held. + */ +static void cc_rejoin_superior_master(cc_cluster_t *cl, const char *superior_ip) +{ + int was_master; + + lock_start_write(cl->peers->lock); + was_master = cc_i_am_master_locked(cl); + cc_upsert_peer_locked(superior_ip, cl); + cc_apply_master_from_list_locked(superior_ip, cl); + cl->peers->node_state = CC_NODE_ACTIVE; + lock_stop_write(cl->peers->lock); + + if (was_master) { + LM_INFO("clusterer_controller: [cluster %d] superior master %s found via " + "beacon - demoting and merging (split-brain resolution)\n", + cl->cluster_id, superior_ip); + cc_arm_master_timers(cl, 0); /* stop MASTER_ALIVE, arm dead watchdog */ + } else { + LM_INFO("clusterer_controller: [cluster %d] moving to superior master %s " + "via beacon (split-brain merge)\n", cl->cluster_id, superior_ip); + } + + /* Drop any locally-held active shtag now that we are no longer master. */ + cc_apply_shtags(cl); + + /* Fetch the superior master's session key. join_pending guards the nonce + * so a beacon storm cannot stomp an exchange already in flight. */ + if (!cl->join_pending) { + cl->join_pending = 1; + cc_send_join_req(cl->sock, cl); + } + cc_arm_tfd(cl->master_dead_tfd, CC_MASTER_KA_TIMEOUT, 0); +} + +/** + * cc_handle_master_beacon() - process a CC_PKT_MASTER_BEACON (bootstrap key). + * + * A master announced itself on the bootstrap layer. If it outranks our own + * partition we merge into it; otherwise we ignore it (that master will yield to + * us when it hears our beacon). Ranking: larger member count wins, ties broken + * by higher IP - the same deterministic tiebreak used elsewhere, so exactly one + * master survives. A healthy single-master cluster sees only its own master's + * beacon (sender == our master) and never acts, so this adds no churn. + */ +static void cc_handle_master_beacon(const char *sender_ip, uint16_t sender_count, + cc_cluster_t *cl) +{ + int i_am_master, is_new, our_count, superior; + char our_master[CC_MAX_IP_LEN + 1]; + + if (strcmp(sender_ip, my_ip) == 0) + return; /* our own beacon looped back */ + + lock_start_read(cl->peers->lock); + is_new = (cl->peers->node_state == CC_NODE_NEW); + i_am_master = cc_i_am_master_locked(cl); + our_count = cl->peers->count; + if (i_am_master) { + size_t l = strnlen(my_ip, CC_MAX_IP_LEN); + memcpy(our_master, my_ip, l); our_master[l] = '\0'; + } else { + size_t l = strnlen(cl->peers->last_master, CC_MAX_IP_LEN); + memcpy(our_master, cl->peers->last_master, l); our_master[l] = '\0'; + } + lock_stop_read(cl->peers->lock); + + /* Still running the join protocol - a JOIN_REQ is already outstanding. */ + if (is_new) + return; + + /* Already following the beacon's sender: normal steady state, nothing to do. */ + if (our_master[0] != '\0' && strcmp(sender_ip, our_master) == 0) + return; + + /* Rank the sender's partition against ours. */ + if (our_master[0] == '\0') + superior = 1; /* we have no master yet */ + else if (sender_count != (uint16_t)our_count) + superior = (sender_count > (uint16_t)our_count);/* larger partition wins */ + else + superior = (ip_to_num(sender_ip) > ip_to_num(our_master)); /* IP tiebreak */ + + if (!superior) + return; /* we outrank the sender; it will yield to us on our beacon */ + + cc_rejoin_superior_master(cl, sender_ip); +} + +/** + * cc_handle_key_grant() - process CC_PKT_KEY_GRANT from master. + * Payload: [target_ip NUL][master_pubkey 32B][join_nonce 16B][wrapped_salt 32B] + * + * Recover master_salt via ECDH unwrap, derive session_key, update shm. + * Only processed if target_ip == my_ip (multicast - all nodes receive it). + */ +static void cc_handle_key_grant(const char *payload, int payload_len, + const char *sender_ip, cc_cluster_t *cl) +{ + const char *p = payload; + const char *end = payload + payload_len; + char target_ip[CC_MAX_IP_LEN + 1]; + unsigned char master_pubkey[CC_PUBKEY_SZ]; + unsigned char echoed_nonce[CC_JOIN_NONCE_SZ]; + unsigned char wrapped[CC_MASTER_SALT_SZ]; + unsigned char ss[CC_PUBKEY_SZ]; + unsigned char ikm[CC_PUBKEY_SZ + 1024 + CC_JOIN_NONCE_SZ]; + unsigned char wrap_key[32]; + unsigned char new_salt[CC_MASTER_SALT_SZ]; + size_t pass_len, ikm_len; + int ip_len, i; + + /* Parse target_ip */ + ip_len = (int)strnlen(p, CC_MAX_IP_LEN); + if (p + ip_len >= end) return; + memcpy(target_ip, p, ip_len); target_ip[ip_len] = '\0'; + p += ip_len + 1; + + /* Only process if addressed to us */ + if (strcmp(target_ip, my_ip) != 0) return; + + /* Masters are the key source - they never accept KEY_GRANTs. + * Without this guard a stale KEY_GRANT (from the previous master, + * responding to a JOIN_REQ we sent while CC_NODE_NEW) can arrive + * after we self-promoted via cc_on_join_tfd and overwrite the + * session key we just generated, causing a key-mismatch loop. */ + { + int _im; + lock_start_read(cl->peers->lock); + _im = cc_i_am_master_locked(cl); + lock_stop_read(cl->peers->lock); + if (_im) { + LM_DBG("clusterer_controller: KEY_GRANT from %s ignored - " + "I am master\n", sender_ip); + return; + } + } + + if (p + CC_PUBKEY_SZ + CC_JOIN_NONCE_SZ + CC_MASTER_SALT_SZ > end) { + LM_WARN("clusterer_controller: KEY_GRANT too short\n"); + return; + } + memcpy(master_pubkey, p, CC_PUBKEY_SZ); p += CC_PUBKEY_SZ; + memcpy(echoed_nonce, p, CC_JOIN_NONCE_SZ); p += CC_JOIN_NONCE_SZ; + memcpy(wrapped, p, CC_MASTER_SALT_SZ); + + /* Verify echoed nonce matches what we sent in JOIN_REQ */ + if (memcmp(echoed_nonce, cl->my_join_nonce, CC_JOIN_NONCE_SZ) != 0) { + LM_WARN("clusterer_controller: KEY_GRANT join_nonce mismatch - dropping\n"); + /* Clear join_pending so the next decryption failure or rejoin_tfd + * can issue a fresh JOIN_REQ. Without this the node stays blocked + * on a nonce that will never match (e.g. overwritten by rejoin + * timer or a master-election cycle between send and receipt). */ + cl->join_pending = 0; + return; + } + + /* Recover master_salt: XOR wrapped with HKDF(ECDH(my_priv, master_pub) || password || nonce) */ + if (cc_ecdh_shared(cl->my_privkey, master_pubkey, ss) < 0) return; + pass_len = strlen(cl->password); + if (pass_len > 1024) pass_len = 1024; + memcpy(ikm, ss, CC_PUBKEY_SZ); + memcpy(ikm + CC_PUBKEY_SZ, cl->password, pass_len); + memcpy(ikm + CC_PUBKEY_SZ + pass_len, echoed_nonce, CC_JOIN_NONCE_SZ); + ikm_len = CC_PUBKEY_SZ + pass_len + CC_JOIN_NONCE_SZ; + if (cc_hkdf_sha256(ikm, ikm_len, + ss, CC_PUBKEY_SZ, "cc_key_grant", wrap_key) < 0) return; + for (i = 0; i < CC_MASTER_SALT_SZ; i++) + new_salt[i] = wrapped[i] ^ wrap_key[i]; + + /* Apply: write to shm, derive session_key */ + lock_start_write(cl->peers->lock); + memcpy(cl->peers->master_salt, new_salt, CC_MASTER_SALT_SZ); + cc_derive_session_key(cl); + lock_stop_write(cl->peers->lock); + + cl->join_pending = 0; + cl->bootstrap_auth_fails = 0; + cl->join_attempt_count = 0; + cl->auth_fail_pkts = 0; /* authenticated: clear wrong-password evidence */ + cl->auth_defer_count = 0; + LM_INFO("clusterer_controller: [cluster %d] KEY_GRANT from %s - " + "session key updated\n", cl->cluster_id, sender_ip); +} + +/** + * cc_handle_key_handoff() - process CC_PKT_KEY_HANDOFF. + * Only acted on by the node that wins the next election (highest IP). + * Payload: [next_master_ip NUL][sender_pubkey 32B][wrapped_salt 32B] + */ +static void cc_handle_key_handoff(const char *payload, int payload_len, + const char *sender_ip, cc_cluster_t *cl) +{ + const char *p = payload; + const char *end = payload + payload_len; + char target_ip[CC_MAX_IP_LEN + 1]; + unsigned char sender_pubkey[CC_PUBKEY_SZ]; + unsigned char wrapped[CC_MASTER_SALT_SZ]; + unsigned char ss[CC_PUBKEY_SZ]; + unsigned char ikm[CC_PUBKEY_SZ + 1024]; + unsigned char wrap_key[32]; + unsigned char new_salt[CC_MASTER_SALT_SZ]; + size_t pass_len; + int ip_len, i; + + ip_len = (int)strnlen(p, CC_MAX_IP_LEN); + if (p + ip_len >= end) return; + memcpy(target_ip, p, ip_len); target_ip[ip_len] = '\0'; + p += ip_len + 1; + + /* Only the intended next master unwraps this */ + if (strcmp(target_ip, my_ip) != 0) return; + + if (p + CC_PUBKEY_SZ + CC_MASTER_SALT_SZ > end) { + LM_WARN("clusterer_controller: KEY_HANDOFF too short\n"); + return; + } + memcpy(sender_pubkey, p, CC_PUBKEY_SZ); p += CC_PUBKEY_SZ; + memcpy(wrapped, p, CC_MASTER_SALT_SZ); + + if (cc_ecdh_shared(cl->my_privkey, sender_pubkey, ss) < 0) return; + pass_len = strlen(cl->password); + if (pass_len > 1024) pass_len = 1024; + memcpy(ikm, ss, CC_PUBKEY_SZ); + memcpy(ikm + CC_PUBKEY_SZ, cl->password, pass_len); + if (cc_hkdf_sha256(ikm, CC_PUBKEY_SZ + pass_len, + ss, CC_PUBKEY_SZ, "cc_key_handoff", wrap_key) < 0) return; + for (i = 0; i < CC_MASTER_SALT_SZ; i++) + new_salt[i] = wrapped[i] ^ wrap_key[i]; + + /* Store salt and re-derive so session_key is guaranteed consistent with + * the adopted salt (the incoming master's salt may differ from ours). */ + lock_start_write(cl->peers->lock); + memcpy(cl->peers->master_salt, new_salt, CC_MASTER_SALT_SZ); + cc_derive_session_key(cl); /* sets have_session_key = 1 */ + lock_stop_write(cl->peers->lock); + + LM_INFO("clusterer_controller: [cluster %d] KEY_HANDOFF from %s - " + "master_salt preserved for seamless transition\n", + cl->cluster_id, sender_ip); +} + +/* ========================================================================= + * Receive dispatcher + * ========================================================================= */ + +/** + * cc_rate_check() - per-source-IP rate limiter, called before decryption. + * Finds or creates a 1-second sliding-window counter for src_ip. + * @return 0 if within CC_RATE_LIMIT packets/s, -1 to drop. + */ +static int cc_rate_check(cc_cluster_t *cl, uint32_t src_ip) +{ + time_t now = time(NULL); + cc_rate_entry_t *oldest = NULL; + int i; + + for (i = 0; i < CC_RATE_TBL_SZ; i++) { + cc_rate_entry_t *e = &cl->rate_tbl[i]; + if (e->ip == 0) { + if (!oldest) oldest = e; /* prefer empty slot */ + continue; + } + if (e->ip == src_ip) { + if (now > e->window_start) { /* new second */ + e->window_start = now; + e->count = 1; + return 0; + } + if (++e->count > CC_RATE_LIMIT) + return -1; + return 0; + } + /* track oldest entry for eviction when table is full */ + if (!oldest || e->window_start < oldest->window_start) + oldest = e; + } + + /* new source IP - claim oldest/empty slot */ + oldest->ip = src_ip; + oldest->window_start = now; + oldest->count = 1; + return 0; +} + +/* -- Join-reject helpers ---------------------------------------------------- + * + * Security model: JOIN_REJECT is sent as a normal CC_BOOTSTRAP_MAGIC packet + * (AES-256-GCM authenticated with the bootstrap key). An attacker without + * the cluster password cannot forge a GCM-authenticated reject, so they + * cannot kick nodes out or block joins. cc_handle_join_reject() also guards + * on CC_NODE_NEW state so that even a legitimate cluster member with the + * correct password cannot send a JOIN_REJECT to an already-active node. + * + * cc_join_fail_check() - master: track per-IP BOOTSTRAP_MAGIC decrypt failures. + * Returns 1 the first time a source IP reaches CC_JOIN_FAIL_LIMIT failures. + * + * cc_send_join_reject() - master: send encrypted JOIN_REJECT via BOOTSTRAP_MAGIC. + * Wire payload: [target_ip NUL] + * + * cc_handle_join_reject() - joiner: stop OpenSIPS if the reject is for us and + * we are still in CC_NODE_NEW (i.e., have not successfully joined yet). + * + * WRONG-PASSWORD fallback: if the joiner has the wrong password it cannot + * decrypt the encrypted JOIN_REJECT. Instead it detects the situation + * through bootstrap_auth_fails (incremented on any BOOTSTRAP_MAGIC decrypt + * failure during CC_NODE_NEW) combined with join_attempt_count. After + * CC_JOIN_FAIL_LIMIT rejoin retries with at least one bootstrap failure + * observed, the joiner concludes the master rejected it and exits. + */ + +static int cc_join_fail_check(const char *src_ip, cc_cluster_t *cl) +{ + uint32_t ip_num = ip_to_num(src_ip); + int evict = 0; /* index of lowest-count slot for eviction */ + int i; + + if (ip_num == 0) + return 0; + + for (i = 0; i < CC_JOIN_FAIL_TABLE_SZ; i++) { + if (cl->join_fail_tbl[i].ip_num != ip_num) + continue; + if (cl->join_fail_tbl[i].rejected) + return 0; /* reject already sent; don't repeat */ + cl->join_fail_tbl[i].count++; + if (cl->join_fail_tbl[i].count >= CC_JOIN_FAIL_LIMIT) { + cl->join_fail_tbl[i].rejected = 1; + return 1; + } + return 0; + } + + /* Not found - insert, evicting the slot with the smallest count */ + for (i = 1; i < CC_JOIN_FAIL_TABLE_SZ; i++) { + if (cl->join_fail_tbl[i].count < cl->join_fail_tbl[evict].count) + evict = i; + } + memset(&cl->join_fail_tbl[evict], 0, sizeof(cl->join_fail_tbl[evict])); + cl->join_fail_tbl[evict].ip_num = ip_num; + snprintf(cl->join_fail_tbl[evict].ip, sizeof(cl->join_fail_tbl[evict].ip), + "%s", src_ip); + cl->join_fail_tbl[evict].count = 1; + return 0; +} + +static void cc_send_join_reject(int sock, const char *target_ip, cc_cluster_t *cl, + int reason) +{ + char pkt[CC_SMALL_PKT_SZ + 1]; /* +1 for the reason byte */ + uint32_t seq = htonl(++cl->peers->my_seq); + int ip_len, plain_len, total_len; + struct sockaddr_in dest; + + ip_len = (int)strnlen(target_ip, CC_MAX_IP_LEN); + + memcpy(pkt, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ); + pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_JOIN_REJECT; + memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + memcpy(pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ, target_ip, ip_len); + pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + ip_len] = '\0'; + /* reason byte follows the NUL-terminated target IP */ + pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + ip_len + 1] = (char)reason; + + plain_len = CC_PLAIN_HDR_SZ + ip_len + 1 + 1; + total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->key, cl->cluster_id); + if (total_len < 0) return; + + memset(&dest, 0, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons((uint16_t)cl->multicast_port); + dest.sin_addr.s_addr = inet_addr(cl->multicast_address); + + if (sendto(sock, pkt, total_len, 0, + (struct sockaddr *)&dest, sizeof(dest)) < 0) + LM_ERR("clusterer_controller: [cluster %d] sendto JOIN_REJECT failed: %s\n", + cl->cluster_id, strerror(errno)); + else + LM_WARN("clusterer_controller: [cluster %d] sent JOIN_REJECT to %s (%s)\n", + cl->cluster_id, target_ip, + reason == CC_REJECT_CONFIG ? "different cluster settings" + : "repeated auth failure - wrong password?"); +} + +static void cc_handle_join_reject(const char *payload, int payload_len, + const char *sender_ip, cc_cluster_t *cl) +{ + char target_ip[CC_MAX_IP_LEN + 1]; + int l, still_new; + + /* Only act during the initial join phase. An active member receiving a + * JOIN_REJECT (e.g. from a cluster peer with the correct password who + * wanted to test the mechanism, or a stale in-flight packet) must ignore + * it - this prevents any cluster member from silently evicting another. */ + lock_start_read(cl->peers->lock); + still_new = (cl->peers->node_state == CC_NODE_NEW); + lock_stop_read(cl->peers->lock); + if (!still_new) return; + + l = (int)strnlen(payload, CC_MAX_IP_LEN); + if (l >= payload_len) return; + memcpy(target_ip, payload, l); + target_ip[l] = '\0'; + + if (strcmp(target_ip, my_ip) != 0) + return; /* not addressed to this node */ + + /* reason byte follows the NUL-terminated target IP (older senders omit it) */ + { + int reason = CC_REJECT_GENERIC; + if (payload_len > l + 1) + reason = (unsigned char)payload[l + 1]; + if (reason == CC_REJECT_CONFIG) + LM_CRIT("clusterer_controller: [cluster %d] JOIN_REJECT from %s - the " + "running cluster has different settings than this node; fix the " + "local config (manage_shtags/master_stickiness/query_time) to " + "match and restart; shutting down\n", + cl->cluster_id, sender_ip); + else + LM_CRIT("clusterer_controller: [cluster %d] JOIN_REJECT from %s - " + "wrong password or unauthorized node; shutting down\n", + cl->cluster_id, sender_ip); + } + exit(-1); +} + +/** + * cc_recv_one() - read one datagram, validate header, dispatch by type. + */ +static void cc_recv_one(int sock, cc_cluster_t *cl) +{ + /* Static buffer: avoids a 64 KB stack frame; safe because cc_recv_one + * is called only from the single-threaded cc_worker process. */ + static char buf[CC_RECV_BUF_SZ]; + struct sockaddr_in src_addr; + socklen_t src_len = sizeof(src_addr); + ssize_t n; + unsigned char pkt_type; + const char *payload; + int payload_len; + + n = recvfrom(sock, buf, sizeof(buf) - 1, 0, + (struct sockaddr *)&src_addr, &src_len); + if (n < 0) { + if (errno != EAGAIN && errno != EWOULDBLOCK) + LM_ERR("clusterer_controller: recvfrom(): %s\n", strerror(errno)); + return; + } + + /* Minimum: magic(2)+cluster_id(2)+nonce(12)+type(1)+seq(4)+tag(16) = 37 */ + if (n < CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_TAG_SZ) { + LM_WARN("clusterer_controller: short packet (%zd bytes), dropping\n", + n); + return; + } + + if (memcmp(buf, CC_PACKET_MAGIC, CC_MAGIC_SZ) != 0 && + memcmp(buf, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ) != 0) { + LM_DBG("clusterer_controller: bad magic, dropping\n"); + return; + } + + /* Cleartext cluster_id filter: silently ignore packets for a different + * cluster sharing this multicast group. Done BEFORE decryption so foreign + * traffic never counts as an auth failure (auth_fail_pkts) - a wrong-cluster + * packet is not a wrong-password attempt. */ + { + uint16_t pkt_cid_be; + memcpy(&pkt_cid_be, buf + CC_MAGIC_SZ, CC_CLUSTER_ID_SZ); + if (ntohs(pkt_cid_be) != (uint16_t)cl->cluster_id) { + LM_DBG("clusterer_controller: [cluster %d] ignoring packet for " + "cluster %u on shared group\n", cl->cluster_id, + ntohs(pkt_cid_be)); + return; + } + } + + /* Rate-limit before any crypto work to shed floods cheaply. */ + if (cc_rate_check(cl, src_addr.sin_addr.s_addr) < 0) + return; + + /* Resolve sender IP once - used for HMAC warning and MEMBER_LIST dispatch */ + { + char sender_ip_buf[INET_ADDRSTRLEN]; + const unsigned char *dec_key; + inet_ntop(AF_INET, &src_addr.sin_addr, + sender_ip_buf, sizeof(sender_ip_buf)); + + /* Select decryption key by magic: + * CC_BOOTSTRAP_MAGIC -> bootstrap key (JOIN_REQ, KEY_GRANT) + * CC_PACKET_MAGIC -> session key (all normal traffic) + * If session key decryption fails, schedule a re-JOIN to refresh it. */ + int is_bootstrap = (memcmp(buf, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ) == 0); + dec_key = is_bootstrap ? cl->key : cl->session_key; + + if (cc_decrypt_pkt(buf, n, sender_ip_buf, dec_key, is_bootstrap) < 0) { + int _im, _new; + char _lm[CC_MAX_IP_LEN + 1]; + lock_start_read(cl->peers->lock); + _im = cc_i_am_master_locked(cl); + _new = (cl->peers->node_state == CC_NODE_NEW); + memcpy(_lm, cl->peers->last_master, sizeof(_lm)); + lock_stop_read(cl->peers->lock); + + /* A packet from another peer we cannot decrypt, seen while still + * joining, means a cluster (or rogue) whose key we do not share is + * present on this group. We count it (any magic) as evidence that we + * may not belong here: session-key failures matter too, because a + * running cluster's steady traffic (MASTER_ALIVE/MEMBER_LIST) is + * session-encrypted, so a wrong-password joiner would otherwise see too + * little bootstrap traffic to notice the cluster and would self-promote + * into a split-brain lone master. This is only *evidence*, never an + * immediate death sentence: cc_on_join_tfd defers and keeps re-joining, + * and a KEY_GRANT arriving in the grace window resets this counter, so a + * healthy node whose key is merely slow is not affected. Our own + * loopback decrypts fine, so guard on my_ip. */ + if (_new && strcmp(sender_ip_buf, my_ip) != 0) + cl->auth_fail_pkts++; + + if (memcmp(buf, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ) == 0) { + + /* Master: track per-IP bootstrap failures; send JOIN_REJECT on limit. + * JOIN_REJECT is encrypted (BOOTSTRAP_MAGIC/GCM) so only nodes with + * the correct password can read it. Forgeries are impossible without + * the bootstrap key. */ + if (_im && cc_join_fail_check(sender_ip_buf, cl)) + cc_send_join_reject(sock, sender_ip_buf, cl, CC_REJECT_GENERIC); + + /* Joiner fallback: count bootstrap decrypt failures during CC_NODE_NEW + * only when the packet came from the known master. This filters out + * rogue nodes on the multicast group whose JOIN_REQs (encrypted with + * their own wrong key) would otherwise increment this counter and + * trigger a spurious exit on a legitimate joining node. + * If last_master is empty we have no master reference yet, so we + * conservatively skip counting - no master means no rejection. */ + if (_new && _lm[0] != '\0' && strcmp(sender_ip_buf, _lm) == 0) + cl->bootstrap_auth_fails++; + } + if (memcmp(buf, CC_PACKET_MAGIC, CC_MAGIC_SZ) == 0 && !cl->join_pending) { + /* Session key mismatch: request a re-key ONLY when the packet we + * could not decrypt came from OUR current master (a legitimate key + * rotation). Undecryptable session packets from any other source + * are rogue or stale traffic (e.g. a wrong-password node broadcasting + * on the group); reacting to them would drive an endless re-JOIN + * churn across the whole cluster. Masters never re-JOIN. */ + if (!_im && _lm[0] != '\0' && strcmp(sender_ip_buf, _lm) == 0) { + LM_INFO("clusterer_controller: [cluster %d] session key mismatch " + "from master %s - sending JOIN_REQ to re-key\n", + cl->cluster_id, sender_ip_buf); + cl->join_pending = 1; + cc_send_join_req(cl->sock, cl); + } + } + return; + } + + /* Sequence check: reject replays for all session-key packets including + * GOODBYE. my_seq lives in shm so mod_destroy increments the same + * counter the worker uses - GOODBYE gets a valid monotonic seq number + * without any special-casing. + * Bootstrap packets (CC_BOOTSTRAP_MAGIC) use join_nonce instead. */ + if (memcmp(buf, CC_PACKET_MAGIC, CC_MAGIC_SZ) == 0) { + uint32_t pkt_seq; + memcpy(&pkt_seq, buf + CC_WIRE_HDR_SZ + 1, CC_SEQ_SZ); + pkt_seq = ntohl(pkt_seq); + if (cc_check_and_update_seq(sender_ip_buf, pkt_seq, cl) < 0) + return; + } + + pkt_type = (unsigned char)buf[CC_WIRE_HDR_SZ]; + payload = buf + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + payload_len = (int)(n - CC_WIRE_HDR_SZ - CC_PLAIN_HDR_SZ - CC_TAG_SZ); + + if (payload_len < 0) { + LM_WARN("clusterer_controller: empty payload from %s, dropping\n", + sender_ip_buf); + return; + } + + + switch (pkt_type) { + + case CC_PKT_ALIVE: { + char ip_buf[CC_MAX_IP_LEN + 1]; + int ip_len = (int)strnlen(payload, CC_MAX_IP_LEN); + const unsigned char *pubkey = NULL; + int cfg_present = 0, p_manage = 0, p_stick = 0, p_qt = 0; + memcpy(ip_buf, payload, ip_len); + ip_buf[ip_len] = '\0'; + /* Pubkey appended after NUL-terminated IP */ + if (payload_len >= ip_len + 1 + (int)CC_PUBKEY_SZ) + pubkey = (const unsigned char *)payload + ip_len + 1; + /* Config descriptor appended after the pubkey (optional) */ + if (payload_len >= ip_len + 1 + (int)CC_PUBKEY_SZ + CC_CONFIG_SZ) { + const char *c = payload + ip_len + 1 + CC_PUBKEY_SZ; + uint16_t qt_be; + p_manage = (unsigned char)c[0]; + p_stick = (unsigned char)c[1]; + memcpy(&qt_be, c + 2, 2); + p_qt = ntohs(qt_be); + cfg_present = 1; + } + cc_handle_alive(ip_buf, pubkey, cfg_present, p_manage, p_stick, p_qt, cl); + break; + } + + case CC_PKT_JOIN_REQ: + cc_handle_join_req(sock, payload, payload_len, cl); + break; + + case CC_PKT_MEMBER_LIST: + cc_handle_member_list(payload, payload_len, sender_ip_buf, cl); + break; + + case CC_PKT_GOODBYE: { + char ip_buf[CC_MAX_IP_LEN + 1]; + int ip_len = payload_len > CC_MAX_IP_LEN ? CC_MAX_IP_LEN : payload_len; + memcpy(ip_buf, payload, ip_len); + ip_buf[ip_len] = '\0'; + cc_handle_goodbye(sock, ip_buf, cl); + break; + } + + case CC_PKT_NODE_ASSIGN: + cc_handle_node_assign(payload, payload_len, sender_ip_buf, cl); + break; + + case CC_PKT_MASTER_ALIVE: + cc_handle_master_alive(sender_ip_buf, cl); + break; + + case CC_PKT_KEY_GRANT: + cc_handle_key_grant(payload, payload_len, sender_ip_buf, cl); + break; + + case CC_PKT_KEY_HANDOFF: + cc_handle_key_handoff(payload, payload_len, sender_ip_buf, cl); + break; + + case CC_PKT_JOIN_REJECT: + cc_handle_join_reject(payload, payload_len, sender_ip_buf, cl); + break; + + case CC_PKT_MASTER_BEACON: { + uint16_t cnt_be, sender_count = 0; + if (payload_len >= 2) { + memcpy(&cnt_be, payload, 2); + sender_count = ntohs(cnt_be); + } + cc_handle_master_beacon(sender_ip_buf, sender_count, cl); + break; + } + + default: + LM_WARN("clusterer_controller: unknown packet type 0x%02x " + "from %s, dropping\n", pkt_type, sender_ip_buf); + break; + } + } +} + +/* ========================================================================= + * Background worker process + * ========================================================================= */ + +/* ========================================================================= + * Reactor callbacks - one per event source + * ========================================================================= */ + +static int cc_on_sock(int fd, void *param, int was_timeout) +{ + cc_recv_one(fd, (cc_cluster_t *)param); + return 0; +} + +static int cc_on_alive_tfd(int fd, void *param, int was_timeout) +{ + cc_cluster_t *cl = (cc_cluster_t *)param; + int prev_master, now_master; + cc_drain_tfd(fd); + cc_send_alive(cl->sock, cl); + lock_start_write(cl->peers->lock); + prev_master = cc_i_am_master_locked(cl); + cc_prune_stale(cl); + cc_elect_master(cl); + now_master = cc_i_am_master_locked(cl); + lock_stop_write(cl->peers->lock); + /* Do not broadcast MASTER_ALIVE before we hold the cluster key - + * request a re-key from the current key-holder instead. */ + if (now_master && !cl->have_session_key) { + cc_request_rekey(cl); + return 0; + } + if (prev_master != now_master) + cc_arm_master_timers(cl, now_master); + + /* Belt-and-suspenders identity registration (covers paths where + * cc_handle_node_assign has not yet run). */ + if (!cl->identity_registered && clctl_loaded && my_node_id > 0) { + str url = {cl->bin_socket, (int)strlen(cl->bin_socket)}; + clctl.update_identity(cl->cluster_id, my_node_id, &url); + cl->identity_registered = 1; + } + + /* Bootstrap shtag activation: only for fresh clusters + * (shtag_bootstrapped == -1). Retries until we are master. */ + if (cl->manage_shtags && clctl_loaded && cl->shtag_bootstrapped == -1 + && clctl.activate_backup_shtags) { + int _im; + lock_start_read(cl->peers->lock); + _im = cc_i_am_master_locked(cl); + lock_stop_read(cl->peers->lock); + if (_im) { + cc_apply_shtags(cl); /* override-aware */ + cl->shtag_bootstrapped = 1; + } + } + return 0; +} + +static int cc_on_join_tfd(int fd, void *param, int was_timeout) +{ + cc_cluster_t *cl = (cc_cluster_t *)param; + cc_drain_tfd(fd); + + int was_new, auth_fails; + lock_start_write(cl->peers->lock); + was_new = (cl->peers->node_state == CC_NODE_NEW); + auth_fails = cl->auth_fail_pkts; + + /* Wrong-password / unauthorized guard: if we are still joining at the + * deadline AND we received packets from peers we could not decrypt, then a + * cluster whose key we do not share exists on this multicast group. + * Self-promoting would create a split-brain lone master (and, with managed + * shtags, a duplicate active tag). A wrong-password node also cannot read + * the master's JOIN_REJECT, so this is where it must shut down. */ + if (was_new && auth_fails >= CC_JOIN_FAIL_LIMIT) { + /* We could not decrypt traffic from peers on our cluster_id: a cluster + * using a different password (or we do) shares this group. Do NOT + * self-terminate on the first deadline - a KEY_GRANT that is merely slow, a + * brief burst of start-up noise, or a flood of crafted garbage would + * otherwise kill a healthy node. Defer and keep re-joining: a real + * KEY_GRANT arriving in the grace window resets auth_fail_pkts + * (cc_handle_key_grant) and we join normally. Only give up after + * CC_JOIN_DEFER_MAX rounds still short of authentication - by then the + * wrong-password / foreign-cluster condition is sustained, not transient, + * and self-promoting would create a split-brain lone master. */ + if (cl->auth_defer_count < CC_JOIN_DEFER_MAX) { + cl->auth_defer_count++; + lock_stop_write(cl->peers->lock); + LM_WARN("clusterer_controller: [cluster %d] %d undecryptable packet(s) " + "on cluster_id %d (%s:%d) - cannot authenticate yet, deferring " + "self-promotion (%d/%d); a slow KEY_GRANT would clear this\n", + cl->cluster_id, auth_fails, cl->cluster_id, + cl->multicast_address, cl->multicast_port, + cl->auth_defer_count, CC_JOIN_DEFER_MAX); + cl->join_pending = 0; + cc_send_join_req(cl->sock, cl); + cc_arm_tfd(cl->join_tfd, CC_JOIN_DEFER_SECS, 0); + return 0; + } + lock_stop_write(cl->peers->lock); + LM_CRIT("clusterer_controller: [cluster %d] cannot authenticate on %s:%d - " + "%d packet(s) stayed undecryptable and no KEY_GRANT arrived across %d " + "re-join round(s) (wrong password, or a foreign cluster on " + "cluster_id %d). Shutting down.\n", + cl->cluster_id, cl->multicast_address, cl->multicast_port, + auth_fails, cl->auth_defer_count, cl->cluster_id); + exit(-1); + } + + /* Split-brain PREVENTION: if a higher-IP node is also still joining (we + * learned it from its JOIN_REQ), defer self-promotion so it becomes the + * single master and we join it, rather than both self-promoting with + * independent session keys. Bounded by CC_JOIN_DEFER_MAX so a higher-IP + * node that was heard but never finished starting cannot stall us. */ + if (was_new && cl->join_defer_count < CC_JOIN_DEFER_MAX + && cl->join_defer_total < CC_JOIN_DEFER_HARDMAX) { + unsigned int my_ipn = ip_to_num(my_ip); + int higher_seen = 0, _i; + for (_i = 0; _i < cl->peers->count; _i++) { + if (strcmp(cl->peers->entries[_i].ip, my_ip) == 0) + continue; + if (cl->peers->entries[_i].ip_num > my_ipn) { + higher_seen = 1; + break; + } + } + if (higher_seen) { + cl->join_defer_count++; + cl->join_defer_total++; + lock_stop_write(cl->peers->lock); + LM_INFO("clusterer_controller: [cluster %d] join deadline: a higher-IP " + "node is still joining - deferring self-promotion (%d/%d) to " + "avoid split brain\n", + cl->cluster_id, cl->join_defer_count, CC_JOIN_DEFER_MAX); + /* Re-send a JOIN_REQ now (clear join_pending so it is not suppressed) + * so the higher-IP node answers as soon as it becomes master, then + * extend the join window for one more short round. */ + cl->join_pending = 0; + cc_send_join_req(cl->sock, cl); + cc_arm_tfd(cl->join_tfd, CC_JOIN_DEFER_SECS, 0); + return 0; + } + } + + if (was_new) { + LM_INFO("clusterer_controller: [cluster %d] join deadline expired, " + "no master found - transitioning to CC_NODE_ACTIVE\n", + cl->cluster_id); + cl->join_defer_count = 0; /* leaving NEW state */ + cl->join_defer_total = 0; + cl->shtag_bootstrapped = -1; /* fresh cluster: eligible to claim active */ + cc_upsert_peer_locked(my_ip, cl); + my_node_id = cc_alloc_node_id_locked(cl); + { + char self_sock[1][CC_MAX_BIN_SOCK_LEN]; + memcpy(self_sock[0], cl->bin_socket, CC_MAX_BIN_SOCK_LEN); + cc_update_peer_bin_locked(my_ip, my_node_id, 1, + (const char (*)[CC_MAX_BIN_SOCK_LEN]) + self_sock, cl); + } + cl->peers->node_state = CC_NODE_ACTIVE; + /* Run election first so is_master and last_master are set before + * cc_on_became_master. Without this, cc_handle_alive would see + * is_master=0 on the loopback ALIVE and call cc_on_became_master + * a second time, regenerating the session key unnecessarily. */ + cc_elect_master(cl); + cc_on_became_master(cl); + } + lock_stop_write(cl->peers->lock); + if (!was_new) + return 0; /* already active via MEMBER_LIST - nothing to do */ + cc_transition_to_active(cl); + cc_arm_master_timers(cl, 1); + + if (!cl->identity_registered && clctl_loaded && my_node_id > 0) { + str url = {cl->bin_socket, (int)strlen(cl->bin_socket)}; + clctl.update_identity(cl->cluster_id, my_node_id, &url); + cl->identity_registered = 1; + } + return 0; +} + +static int cc_on_rejoin_tfd(int fd, void *param, int was_timeout) +{ + cc_cluster_t *cl = (cc_cluster_t *)param; + int still_new; + + cc_drain_tfd(fd); + lock_start_read(cl->peers->lock); + still_new = (cl->peers->node_state == CC_NODE_NEW); + lock_stop_read(cl->peers->lock); + if (still_new && !cl->join_pending) { + cl->join_attempt_count++; + + /* Wrong-password fallback: if the master keeps sending bootstrap + * packets we cannot decrypt (bootstrap_auth_fails > 0) and we have + * already retried CC_JOIN_FAIL_LIMIT times, give up. This fires when + * the encrypted JOIN_REJECT from the master was lost in transit (if it + * arrived intact, cc_handle_join_reject would have already exited). */ + if (cl->join_attempt_count >= CC_JOIN_FAIL_LIMIT + && cl->bootstrap_auth_fails > 0) { + LM_CRIT("clusterer_controller: [cluster %d] join failed after %d " + "attempts with %d bootstrap auth error(s) - wrong password? " + "Shutting down.\n", + cl->cluster_id, cl->join_attempt_count, + cl->bootstrap_auth_fails); + exit(-1); + } + + cc_send_join_req(cl->sock, cl); + cl->join_pending = 1; + LM_DBG("clusterer_controller: [cluster %d] resending JOIN_REQ " + "(attempt %d)\n", cl->cluster_id, cl->join_attempt_count); + } + return 0; +} + +static int cc_on_master_alive_tfd(int fd, void *param, int was_timeout) +{ + cc_cluster_t *cl = (cc_cluster_t *)param; + cc_drain_tfd(fd); + cc_send_master_alive(cl->sock, cl); + /* Emit a bootstrap-key beacon every CC_MASTER_BEACON_EVERY ticks so any + * peer master holding a different session key can find us and merge. */ + if (++cl->beacon_tick >= CC_MASTER_BEACON_EVERY) { + cl->beacon_tick = 0; + cc_send_master_beacon(cl->sock, cl); + } + return 0; +} + +static int cc_on_master_dead_tfd(int fd, void *param, int was_timeout) +{ + cc_cluster_t *cl = (cc_cluster_t *)param; + int now_master; + char dead_master[CC_MAX_IP_LEN + 1]; + + cc_drain_tfd(fd); + + dead_master[0] = '\0'; + + lock_start_write(cl->peers->lock); + /* The master has been silent for CC_MASTER_KA_TIMEOUT (3s). Age the silent + * master OUT of the election window before re-electing, otherwise + * cc_elect_master would just re-select it: the election window is + * query_time * CC_ELECT_FACTOR (~15s), far longer than the keepalive + * timeout, so a just-declared-dead master stays "eligible" and keeps + * winning - delaying real failover by ~12s. Zeroing its last_seen makes + * the immediate re-election pick the next-highest LIVE peer. If the + * master was only briefly unreachable, its next MASTER_ALIVE/ALIVE + * refreshes last_seen and the normal highest-IP election restores it. */ + { + int _i; + for (_i = 0; _i < cl->peers->count; _i++) { + if (cl->peers->entries[_i].is_master && + strcmp(cl->peers->entries[_i].ip, my_ip) != 0) { + size_t _l = strnlen(cl->peers->entries[_i].ip, CC_MAX_IP_LEN); + memcpy(dead_master, cl->peers->entries[_i].ip, _l); + dead_master[_l] = '\0'; + cl->peers->entries[_i].last_seen = 0; + cl->peers->entries[_i].in_election = 0; + } + cl->peers->entries[_i].is_master = 0; + } + cl->peers->last_master[0] = '\0'; + } + + LM_INFO("clusterer_controller: [cluster %d] master %s went silent " + "(no keepalive for %ds) - re-electing\n", + cl->cluster_id, + dead_master[0] ? dead_master : "(unknown)", + CC_MASTER_KA_TIMEOUT); + + /* cc_elect_master logs the resulting MASTER/BACKUP roles and why. */ + cc_elect_master(cl); + now_master = cc_i_am_master_locked(cl); + lock_stop_write(cl->peers->lock); + + /* Preserve-key recovery: every surviving member already holds the session + * key, so the winner has it too. Guard defensively against the impossible + * keyless-winner case rather than broadcasting an undecryptable MEMBER_LIST. */ + if (now_master && !cl->have_session_key) { + LM_WARN("clusterer_controller: [cluster %d] elected master after " + "keepalive timeout but hold no session key - deferring\n", + cl->cluster_id); + return 0; + } + + cc_arm_master_timers(cl, now_master); + + /* The new master announces itself; all members already hold the session + * key so no re-keying is needed. */ + if (now_master) + cc_send_member_list(cl->sock, cl); + return 0; +} + +/** + * cc_worker() - the single dedicated background process. + * + * JOIN PROTOCOL: + * 1. Open socket, join multicast group. + * 2. Send CC_PKT_JOIN_REQ and set state = CC_NODE_NEW with a deadline + * of (now + query_time). + * 3. Listen for incoming packets. If CC_PKT_MEMBER_LIST arrives: + * -> cc_handle_member_list() sets state = CC_NODE_ACTIVE. + * If deadline expires with no MEMBER_LIST: + * -> no master exists yet; transition to CC_NODE_ACTIVE and + * join the normal election cycle. + * + * ACTIVE LOOP: + * Fully event-driven via OpenSIPS reactor (epoll by default). + * Each event source is a registered fd with a dedicated callback: + * cc_on_sock - incoming UDP packet + * cc_on_alive_tfd - periodic ALIVE heartbeat (query_time seconds) + * cc_on_join_tfd - one-shot join deadline + * cc_on_rejoin_tfd - 1-second JOIN_REQ retry while in CC_NODE_NEW + * reactor_proc_init() also wires in IPC (shutdown, load stats, reload). + */ +static void cc_worker(int rank) +{ + cc_cluster_t *cl; + + if (rank >= cc_cluster_count) { + /* Extra process slot - no cluster assigned, exit cleanly */ + return; + } + cl = &cc_clusters[rank]; + + LM_INFO("clusterer_controller: [cluster %d] worker started (pid=%d)\n", + cl->cluster_id, getpid()); + + /* Publish our process index so MI handlers (other processes) can reach us + * via ipc_send_rpc() to apply operator-driven shtag overrides promptly. */ + cl->peers->worker_proc_no = process_no; + + cl->shtag_last_active = -1; /* unknown - first decision logs its reason */ + cl->shtag_last_forced = 0; + + cl->sock = cc_setup_socket(cl); + if (cl->sock < 0) { + LM_CRIT("clusterer_controller: [cluster %d] cannot open multicast socket, " + "worker exits\n", cl->cluster_id); + exit(-1); + } + + /* Generate ephemeral X25519 keypair - private key never leaves this process */ + if (cc_gen_ecdh_keypair(cl->my_privkey, cl->my_pubkey) < 0) { + LM_CRIT("clusterer_controller: [cluster %d] ECDH keypair generation failed\n", + cl->cluster_id); + exit(-1); + } + + /* Store our pubkey in peer table entry for ourselves so MEMBER_LIST broadcasts it */ + lock_start_write(cl->peers->lock); + { + int _i; + for (_i = 0; _i < cl->peers->count; _i++) { + if (strcmp(cl->peers->entries[_i].ip, my_ip) == 0) { + memcpy(cl->peers->entries[_i].pubkey, cl->my_pubkey, CC_PUBKEY_SZ); + break; + } + } + } + lock_stop_write(cl->peers->lock); + + cl->alive_tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); + cl->join_tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); + cl->rejoin_tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); + cl->master_alive_tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); + cl->master_dead_tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); + if (cl->alive_tfd < 0 || cl->join_tfd < 0 || cl->rejoin_tfd < 0 || + cl->master_alive_tfd < 0 || cl->master_dead_tfd < 0) { + LM_CRIT("clusterer_controller: [cluster %d] timerfd_create: %s\n", + cl->cluster_id, strerror(errno)); + exit(-1); + } + + cl->rate_tbl = pkg_malloc(CC_RATE_TBL_SZ * sizeof(cc_rate_entry_t)); + if (!cl->rate_tbl) { + LM_CRIT("clusterer_controller: [cluster %d] no pkg memory for rate table\n", + cl->cluster_id); + exit(-1); + } + memset(cl->rate_tbl, 0, CC_RATE_TBL_SZ * sizeof(cc_rate_entry_t)); + + /* ---- Phase 1: join protocol ---- */ + lock_start_write(cl->peers->lock); + cl->peers->node_state = CC_NODE_NEW; + cl->peers->join_deadline = time(NULL) + (time_t)query_time; + lock_stop_write(cl->peers->lock); + + cc_send_join_req(cl->sock, cl); + cc_arm_tfd(cl->join_tfd, (time_t)query_time, 0); /* one-shot deadline */ + cc_arm_tfd(cl->rejoin_tfd, 1, 1); /* retry every 1 s */ + /* alive_tfd left disarmed - armed by cc_transition_to_active() */ + + LM_INFO("clusterer_controller: [cluster %d] sent JOIN_REQ, waiting up to %ds " + "for master response\n", cl->cluster_id, query_time); + + /* ---- Register with reactor and enter event loop ---- */ + if (reactor_proc_init("clusterer_controller worker") < 0) { + LM_CRIT("clusterer_controller: [cluster %d] reactor_proc_init failed\n", + cl->cluster_id); + exit(-1); + } + + if (reactor_proc_add_fd(cl->sock, cc_on_sock, cl) < 0 || + reactor_proc_add_fd(cl->alive_tfd, cc_on_alive_tfd, cl) < 0 || + reactor_proc_add_fd(cl->join_tfd, cc_on_join_tfd, cl) < 0 || + reactor_proc_add_fd(cl->rejoin_tfd, cc_on_rejoin_tfd, cl) < 0 || + reactor_proc_add_fd(cl->master_alive_tfd, cc_on_master_alive_tfd, cl) < 0 || + reactor_proc_add_fd(cl->master_dead_tfd, cc_on_master_dead_tfd, cl) < 0) { + LM_CRIT("clusterer_controller: [cluster %d] reactor_proc_add_fd failed\n", + cl->cluster_id); + exit(-1); + } + + reactor_proc_loop(); + + /* ---- Graceful shutdown epilogue ---- */ + { + int i_am_master; + lock_start_read(cl->peers->lock); + i_am_master = cc_i_am_master_locked(cl); + lock_stop_read(cl->peers->lock); + + if (i_am_master && cl->peers->count > 1) { + /* Find the peer that will win the next election (highest IP, not us) */ + unsigned int best_ip = 0; + int best_idx = -1; + int _i; + lock_start_read(cl->peers->lock); + for (_i = 0; _i < cl->peers->count; _i++) { + cc_peer_t *e = &cl->peers->entries[_i]; + if (strcmp(e->ip, my_ip) == 0) continue; + if (e->ip_num > best_ip) { best_ip = e->ip_num; best_idx = _i; } + } + if (best_idx >= 0) { + char next_ip[CC_MAX_IP_LEN + 1]; + unsigned char next_pub[CC_PUBKEY_SZ]; + memcpy(next_ip, cl->peers->entries[best_idx].ip, CC_MAX_IP_LEN + 1); + memcpy(next_pub, cl->peers->entries[best_idx].pubkey, CC_PUBKEY_SZ); + lock_stop_read(cl->peers->lock); + cc_send_key_handoff(cl->sock, next_ip, next_pub, cl); + } else { + lock_stop_read(cl->peers->lock); + } + } + } + + close(cl->sock); + close(cl->alive_tfd); + close(cl->join_tfd); + close(cl->rejoin_tfd); + close(cl->master_alive_tfd); + close(cl->master_dead_tfd); +} + +/* ========================================================================= + * MI command handlers + * ========================================================================= */ + +/** + * mi_cl_ctr_members() - list active cluster members with their role. + * + * opensips-cli -x mi cl_ctr_list_members + * + * [ + * {"ip": "10.0.0.3", "status": "master"}, + * {"ip": "10.0.0.1", "status": "member"}, + * {"ip": "10.0.0.2", "status": "member"} + * ] + * + * Only peers within the current quantized election window are shown, + * consistent with what cc_elect_master(cl) considers. + */ +static mi_response_t *mi_cl_ctr_members(const mi_params_t *params, + struct mi_handler *hdl) +{ + mi_response_t *resp; + mi_item_t *arr, *cl_obj, *members_arr, *peer_obj, *bin_arr; + int i, j, ci; + cc_cluster_t *cl; + + resp = init_mi_result_array(&arr); + if (!resp) + return NULL; + + for (ci = 0; ci < cc_cluster_count; ci++) { + cl = &cc_clusters[ci]; + if (!cl->peers) + continue; + + cl_obj = add_mi_object(arr, NULL, 0); + if (!cl_obj) goto error; + + if (add_mi_number(cl_obj, MI_SSTR("cluster_id"), cl->cluster_id) < 0) + goto error; + + members_arr = add_mi_array(cl_obj, MI_SSTR("members")); + if (!members_arr) goto error; + + lock_start_read(cl->peers->lock); + + for (i = 0; i < cl->peers->count; i++) { + cc_peer_t *e = &cl->peers->entries[i]; + + peer_obj = add_mi_object(members_arr, NULL, 0); + if (!peer_obj) { + lock_stop_read(cl->peers->lock); + goto error; + } + if (add_mi_string(peer_obj, MI_SSTR("ip"), + e->ip, strlen(e->ip)) < 0 || + add_mi_number(peer_obj, MI_SSTR("node_id"), e->node_id) < 0 || + add_mi_string(peer_obj, MI_SSTR("status"), + e->is_master ? "master" : (e->is_backup ? "backup" : "member"), + 6) < 0) { + lock_stop_read(cl->peers->lock); + goto error; + } + + bin_arr = add_mi_array(peer_obj, MI_SSTR("bin_sockets")); + if (!bin_arr) { + lock_stop_read(cl->peers->lock); + goto error; + } + for (j = 0; j < (int)e->bin_count; j++) { + if (add_mi_string(bin_arr, NULL, 0, + e->bin_sockets[j], + strlen(e->bin_sockets[j])) < 0) { + lock_stop_read(cl->peers->lock); + goto error; + } + } + } + + lock_stop_read(cl->peers->lock); + } + + return resp; + +error: + LM_ERR("clusterer_controller: mi_cl_ctr_members: failed to build response\n"); + free_mi_response(resp); + return NULL; +} + +/** + * mi_cl_ctr_node_info() - return full info for a specific node_id. + * + * opensips-cli -x mi cl_ctr_node_info node_id=2 + */ +static mi_response_t *mi_cl_ctr_node_info(const mi_params_t *params, + struct mi_handler *hdl) +{ + mi_response_t *resp; + mi_item_t *root, *bin_arr; + int target_id; + int i, j, ci; + cc_cluster_t *cl; + cc_peer_t *e; + + if (get_mi_int_param(params, "node_id", &target_id) < 0) + return init_mi_param_error(); + + for (ci = 0; ci < cc_cluster_count; ci++) { + cl = &cc_clusters[ci]; + if (!cl->peers) + continue; + + lock_start_read(cl->peers->lock); + for (i = 0; i < cl->peers->count; i++) { + e = &cl->peers->entries[i]; + if ((int)e->node_id != target_id) + continue; + + resp = init_mi_result_object(&root); + if (!resp) { + lock_stop_read(cl->peers->lock); + return NULL; + } + if (add_mi_number(root, MI_SSTR("node_id"), e->node_id) < 0 || + add_mi_string(root, MI_SSTR("ip"), e->ip, strlen(e->ip)) < 0 || + add_mi_number(root, MI_SSTR("cluster_id"), cl->cluster_id) < 0 || + add_mi_string(root, MI_SSTR("status"), + e->is_master ? "master" : (e->is_backup ? "backup" : "member"), + 6) < 0) { + lock_stop_read(cl->peers->lock); + goto error_node; + } + bin_arr = add_mi_array(root, MI_SSTR("bin_sockets")); + if (!bin_arr) { + lock_stop_read(cl->peers->lock); + goto error_node; + } + for (j = 0; j < (int)e->bin_count; j++) { + if (add_mi_string(bin_arr, NULL, 0, + e->bin_sockets[j], + strlen(e->bin_sockets[j])) < 0) { + lock_stop_read(cl->peers->lock); + goto error_node; + } + } + lock_stop_read(cl->peers->lock); + return resp; + } + lock_stop_read(cl->peers->lock); + } + + return init_mi_error(404, MI_SSTR("node_id not found")); + +error_node: + LM_ERR("clusterer_controller: mi_cl_ctr_node_info: failed to build response\n"); + free_mi_response(resp); + return NULL; +} + +/** + * mi_cl_ctr_config() - list all configured clusters and their resolved settings. + * + * opensips-cli -x mi cl_ctr_list_config + * + * Reports per cluster: id, multicast endpoint, master_stickiness, + * manage_shtags, query_time, this node's BIN socket and current member count. + * The password is intentionally NOT exposed. + */ +static mi_response_t *mi_cl_ctr_config(const mi_params_t *params, + struct mi_handler *hdl) +{ + mi_response_t *resp; + mi_item_t *arr, *cl_obj; + int ci, members; + char mcast[INET_ADDRSTRLEN + 8]; /* "IP:PORT" */ + char shtag_mode[24]; /* "auto" / "override:" */ + cc_cluster_t *cl; + + resp = init_mi_result_array(&arr); + if (!resp) + return NULL; + + for (ci = 0; ci < cc_cluster_count; ci++) { + cl = &cc_clusters[ci]; + + cl_obj = add_mi_object(arr, NULL, 0); + if (!cl_obj) goto error; + + snprintf(mcast, sizeof(mcast), "%s:%d", + cl->multicast_address, cl->multicast_port); + + members = 0; + /* Report the EFFECTIVE (possibly adopted) settings from shm, so the + * values reflect what is actually in force after on_config_mismatch= + * adopt (the worker mirrors them there). Fall back to the configured + * values if the peer table is not up yet. */ + int eff_manage = cl->manage_shtags, eff_stick = cl->master_stickiness, + eff_qt = query_time; + { + uint16_t forced = 0; + if (cl->peers) { + lock_start_read(cl->peers->lock); + members = cl->peers->count; + forced = cl->peers->shtag_forced_node_id; + eff_manage = cl->peers->eff_manage_shtags; + eff_stick = cl->peers->eff_master_stickiness; + eff_qt = cl->peers->eff_query_time; + lock_stop_read(cl->peers->lock); + } + /* Current shtag allocation mode: "auto" = master-driven automatic + * allocation; "override:" = operator forced a fixed holder + * via cl_ctr_shtag_force. ("manual" is reserved for the future + * maintenance mode.) */ + if (forced) + snprintf(shtag_mode, sizeof(shtag_mode), "override:%u", forced); + else + snprintf(shtag_mode, sizeof(shtag_mode), "auto"); + } + + if (add_mi_number(cl_obj, MI_SSTR("cluster_id"), cl->cluster_id) < 0 || + add_mi_string(cl_obj, MI_SSTR("multicast"), mcast, strlen(mcast)) < 0 || + add_mi_string(cl_obj, MI_SSTR("my_ip"), my_ip, strlen(my_ip)) < 0 || + add_mi_string(cl_obj, MI_SSTR("bin_socket"), + cl->bin_socket, strlen(cl->bin_socket)) < 0 || + add_mi_number(cl_obj, MI_SSTR("query_time"), eff_qt) < 0 || + add_mi_number(cl_obj, MI_SSTR("master_stickiness"), eff_stick) < 0 || + add_mi_number(cl_obj, MI_SSTR("manage_shtags"), eff_manage) < 0 || + add_mi_string(cl_obj, MI_SSTR("shtag_mode"), + shtag_mode, strlen(shtag_mode)) < 0 || + add_mi_number(cl_obj, MI_SSTR("member_count"), members) < 0) + goto error; + } + + return resp; + +error: + LM_ERR("clusterer_controller: mi_cl_ctr_config: failed to build response\n"); + free_mi_response(resp); + return NULL; +} + +/** + * cc_rpc_apply_shtags() - IPC job run inside the cc_worker process. + * + * An MI handler (running in a different process) has already updated + * cl->peers->shtag_forced_node_id in shm; this job makes the change take + * effect promptly: it re-applies the local shtag decision and, if we are the + * master, re-broadcasts the MEMBER_LIST so every member learns the new + * override without waiting for the next periodic announcement. + */ +static void cc_rpc_apply_shtags(int sender, void *param) +{ + cc_cluster_t *cl = (cc_cluster_t *)param; + int i_am_master; + + (void)sender; + if (!cl || !cl->peers) + return; + + cc_apply_shtags(cl); + + lock_start_read(cl->peers->lock); + i_am_master = cc_i_am_master_locked(cl); + lock_stop_read(cl->peers->lock); + + if (i_am_master && cl->sock >= 0) + cc_send_member_list(cl->sock, cl); +} + +/** + * cc_mi_find_cluster() - locate a configured cluster by its cluster_id. + * Returns NULL if no cluster matches. + */ +static cc_cluster_t *cc_mi_find_cluster(int cluster_id) +{ + int ci; + for (ci = 0; ci < cc_cluster_count; ci++) + if (cc_clusters[ci].cluster_id == cluster_id) + return &cc_clusters[ci]; + return NULL; +} + +/** + * mi_cl_ctr_shtag_force() - force a specific node to hold the active sharing tag. + * + * opensips-cli -x mi cl_ctr_shtag_force cluster_id=1 node_id=2 + * + * Must be issued on the current master. Suspends automatic (master-driven) + * shtag allocation until cl_ctr_shtag_auto is called; the chosen node becomes the + * sole active shtag holder cluster-wide. The override is propagated to every + * member via the MEMBER_LIST and survives master fail-over, but is cleared + * automatically if the forced node leaves or times out. + */ +static mi_response_t *mi_cl_ctr_shtag_force(const mi_params_t *params, + struct mi_handler *hdl) +{ + int cluster_id, node_id; + cc_cluster_t *cl; + int i_am_master, found = 0, proc_no; + + if (get_mi_int_param(params, "cluster_id", &cluster_id) < 0 || + get_mi_int_param(params, "node_id", &node_id) < 0) + return init_mi_param_error(); + + if (node_id <= 0 || node_id > 0xFFFF) + return init_mi_error(400, MI_SSTR("node_id out of range")); + + cl = cc_mi_find_cluster(cluster_id); + if (!cl || !cl->peers) + return init_mi_error(404, MI_SSTR("cluster_id not found")); + + if (!cl->manage_shtags) + return init_mi_error(409, + MI_SSTR("shtag management is disabled for this cluster")); + + lock_start_write(cl->peers->lock); + i_am_master = cc_i_am_master_locked(cl); + if (i_am_master) { + int i; + for (i = 0; i < cl->peers->count; i++) { + if ((int)cl->peers->entries[i].node_id == node_id) { + found = 1; + break; + } + } + /* TODO(maintenance-mode): once node maintenance mode lands, reject + * forcing the shtag onto a node that is currently in maintenance. */ + if (found) + cl->peers->shtag_forced_node_id = (uint16_t)node_id; + } + proc_no = cl->peers->worker_proc_no; + lock_stop_write(cl->peers->lock); + + if (!i_am_master) + return init_mi_error(409, + MI_SSTR("not the master - issue cl_ctr_shtag_force on the master node")); + if (!found) + return init_mi_error(404, MI_SSTR("node_id not a member of this cluster")); + + /* Apply locally and broadcast the new override from the worker process. */ + if (proc_no >= 0) + ipc_send_rpc(proc_no, cc_rpc_apply_shtags, cl); + + LM_INFO("clusterer_controller: [cluster %d] operator forced shtag onto " + "node %d\n", cluster_id, node_id); + return init_mi_result_ok(); +} + +/** + * mi_cl_ctr_shtag_auto() - resume automatic (master-driven) shtag allocation. + * + * opensips-cli -x mi cl_ctr_shtag_auto cluster_id=1 + * + * Clears any operator override set by cl_ctr_shtag_force so the active shtag + * follows the master again. Must be issued on the current master. + */ +static mi_response_t *mi_cl_ctr_shtag_auto(const mi_params_t *params, + struct mi_handler *hdl) +{ + int cluster_id; + cc_cluster_t *cl; + int i_am_master, proc_no; + + if (get_mi_int_param(params, "cluster_id", &cluster_id) < 0) + return init_mi_param_error(); + + cl = cc_mi_find_cluster(cluster_id); + if (!cl || !cl->peers) + return init_mi_error(404, MI_SSTR("cluster_id not found")); + + lock_start_write(cl->peers->lock); + i_am_master = cc_i_am_master_locked(cl); + if (i_am_master) + cl->peers->shtag_forced_node_id = 0; + proc_no = cl->peers->worker_proc_no; + lock_stop_write(cl->peers->lock); + + if (!i_am_master) + return init_mi_error(409, + MI_SSTR("not the master - issue cl_ctr_shtag_auto on the master node")); + + if (proc_no >= 0) + ipc_send_rpc(proc_no, cc_rpc_apply_shtags, cl); + + LM_INFO("clusterer_controller: [cluster %d] operator resumed automatic " + "shtag allocation\n", cluster_id); + return init_mi_result_ok(); +} +/* ========================================================================= + * Lifecycle + * ========================================================================= */ + +/** + * cc_resolve_local_identity() - determine my_ip and my_interface_buf. + * + * Three modes depending on which modparams were provided: + * + * Mode 1 - ip= only: + * Walk getifaddrs() to find the interface that owns the given IP. + * Fails if no interface owns it. + * + * Mode 2 - interface= only: + * Walk getifaddrs() to find the interface and take its first IPv4 address. + * Warns if the interface has multiple IPv4 addresses; uses the first one + * (the kernel's enumeration order matches `ip addr show`). + * + * Mode 3 - neither: + * Connect a throw-away UDP socket to the multicast group (no data sent). + * getsockname() returns the source IP the kernel would select. + * Reverse-look up the interface name via getifaddrs(). + * + * On success: my_ip points to a valid dotted-decimal IPv4 string and + * my_interface_buf holds the interface name (may be empty if the + * reverse lookup failed in mode 3 - non-fatal). + */ +/** + * cc_parse_cluster_str() - parse one "cluster" modparam string into cl. + * + * Format: "id=N,multicast=A.B.C.D:PORT[,password=STRING][,bin_socket=bin:IP:PORT]" + * - id= required, positive integer + * - multicast= required, IPv4:port + * - password= optional, falls back to global password modparam + * - bin_socket= optional, BIN socket for this cluster; falls back to + * first discovered socket (or only socket if one exists) + */ +static int cc_parse_cluster_str(const char *str, cc_cluster_t *cl) +{ + char buf[2048]; + char *p, *tok, *key, *val, *colon; + struct in_addr addr; + unsigned char first_octet; + int has_id = 0, has_mcast = 0; + + strncpy(buf, str, sizeof(buf) - 1); + buf[sizeof(buf) - 1] = '\0'; + + /* Defaults */ + cl->multicast_port = 3333; + strncpy(cl->password, password, sizeof(cl->password) - 1); + cl->password[sizeof(cl->password) - 1] = '\0'; + cl->manage_shtags = -1; /* sentinel: inherit global default in mod_init */ + cl->master_stickiness = -1; /* sentinel: inherit global default in mod_init */ + + for (tok = strtok_r(buf, ",", &p); tok; tok = strtok_r(NULL, ",", &p)) { + while (*tok == ' ' || *tok == '\t') tok++; + key = tok; + val = strchr(tok, '='); + if (!val) continue; + *val++ = '\0'; + + if (strcmp(key, "id") == 0) { + cl->cluster_id = atoi(val); + if (cl->cluster_id <= 0 || cl->cluster_id > 65535) { + LM_ERR("clusterer_controller: cluster id must be 1..65535 " + "(carried as a 2-byte field on the wire)\n"); + return -1; + } + has_id = 1; + + } else if (strcmp(key, "multicast") == 0) { + colon = strrchr(val, ':'); + if (colon) { + *colon = '\0'; + cl->multicast_port = atoi(colon + 1); + if (cl->multicast_port <= 0 || cl->multicast_port > 65535) { + LM_ERR("clusterer_controller: invalid port in cluster '%s'\n", str); + return -1; + } + } + strncpy(cl->multicast_address, val, INET_ADDRSTRLEN - 1); + cl->multicast_address[INET_ADDRSTRLEN - 1] = '\0'; + if (inet_aton(cl->multicast_address, &addr) == 0) { + LM_ERR("clusterer_controller: invalid multicast address '%s'\n", val); + return -1; + } + first_octet = (unsigned char)((ntohl(addr.s_addr) >> 24) & 0xFF); + if (first_octet < 224 || first_octet > 239) { + LM_ERR("clusterer_controller: '%s' is not a multicast address\n", val); + return -1; + } + has_mcast = 1; + + } else if (strcmp(key, "password") == 0) { + strncpy(cl->password, val, sizeof(cl->password) - 1); + cl->password[sizeof(cl->password) - 1] = '\0'; + + } else if (strcmp(key, "bin_socket") == 0) { + if (strlen(val) >= CC_MAX_BIN_SOCK_LEN) { + LM_ERR("clusterer_controller: bin_socket value too long\n"); + return -1; + } + strncpy(cl->bin_socket, val, CC_MAX_BIN_SOCK_LEN - 1); + cl->bin_socket[CC_MAX_BIN_SOCK_LEN - 1] = '\0'; + + } else if (strcmp(key, "manage_shtags") == 0) { + cl->manage_shtags = atoi(val) ? 1 : 0; + + } else if (strcmp(key, "master_stickiness") == 0) { + cl->master_stickiness = atoi(val) ? 1 : 0; + } + } + + if (!has_id) { + LM_ERR("clusterer_controller: cluster string missing id= in '%s'\n", str); + return -1; + } + if (!has_mcast) { + LM_ERR("clusterer_controller: cluster string missing multicast= in '%s'\n", str); + return -1; + } + return 0; +} + +static int cc_resolve_local_identity(void) +{ + struct ifaddrs *ifap = NULL, *ifa; + int found = 0; + + if (getifaddrs(&ifap) < 0) { + LM_ERR("clusterer_controller: getifaddrs() failed: %s\n", + strerror(errno)); + return -1; + } + + if (my_ip && *my_ip) { + /* ---- Mode 1: ip= explicitly provided - find owning interface ---- */ + struct in_addr target; + + if (inet_aton(my_ip, &target) == 0) { + LM_ERR("clusterer_controller: cannot parse 'my_ip' '%s'\n", my_ip); + freeifaddrs(ifap); + return -1; + } + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (!ifa->ifa_addr || ifa->ifa_addr->sa_family != AF_INET) + continue; + if (((struct sockaddr_in *)ifa->ifa_addr)->sin_addr.s_addr + == target.s_addr) { + strncpy(my_interface_buf, ifa->ifa_name, IF_NAMESIZE - 1); + my_interface_buf[IF_NAMESIZE - 1] = '\0'; + found = 1; + break; + } + } + if (!found) { + LM_ERR("clusterer_controller: no local interface owns IP '%s'\n", + my_ip); + freeifaddrs(ifap); + return -1; + } + LM_INFO("clusterer_controller: using IP %s on interface %s\n", + my_ip, my_interface_buf); + + } else if (my_interface && *my_interface) { + /* ---- Mode 2: interface= provided - derive IP from it ---- */ + int addr_count = 0; + + strncpy(my_interface_buf, my_interface, IF_NAMESIZE - 1); + my_interface_buf[IF_NAMESIZE - 1] = '\0'; + + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (!ifa->ifa_addr || ifa->ifa_addr->sa_family != AF_INET) + continue; + if (strcmp(ifa->ifa_name, my_interface) != 0) + continue; + addr_count++; + if (addr_count == 1) { + struct in_addr a = + ((struct sockaddr_in *)ifa->ifa_addr)->sin_addr; + inet_ntop(AF_INET, &a, my_ip_buf, sizeof(my_ip_buf)); + my_ip = my_ip_buf; + found = 1; + } + } + if (!found) { + LM_ERR("clusterer_controller: interface '%s' not found or " + "has no IPv4 address\n", my_interface); + freeifaddrs(ifap); + return -1; + } + if (addr_count > 1) + LM_WARN("clusterer_controller: interface '%s' has %d IPv4 " + "addresses, using %s (first returned by kernel) - " + "use 'my_ip' modparam to override\n", + my_interface, addr_count, my_ip); + else + LM_INFO("clusterer_controller: using IP %s on interface %s\n", + my_ip, my_interface_buf); + + } else { + /* ---- Mode 3: neither - auto-detect via kernel routing table ---- */ + struct sockaddr_in dest, local; + socklen_t local_len = sizeof(local); + int probe; + + memset(&dest, 0, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons((uint16_t)cc_clusters[0].multicast_port); + dest.sin_addr.s_addr = inet_addr(cc_clusters[0].multicast_address); + + probe = socket(AF_INET, SOCK_DGRAM, 0); + if (probe < 0) { + LM_ERR("clusterer_controller: auto-detect socket: %s\n", + strerror(errno)); + freeifaddrs(ifap); + return -1; + } + if (connect(probe, (struct sockaddr *)&dest, sizeof(dest)) < 0) { + LM_ERR("clusterer_controller: auto-detect connect: %s\n", + strerror(errno)); + close(probe); + freeifaddrs(ifap); + return -1; + } + memset(&local, 0, sizeof(local)); + if (getsockname(probe, (struct sockaddr *)&local, &local_len) < 0) { + LM_ERR("clusterer_controller: auto-detect getsockname: %s\n", + strerror(errno)); + close(probe); + freeifaddrs(ifap); + return -1; + } + close(probe); + + inet_ntop(AF_INET, &local.sin_addr, my_ip_buf, sizeof(my_ip_buf)); + my_ip = my_ip_buf; + + /* Reverse-look up the interface name */ + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (!ifa->ifa_addr || ifa->ifa_addr->sa_family != AF_INET) + continue; + if (((struct sockaddr_in *)ifa->ifa_addr)->sin_addr.s_addr + == local.sin_addr.s_addr) { + strncpy(my_interface_buf, ifa->ifa_name, IF_NAMESIZE - 1); + my_interface_buf[IF_NAMESIZE - 1] = '\0'; + found = 1; + break; + } + } + if (found) + LM_INFO("clusterer_controller: auto-detected IP %s on " + "interface %s\n", my_ip, my_interface_buf); + else + LM_WARN("clusterer_controller: auto-detected IP %s but could " + "not determine interface name\n", my_ip); + } + + freeifaddrs(ifap); + return 0; +} + +/** + * cc_discover_bin_sockets() - enumerate BIN listeners via proto_bin. + * + * Walks the protos[PROTO_BIN].listeners list and collects every + * entry with proto == PROTO_BIN. proto_bin must be loaded before this + * module so the listeners are already registered when mod_init() runs. + * + * Populates my_bin_sockets[] and my_bin_count. + * Returns 0 on success, -1 if no BIN sockets are found. + */ +static int cc_discover_bin_sockets(void) +{ + struct socket_info_full *sif; + struct socket_info *si; + char buf[CC_MAX_BIN_SOCK_LEN]; + int len; + + /* On devel, protos[].listeners is a list of socket_info_full (next/prev), + * each embedding the socket_info as its first member. */ + for (sif = protos[PROTO_BIN].listeners; sif; sif = sif->next) { + si = &sif->socket_info; + /* all entries here are PROTO_BIN by construction */ + if (si->proto != PROTO_BIN) + continue; + /* Reject wildcard - clusterer needs an explicit IP to set send_sock. + * Use socket=bin:IP:PORT instead of socket=bin:*:PORT. */ + if (si->address_str.len == 0 + || (si->address_str.len == 1 && si->address_str.s[0] == '*') + || (si->address_str.len == 7 + && memcmp(si->address_str.s, "0.0.0.0", 7) == 0)) { + LM_ERR("clusterer_controller: wildcard BIN socket " + "(bin:*:%u) is not allowed - use an explicit IP " + "(e.g. socket=bin:%s:%u)\n", + si->port_no, my_ip ? my_ip : "YOUR_IP", si->port_no); + return -1; + } + if (my_bin_count >= CC_MAX_BIN_SOCKETS) { + LM_WARN("clusterer_controller: more than %d BIN sockets, " + "ignoring the rest\n", CC_MAX_BIN_SOCKETS); + break; + } + len = snprintf(buf, sizeof(buf), "bin:%.*s:%u", + si->address_str.len, si->address_str.s, + si->port_no); + if (len <= 0 || len >= CC_MAX_BIN_SOCK_LEN) { + LM_WARN("clusterer_controller: BIN socket name too long, " + "skipping\n"); + continue; + } + memcpy(my_bin_sockets[my_bin_count], buf, len + 1); + LM_INFO("clusterer_controller: found BIN socket: %s\n", buf); + my_bin_count++; + } + + if (my_bin_count == 0) { + LM_ERR("clusterer_controller: no BIN sockets found - " + "is proto_bin loaded and socket=bin: configured?\n"); + return -1; + } + + return 0; +} + +static int mod_init(void) +{ + struct in_addr addr; + int i, j; + + LM_INFO("clusterer_controller: initialising\n"); + + if (wc_InitRng(&cc_rng) != 0) { + LM_ERR("clusterer_controller: wc_InitRng failed\n"); + return -1; + } + +#ifdef CC_HAVE_SODIUM + if (sodium_init() < 0) { + LM_ERR("clusterer_controller: sodium_init() failed\n"); + return -1; + } +#endif + + /* Resolve the on_config_mismatch policy string. */ + if (on_config_mismatch_s) { + if (strcasecmp(on_config_mismatch_s, "warn") == 0) + on_config_mismatch = CC_CFGMISMATCH_WARN; + else if (strcasecmp(on_config_mismatch_s, "reject") == 0) + on_config_mismatch = CC_CFGMISMATCH_REJECT; + else if (strcasecmp(on_config_mismatch_s, "adopt") == 0) + on_config_mismatch = CC_CFGMISMATCH_ADOPT; + else { + LM_ERR("clusterer_controller: invalid on_config_mismatch '%s' " + "(expected warn|reject|adopt)\n", on_config_mismatch_s); + return -1; + } + } + + /* ---- Require at least one cluster ---------------------------------- */ + + if (cc_cluster_str_count == 0) { + LM_ERR("clusterer_controller: no 'cluster' modparam defined\n"); + return -1; + } + + /* ---- Global param validation --------------------------------------- */ + + if (query_time < 1) { + LM_WARN("clusterer_controller: 'query_time' %d below min, clamping to 1s\n", + query_time); + query_time = 1; + } else if (query_time > 60) { + LM_WARN("clusterer_controller: 'query_time' %d exceeds max, clamping to 60s\n", + query_time); + query_time = 60; + } + + /* ---- Parse and validate all cluster strings ------------------------ */ + + for (i = 0; i < cc_cluster_str_count; i++) { + if (cc_parse_cluster_str(cc_cluster_strs[i], &cc_clusters[i]) < 0) + return -1; + cc_cluster_count++; + pkg_free(cc_cluster_strs[i]); + cc_cluster_strs[i] = NULL; + } + + /* Validate cluster_id uniqueness and (multicast, port) uniqueness */ + for (i = 0; i < cc_cluster_count; i++) { + for (j = i + 1; j < cc_cluster_count; j++) { + if (cc_clusters[i].cluster_id == cc_clusters[j].cluster_id) { + LM_ERR("clusterer_controller: duplicate cluster_id %d\n", + cc_clusters[i].cluster_id); + return -1; + } + if (strcmp(cc_clusters[i].multicast_address, + cc_clusters[j].multicast_address) == 0 && + cc_clusters[i].multicast_port == cc_clusters[j].multicast_port) { + LM_ERR("clusterer_controller: duplicate multicast %s:%d\n", + cc_clusters[i].multicast_address, + cc_clusters[i].multicast_port); + return -1; + } + } + } + + /* ---- Discover BIN sockets from opensips config file --------------- */ + /* Called after cc_resolve_local_identity() so my_ip is available for */ + /* wildcard substitution (bin:*:PORT -> bin:my_ip:PORT). */ + + /* ---- Resolve local identity using first cluster for Mode 3 probe --- */ + + if (cc_resolve_local_identity() < 0) + return -1; + + if (cc_discover_bin_sockets() < 0) + return -1; + + if (strlen(my_ip) > CC_MAX_IP_LEN) { + LM_ERR("clusterer_controller: resolved my_ip too long\n"); + return -1; + } + if (inet_aton(my_ip, &addr) == 0) { + LM_ERR("clusterer_controller: cannot parse resolved my_ip '%s'\n", my_ip); + return -1; + } + + /* ---- Multi-cluster: each cluster must name its BIN socket ---------- */ + + if (cc_cluster_count > 1) { + for (i = 0; i < cc_cluster_count; i++) { + if (cc_clusters[i].bin_socket[0] == '\0') { + LM_ERR("clusterer_controller: cluster %d has no bin_socket= " + "defined - required when multiple clusters are configured " + "(e.g. id=%d,multicast=...,bin_socket=bin:IP:PORT)\n", + cc_clusters[i].cluster_id, cc_clusters[i].cluster_id); + return -1; + } + } + } + + /* ---- Per-cluster: resolve BIN socket, derive key, allocate peers --- */ + + for (i = 0; i < cc_cluster_count; i++) { + cc_cluster_t *cl = &cc_clusters[i]; + + /* Resolve sentinels to the global default when the cluster string did + * not set them explicitly. Done here (unconditionally, before workers + * fork and before any MI query) so every cl->* setting always holds a + * concrete 0/1 value regardless of whether clusterer is loaded - a mix + * of global and per-cluster overrides always reports correctly. */ + if (cl->master_stickiness == -1) + cl->master_stickiness = master_stickiness ? 1 : 0; + if (cl->manage_shtags == -1) + cl->manage_shtags = manage_shtags ? 1 : 0; + + /* Resolve which BIN socket to use for this cluster. + * Priority: explicit bin_socket= in cluster string > + * sole discovered socket > + * first discovered socket (warn if multiple) */ + if (cl->bin_socket[0] != '\0') { + /* Explicit override - validate it was actually discovered */ + int found_bs = 0, bi; + for (bi = 0; bi < my_bin_count; bi++) { + if (strcmp(my_bin_sockets[bi], cl->bin_socket) == 0) { + found_bs = 1; + break; + } + } + if (!found_bs) { + char _disc[CC_MAX_BIN_SOCKETS * (CC_MAX_BIN_SOCK_LEN + 2)]; + int _o = 0, _b; + _disc[0] = '\0'; + for (_b = 0; _b < my_bin_count; _b++) + _o += snprintf(_disc + _o, sizeof(_disc) - _o, "%s%s", + _b ? ", " : "", my_bin_sockets[_b]); + LM_ERR("clusterer_controller: cluster %d bin_socket='%s' does not " + "match any configured BIN listener (discovered: %s) - peers " + "cannot connect and clusterer replication would silently " + "fail; fix bin_socket= or the socket=bin: line\n", + cl->cluster_id, cl->bin_socket, _disc); + return -1; + } + } else if (my_bin_count == 1) { + /* Only one socket - unambiguous */ + { + size_t _l = strnlen(my_bin_sockets[0], CC_MAX_BIN_SOCK_LEN - 1); + memcpy(cl->bin_socket, my_bin_sockets[0], _l); + cl->bin_socket[_l] = '\0'; + } + } else { + /* Multiple sockets, no explicit override - use first, warn */ + { + size_t _l = strnlen(my_bin_sockets[0], CC_MAX_BIN_SOCK_LEN - 1); + memcpy(cl->bin_socket, my_bin_sockets[0], _l); + cl->bin_socket[_l] = '\0'; + } + LM_WARN("clusterer_controller: cluster %d has no bin_socket= override " + "and multiple BIN sockets exist - using %s; add bin_socket= " + "to the cluster string to be explicit\n", + cl->cluster_id, cl->bin_socket); + } + LM_INFO("clusterer_controller: cluster %d: bin_socket=%s\n", + cl->cluster_id, cl->bin_socket); + + if (cc_derive_key(cl) < 0) + return -1; + + cl->peers = shm_malloc(sizeof(cc_peers_t)); + if (!cl->peers) { + LM_ERR("clusterer_controller: no shm memory for cluster %d peer table\n", + cl->cluster_id); + return -1; + } + memset(cl->peers, 0, sizeof(cc_peers_t)); + cl->peers->node_state = CC_NODE_NEW; + cl->peers->worker_proc_no = -1; /* published by cc_worker after fork */ + cl->peers->eff_manage_shtags = cl->manage_shtags; + cl->peers->eff_master_stickiness = cl->master_stickiness; + cl->peers->eff_query_time = query_time; + + cl->peers->lock = lock_init_rw(); + if (!cl->peers->lock) { + LM_ERR("clusterer_controller: lock_init_rw() failed for cluster %d\n", + cl->cluster_id); + shm_free(cl->peers); + cl->peers = NULL; + return -1; + } + + LM_INFO("clusterer_controller: cluster %d: multicast=%s:%d bin=%s\n", + cl->cluster_id, cl->multicast_address, cl->multicast_port, + cl->bin_socket); + } + + LM_INFO("clusterer_controller: my_ip=%s interface=%s query_time=%ds " + "clusters=%d bin_sockets=%d crypto=%s\n", + my_ip, my_interface_buf[0] ? my_interface_buf : "(unknown)", + query_time, cc_cluster_count, my_bin_count, CC_CRYPTO_SUITE); + + /* Set worker process count dynamically - one per cluster */ + procs[0].no = cc_cluster_count; + + /* Load clusterer controller API if clusterer.so is present and + * use_controller=1 is set. Soft dependency - controller works + * standalone even without clusterer loaded. */ + { + load_clusterer_ctrl_binds_f load_fn; + load_fn = (load_clusterer_ctrl_binds_f) + find_export("load_clusterer_ctrl_binds", 0); + if (load_fn && load_fn(&clctl) == 0) { + clctl_loaded = 1; + LM_INFO("clusterer_controller: clusterer API loaded - " + "topology will be driven dynamically\n"); + + /* When we manage sharing tags, start every tag as BACKUP and + * lock out MI/script changes - the controller master decides + * who becomes active. (manage_shtags sentinels were already + * resolved to concrete 0/1 in the unconditional loop above.) */ + { + int _ci; + for (_ci = 0; _ci < cc_cluster_count; _ci++) { + if (!cc_clusters[_ci].manage_shtags) + continue; + if (clctl.force_backup_shtags) + clctl.force_backup_shtags(cc_clusters[_ci].cluster_id); + if (clctl.set_shtag_managed) + clctl.set_shtag_managed(cc_clusters[_ci].cluster_id); + } + } + + } else { + LM_DBG("clusterer_controller: clusterer not loaded or " + "use_controller not set - running standalone\n"); + } + } + + return 0; +} + +static int cc_child_init(int rank) +{ + /* Re-seed RNG after fork - each worker must have independent state. */ + wc_FreeRng(&cc_rng); + if (wc_InitRng(&cc_rng) != 0) { + LM_ERR("clusterer_controller: wc_InitRng failed in child\n"); + return -1; + } + + /* Sync current_id from shared memory in every child process. + * The global current_id diverges after fork - each process needs + * to re-read the correct value from cluster->current_node. */ + if (clctl_loaded && clctl.sync_current_id) + clctl.sync_current_id(); + return 0; +} + +static void mod_destroy(void) +{ + int i, sock; + unsigned char ttl = 32; + cc_cluster_t *cl; + + if (!my_ip) + goto cleanup; + + /* Send GOODBYE on each cluster's multicast group so peers re-elect */ + for (i = 0; i < cc_cluster_count; i++) { + cl = &cc_clusters[i]; + if (!cl->peers) + continue; + if (cl->peers->count <= 1) { + LM_INFO("clusterer_controller: [cluster %d] sole node, " + "skipping GOODBYE\n", cl->cluster_id); + continue; + } + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock < 0) { + LM_ERR("clusterer_controller: [cluster %d] goodbye socket(): %s\n", + cl->cluster_id, strerror(errno)); + continue; + } + setsockopt(sock, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)); + /* Derive session_key locally from master_salt in shm so we can encrypt + * GOODBYE - the worker's cl->session_key is in a different process. */ + { + size_t plen = strlen(cl->password); + cc_hkdf_sha256((unsigned char *)cl->password, plen, + cl->peers->master_salt, CC_MASTER_SALT_SZ, + "cc_session", cl->session_key); + } + cc_send_pkt_with_ip(sock, CC_PKT_GOODBYE, cl); + close(sock); + LM_INFO("clusterer_controller: [cluster %d] GOODBYE sent\n", + cl->cluster_id); + } + +cleanup: + for (i = 0; i < cc_cluster_count; i++) { + cl = &cc_clusters[i]; + if (!cl->peers) + continue; + if (cl->peers->lock) { + lock_destroy_rw(cl->peers->lock); + cl->peers->lock = NULL; + } + shm_free(cl->peers); + cl->peers = NULL; + } + LM_INFO("clusterer_controller: shut down\n"); +} diff --git a/modules/clusterer_controller/doc/clusterer_controller.xml b/modules/clusterer_controller/doc/clusterer_controller.xml new file mode 100644 index 00000000000..af2ee984ad0 --- /dev/null +++ b/modules/clusterer_controller/doc/clusterer_controller.xml @@ -0,0 +1,22 @@ + + + + + + +%docentities; +]> + + + CLUSTERER_CONTROLLER Module + &osipsname; + + + &admin; + &tests; + &contrib; + &docCopyrights; + ©right; 2026 VoIPLine Telecom + \ No newline at end of file diff --git a/modules/clusterer_controller/doc/clusterer_controller_admin.xml b/modules/clusterer_controller/doc/clusterer_controller_admin.xml new file mode 100644 index 00000000000..17df402df7d --- /dev/null +++ b/modules/clusterer_controller/doc/clusterer_controller_admin.xml @@ -0,0 +1,1813 @@ + + + + + &adminguide; + +
+ Overview + + The clusterer_controller module provides + automatic peer discovery and topology management for the + clusterer module via authenticated, encrypted + UDP multicast. It eliminates the need for static node configuration + or a database — nodes discover each other automatically at startup + and the cluster topology is maintained dynamically at runtime. + + + When use_controller=1 is set in the + clusterer module, the + clusterer_controller module takes over all + topology management: it allocates unique node IDs, discovers peer + BIN socket addresses, and calls the clusterer internal API to add + or remove nodes as they join or leave the cluster. + + + By default (manage_shtags=1), the module also + provides fully automatic sharing tag failover. The controller + master node is the single decision point for which node holds the + active tag — no event routes, MI commands, or + seed_fallback_interval configuration is needed. + Sharing tags are forced to backup at startup regardless of the + =active config value, and the active tag is + claimed by the controller master automatically when the cluster + forms or when the active node departs. An operator can override this + automatic allocation and pin the active tag to a chosen node with the + cl_ctr_shtag_force MI command, reverting to automatic + allocation with cl_ctr_shtag_auto. + + + The minimal configuration per node is a single multicast group + address. No IP addresses, node IDs, BIN URLs, or sharing tag + management scripts need to be hardcoded or maintained. Any number + of nodes can join or leave without any configuration change on + the remaining nodes. + +
+ +
+ Discovery Protocol + + All traffic uses UDP multicast to the configured + multicast address and port. Clusters are kept + apart in two independent ways: by the multicast endpoint (two + clusters may use different IP addresses, or the same address with + different UDP ports), and by the id + (cluster_id) carried in the cleartext of every + packet. A node silently ignores any packet whose cluster_id differs + from its own, so several clusters can safely share one multicast + group and port. Two clusters merge only if they share + all of the multicast address, the UDP port, the + cluster_id and the password — i.e. they are configured identically, + which is the operator's responsibility to avoid. + + + Every packet is encrypted and authenticated with an AEAD — + AES-256-GCM by default, or XChaCha20-Poly1305 when the module is built + against libsodium (see the Security Architecture + section). Two distinct encryption keys are used depending on the + communication phase: + + + Bootstrap key — derived from + the configured password with a memory-hard KDF + (scrypt N=216, r=8, + p=1, or Argon2id in the libsodium build, with a + per-cluster salt), so a password captured from a bootstrap packet + cannot be brute-forced cheaply offline. Derived once at startup. + Used for the admission handshake (JOIN_REQ, KEY_GRANT, JOIN_REJECT) + and the split-brain MASTER_BEACON — traffic that must be readable + before a session key exists, or by masters holding different + session keys. + + + Session key — derived via + HKDF-SHA256 from the password and a 32-byte master salt + generated once when the cluster first bootstraps. All normal + cluster traffic uses this key. It is preserved across master + changes (a new master reuses the key every member already + holds), so failover needs no re-keying. + + + + + Each packet wire format begins with a 2-byte magic value + identifying the key type and a 2-byte cluster_id, both in cleartext + (the magic and cluster_id must be readable before decryption to + select the key and to filter foreign clusters), followed by a random + AEAD nonce (12 bytes for AES-256-GCM, 24 for XChaCha20-Poly1305) and + the ciphertext. The magic and cluster_id are additionally bound into + the authentication tag as AAD, so they cannot be altered undetected. + Only the payload is encrypted; the authenticated plaintext begins with + a 1-byte packet type and a 4-byte monotonic sequence number used for + replay protection. A 16-byte authentication tag follows the ciphertext. + Packets for a different cluster_id are dropped before decryption; + packets encrypted with a different password fail authentication and + are silently discarded. + + + The following packet types are defined: + + + ALIVE — periodic heartbeat + sent by every active node every query_time + seconds. Carries the sender IP, its X25519 public key (so peers + can prepare for key agreement) and a small descriptor of the + sender's consistency-critical settings + (manage_shtags, + master_stickiness, + query_time) used for configuration-drift + detection (see Security Architecture). Encrypted with the session + key. + + + JOIN_REQ — sent at startup + by a new node, carrying its IP, BIN socket list, X25519 + ephemeral public key, and a 16-byte random join nonce. + Encrypted with the bootstrap key so it can be sent before a + session key exists. + + + MEMBER_LIST — sent by the + master in response to a JOIN_REQ, carrying the member count, the + operator-forced sharing-tag holder node_id (0 = automatic), and + the full peer IP list so the joining node can participate in + elections. Only accepted from the current master (except during + initial join when no master is yet known). + + + NODE_ASSIGN — sent by the + master to multicast, allocating a node_id and BIN socket + record for a joining node. All cluster members receive and + apply it. + + + GOODBYE — sent on graceful + shutdown so peers can remove the node immediately without + waiting for timeout. Uses the sender's monotonic sequence + counter to prevent forgery. + + + MASTER_ALIVE — keepalive + sent by the master every 1 second (independent of + query_time). Used by all peers to detect + master failure quickly (3-second timeout). Encrypted with + the session key. + + + KEY_GRANT — unicast response + to a JOIN_REQ, sent by the master directly to the joining + node. Contains the master's X25519 public key, the original + join nonce, and the master salt wrapped with a per-exchange + key derived from ECDH(master_priv, joiner_pub) + password + + join_nonce via HKDF-SHA256. Encrypted with the bootstrap key. + + + KEY_HANDOFF — unicast packet + sent by the outgoing master on graceful shutdown to the + next-highest-IP peer, delivering the master salt so that peer + can become the new master without a full re-join cycle. + Encrypted with the session key. + + + JOIN_REJECT — sent by the + master to a joining node whose JOIN_REQ repeatedly fails + authentication (wrong password). After + CC_JOIN_FAIL_LIMIT (3) consecutive + bootstrap-key decryption failures from the same source IP the + master sends a JOIN_REJECT to that IP. Encrypted with the + bootstrap key so it cannot be forged by a node that does not + know the cluster password. The joining node logs a critical + error and shuts down OpenSIPS on receipt. + + + MASTER_BEACON — a master-only + announcement multicast every few MASTER_ALIVE ticks, carrying this + partition's member count. Unlike MASTER_ALIVE it is encrypted with + the bootstrap key, so it is readable even by a + master that holds a different session key. This is how a split + brain between two independently bootstrapped partitions is detected + and merged (see Master Election). + + + +
+ +
+ Master Election + + Each cluster has three roles: master + (the active coordinator), backup (the + standby promoted when the master fails, always the highest-IP + non-master) and member. The election + uses a quantized time window so that all nodes evaluate the same + eligible peer set and reach the same result deterministically. No NTP + synchronisation between nodes is required for correct election results. + + + The master_stickiness parameter (default 1) controls + whether a live master is kept when a higher-IP node joins. With + stickiness enabled, the master stays put and the higher-IP joiner + becomes the backup, minimising handovers; with stickiness disabled the + highest-IP node always becomes master. In either mode two live masters + are reconciled deterministically (see Split-brain + handling below). See the master_stickiness + parameter for details. + + + Only the master handles JOIN_REQ packets, allocates node_ids, and + sends NODE_ASSIGN and MEMBER_LIST packets. Non-master nodes are + passive during join events. A joining node receives the current + session key from the master (via KEY_GRANT) and joins as a member or + backup; it never seizes mastership during the join handshake. + + + Preserved session key: the session key + is generated once, when the first node bootstraps the cluster, and is + then preserved across every master change. A new master does not + re-key; because every member already holds the key (obtained when it + joined), master transitions require no re-keying and no re-JOIN cycle. + + + Fast master failure detection: + the master sends MASTER_ALIVE packets every 1 second. All non-master + peers maintain a 3-second watchdog timer that fires if no MASTER_ALIVE + is received. On expiry the silent master is aged out of the election + window and each peer immediately re-elects, promoting the backup + (highest-IP survivor) — which already holds the session key, so it + starts serving within one keepalive interval. + + + Graceful master handoff: when the + current master shuts down cleanly, it sends a KEY_HANDOFF packet + directly to the next-highest-IP peer before sending GOODBYE to + multicast. This confirms the master salt to the incoming master so it + can assume control immediately. + + + Split-brain handling. A split brain + (more than one node believing it is master) is prevented and, if it + still occurs, healed by three cooperating mechanisms: + + + Prevention at join time. When + several nodes start simultaneously they all exchange + (bootstrap-decryptable) JOIN_REQs and thus learn about each other. + At the join deadline, a node that has seen a higher-IP node also + still joining defers its own self-promotion (for a few bounded + rounds) and joins that node instead, so only the highest-IP starter + becomes master and no independent-key lone masters are created. + + + Same-key yield. Two masters that + share a session key (for example after a network partition heals) + can read each other's MASTER_ALIVE; the lower-IP master immediately + yields to the higher-IP one. + + + Divergent-key merge. Two masters + that were bootstrapped independently hold different session keys and + so cannot read each other's MASTER_ALIVE. Each therefore emits a + MASTER_BEACON encrypted with the shared bootstrap key. On hearing a + beacon from a superior partition — larger member count, ties broken + by higher IP — a node abandons its partition, re-joins the superior + master and adopts its session key, converging the whole cluster onto + a single master and key. + + + +
+ +
+ Security Architecture + + The module uses a two-phase key agreement to provide forward + secrecy and replay protection for all cluster traffic. + + + Payload encryption and header binding: + every packet's payload is sealed with an AEAD. The 2-byte magic (a key + selector that must be readable before decryption) and the 2-byte + cluster_id that precede the nonce are cleartext framing, but they are + bound into the AEAD tag as additional authenticated data (AAD): a + captured packet cannot be re-stamped with a different cluster_id and + still authenticate, which matters when two clusters share one multicast + group and password. A node also drops any packet whose cluster_id does + not match its own before attempting decryption, so + foreign-cluster traffic on the group never counts as an authentication + failure. + + + Crypto suite (selected at build time): + by default the module uses AES-256-GCM (12-byte + nonce) for the payload AEAD and scrypt + (N=216, r=8, p=1) for the bootstrap-key + derivation, through WolfSSL. If libsodium is + detected on the build host, the module is compiled instead with + XChaCha20-Poly1305 (24-byte nonce, whose 192-bit + nonce space removes any random-nonce collision concern) and + Argon2id. The two wire formats are not + interoperable, so every node in a cluster must be built with the same + suite; the active suite is reported in the startup log + (crypto=...). + + + Phase 1 — bootstrap (JOIN_REQ / + KEY_GRANT): each worker process generates an ephemeral + X25519 keypair on startup. When joining, the node sends its + public key and a 16-byte random join nonce in the JOIN_REQ, + encrypted with the bootstrap key (a memory-hard KDF of the password — + scrypt, or Argon2id in the libsodium build). The + master responds with a KEY_GRANT encrypted with the same bootstrap + key, containing its own public key, the echoed join nonce, and + the master salt wrapped with a per-exchange key derived via + HKDF-SHA256 from: + IKM = ECDH(master_priv, joiner_pub) || password || join_nonce + This ensures that even if the password is compromised, previously + recorded exchanges cannot be decrypted without both the ephemeral + private key and the per-exchange join nonce. + + + Phase 2 — session (all other + packets): once the master salt is known, all nodes + derive the session key as: + session_key = HKDF-SHA256(IKM=password, salt=master_salt, info="cc-session-key") + The session key is generated once, when the first node bootstraps the + cluster, and preserved across every master change: a new master reuses + the key that every member already holds, so master transitions require + no re-keying. All normal cluster traffic (ALIVE, MEMBER_LIST, + NODE_ASSIGN, GOODBYE, MASTER_ALIVE, KEY_HANDOFF) is encrypted with this + key. + + + Replay protection: each sender + maintains a monotonically increasing 32-bit sequence number + embedded in the authenticated plaintext of every session-key + packet. Each receiver tracks the last accepted sequence number + per source IP and rejects any packet whose sequence is not + strictly greater than the last accepted value. Sequence counters + are reset to zero whenever the session key is (re)derived — at cluster + bootstrap and when a joiner adopts the key via KEY_GRANT/KEY_HANDOFF — + and on peer restart detection (JOIN_REQ received from a known IP resets + that peer's counter; MEMBER_LIST upsert resets all listed peers). + This protection does not depend on clock synchronisation. + + + Rate limiting: a per-source rate + limiter (256 slots, 20 packets/second limit) is applied before any + decryption attempt. This prevents CPU exhaustion from packet floods + directed at the multicast group. + + + Join authentication and rejection: + the master tracks consecutive bootstrap-key decryption failures per + source IP in a small worker-local table + (CC_JOIN_FAIL_TABLE_SZ = 8 slots). When any + source IP accumulates CC_JOIN_FAIL_LIMIT (3) + consecutive failures — indicating a node attempting to join with the + wrong password — the master sends an encrypted JOIN_REJECT packet + and stops responding to further JOIN_REQs from that IP. + + + On the joining side, a received JOIN_REJECT is only acted on while + the node is still in the initial join phase + (CC_NODE_NEW state) and is addressed to this + node; an already-active cluster member ignores any JOIN_REJECT + unconditionally, so a node with the correct password can never be + evicted by a peer. + + + A node joining with the wrong password cannot + decrypt the JOIN_REJECT (it is encrypted with the master's bootstrap + key), so it relies on a self-contained signal instead: while joining + it counts packets received from other peers that it cannot decrypt. + If, at the join deadline, the node is still unjoined and has seen + CC_JOIN_FAIL_LIMIT or more such undecryptable + packets, it concludes that a cluster it cannot authenticate to exists + on the group and shuts down OpenSIPS with a critical log message — + rather than promoting itself into a lone, split-brain master (which, + with managed sharing tags, would create a duplicate active tag). This + counter is reset the moment a KEY_GRANT is successfully processed, so + a legitimate joiner that briefly saw an undecryptable packet before + receiving its key is never affected. + + + Rogue traffic isolation: a node + requests a re-key in response to an undecryptable session-key packet + only when that packet came from its current master (a legitimate key + rotation). Undecryptable session packets from any other source — for + example a wrong-password or malicious node broadcasting on the + multicast group — are ignored, so such traffic cannot drive the + cluster into a re-JOIN churn. + + + Peer table exhaustion defence: the + peer table is bounded at CC_MAX_PEERS (256) + entries. When the table is full, the master rejects JOIN_REQ + packets from unknown IPs with a JOIN_REJECT response. Known peers + that are reconnecting after a restart continue to be admitted + regardless of the table count, since they already own a slot. + This prevents an attacker with the cluster password from exhausting + the peer table by flooding JOIN_REQs from spoofed source addresses. + + + Configuration-consistency enforcement: + all nodes of a cluster must use identical consistency-critical settings + (manage_shtags, master_stickiness + and query_time); a per-node mismatch would otherwise + cause silent, inconsistent failover and sharing-tag behaviour (for + example, a master with manage_shtags=0 would leave no + node holding the active tag). Each node advertises these effective + settings in its ALIVE heartbeat and in its JOIN_REQ, so mismatches are + detected. What happens then is controlled by the + on_config_mismatch modparam: + + + reject (default) - when a node tries to + join an established cluster (a master is alive) with different + settings, the master logs the attempt and returns a JOIN_REJECT; + the joining node logs the offending settings and shuts down, so a + misconfigured node never joins. + + + warn - the node is allowed to join, but any + peer that observes a different value logs a single loud + CONFIG MISMATCH warning (repeated only if the + peer's advertised configuration changes, cleared once it matches). + + + adopt - the joining node adopts the running + cluster's (master's) settings at runtime and continues; the + adopted values are what cl_ctr_list_config + reports. + + + This turns an easy-to-miss misconfiguration into an obvious log line, a + refused join, or a self-correction rather than a hard-to-diagnose HA + failure. + + + Node identity and node_id allocation: + node_id values are allocated exclusively by the current + master, serialised under the peer-table lock, and a joining node never + picks its own id. The master hands out the lowest unused id by scanning + the live peer table, so a node that has failed but is not yet timed out + still occupies its slot and its id is never handed to a different joiner — + new nodes always receive a distinct id even during the failure-detection + window. A node that restarts and rejoins from the same address reuses its + previous id (and has its replay counter reset), so ids stay stable across + restarts. Because peers are keyed by source IP address, every node in a + cluster must present a stable, unique source IP: two distinct nodes that + appear behind the same address (for example through NAT) would share a + single peer slot and node_id. Deploy the cluster on a + network where each member has its own routable address on the + BIN/multicast interface. + + + Trust model — shared secret, not per-node + identity: the cluster is a single shared-secret trust domain. + Authentication proves only that a peer holds the cluster password; it does + not bind a cryptographic identity to an individual node, and there is no + per-node authorisation or revocation. Consequently any party in possession + of the password is a fully trusted member and can legitimately win the + highest-IP master election and assume the master role — there is no + distinction between "may be a member" and "may be master". An attacker + without the password cannot affect the election at + all: forged or replayed MASTER_ALIVE and beacon + packets fail AEAD authentication (or the strict per-source sequence check) + and are dropped before any election logic runs. The residual exposure is + therefore a malicious or compromised insider that + already holds the shared key. Protect the password accordingly, and rotate + it if a node is decommissioned or suspected compromised. Removing this + limitation — per-node keypairs with enrolment and revocation, so a single + node can be distrusted without re-keying the whole cluster — is planned + future work. + +
+ +
+ Dependencies +
+ &osips; Modules + + The following modules are required by this module: + + + + proto_bin — required so that + BIN listeners are registered and available for + discovery when clusterer_controller + initialises and scans the proto_bin listener list. + + + + + clusterer — required with + use_controller=1 set, so that + the clusterer internal API + (load_clusterer_ctrl_binds) is + exported and available at initialisation time. + + + + + + Both dependencies are declared in the module's + dep_export_t. &osips; will refuse to + start if either dependency is not satisfied. This does + not imply that the modules must appear + in a particular order in the configuration file — &osips; + resolves the dependency at runtime and will initialize the + required modules first regardless of + loadmodule order. + + + All other modules that use the clusterer interface + (tm, dialog, + dispatcher, usrloc + etc.) may be loaded in any order relative to + clusterer_controller. The clusterer + module automatically creates a cluster stub when + use_controller=1 is set and a module + attempts to register a capability for an unknown cluster. + +
+ +
+ External Libraries or Applications + + The following libraries must be installed: + + + + tls_wolfssl — the + clusterer_controller module links + statically against the WolfSSL library built by the + tls_wolfssl module + (modules/tls_wolfssl/lib/lib/libwolfssl.a). + WolfSSL provides AES-256-GCM authenticated encryption, + X25519 ECDH key agreement, HKDF-SHA256 key derivation, + the scrypt password KDF, and SHA-256. The tls_wolfssl module + must be present in the source tree; its static library + is built automatically as a dependency if not already + present. + + + + + libsodium (optional) — if the build host + has libsodium development files (detected via + pkg-config), the module is compiled with + the stronger crypto suite: XChaCha20-Poly1305 for the payload + AEAD and Argon2id for the bootstrap-key derivation, in place of + AES-256-GCM and scrypt. X25519 ECDH and HKDF continue to use + WolfSSL. libsodium is linked dynamically, so each target host + then also needs the libsodium runtime package (for example + libsodium23). Because the two suites produce + incompatible wire formats, every node in a cluster must be built + the same way; the active suite is printed in the startup log. + + + + +
+
+ +
+ Exported Parameters + +
+ <varname>cluster</varname> (string) + + Define a cluster to participate in. The value is a + comma-separated key=value string with the following fields: + + + id (required) — + positive integer cluster identifier, must match the + cluster_id used by clusterer + consumers (dialog, usrloc, dispatcher, etc.). + + + multicast (required) + — IPv4 multicast address and UDP port in the form + A.B.C.D:PORT. The address must + be in the 224.0.0.0/4 range. + + + password (optional) + — AES-256 encryption key material. All nodes in the + same cluster must use the same password. Falls back + to the global password modparam + if not set. + + + bin_socket + (optional) — BIN socket to advertise for this + cluster, in the form + bin:IP:PORT. Required when + multiple clusters are defined (so the controller knows + which listener to advertise), but it need + not be distinct — several clusters + may share the same BIN socket. When only one cluster + is defined and only one BIN socket exists, the + socket is auto-detected from the + proto_bin listeners. + + + manage_shtags + (optional) — per-cluster override for the global + manage_shtags modparam. Set to + 1 to enable automatic sharing + tag failover for this cluster, or + 0 to disable it. When omitted, + the global manage_shtags value + applies, regardless of the order in which + cluster and + manage_shtags modparams appear + in the config file. + + + + + This parameter may be set multiple times to participate in + multiple clusters simultaneously. Each cluster runs its own + independent worker process. + + + No default value. At least one cluster must be + defined. + + + Set <varname>cluster</varname> parameter — single cluster + +... +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333") +... + + + + Set <varname>cluster</varname> parameter — multiple clusters + on separate networks (one BIN socket per network) + +... +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:10.0.1.10:5566") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.2:3333,bin_socket=bin:10.0.2.10:5566") +... + + + + Set <varname>cluster</varname> parameter — multiple clusters + sharing a single BIN socket (recommended default) + +... +# one proto_bin listener serves both clusters; the cluster_id in each +# BIN packet keeps their replication traffic separate +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:10.0.0.10:5566") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.2:3333,bin_socket=bin:10.0.0.10:5566") +... + + + + One BIN socket can serve any number of + clusters. Every clusterer BIN packet carries its + cluster_id, so a single proto_bin + listener demultiplexes traffic for all clusters. Unless you specifically + want different clusters on different interfaces or networks, set the + same bin_socket value on every + cluster entry (it is mandatory once more than one + cluster is defined, but need not be distinct). Using multiple BIN sockets + is for network/interface segregation, not throughput: + the replication data plane runs over BIN (TCP) and scales with the shared + TCP worker pool (tcp_children) and IPC dispatch, + independent of the number of BIN sockets. What does scale with the number + of clusters is the controller's own worker set — it spawns one lightweight + control worker (UDP multicast: discovery, election, keepalives) per + cluster — so it is the cluster count, not the + BIN-socket count, that adds processes. + + + Each cluster needs a distinct multicast + endpoint, but clusters may share one BIN socket. The two + sockets play opposite roles. The controller's multicast + endpoint is its control plane: every cluster defined in one instance must + use a different multicast address:port — two + cluster entries with the same + multicast value are rejected at startup + (duplicate multicast). Use a different port + (e.g. :3333 and :3334) or a + different group per cluster. The bin_socket is the + replication data plane and is the opposite: it may be freely shared + across clusters, since every BIN packet carries its + cluster_id. (The cluster_id filter + on the multicast wire header is for a different purpose: letting + separate deployments coexist on a shared multicast + group, not two clusters inside one instance.) + +
+ +
+ <varname>my_ip</varname> (string) + + Explicitly set the local IPv4 address used by the controller + for its own node identity and master election. This is the IP + that the controller advertises to peers in JOIN_REQ and + NODE_ASSIGN packets and uses for the highest-IP master + election algorithm. + + + Note: this parameter controls the controller's identity only. + The BIN socket address advertised to clusterer peers is + discovered separately from the proto_bin + listener list and is independent of this setting. Do not + confuse my_ip with the BIN socket IP + defined by the socket=bin:IP:PORT core + parameter. + + + When set, the module walks the interface list to find which + local interface owns this address and uses that interface for + multicast traffic. Startup fails if no local interface owns + the given address. + + + The module supports three identity resolution modes depending + on which modparams are provided: + + + Mode 1 — my_ip set: + The given IP is used directly. The owning interface is + resolved automatically from the system interface list. + Use this mode on multi-homed hosts where you want to + pin the controller identity to a specific IP. + + + Mode 2 — interface set, my_ip not set: + The first IPv4 address on the named interface is used as + the controller identity IP. A warning is logged if the + interface has multiple IPv4 addresses. + + + Mode 3 — neither set (default): + A throw-away UDP socket is connected to the multicast + group and getsockname() is called + to determine which source IP the kernel would select. + The interface name is resolved from the returned IP. + Suitable for single-homed hosts. + + + Default: auto-detected (Mode 3). + + + Set <varname>my_ip</varname> parameter + +... +modparam("clusterer_controller", "my_ip", "10.22.23.191") +... + + +
+ +
+ <varname>interface</varname> (string) + + Explicitly set the network interface name to use for + multicast traffic (e.g. eth0, + enp6s18). The module takes the first + IPv4 address assigned to this interface as the controller's + identity IP. This corresponds to Mode 2 described in the + my_ip parameter documentation above. + + + Like my_ip, this parameter affects the + controller's own identity only and has no effect on the + BIN socket addresses advertised to clusterer peers. + + + If the interface has more than one IPv4 address, a warning + is logged and the first address (in the order returned by + the kernel) is used. Set my_ip explicitly + to avoid ambiguity on multi-address interfaces. + + + Ignored if my_ip is also set — + my_ip takes precedence. + + + Default: auto-detected (Mode 3 — see + my_ip). + + + Set <varname>interface</varname> parameter + +... +modparam("clusterer_controller", "interface", "eth0") +... + + +
+ +
+ <varname>query_time</varname> (integer) + + How often (in seconds) each active node sends an ALIVE + heartbeat to the multicast group. This value also controls + the election window + (3 × query_time) and the peer purge + window (6 × query_time). + + + Smaller values mean faster failure detection but higher + multicast traffic. Valid range: 1–60. + + + Default value is 5. + + + Set <varname>query_time</varname> parameter + +... +modparam("clusterer_controller", "query_time", 5) +... + + +
+ +
+ <varname>password</varname> (string) + + Global default encryption password for all clusters. All + nodes in a cluster must use the same password. The password + serves two purposes: + + + Bootstrap key — + the password is stretched with scrypt + (memory-hard, per-cluster salt) to encrypt JOIN_REQ and + KEY_GRANT packets during the initial key exchange phase + before a session key is established. + + + Session key material — + the password is fed into HKDF-SHA256 together with the + master salt to derive the session key used for all + normal cluster traffic. + + + Can be overridden per cluster using the + password= key in the + cluster parameter. + + + Default value is 3eCrEt*5629. + Change this in production. Use a long, high-entropy + secret rather than a memorable phrase — scrypt raises the cost of + an offline guess, but only a strong secret removes the risk. A + generated key is ideal, e.g. openssl rand -base64 32. + The module logs a startup warning if the configured password is the + default or has an estimated entropy below 80 bits. + + + Set <varname>password</varname> parameter + +... +modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") +... + + +
+ +
+ <varname>manage_shtags</varname> (integer) + + When set to 1 (the default), + the controller master node automatically manages sharing tag + failover for all clusters. The controller becomes the single + decision point for which node holds the active tag, eliminating + races between nodes and requiring no script-level event routes + or MI commands to handle failover. While active, the + clusterer_set_tag_active MI command and + the $shtag() script variable setter are + blocked for controller-managed clusters, returning an error + to the caller. + + + Behaviour when manage_shtags=1: + + + Startup: all local sharing tags + are forced to backup state during module + initialisation, regardless of the =active + value in the clusterer sharing_tag modparam. + The deferred BIN broadcast flag is also cleared so no + SHTAG_ACTIVE packet is ever sent at + startup. This ensures that no node can steal the active tag from + an existing cluster member simply by restarting. + + + Bootstrap: when the first node + starts alone and no existing master responds within + query_time seconds (join deadline), it elects + itself master and activates all local backup tags exactly once. + Nodes that join an existing cluster are never eligible for this + bootstrap path and never self-activate. + + + Failover: when any node departs + (graceful shutdown via GOODBYE packet, or timeout-based removal), + the controller master activates its own backup tags for that + cluster. This covers all departure scenarios: last node standing, + master still present, and post re-election. + + + Rejoin: a node rejoining an + existing cluster always starts in backup state and never reclaims + the active tag from the current holder, even if + =active appears in its config. + + + When set to 0, the controller + does not touch sharing tag state at all. The + =active config value, + seed_fallback_interval, and external + MI/event-route scripts behave exactly as in stock clusterer + without the controller. Use this when you have existing tag + management scripts and want to opt out of automatic failover. + + + Default value is 1. + + + Global vs per-cluster scope: + This modparam sets a global default that applies to every cluster + defined via the cluster modparam. Individual + clusters can override it by including + manage_shtags=0 or + manage_shtags=1 directly in the cluster + string. The global default is resolved at startup after all + modparams are processed, so the order of + manage_shtags and cluster + lines in the config file does not matter. + + + Global <varname>manage_shtags</varname> — applies to all clusters + +... +# Enable automatic failover for every cluster (this is also the default) +modparam("clusterer_controller", "manage_shtags", 1) +modparam("clusterer_controller", "cluster", "id=1,multicast=239.0.90.1:3333") +modparam("clusterer_controller", "cluster", "id=2,multicast=239.0.90.2:3333") +... + + + + Per-cluster override — opt one cluster out of automatic failover + +... +# manage_shtags=1 globally, but cluster 2 uses its own MI/event-route scripts +modparam("clusterer_controller", "manage_shtags", 1) +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.2:3333,manage_shtags=0") +... + + + + Global opt-out with one cluster opting in + +... +# Disable automatic failover globally; enable it only for cluster 1 +modparam("clusterer_controller", "manage_shtags", 0) +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,manage_shtags=1") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.2:3333") +... + + + + Typical full configuration with manage_shtags=1 (default) + +# All nodes use identical config — only the BIN socket IP differs per node. +# The =active tag value in sharing_tag is ignored by the controller; +# it is kept in the config only for compatibility with manage_shtags=0. + +socket=bin:10.22.23.191:3857 + +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "sharing_tag", "vip1/1=active") +modparam("clusterer", "ping_interval", 4) +modparam("clusterer", "ping_timeout", 1500) + +loadmodule "clusterer_controller.so" +modparam("clusterer_controller", "cluster", "id=1,multicast=239.0.90.1:3333") +modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") +# manage_shtags defaults to 1 — no need to set it explicitly + + +
+ +
+ <varname>master_stickiness</varname> (integer) + + Controls whether a live master keeps its role when a higher-IP + node joins. Default 1 (sticky). + + + The module recognises three roles per cluster: + master (the active coordinator), + backup (the standby that takes + over when the master fails), and + member (all other nodes). The + backup is always the highest-IP node that is not the master. + + + + master_stickiness=1 (default): + the master is sticky — a live master keeps + the role and is not displaced when a higher-IP node joins. The + newly joined node becomes the backup (replacing the previous + backup if it has a higher IP); the master only changes when the + current master actually fails, at which point the backup is + promoted. This minimises the number of master handovers. + + + master_stickiness=0: pure + highest-IP election — a higher-IP node takes over as master as + soon as it appears. This produces more handovers but always + keeps the highest-IP node as master. + + + + In both modes a split-brain (two nodes each believing they are + master, e.g. after a network partition heals) is resolved + deterministically: the lower-IP master yields to the higher-IP one. + + + Global vs per-cluster scope: + like manage_shtags, this sets a global default + that individual clusters can override with + master_stickiness=0 or + master_stickiness=1 in the + cluster string. Resolution happens at startup + regardless of modparam order. + + + Set <varname>master_stickiness</varname> parameter + +... +# Global default (sticky) — omit entirely for the same effect +modparam("clusterer_controller", "master_stickiness", 1) + +# Per-cluster override: cluster 2 always promotes the highest-IP node +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.2:3333,master_stickiness=0") +... + + +
+ +
+ <varname>on_config_mismatch</varname> (string) + + Policy applied when a node's consistency-critical settings + (manage_shtags, + master_stickiness, query_time) + differ from those of the running cluster. All nodes of a cluster are + expected to use identical values; this parameter decides what happens + when they do not. One of: + + reject (default) - the + master refuses the join with a JOIN_REJECT and the joining node + shuts down after logging which settings differ. + warn - the node is allowed + to join; a single CONFIG MISMATCH warning is + logged per mismatching peer. + adopt - the joining node + adopts the running cluster's settings at runtime and + continues. + + Global only (not per-cluster). Default: reject. + + + Set <varname>on_config_mismatch</varname> parameter + +... +modparam("clusterer_controller", "on_config_mismatch", "reject") +... + + +
+ +
+ +
+ Exported MI Functions + +
+ <function moreinfo="none">cl_ctr_list_members</function> + + List all current cluster members with their node_id, status and + BIN socket addresses. Status is one of master, + backup (the standby that will be promoted if + the master fails) or member. Only peers within + the current election window are shown. + + Parameters: none + + <function>cl_ctr_list_members</function> usage + +opensips-cli -x mi cl_ctr_list_members +[ + { + "cluster_id": 1, + "members": [ + { + "ip": "10.22.23.191", + "node_id": 1, + "status": "master", + "bin_sockets": [ "bin:10.22.23.191:3857" ] + }, + { + "ip": "10.22.23.193", + "node_id": 2, + "status": "backup", + "bin_sockets": [ "bin:10.22.23.193:3857" ] + }, + { + "ip": "10.22.23.192", + "node_id": 3, + "status": "member", + "bin_sockets": [ "bin:10.22.23.192:3857" ] + } + ] + } +] + + +
+ +
+ <function moreinfo="none">cl_ctr_node_info</function> + + Return full information for a specific node identified by + its allocated node_id. + + Parameters: + + + node_id — the integer node_id to + look up. + + + + <function>cl_ctr_node_info</function> usage + +opensips-cli -x mi cl_ctr_node_info node_id=2 +{ + "node_id": 2, + "ip": "10.22.23.192", + "cluster_id": 1, + "status": "backup", + "bin_sockets": [ "bin:10.22.23.192:3857" ] +} + + +
+ +
+ <function moreinfo="none">cl_ctr_list_config</function> + + List all configured clusters and their resolved settings — the + effective values actually in use after global defaults and + per-cluster overrides have been applied. Useful for confirming that + a per-cluster master_stickiness or + manage_shtags override took effect. The cluster + password is never exposed. + + + The shtag_mode field reports the current + sharing-tag allocation policy: auto when the + active tag follows the master automatically, or + override:<node_id> when an operator has + pinned a fixed holder with cl_ctr_shtag_force. + + Parameters: none + + <function>cl_ctr_list_config</function> usage + +opensips-cli -x mi cl_ctr_list_config +[ + { + "cluster_id": 1, + "multicast": "239.0.90.1:3333", + "my_ip": "10.22.23.191", + "bin_socket": "bin:10.22.23.191:3857", + "query_time": 5, + "master_stickiness": 1, + "manage_shtags": 1, + "shtag_mode": "auto", + "member_count": 3 + } +] + + +
+ +
+ <function moreinfo="none">cl_ctr_shtag_force</function> + + Force a specific node to hold the active sharing tag, overriding the + normal master-driven allocation. This is useful for planned + maintenance or manual traffic steering: the chosen node becomes the + sole active shtag holder cluster-wide while every other node — + including the master — is put into backup for that tag. + + + The command must be issued on the current master + (it returns an error otherwise). The override is propagated to all + members in the MEMBER_LIST and survives master + fail-over: a newly elected master keeps honouring it rather + than reclaiming the tag. Automatic allocation stays suspended until + cl_ctr_shtag_auto is called. If the forced node + leaves the cluster or times out, the override is cleared + automatically and automatic allocation resumes. + + Parameters: + + cluster_id — the target cluster. + node_id — the node that must hold the active tag; it must be a current member of the cluster. + + + <function>cl_ctr_shtag_force</function> usage + +opensips-cli -x mi cl_ctr_shtag_force cluster_id=1 node_id=3 + + +
+ +
+ <function moreinfo="none">cl_ctr_shtag_auto</function> + + Clear any override set by cl_ctr_shtag_force and + resume automatic, master-driven sharing-tag allocation — the active + tag follows the master again. Must be issued on the current master. + + Parameters: + + cluster_id — the target cluster. + + + <function>cl_ctr_shtag_auto</function> usage + +opensips-cli -x mi cl_ctr_shtag_auto cluster_id=1 + + +
+ +
+ +
+ Exported Pseudo-Variables + + These read-only pseudo-variables expose live cluster state to the + routing script, so a decision such as only the master runs this + job can be made without an MI call. They read directly from + shared memory and are therefore available in every process (SIP + workers included). + + + Each variable optionally takes a cluster id as its argument, e.g. + $cl_ctr_role(2). The bare form + ($cl_ctr_role) resolves to the only configured + cluster; when several clusters are defined the bare form returns NULL + and logs a one-time warning, so the cluster must be named explicitly. + An unknown cluster id, or a value that is not currently known (e.g. no + master yet), returns NULL. All of them are read-only - assigning to + them fails. + + + + $cl_ctr_role — this node's role + in the cluster: master, + backup, member, or + joining (still authenticating / before the + first election). + + + $cl_ctr_is_master — 1 if this + node is the cluster master, 0 otherwise. A fast path for the most + common check. + + + $cl_ctr_master_ip — IP of the + current master (NULL if none is elected yet). + + + $cl_ctr_backup_ip — IP of the + current backup / standby master (NULL if none). + + + $cl_ctr_node_id — this node's + node_id within that cluster (NULL until + assigned). A node may hold different ids in different clusters. + + + $cl_ctr_my_ip — the controller + identity IP of this node. + + + $cl_ctr_members — number of live + members currently in the cluster. + + + $cl_ctr_shtag_mode — + auto (tags follow the elected master) or + forced (an operator pinned them with + cl_ctr_shtag_force). + + + $cl_ctr_forced_node — the + node_id the active sharing tag is pinned to + (NULL when in auto mode). + + + + Using <varname>$cl_ctr_*</varname> in the script + +# single cluster: run a periodic job only on the master +if ($cl_ctr_is_master) + route(do_master_only_work); + +# several clusters: name the one you mean +xlog("cluster 2 master is $cl_ctr_master_ip(2), I am $cl_ctr_role(2)\n"); + + + +
+ +
+ Exported Functions + + Per-peer lookups take two arguments (cluster_id, node_id) + and are therefore script functions, not + pseudo-variables (a comma inside a variable's parentheses is ambiguous when + the variable is used as a function argument). Boolean checks return + true/false for use directly in an if; value lookups + write into an output variable. All are usable from any route. + + + + cl_ctr_node_is_master(cluster_id, node_id) + — true if that node is the cluster master. + + + cl_ctr_node_present(cluster_id, node_id) + — true if that node_id is a live member. + + + cl_ctr_get_node_role(cluster_id, node_id, out_var) + — write that node's role + (master/backup/member) + into out_var; returns false if it is not a member. + + + cl_ctr_get_node_ip(cluster_id, node_id, out_var) + — write that node's IP into out_var. + + + + Per-peer lookup functions + +if (cl_ctr_node_present(1, 3) && cl_ctr_node_is_master(1, 3)) { + cl_ctr_get_node_ip(1, 3, $var(ip)); + xlog("node 3 ($var(ip)) leads cluster 1\n"); +} + + +
+ +
+ Multiple Clusters + + A single &osips; instance can participate in multiple clusters + simultaneously by repeating the cluster + modparam. Each cluster runs an independent worker process with + its own multicast socket, peer table, master election, and + node_id space. + + + Clusters are distinguished by the combination of their multicast + IP address and UDP port. Two useful topologies are possible: + + + + Different ports, same multicast IP + — convenient when all clusters share the same L2 segment. + The port number alone separates traffic for each cluster. + Each cluster's password should also + differ to provide an additional encryption barrier. + + + Different multicast IPs + — useful when clusters span different network segments or + when multicast routing is scoped differently per cluster. + + + + When multiple clusters are defined and the node has more than + one BIN socket, the bin_socket= key must be + specified in each cluster string to indicate which BIN socket + to advertise for that cluster. If only one BIN socket exists, + it is used for all clusters automatically. + + + Each cluster has its own independent clusterer + cluster_id, allowing different &osips; + subsystems to replicate on different clusters: + + + Multiple clusters — dialog on cluster 1, usrloc on cluster 2 + +# Two BIN sockets, one per cluster +socket=bin:10.0.1.10:5566 +socket=bin:10.0.2.10:5566 + +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +modparam("clusterer", "use_controller", 1) + +loadmodule "clusterer_controller.so" +# Cluster 1 — dialog replication group, LAN segment 10.0.1.0/24 +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:10.0.1.10:5566") +# Cluster 2 — usrloc replication group, LAN segment 10.0.2.0/24 +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.1:3334,bin_socket=bin:10.0.2.10:5566") + +loadmodule "dialog.so" +modparam("dialog", "dialog_replication_cluster", 1) + +loadmodule "usrloc.so" +modparam("usrloc", "cluster_id", 2) + + + + Multiple clusters — same multicast IP, different ports + +socket=bin:10.22.23.191:3857 + +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +modparam("clusterer", "use_controller", 1) + +loadmodule "clusterer_controller.so" +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,password=ClusterOneSecret") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.1:3334,password=ClusterTwoSecret") + +loadmodule "dialog.so" +modparam("dialog", "dialog_replication_cluster", 1) + +loadmodule "dispatcher.so" +modparam("dispatcher", "cluster_id", 2) + + +
+ +
+ Hybrid Topologies (native + controller clusters) + + A single &osips; instance can run native clusterer + clusters (defined in the database or statically via + my_node_info/neighbor_node_info) + and controller-managed clusters side by side. + This allows, for example, a fixed, DB-provisioned replication cluster + to coexist with a zero-config controller-driven HA cluster on the same + node. + + + Which kind a cluster is follows from how it is declared: + + + + Controller-managed — every + cluster_id registered with the clusterer module + via modparam("clusterer", "cluster_id", N) + (and matched by a cluster entry in this module). + Its topology and this node's node_id are driven + at runtime by the controller; it never touches the database and + always behaves as db_mode=0, regardless of the + global db_mode. + + + Native — every cluster loaded + from the clusterer database (db_mode!=0) or + provisioned statically. It behaves exactly as classic clusterer: + fixed my_node_id, DB persistence (when + DB-backed), script/MI-managed sharing tags. + + + + The rules that keep the two kinds apart: + + + + A cluster_id is exclusively + controller-managed or native — declaring the same id both ways is + rejected at startup. + + + my_node_id identifies this node in its + native clusters only and is required only when + native clusters exist; controller clusters get their node id + assigned at runtime, and this node may well hold + different node ids in different clusters. + + + db_url/db_mode apply to + native clusters only. A controller-only deployment needs neither. + + + Sharing tags: only tags of controller-managed clusters are forced + to backup at startup and driven by the controller master; tags of + native clusters keep their configured state (=active + included) and stay script/MI-managed. + + + Native and controller clusters may share the same BIN socket - the + cluster_id carried in every BIN packet keeps + their traffic apart. + + + + Hybrid — DB-native cluster 10 + controller cluster 1 + +socket=bin:10.0.0.10:5566 + +loadmodule "db_mysql.so" +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +# native side: cluster 10 lives in the DB (nodes provisioned there) +modparam("clusterer", "db_mode", 1) +modparam("clusterer", "db_url", "mysql://opensips:pass@localhost/opensips") +modparam("clusterer", "my_node_id", 5) # this node's id in cluster 10 +# controller side: cluster 1 is dynamic +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) + +loadmodule "clusterer_controller.so" +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:10.0.0.10:5566,password=S3cret") + +loadmodule "dialog.so" +modparam("dialog", "dialog_replication_cluster", 1) # controller-managed + +loadmodule "dispatcher.so" +modparam("dispatcher", "cluster_id", 10) # DB-native + + +
+ +
+ Configuration Example + + The following example shows a minimal two-module configuration + for zero-config HA clustering with dialog replication. The + loadmodule order does not matter — the + dependency system enforces correct initialization order + automatically. + + + Minimal HA cluster configuration + +# Each node needs an explicit BIN socket (no wildcard) +socket=bin:10.22.23.191:3857 + +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "sharing_tag", "vip1/1=active") +modparam("clusterer", "ping_interval", 4) +modparam("clusterer", "ping_timeout", 1500) + +loadmodule "clusterer_controller.so" +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333") +modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") + +loadmodule "tm.so" +modparam("tm", "tm_replication_cluster", 1) + +loadmodule "dialog.so" +modparam("dialog", "dialog_replication_cluster", 1) +modparam("dialog", "cluster_auto_sync", 1) + +loadmodule "dispatcher.so" +modparam("dispatcher", "cluster_id", 1) +modparam("dispatcher", "cluster_probing_mode", "distributed") + + + + The same configuration file (with the node-specific + socket=bin:IP:PORT line changed per node) + is used on every node. No other per-node customization is + required. + +
+ +
+ Limitations + + + IPv4 only. IPv6 multicast is not currently supported. + + + The module pre-allocates approximately 152 KB of shared + memory (-m) per cluster and 6 KB of + private memory (-M) per worker process + at startup. If either allocation fails, OpenSIPS will refuse + to start with an error in the log. + + + Wildcard BIN sockets (bin:*:PORT) are + rejected at startup. An explicit IP address must be used in + the socket= line. + + + Node IDs are not persistent across a full cluster restart + (all nodes down simultaneously). IDs are reallocated + starting from 1 when the cluster reforms. This has no + operational impact as long as at least one node remains up + during rolling restarts. + + + The multicast network must support IP multicast routing + between all cluster nodes. Nodes on different L3 segments + require PIM or similar multicast routing. + + + PIM-DM networks: PIM Dense + Mode periodically re-floods multicast traffic (prune state + typically expires every 3 minutes). On slow or congested + networks this brief re-flood/prune cycle could cause a gap + in MASTER_ALIVE delivery and trigger a spurious master + re-election. If this occurs, increase the effective timeout + by raising CC_MASTER_KA_MISSED in the + source from 3 to 5 or higher. PIM Sparse Mode (PIM-SM) or + networks with IGMP snooping do not have this issue. + + + L2 overlay tunnels (Geneve, VXLAN, + GRE): if the overlay presents a flat L2 segment + with multicast support, the controller works transparently. + However, some VXLAN deployments disable multicast entirely + (no underlay multicast group, no BUM replication) — in that + case the controller will not function, as it has no unicast + fallback. Encapsulation overhead also adds latency and + jitter; on high-latency overlays consider raising + CC_MASTER_KA_MISSED to avoid spurious + re-elections. + + + IPsec-protected links: + native IPsec multicast requires GDOI/GET VPN (RFC 6407), + which is rarely deployed. The recommended approach is to + run multicast inside an inner tunnel (GRE-over-IPsec, + Geneve-over-IPsec) that presents a multicast-capable + interface. Running the controller over such a setup results + in double encryption (application-layer AES-256-GCM plus + IPsec ESP), which is harmless but adds minor CPU overhead. + IPsec ESP tunnel mode also reduces the effective MTU by + approximately 50 bytes, which compounds the MEMBER_LIST + fragmentation issue described below. + + + MEMBER_LIST fragmentation: + the MEMBER_LIST packet grows with cluster size and reaches + approximately 4395 bytes at the maximum of 256 nodes. This + exceeds the 1472-byte UDP payload budget of a standard + 1500-byte MTU Ethernet link and requires IP fragmentation: + + Standard Ethernet (1500 MTU): 3 fragments + IPsec ESP tunnel (~1400 MTU): 4 fragments + GRE-over-IPsec (~1350 MTU): 4–5 fragments + + All other packet types (ALIVE, JOIN_REQ, KEY_GRANT, GOODBYE, + etc.) fit comfortably within a single datagram on any of + these links. The DF bit is not set, so IP fragmentation + occurs transparently where the network allows it. However, + firewalls or stateless middleboxes that silently drop + fragmented UDP will prevent new nodes from joining, since + MEMBER_LIST is required to complete the join sequence. + Verify that fragmented UDP is permitted on all paths between + cluster nodes, particularly over VPN tunnels and across + datacenter firewalls. + + +
+ +
+ Planned Features + + The following features are planned for future releases: + + + + Node maintenance mode — take a node + out of duty for a rolling upgrade while it stays in the cluster. A + node in maintenance keeps replicating and answering pings and stays + visible in cl_ctr_list_members, but is excluded + from election (never master or backup; if it is master it hands over + gracefully first) and sheds its sharing tags (a + cl_ctr_shtag_force pin on it auto-clears). Two + levels are planned: + + evicted — out of election + and tags; the routing script refuses new work while established + dialogs finish. + full — additionally marks + the node down for clusterer consumers so peers stop routing + replication work to it. + + The state is cluster-wide (carried in the ALIVE/MEMBER_LIST control + plane, so every node agrees and it survives master failover) and + runtime-only — a restart brings the node back in service. + + + Interfaces (following the read-only variables and MI commands above): + + + MI cl_ctr_maintenance (any node, the + master propagates it) sets a target node's state + evicted/full/off; + cl_ctr_list_members gains a maintenance + column. + + + Script function cl_ctr_set_maintenance() + (a verb - an action) for a node to put itself in or out of + maintenance from the routing logic. + + + Read-only variables $cl_ctr_maintenance + (this node: none/evicted/full) and + $cl_ctr_node_maint(cluster_id, node_id) + (any peer); $cl_ctr_role gains a + maintenance value. + + + Event E_CL_CTR_MAINTENANCE raised on + every node when any member's maintenance state changes, so an + event_route can react (e.g. shift + dispatcher weights). + + + + + Statistics — module statistics + (current role, member count, master changes, nodes joined/left, + JOIN_REJECTs, decrypt failures, config mismatches, split-brain + merges) exposed via get_statistics and + monitoring exporters. + + + Events — events raised on state + transitions (became master, demoted, node joined/left, split-brain + merged, config mismatch, authentication reject), named + E_CL_CTR_* and consumable from an + event_route or any event subscriber transport. + + + IPv6 multicast — the control plane + currently uses IPv4 multicast (groups in 224.0.0.0/4). + Add IPv6 multicast support (ff00::/8 groups, + AF_INET6 sockets and membership) so the controller + can run on IPv6-only or dual-stack deployments. + + + + Read-only script variables for cluster state + ($cl_ctr_role, + $cl_ctr_is_master, and the rest) are already + available - see . + +
+ + diff --git a/modules/clusterer_controller/doc/clusterer_controller_tests.xml b/modules/clusterer_controller/doc/clusterer_controller_tests.xml new file mode 100644 index 00000000000..219306e537d --- /dev/null +++ b/modules/clusterer_controller/doc/clusterer_controller_tests.xml @@ -0,0 +1,266 @@ + + + + HA Behaviour Tests + + The following tests were performed on a three-node cluster + (nodes A=10.22.23.191, + B=10.22.23.192, + C=10.22.23.193) to verify correct failover, + tag stability, and no-steal-on-join behaviour. The sharing tag + under test is vip1 in cluster 1. All nodes run + with manage_shtags=1. + + + Tag state was queried after each operation via: + opensips-cli -x mi clusterer_list_shtags + + +
+ Baseline + + All three nodes running. B holds the active tag; A and C are + backup. + + +A (10.22.23.191) svc=active tag=backup +B (10.22.23.192) svc=active tag=active +C (10.22.23.193) svc=active tag=backup + +
+ +
+ Test 1 — Stop the active node + + B (active) is stopped. The remaining nodes must elect a new + active holder. B must rejoin as backup and must not steal the + tag from whichever node became active. + + +# Stop B +A svc=active tag=backup +B svc=inactive tag=(down) +C svc=active tag=active <-- C promoted + +# Start B +A svc=active tag=backup +B svc=active tag=backup <-- rejoined as backup +C svc=active tag=active <-- C retains active + + + Result: PASS. + Failover within the dead-node detection window; rejoining node + did not steal the active tag. + +
+ +
+ Test 2 — Stop a backup node + + A (backup) is stopped. The active tag must remain on C without + any transition. A must rejoin as backup. + + +# Stop A +A svc=inactive tag=(down) +B svc=active tag=backup +C svc=active tag=active <-- unchanged + +# Start A +A svc=active tag=backup <-- rejoined as backup +B svc=active tag=backup +C svc=active tag=active <-- still active + + + Result: PASS. + Removing a backup node causes no tag movement; rejoining node + started in backup state. + +
+ +
+ Test 3 — Stop both backup nodes + + A and B (both backup) are stopped simultaneously. The lone + remaining node C must retain the active tag. A and B must + rejoin as backup. + + +# Stop A and B +A svc=inactive tag=(down) +B svc=inactive tag=(down) +C svc=active tag=active <-- unchanged, lone node + +# Start A, then B +A svc=active tag=backup <-- rejoined as backup +B svc=active tag=backup <-- rejoined as backup +C svc=active tag=active <-- still active + + + Result: PASS. + Active node remained stable while running alone; both rejoining + nodes came up in backup state. + +
+ +
+ Test 4 — Stop active node and one backup + + B (active) and C (backup) are stopped. The sole remaining node + A must become active. B and C must rejoin as backup. + + +# Stop B and C +A svc=active tag=active <-- A promoted, now lone node +B svc=inactive tag=(down) +C svc=inactive tag=(down) + +# Start B +A svc=active tag=active <-- retains active +B svc=active tag=backup <-- rejoined as backup +C svc=inactive tag=(down) + +# Start C +A svc=active tag=active <-- retains active +B svc=active tag=backup +C svc=active tag=backup <-- rejoined as backup + + + Result: PASS. + The surviving node correctly claimed the active tag; each + rejoining node started in backup state without challenging the + active holder. + +
+ +
+ Test 5 — Full cluster restart + + All three nodes are stopped (full outage). Nodes are then + started one at a time. The first node up must self-elect as + active (no peers available to sync from). Subsequent nodes must + join as backup and must not steal the active tag. + + +# All stopped +A svc=inactive tag=(down) +B svc=inactive tag=(down) +C svc=inactive tag=(down) + +# Start A first +A svc=active tag=active <-- first/lone seed, self-synced +B svc=inactive tag=(down) +C svc=inactive tag=(down) + +# Start B +A svc=active tag=active <-- retains active +B svc=active tag=backup <-- joined as backup, did not steal +C svc=inactive tag=(down) + +# Start C +A svc=active tag=active <-- retains active +B svc=active tag=backup +C svc=active tag=backup <-- joined as backup, did not steal + + + Result: PASS. + The first node to start elected itself active and no spurious + sync errors were logged. Each subsequent node joined as backup + without challenging the active holder. + +
+ +
+ Test 6 — Multiple clusters over one BIN socket + + Two controller clusters (id=1 and + id=2) are configured on all three nodes. Each + cluster has its own multicast endpoint (a distinct port is required - + two clusters on the same multicast address:port are rejected at + startup with duplicate multicast), but both + advertise the same BIN socket + (bin:IP:3857). Each cluster must form independently + and its replication must stay isolated over the shared socket. + + +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "cluster_id", 2) +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:IP:3857") +modparam("clusterer_controller", "cluster", + "id=2,multicast=239.0.90.1:3334,bin_socket=bin:IP:3857") + +# both clusters converge, each with its own master/backup/member roles; +# clusterer_list shows cluster 1 and cluster 2 both using bin:IP:3857 + + + Result: PASS. + Both clusters formed independently (each member_count=3 + with consistent roles), the BIN links of both were Up + over the single shared socket, and there were zero decrypt / cross-talk / + foreign-cluster errors on any node - the cluster_id in + each BIN packet keeps the two clusters' replication traffic separate. A + control test confirmed that configuring both clusters on the same + multicast address:port is refused at startup. + +
+ +
+ Summary + + + + + + + + Test + Scenario + Result + + + + + 1 + Stop active node; rejoin + PASS + + + 2 + Stop backup node; rejoin + PASS + + + 3 + Stop both backups simultaneously; rejoin + PASS + + + 4 + Stop active + one backup; rejoin + PASS + + + 5 + Full cluster restart; sequential startup + PASS + + + 6 + Two clusters sharing one BIN socket (distinct multicast) + PASS + + + + + + In all five failover tests (1–5): exactly one node held the active + sharing tag at all times (including during the failure window), and no + rejoining node stole the active tag from the current holder. Test 6 + additionally confirmed that two clusters can share a single BIN socket + with fully isolated replication. + +
+ + diff --git a/modules/clusterer_controller/doc/contributors.xml b/modules/clusterer_controller/doc/contributors.xml new file mode 100644 index 00000000000..7d19b210dc0 --- /dev/null +++ b/modules/clusterer_controller/doc/contributors.xml @@ -0,0 +1,25 @@ + + Contributors +
+ Contributors + + Definition, design and implementation of this module was made by: + + + Yury Kirsanov — VoIPLine Telecom + + + +
+
+ Documentation Contributors + + Documentation was written by: + + + Yury Kirsanov — VoIPLine Telecom + + + +
+ diff --git a/modules/clusterer_controller/test/cc_join_reject_test.py b/modules/clusterer_controller/test/cc_join_reject_test.py new file mode 100755 index 00000000000..6096a1fdc2e --- /dev/null +++ b/modules/clusterer_controller/test/cc_join_reject_test.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +""" +cc_join_reject_test.py - rogue-joiner security test for clusterer_controller. + +Simulates an unauthorized node on the cluster's multicast group WITHOUT touching +any node config. It exercises two defences: + + 1. JOIN_REJECT path: send JOIN_REQ packets with the correct bootstrap magic but + a bogus (wrong-key) body. The master cannot decrypt them; after a few it is + expected to emit an encrypted JOIN_REJECT (a bootstrap-magic packet back on + the group). We can't decrypt that reject (wrong key), but observing a + bootstrap-magic packet from a cluster node in response confirms the master's + reject logic fired. + + 2. Rogue-traffic isolation (anti-churn): send bogus SESSION-magic packets + (a fake MASTER_ALIVE). A correct cluster must IGNORE these. We measure the + rate of real session traffic before vs during the flood: a large spike would + mean the flood pushed non-master nodes into a re-JOIN churn (the old bug). + +Run this on a host on the multicast segment that is NOT a live cluster member +(e.g. one node with `systemctl stop opensips`). Requires only python3. + + ./cc_join_reject_test.py --group 239.0.90.1 --port 3333 +""" +import argparse, collections, os, socket, struct, threading, time + +# --- wire constants (must match clusterer_controller.c) --- +BOOTSTRAP_MAGIC = bytes([0xCC, 0x01]) +SESSION_MAGIC = bytes([0xCC, 0x00]) +MAGIC_SZ, CLUSTER_ID_SZ, NONCE_SZ, TAG_SZ = 2, 2, 12, 16 + + +def bogus_packet(magic, cluster_id): + # [magic 2B][cluster_id 2B BE][nonce 12B][~60B random 'ciphertext'][tag 16B] + return (magic + struct.pack("!H", cluster_id & 0xFFFF) + + os.urandom(NONCE_SZ) + os.urandom(60) + os.urandom(TAG_SZ)) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--group", default="239.0.90.1") + ap.add_argument("--port", type=int, default=3333) + ap.add_argument("--count", type=int, default=6, help="packets per phase") + ap.add_argument("--interval", type=float, default=1.0, help="seconds between packets") + ap.add_argument("--cluster-id", type=int, default=1, + help="cluster_id to stamp on packets; use a value the target " + "cluster does NOT use to verify foreign packets are filtered") + args = ap.parse_args() + + # discover our own source IP (to filter our own loopback out) + p = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + p.connect((args.group, args.port)); my_ip = p.getsockname()[0]; p.close() + + rx = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + rx.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + rx.bind(("", args.port)) + rx.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, + struct.pack("4s4s", socket.inet_aton(args.group), + socket.inet_aton(my_ip))) + rx.settimeout(0.3) + + tx = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + tx.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 1) + + boot_from = collections.Counter() # bootstrap-magic pkts from peers (JOIN_REJECT candidates) + sess_from = collections.Counter() # session-magic pkts from peers + stop = threading.Event() + + def receiver(): + while not stop.is_set(): + try: + data, addr = rx.recvfrom(65535) + except socket.timeout: + continue + if addr[0] == my_ip or len(data) < MAGIC_SZ: + continue + if data[:MAGIC_SZ] == BOOTSTRAP_MAGIC: + boot_from[addr[0]] += 1 + elif data[:MAGIC_SZ] == SESSION_MAGIC: + sess_from[addr[0]] += 1 + + threading.Thread(target=receiver, daemon=True).start() + print(f"[*] rogue joiner {my_ip} -> {args.group}:{args.port}") + print(f" cluster_id={args.cluster_id} bootstrap={BOOTSTRAP_MAGIC.hex()} session={SESSION_MAGIC.hex()}") + + # baseline: 3s of quiet listening to learn the normal session-traffic rate + print("[*] measuring baseline cluster traffic for 3s ...") + t0 = time.time(); base0 = sum(sess_from.values()); time.sleep(3.0) + base_rate = (sum(sess_from.values()) - base0) / (time.time() - t0) + print(f" baseline session rate: {base_rate:.1f} pkt/s") + + # phase 1 - JOIN_REJECT probe + boot_before = sum(boot_from.values()) + print(f"\n[*] PHASE 1: sending {args.count} bogus JOIN_REQ (bootstrap magic) ...") + for i in range(args.count): + tx.sendto(bogus_packet(BOOTSTRAP_MAGIC, args.cluster_id), (args.group, args.port)) + print(f" -> JOIN_REQ #{i+1}") + time.sleep(args.interval) + time.sleep(2.0) + rejects = sum(boot_from.values()) - boot_before + + # phase 2 - anti-churn probe (fake MASTER_ALIVE flood) + print(f"\n[*] PHASE 2: flooding {args.count*3} bogus SESSION packets (fake MASTER_ALIVE) ...") + t1 = time.time(); sess1 = sum(sess_from.values()) + for i in range(args.count * 3): + tx.sendto(bogus_packet(SESSION_MAGIC, args.cluster_id), (args.group, args.port)) + time.sleep(args.interval / 3.0) + flood_rate = (sum(sess_from.values()) - sess1) / (time.time() - t1) + + time.sleep(1.0); stop.set() + + # --- report --- + print("\n===================== RESULT =====================") + print(f"JOIN_REQ sent (phase 1): {args.count}") + print(f"JOIN_REJECT-candidate replies from master: {rejects} " + f"from {sorted(k for k,v in boot_from.items() if v)}") + print(f"real session rate baseline={base_rate:.1f}/s during-flood={flood_rate:.1f}/s") + print("--------------------------------------------------") + ok = True + if rejects > 0: + print("PASS master answered unauthenticated JOIN_REQ with a JOIN_REJECT") + else: + print("WARN no JOIN_REJECT seen (rejection may rely on joiner-side shutdown)") + # a >3x spike over baseline indicates the flood induced re-JOIN churn + if flood_rate <= max(base_rate * 3.0, base_rate + 5): + print("PASS cluster ignored the rogue session flood (no churn spike)") + else: + print("FAIL session traffic spiked under the flood -> cluster was disrupted"); ok = False + print("==================================================") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 4348580f07b419536afbd39d0a3576da869d9cdd Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Wed, 15 Jul 2026 01:13:08 +1000 Subject: [PATCH 02/21] clusterer/clusterer_controller: enforce use_controller vs module-load consistency Two startup sanity checks for a previously silent misconfiguration: - clusterer_controller is only meaningful when the clusterer module has use_controller=1 (the global switch that pre-creates the controller-managed cluster stubs, sets each one's controller_managed flag so they never touch the DB, and arms the guard that stops the controller from hijacking a native cluster of the same id). If use_controller=0 there is no controller-managed cluster and those safety mechanisms are off, so mod_init now FAILS (refuse to start) rather than driving clusters clusterer never authorised. Hybrid setups keep use_controller=1 - only the per-cluster kind differs - so this never trips them. - The mirror case, use_controller=1 but no clusterer_controller module bound the controller API, logs an ERROR at clusterer child_init (the pre-created controller-managed stubs would never obtain an identity). clusterer does NOT abort here: its native/hybrid clusters still work; only the controller-stub clusters are dead. Plumbing: the clusterer_ctrl binds struct now carries clusterer's use_controller value, and load_clusterer_ctrl_binds() sets clusterer_ctrl_bound so clusterer can tell whether a controller registered. Verified on all permutations: pure controller, hybrid, native-only, and both mismatches. --- modules/clusterer/clusterer_ctrl.c | 5 +++ modules/clusterer/clusterer_ctrl.h | 10 ++++++ modules/clusterer/clusterer_mod.c | 12 +++++++ modules/clusterer_controller/README | 28 +++++++++++++-- .../clusterer_controller.c | 19 ++++++++++ .../doc/clusterer_controller_admin.xml | 36 ++++++++++++++++--- 6 files changed, 102 insertions(+), 8 deletions(-) diff --git a/modules/clusterer/clusterer_ctrl.c b/modules/clusterer/clusterer_ctrl.c index e27ef114aa3..0b9c354b401 100644 --- a/modules/clusterer/clusterer_ctrl.c +++ b/modules/clusterer/clusterer_ctrl.c @@ -372,12 +372,17 @@ int clusterer_ctrl_unset_shtag_managed(int cluster_id) return 0; } +/* 1 once a controller module has bound this API (see clusterer_ctrl.h). */ +int clusterer_ctrl_bound = 0; + int load_clusterer_ctrl_binds(clusterer_ctrl_binds_t *binds) { if (!binds) { LM_ERR("clusterer: load_clusterer_ctrl_binds: NULL binds\n"); return -1; } + clusterer_ctrl_bound = 1; + binds->use_controller = use_controller; binds->set_my_identity = clusterer_ctrl_set_identity; binds->add_node = clusterer_ctrl_add_node; binds->remove_node = clusterer_ctrl_remove_node; diff --git a/modules/clusterer/clusterer_ctrl.h b/modules/clusterer/clusterer_ctrl.h index af4f8b9f47b..270f8a3dcaf 100644 --- a/modules/clusterer/clusterer_ctrl.h +++ b/modules/clusterer/clusterer_ctrl.h @@ -157,8 +157,18 @@ typedef struct clusterer_ctrl_binds { * @return 0 on success, -1 if cluster not found */ int (*unset_shtag_managed)(int cluster_id); + + /* Introspection: the value of the clusterer 'use_controller' modparam at + * bind time. Lets clusterer_controller detect the mirror misconfiguration + * (controller loaded but clusterer has use_controller=0, so no stubs were + * pre-created). */ + int use_controller; } clusterer_ctrl_binds_t; +/* Set to 1 by load_clusterer_ctrl_binds() when a controller module binds the + * API, so clusterer can warn if use_controller=1 yet no controller registered. */ +extern int clusterer_ctrl_bound; + /** * load_clusterer_ctrl_binds() — fill a clusterer_ctrl_binds_t struct. * diff --git a/modules/clusterer/clusterer_mod.c b/modules/clusterer/clusterer_mod.c index a837da4192b..bfc60872a9a 100644 --- a/modules/clusterer/clusterer_mod.c +++ b/modules/clusterer/clusterer_mod.c @@ -657,6 +657,18 @@ static int child_init(int rank) return -1; } } + + /* One-shot config sanity check (rank 1 fires once, and runs after every + * module's mod_init, so a clusterer_controller would already have bound the + * controller API): use_controller=1 is meaningless without that module - + * the pre-created controller-managed cluster stubs would never obtain an + * identity or form. */ + if (rank == 1 && use_controller && !clusterer_ctrl_bound) + LM_ERR("clusterer: use_controller=1 but no clusterer_controller module " + "registered the controller API - controller-managed cluster(s) " + "will never obtain a node identity or form. Load the " + "clusterer_controller module, or unset use_controller.\n"); + return 0; } diff --git a/modules/clusterer_controller/README b/modules/clusterer_controller/README index 5cda8f9f1e5..651e1bc2e18 100644 --- a/modules/clusterer_controller/README +++ b/modules/clusterer_controller/README @@ -496,9 +496,15 @@ on-key") * proto_bin — required so that BIN listeners are registered and available for discovery when clusterer_controller initialises and scans the proto_bin listener list. - * clusterer — required with use_controller=1 set, so that the - clusterer internal API (load_clusterer_ctrl_binds) is - exported and available at initialisation time. + * clusterer — required, and it must have use_controller=1 + set. That global switch is what pre-creates the + controller-managed cluster stubs, marks them so they never + touch the database, and arms the guard that stops the + controller from driving a native cluster of the same id. If + clusterer_controller is loaded while clusterer has + use_controller=0, it refuses to start with a clear error — + there is no controller-managed cluster for it to drive and + those safety mechanisms would be off. Both dependencies are declared in the module's dep_export_t. OpenSIPS will refuse to start if either dependency is not @@ -507,6 +513,22 @@ on-key") resolves the dependency at runtime and will initialize the required modules first regardless of loadmodule order. + The reverse is also checked: if use_controller=1 is set but the + clusterer_controller module is not loaded, clusterer logs an + error at startup, since the controller-managed cluster stubs + would never obtain a node identity or form. clusterer itself + keeps running, so any native or DB-backed clusters are + unaffected. + + Hybrid deployments are unaffected. use_controller is a single + global switch; in a hybrid instance (native and + controller-managed clusters side by side) it is always 1 and + only the per-cluster kind differs (native via DB/static + provisioning versus controller-managed via the cluster_id + list). Both consistency checks above therefore only fire on a + genuine module/config mismatch — never on a hybrid or + pure-controller setup. + All other modules that use the clusterer interface (tm, dialog, dispatcher, usrloc etc.) may be loaded in any order relative to clusterer_controller. The clusterer module automatically diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index 6247b1c4862..bd6e03f8fcb 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -5804,6 +5804,25 @@ static int mod_init(void) LM_INFO("clusterer_controller: clusterer API loaded - " "topology will be driven dynamically\n"); + /* The controller is only meaningful when the clusterer module is told + * to expect it (use_controller=1, a global switch): that is what + * pre-creates the controller-managed cluster stubs, sets each one's + * controller_managed flag (so they never touch the DB), and arms the + * guard that stops the controller from hijacking a native cluster of the + * same id. With use_controller=0 there is no controller-managed cluster + * at all and those safety mechanisms are off, so refuse to start rather + * than run the control plane against clusters clusterer never authorised + * us to drive. (Hybrid setups keep use_controller=1 - only the per- + * cluster kind differs - so this never trips them.) */ + if (!clctl.use_controller) { + LM_ERR("clusterer_controller: loaded but the clusterer module has " + "use_controller=0 - there is no controller-managed cluster and " + "the controller's safety guards are disabled. Set " + "modparam(\"clusterer\", \"use_controller\", 1), or remove the " + "clusterer_controller module.\n"); + return -1; + } + /* When we manage sharing tags, start every tag as BACKUP and * lock out MI/script changes - the controller master decides * who becomes active. (manage_shtags sentinels were already diff --git a/modules/clusterer_controller/doc/clusterer_controller_admin.xml b/modules/clusterer_controller/doc/clusterer_controller_admin.xml index 17df402df7d..ae7132d75f8 100644 --- a/modules/clusterer_controller/doc/clusterer_controller_admin.xml +++ b/modules/clusterer_controller/doc/clusterer_controller_admin.xml @@ -502,11 +502,17 @@ - clusterer — required with - use_controller=1 set, so that - the clusterer internal API - (load_clusterer_ctrl_binds) is - exported and available at initialisation time. + clusterer — required, and it + must have + use_controller=1 set. That global switch + is what pre-creates the controller-managed cluster stubs, + marks them so they never touch the database, and arms the + guard that stops the controller from driving a native cluster + of the same id. If clusterer_controller + is loaded while clusterer has + use_controller=0, it refuses to start with + a clear error — there is no controller-managed cluster for it + to drive and those safety mechanisms would be off. @@ -521,6 +527,26 @@ required modules first regardless of loadmodule order. + + The reverse is also checked: if + use_controller=1 is set but the + clusterer_controller module is not loaded, + clusterer logs an error at startup, since the controller-managed + cluster stubs would never obtain a node identity or form. clusterer + itself keeps running, so any native or DB-backed clusters are + unaffected. + + + Hybrid deployments are unaffected. + use_controller is a single global switch; in a + hybrid instance (native and controller-managed clusters side by side) + it is always 1 and only the + per-cluster kind differs (native via DB/static + provisioning versus controller-managed via the + cluster_id list). Both consistency checks above + therefore only fire on a genuine module/config mismatch — never on a + hybrid or pure-controller setup. + All other modules that use the clusterer interface (tm, dialog, From 8a193106558a08a5b17d9e45d2b77868f6303c11 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Wed, 15 Jul 2026 02:18:52 +1000 Subject: [PATCH 03/21] clusterer_controller docs: make every cluster explicit in config examples Reviewed all config examples so every cluster is clearly and self-containedly defined, per reviewer feedback: - Show modparam("clusterer","cluster_id",N) for each controller-managed cluster in the full-config and multi-cluster examples (previously several relied on the implicit auto-create-on-capability-registration path, so the cluster never appeared "defined" in the snippet). - Hybrid DB example: note that native cluster 10 is defined by rows in the clusterer DB table (not a modparam), and fix the misleading "this node id in cluster 10" comment (my_node_id is a single global id across all DB clusters). - Add the missing "Hybrid, no DB" (static) example to the admin guide, with the required db_mode=0 (without it my_node_info/neighbor_node_info are ignored). - Comment use_controller as the global switch throughout. No behavior change - documentation only. README regenerated. --- modules/clusterer_controller/README | 112 ++++++++++++++++-- .../doc/clusterer_controller_admin.xml | 80 +++++++++++-- 2 files changed, 171 insertions(+), 21 deletions(-) diff --git a/modules/clusterer_controller/README b/modules/clusterer_controller/README index 651e1bc2e18..db7487fa19e 100644 --- a/modules/clusterer_controller/README +++ b/modules/clusterer_controller/README @@ -90,7 +90,10 @@ CLUSTERER_CONTROLLER Module 1.22. Multiple clusters — same multicast IP, different ports 1.23. Hybrid — DB-native cluster 10 + controller cluster 1 - 1.24. Minimal HA cluster configuration + 1.24. Hybrid, no DB — static native cluster 7 + controller + cluster 1 + + 1.25. Minimal HA cluster configuration Chapter 1. Admin Guide @@ -827,6 +830,12 @@ modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") Example 1.8. Global manage_shtags — applies to all clusters ... +# clusters 1 and 2 registered as controller-managed on the clusterer sid +e +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "cluster_id", 2) + # Enable automatic failover for every cluster (this is also the default) modparam("clusterer_controller", "manage_shtags", 1) modparam("clusterer_controller", "cluster", "id=1,multicast=239.0.90.1:3 @@ -838,6 +847,12 @@ modparam("clusterer_controller", "cluster", "id=2,multicast=239.0.90.2:3 Example 1.9. Per-cluster override — opt one cluster out of automatic failover ... +# clusters 1 and 2 registered as controller-managed on the clusterer sid +e +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "cluster_id", 2) + # manage_shtags=1 globally, but cluster 2 uses its own MI/event-route sc ripts modparam("clusterer_controller", "manage_shtags", 1) @@ -849,6 +864,12 @@ modparam("clusterer_controller", "cluster", Example 1.10. Global opt-out with one cluster opting in ... +# clusters 1 and 2 registered as controller-managed on the clusterer sid +e +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "cluster_id", 2) + # Disable automatic failover globally; enable it only for cluster 1 modparam("clusterer_controller", "manage_shtags", 0) modparam("clusterer_controller", "cluster", @@ -869,7 +890,10 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) +modparam("clusterer", "use_controller", 1) # enable the controller p +ath (global switch) +modparam("clusterer", "cluster_id", 1) # register cluster 1 as c +ontroller-managed modparam("clusterer", "sharing_tag", "vip1/1=active") modparam("clusterer", "ping_interval", 4) modparam("clusterer", "ping_timeout", 1500) @@ -914,6 +938,11 @@ modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") Example 1.12. Set master_stickiness parameter ... +# both clusters registered as controller-managed on the clusterer side +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "cluster_id", 2) + # Global default (sticky) — omit entirely for the same effect modparam("clusterer_controller", "master_stickiness", 1) @@ -1181,7 +1210,12 @@ socket=bin:10.0.2.10:5566 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) +modparam("clusterer", "use_controller", 1) # enable the controller pat +h (global switch) +modparam("clusterer", "cluster_id", 1) # cluster 1 is controller- +managed +modparam("clusterer", "cluster_id", 2) # cluster 2 is controller- +managed loadmodule "clusterer_controller.so" # Cluster 1 — dialog replication group, LAN segment 10.0.1.0/24 @@ -1204,7 +1238,12 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) +modparam("clusterer", "use_controller", 1) # enable the controller pat +h (global switch) +modparam("clusterer", "cluster_id", 1) # cluster 1 is controller- +managed +modparam("clusterer", "cluster_id", 2) # cluster 2 is controller- +managed loadmodule "clusterer_controller.so" modparam("clusterer_controller", "cluster", @@ -1265,15 +1304,21 @@ loadmodule "db_mysql.so" loadmodule "proto_bin.so" loadmodule "clusterer.so" -# native side: cluster 10 lives in the DB (nodes provisioned there) +# native side: cluster 10 is defined entirely by rows in the clusterer D +B +# table (one row per node, including a row whose node_id = my_node_id be +low +# for THIS node) - there is no cluster_id modparam for native clusters. modparam("clusterer", "db_mode", 1) modparam("clusterer", "db_url", "mysql://opensips:pass@localhost/opensip s") -modparam("clusterer", "my_node_id", 5) # this node's id in cluster -10 -# controller side: cluster 1 is dynamic -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "my_node_id", 5) # this node's id in its nati +ve/DB clusters (global) +# controller side: cluster 1 is dynamic (no DB, no static rows) +modparam("clusterer", "use_controller", 1) # enable the controller path + (global switch) +modparam("clusterer", "cluster_id", 1) # register cluster 1 as con +troller-managed loadmodule "clusterer_controller.so" modparam("clusterer_controller", "cluster", @@ -1287,6 +1332,46 @@ ed loadmodule "dispatcher.so" modparam("dispatcher", "cluster_id", 10) # DB-native + Example 1.24. Hybrid, no DB — static native cluster 7 + + controller cluster 1 +socket=bin:10.0.0.10:5566 + +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +# native side: cluster 7 is provisioned statically (no DB). +# db_mode=0 is REQUIRED, otherwise my_node_info/neighbor_node_info are i +gnored. +modparam("clusterer", "db_mode", 0) +modparam("clusterer", "my_node_id", 5) # this node's id in + native cluster 7 +modparam("clusterer", "my_node_info", "cluster_id=7, url=bin:10.0. +0.10:5566") +modparam("clusterer", "neighbor_node_info", "cluster_id=7, node_id=6, ur +l=bin:10.0.0.11:5566") +# controller side: cluster 1 is dynamic +modparam("clusterer", "use_controller", 1) # enable the contro +ller path (global switch) +modparam("clusterer", "cluster_id", 1) # register cluster + 1 as controller-managed + +loadmodule "clusterer_controller.so" +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:10.0.0.10:5566,passwo +rd=S3cret") + +loadmodule "dialog.so" +modparam("dialog", "dialog_replication_cluster", 1) # controller-manag +ed (cluster 1) + +loadmodule "dispatcher.so" +modparam("dispatcher", "cluster_id", 7) # native (cluster 7 +) + + Here my_node_id=5 is this node's id in the native cluster 7; in + cluster 1 the controller assigns an id at runtime, which may + differ. + 1.12. Configuration Example The following example shows a minimal two-module configuration @@ -1294,14 +1379,17 @@ modparam("dispatcher", "cluster_id", 10) # DB-native loadmodule order does not matter — the dependency system enforces correct initialization order automatically. - Example 1.24. Minimal HA cluster configuration + Example 1.25. Minimal HA cluster configuration # Each node needs an explicit BIN socket (no wildcard) socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) +modparam("clusterer", "use_controller", 1) # enable the controller p +ath (global switch) +modparam("clusterer", "cluster_id", 1) # register cluster 1 as c +ontroller-managed modparam("clusterer", "sharing_tag", "vip1/1=active") modparam("clusterer", "ping_interval", 4) modparam("clusterer", "ping_timeout", 1500) diff --git a/modules/clusterer_controller/doc/clusterer_controller_admin.xml b/modules/clusterer_controller/doc/clusterer_controller_admin.xml index ae7132d75f8..b663231e511 100644 --- a/modules/clusterer_controller/doc/clusterer_controller_admin.xml +++ b/modules/clusterer_controller/doc/clusterer_controller_admin.xml @@ -984,6 +984,11 @@ modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") Global <varname>manage_shtags</varname> — applies to all clusters ... +# clusters 1 and 2 registered as controller-managed on the clusterer side +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "cluster_id", 2) + # Enable automatic failover for every cluster (this is also the default) modparam("clusterer_controller", "manage_shtags", 1) modparam("clusterer_controller", "cluster", "id=1,multicast=239.0.90.1:3333") @@ -995,6 +1000,11 @@ modparam("clusterer_controller", "cluster", "id=2,multicast=239.0.90.2:3333") Per-cluster override — opt one cluster out of automatic failover ... +# clusters 1 and 2 registered as controller-managed on the clusterer side +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "cluster_id", 2) + # manage_shtags=1 globally, but cluster 2 uses its own MI/event-route scripts modparam("clusterer_controller", "manage_shtags", 1) modparam("clusterer_controller", "cluster", @@ -1008,6 +1018,11 @@ modparam("clusterer_controller", "cluster", Global opt-out with one cluster opting in ... +# clusters 1 and 2 registered as controller-managed on the clusterer side +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "cluster_id", 2) + # Disable automatic failover globally; enable it only for cluster 1 modparam("clusterer_controller", "manage_shtags", 0) modparam("clusterer_controller", "cluster", @@ -1029,7 +1044,8 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) +modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) +modparam("clusterer", "cluster_id", 1) # register cluster 1 as controller-managed modparam("clusterer", "sharing_tag", "vip1/1=active") modparam("clusterer", "ping_interval", 4) modparam("clusterer", "ping_timeout", 1500) @@ -1091,6 +1107,11 @@ modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") Set <varname>master_stickiness</varname> parameter ... +# both clusters registered as controller-managed on the clusterer side +modparam("clusterer", "use_controller", 1) +modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "cluster_id", 2) + # Global default (sticky) — omit entirely for the same effect modparam("clusterer_controller", "master_stickiness", 1) @@ -1473,7 +1494,9 @@ socket=bin:10.0.2.10:5566 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) +modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) +modparam("clusterer", "cluster_id", 1) # cluster 1 is controller-managed +modparam("clusterer", "cluster_id", 2) # cluster 2 is controller-managed loadmodule "clusterer_controller.so" # Cluster 1 — dialog replication group, LAN segment 10.0.1.0/24 @@ -1498,7 +1521,9 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) +modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) +modparam("clusterer", "cluster_id", 1) # cluster 1 is controller-managed +modparam("clusterer", "cluster_id", 2) # cluster 2 is controller-managed loadmodule "clusterer_controller.so" modparam("clusterer_controller", "cluster", @@ -1589,13 +1614,15 @@ loadmodule "db_mysql.so" loadmodule "proto_bin.so" loadmodule "clusterer.so" -# native side: cluster 10 lives in the DB (nodes provisioned there) +# native side: cluster 10 is defined entirely by rows in the clusterer DB +# table (one row per node, including a row whose node_id = my_node_id below +# for THIS node) - there is no cluster_id modparam for native clusters. modparam("clusterer", "db_mode", 1) modparam("clusterer", "db_url", "mysql://opensips:pass@localhost/opensips") -modparam("clusterer", "my_node_id", 5) # this node's id in cluster 10 -# controller side: cluster 1 is dynamic -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) +modparam("clusterer", "my_node_id", 5) # this node's id in its native/DB clusters (global) +# controller side: cluster 1 is dynamic (no DB, no static rows) +modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) +modparam("clusterer", "cluster_id", 1) # register cluster 1 as controller-managed loadmodule "clusterer_controller.so" modparam("clusterer_controller", "cluster", @@ -1608,6 +1635,40 @@ loadmodule "dispatcher.so" modparam("dispatcher", "cluster_id", 10) # DB-native + + Hybrid, no DB — static native cluster 7 + controller cluster 1 + +socket=bin:10.0.0.10:5566 + +loadmodule "proto_bin.so" + +loadmodule "clusterer.so" +# native side: cluster 7 is provisioned statically (no DB). +# db_mode=0 is REQUIRED, otherwise my_node_info/neighbor_node_info are ignored. +modparam("clusterer", "db_mode", 0) +modparam("clusterer", "my_node_id", 5) # this node's id in native cluster 7 +modparam("clusterer", "my_node_info", "cluster_id=7, url=bin:10.0.0.10:5566") +modparam("clusterer", "neighbor_node_info", "cluster_id=7, node_id=6, url=bin:10.0.0.11:5566") +# controller side: cluster 1 is dynamic +modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) +modparam("clusterer", "cluster_id", 1) # register cluster 1 as controller-managed + +loadmodule "clusterer_controller.so" +modparam("clusterer_controller", "cluster", + "id=1,multicast=239.0.90.1:3333,bin_socket=bin:10.0.0.10:5566,password=S3cret") + +loadmodule "dialog.so" +modparam("dialog", "dialog_replication_cluster", 1) # controller-managed (cluster 1) + +loadmodule "dispatcher.so" +modparam("dispatcher", "cluster_id", 7) # native (cluster 7) + + + Here my_node_id=5 is this node's id in the native + cluster 7; in cluster 1 the controller assigns an id at runtime, which + may differ. + +
@@ -1628,7 +1689,8 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) +modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) +modparam("clusterer", "cluster_id", 1) # register cluster 1 as controller-managed modparam("clusterer", "sharing_tag", "vip1/1=active") modparam("clusterer", "ping_interval", 4) modparam("clusterer", "ping_timeout", 1500) From b7a173ee66a1a3cb3c165962887cac99dc673e53 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Wed, 15 Jul 2026 02:32:49 +1000 Subject: [PATCH 04/21] clusterer: declare controller-managed clusters via cluster_options Add a per-cluster 'cluster_options' modparam - the same "key=value, key=value" idiom as my_node_info - so a cluster is marked controller-managed with: modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") cluster_id is required; use_controller is a 0/1 flag defaulting to 0 (native), and only use_controller=1 pre-creates the controller-managed stub (which never touches the DB and is guarded against hijacking a native cluster of the same id). Native clusters need no cluster_options line at all. Every other clusterer setting (db_mode, ping_*, my_node_id, sharing_tag, ...) stays a global modparam. The controller-managed ids and the clusterer_controller 'cluster' entries must match EXACTLY. clusterer_controller aborts at startup, naming the offending id, if either side references a cluster the other does not: - a managed id with no 'cluster' config has no BIN socket or crypto params; - a 'cluster' config for an unmanaged id has nothing legitimate to drive. The controller loads the clusterer's controller-managed id set through the ctrl binds (managed_count/managed_ids) to run this check pre-fork. The interim 'use_controller'/'cluster_id' int modparams (never released) are kept registered only to fail with a migration hint. Docs (admin guide, tests appendix, README) rewritten to the cluster_options form; every example gives each cluster an explicit definition. --- modules/clusterer/clusterer_ctrl.c | 2 + modules/clusterer/clusterer_ctrl.h | 14 +- modules/clusterer/clusterer_mod.c | 131 ++++++++++++--- modules/clusterer/node_info.h | 3 + modules/clusterer_controller/README | 157 +++++++++--------- .../clusterer_controller.c | 70 ++++++-- .../doc/clusterer_controller_admin.xml | 113 +++++++------ .../doc/clusterer_controller_tests.xml | 5 +- 8 files changed, 319 insertions(+), 176 deletions(-) diff --git a/modules/clusterer/clusterer_ctrl.c b/modules/clusterer/clusterer_ctrl.c index 0b9c354b401..72ea99453ed 100644 --- a/modules/clusterer/clusterer_ctrl.c +++ b/modules/clusterer/clusterer_ctrl.c @@ -383,6 +383,8 @@ int load_clusterer_ctrl_binds(clusterer_ctrl_binds_t *binds) } clusterer_ctrl_bound = 1; binds->use_controller = use_controller; + binds->managed_count = cc_stub_count; + binds->managed_ids = cc_stub_ids; binds->set_my_identity = clusterer_ctrl_set_identity; binds->add_node = clusterer_ctrl_add_node; binds->remove_node = clusterer_ctrl_remove_node; diff --git a/modules/clusterer/clusterer_ctrl.h b/modules/clusterer/clusterer_ctrl.h index 270f8a3dcaf..c7a41eb88ca 100644 --- a/modules/clusterer/clusterer_ctrl.h +++ b/modules/clusterer/clusterer_ctrl.h @@ -158,11 +158,17 @@ typedef struct clusterer_ctrl_binds { */ int (*unset_shtag_managed)(int cluster_id); - /* Introspection: the value of the clusterer 'use_controller' modparam at - * bind time. Lets clusterer_controller detect the mirror misconfiguration - * (controller loaded but clusterer has use_controller=0, so no stubs were - * pre-created). */ + /* Introspection: 1 if the clusterer has any controller-managed cluster + * (a cluster_options with use_controller=1), else 0. */ int use_controller; + + /* The set of cluster_ids the clusterer marked controller-managed + * (cluster_options use_controller=1): count, and a pointer to the clusterer's + * own array (valid for the process lifetime; read pre-fork in mod_init). The + * controller checks it has a matching 'cluster' config for every one of these - + * a clusterer-managed cluster with no controller config is a hard error. */ + int managed_count; + int *managed_ids; } clusterer_ctrl_binds_t; /* Set to 1 by load_clusterer_ctrl_binds() when a controller module binds the diff --git a/modules/clusterer/clusterer_mod.c b/modules/clusterer/clusterer_mod.c index bfc60872a9a..9a3d5695b6f 100644 --- a/modules/clusterer/clusterer_mod.c +++ b/modules/clusterer/clusterer_mod.c @@ -51,22 +51,112 @@ int current_id = -1; int *_current_id_shm = NULL; int db_mode = 1; int clusterer_enable_rerouting = 1; -int use_controller = 0; +int use_controller = 0; /* derived: 1 if any cluster_options sets use_controller=1 */ -/* cluster_ids to pre-create when use_controller=1 */ -static int cc_stub_ids[64]; -static int cc_stub_count = 0; +/* cluster_ids marked controller-managed via 'cluster_options' (use_controller=1). + * Not static: the controller loads a pointer to these through the ctrl binds so it + * can verify it has a matching 'cluster' config for each. */ +int cc_stub_ids[64]; +int cc_stub_count = 0; -static int cc_add_cluster_id(modparam_t type, void *val) +/* extract "=" from a "key=value, key=value" cluster_options string. + * Returns 0 and sets *out on success, 1 if the key is absent, -1 if malformed. */ +static int cc_opt_int(str *descr, str *name, int *out) { - if (cc_stub_count >= 64) { - LM_ERR("clusterer: too many cluster_id entries\n"); + char *p, *pe; + str aux; + + p = str_strstr(descr, name); + if (!p) + return 1; + p += name->len; + p = q_memchr(p, '=', descr->s + descr->len - p); + if (!p) { + LM_ERR("clusterer: expected '=' after '%.*s' in cluster_options\n", + name->len, name->s); + return -1; + } + p++; + pe = q_memchr(p, ',', descr->s + descr->len - p); + aux.s = p; + aux.len = pe ? pe - p : descr->s + descr->len - p; + str_trim_spaces_lr(aux); + if (aux.len == 0 || str2int(&aux, (unsigned int *)out)) { + LM_ERR("clusterer: bad value for '%.*s' in cluster_options\n", + name->len, name->s); return -1; } - cc_stub_ids[cc_stub_count++] = (int)(long)val; return 0; } +/* 'cluster_options' modparam: per-cluster settings, same key=value idiom as + * my_node_info. Format: "cluster_id=N, use_controller=0|1". cluster_id is + * required; use_controller is optional and defaults to 0 (native). Only + * use_controller=1 registers the cluster as controller-managed - its identity + * and peer list are then driven at runtime by clusterer_controller. Every other + * clusterer option stays a global modparam. */ +static int cc_add_cluster_options(modparam_t type, void *val) +{ + static str cid_prop = str_init("cluster_id"); + static str uc_prop = str_init("use_controller"); + str descr; + int cid = -1, uc = 0, rc, i; + + if (!val || !*(char *)val) { + LM_ERR("clusterer: empty 'cluster_options'\n"); + return -1; + } + descr.s = (char *)val; + descr.len = strlen(descr.s); + + rc = cc_opt_int(&descr, &cid_prop, &cid); + if (rc < 0) + return -1; + if (rc == 1 || cid < 1) { + LM_ERR("clusterer: 'cluster_options' requires cluster_id >= 1, " + "e.g. \"cluster_id=1, use_controller=1\"\n"); + return -1; + } + + rc = cc_opt_int(&descr, &uc_prop, &uc); /* absent (rc==1) -> uc stays 0 */ + if (rc < 0) + return -1; + if (uc != 0 && uc != 1) { + LM_ERR("clusterer: use_controller in 'cluster_options' must be 0 or 1 " + "(cluster_id %d)\n", cid); + return -1; + } + + if (uc == 0) + return 0; /* native (the default): nothing to register */ + + for (i = 0; i < cc_stub_count; i++) + if (cc_stub_ids[i] == cid) { + LM_ERR("clusterer: cluster_options declares cluster_id %d as " + "controller-managed more than once\n", cid); + return -1; + } + if (cc_stub_count >= (int)(sizeof cc_stub_ids / sizeof cc_stub_ids[0])) { + LM_ERR("clusterer: too many controller-managed clusters\n"); + return -1; + } + cc_stub_ids[cc_stub_count++] = cid; + use_controller = 1; /* at least one controller-managed cluster exists */ + return 0; +} + +/* Retired interim modparams (never released): controller-managed clusters are now + * declared via 'cluster_options'. Kept registered only to fail with a clear + * migration hint instead of a generic "unknown parameter". */ +static int cc_retired_param(modparam_t type, void *val) +{ + LM_ERR("clusterer: the 'use_controller'/'cluster_id' modparams have been " + "replaced; declare each controller-managed cluster with " + "modparam(\"clusterer\", \"cluster_options\", " + "\"cluster_id=N, use_controller=1\")\n"); + return -1; +} + str clusterer_db_url = {NULL, 0}; extern db_con_t *db_hdl; @@ -176,8 +266,9 @@ static const param_export_t params[] = { {"flags_col", STR_PARAM, &flags_col.s }, {"description_col", STR_PARAM, &description_col.s }, {"db_mode", INT_PARAM, &db_mode }, - {"use_controller", INT_PARAM, &use_controller }, - {"cluster_id", INT_PARAM|USE_FUNC_PARAM, (void*)cc_add_cluster_id}, + {"cluster_options", STR_PARAM|USE_FUNC_PARAM, (void*)cc_add_cluster_options}, + {"use_controller", INT_PARAM|USE_FUNC_PARAM, (void*)cc_retired_param}, + {"cluster_id", INT_PARAM|USE_FUNC_PARAM, (void*)cc_retired_param}, {"neighbor_node_info", STR_PARAM|USE_FUNC_PARAM, (void*)&provision_neighbor}, {"my_node_info", STR_PARAM|USE_FUNC_PARAM, @@ -526,8 +617,8 @@ static int mod_init(void) db_hdl = NULL; } - /* Controller-managed clusters: pre-create a stub for each cluster_id the - * controller registered (via the 'cluster_id' modparam), so cl_register_cap() + /* Controller-managed clusters: pre-create a stub for each cluster_id declared + * controller-managed (cluster_options use_controller=1), so cl_register_cap() * succeeds for tm/dialog before clusterer_controller injects the real * identity. Each stub is flagged controller_managed: it never touches the DB * and behaves as db_mode=0, so it coexists with DB-native clusters. A @@ -541,7 +632,7 @@ static int mod_init(void) for (_ex = *cluster_list; _ex; _ex = _ex->next) if (_ex->cluster_id == cc_stub_ids[_ci]) { LM_ERR("clusterer: cluster_id %d is registered as " - "controller-managed (cluster_id modparam) but is " + "controller-managed (cluster_options use_controller=1) but is " "already defined via native config " "(my_node_info/neighbor_node_info/DB) or listed " "twice - a cluster_id must be unique and either " @@ -660,14 +751,14 @@ static int child_init(int rank) /* One-shot config sanity check (rank 1 fires once, and runs after every * module's mod_init, so a clusterer_controller would already have bound the - * controller API): use_controller=1 is meaningless without that module - - * the pre-created controller-managed cluster stubs would never obtain an - * identity or form. */ + * controller API): a controller-managed cluster is meaningless without that + * module - the pre-created stubs would never obtain an identity or form. */ if (rank == 1 && use_controller && !clusterer_ctrl_bound) - LM_ERR("clusterer: use_controller=1 but no clusterer_controller module " - "registered the controller API - controller-managed cluster(s) " - "will never obtain a node identity or form. Load the " - "clusterer_controller module, or unset use_controller.\n"); + LM_ERR("clusterer: cluster_options declares controller-managed cluster(s) " + "but no clusterer_controller module registered the controller API - " + "they will never obtain a node identity or form. Load the " + "clusterer_controller module, or drop use_controller from " + "cluster_options.\n"); return 0; } diff --git a/modules/clusterer/node_info.h b/modules/clusterer/node_info.h index a95661d7ded..b8ff8f7917f 100644 --- a/modules/clusterer/node_info.h +++ b/modules/clusterer/node_info.h @@ -194,6 +194,9 @@ static inline int cluster_self_id(const struct cluster_info *cl) } extern int db_mode; extern int use_controller; +/* cluster_ids declared controller-managed via cluster_options (use_controller=1) */ +extern int cc_stub_ids[]; +extern int cc_stub_count; extern rw_lock_t *cl_list_lock; extern cluster_info_t **cluster_list; diff --git a/modules/clusterer_controller/README b/modules/clusterer_controller/README index db7487fa19e..7d017cde7a3 100644 --- a/modules/clusterer_controller/README +++ b/modules/clusterer_controller/README @@ -106,11 +106,13 @@ Chapter 1. Admin Guide each other automatically at startup and the cluster topology is maintained dynamically at runtime. - When use_controller=1 is set in the clusterer module, the - clusterer_controller module takes over all topology management: - it allocates unique node IDs, discovers peer BIN socket - addresses, and calls the clusterer internal API to add or - remove nodes as they join or leave the cluster. + When a cluster is registered as controller-managed via + modparam("clusterer", "cluster_options", "cluster_id=N, + use_controller=1"), the clusterer_controller module takes over + all topology management: it allocates unique node IDs, + discovers peer BIN socket addresses, and calls the clusterer + internal API to add or remove nodes as they join or leave the + cluster. By default (manage_shtags=1), the module also provides fully automatic sharing tag failover. The controller master node is @@ -499,15 +501,20 @@ on-key") * proto_bin — required so that BIN listeners are registered and available for discovery when clusterer_controller initialises and scans the proto_bin listener list. - * clusterer — required, and it must have use_controller=1 - set. That global switch is what pre-creates the + * clusterer — required, and it must register every + controller-managed cluster with modparam("clusterer", + "cluster_options", "cluster_id=N, use_controller=1"). That + per-cluster parameter is what pre-creates the controller-managed cluster stubs, marks them so they never touch the database, and arms the guard that stops the - controller from driving a native cluster of the same id. If - clusterer_controller is loaded while clusterer has - use_controller=0, it refuses to start with a clear error — - there is no controller-managed cluster for it to drive and - those safety mechanisms would be off. + controller from driving a native cluster of the same id. + The controller-managed ids declared here and the cluster + entries configured in clusterer_controller must match + exactly: if either side names a cluster the other does not, + the controller refuses to start with an error naming the + offending id — a managed id with no controller config has + no BIN socket or crypto parameters, and a controller config + for an unmanaged id has nothing to drive. Both dependencies are declared in the module's dep_export_t. OpenSIPS will refuse to start if either dependency is not @@ -516,28 +523,30 @@ on-key") resolves the dependency at runtime and will initialize the required modules first regardless of loadmodule order. - The reverse is also checked: if use_controller=1 is set but the - clusterer_controller module is not loaded, clusterer logs an - error at startup, since the controller-managed cluster stubs - would never obtain a node identity or form. clusterer itself - keeps running, so any native or DB-backed clusters are - unaffected. - - Hybrid deployments are unaffected. use_controller is a single - global switch; in a hybrid instance (native and - controller-managed clusters side by side) it is always 1 and - only the per-cluster kind differs (native via DB/static - provisioning versus controller-managed via the cluster_id - list). Both consistency checks above therefore only fire on a - genuine module/config mismatch — never on a hybrid or - pure-controller setup. + This is also checked from the other side: if a cluster_options + entry sets use_controller=1 but the clusterer_controller module + is not loaded at all, clusterer logs an error at startup, since + the controller-managed cluster stubs would never obtain a node + identity or form. clusterer itself keeps running, so any native + or DB-backed clusters are unaffected. + + Hybrid deployments are unaffected. use_controller is a + per-cluster option carried in cluster_options, defaulting to 0. + In a hybrid instance (native and controller-managed clusters + side by side), the controller-managed clusters each get a + cluster_options line with use_controller=1, while the native + ones are defined the usual way (DB rows or static + my_node_info/neighbor_node_info) and need no cluster_options + line. The exact-match check above compares only the + controller-managed ids against the controller's cluster + entries, so it never fires on a native cluster. All other modules that use the clusterer interface (tm, dialog, dispatcher, usrloc etc.) may be loaded in any order relative to clusterer_controller. The clusterer module automatically - creates a cluster stub when use_controller=1 is set and a - module attempts to register a capability for an unknown - cluster. + creates a cluster stub when a cluster_options entry sets + use_controller=1 and a module attempts to register a capability + for that cluster. 1.5.2. External Libraries or Applications @@ -832,9 +841,10 @@ modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") ... # clusters 1 and 2 registered as controller-managed on the clusterer sid e -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) -modparam("clusterer", "cluster_id", 2) +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1 +") # Enable automatic failover for every cluster (this is also the default) modparam("clusterer_controller", "manage_shtags", 1) @@ -849,9 +859,10 @@ modparam("clusterer_controller", "cluster", "id=2,multicast=239.0.90.2:3 ... # clusters 1 and 2 registered as controller-managed on the clusterer sid e -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) -modparam("clusterer", "cluster_id", 2) +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1 +") # manage_shtags=1 globally, but cluster 2 uses its own MI/event-route sc ripts @@ -866,9 +877,10 @@ modparam("clusterer_controller", "cluster", ... # clusters 1 and 2 registered as controller-managed on the clusterer sid e -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) -modparam("clusterer", "cluster_id", 2) +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1 +") # Disable automatic failover globally; enable it only for cluster 1 modparam("clusterer_controller", "manage_shtags", 0) @@ -890,10 +902,8 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) # enable the controller p -ath (global switch) -modparam("clusterer", "cluster_id", 1) # register cluster 1 as c -ontroller-managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") modparam("clusterer", "sharing_tag", "vip1/1=active") modparam("clusterer", "ping_interval", 4) modparam("clusterer", "ping_timeout", 1500) @@ -939,9 +949,10 @@ modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") Example 1.12. Set master_stickiness parameter ... # both clusters registered as controller-managed on the clusterer side -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) -modparam("clusterer", "cluster_id", 2) +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1 +") # Global default (sticky) — omit entirely for the same effect modparam("clusterer_controller", "master_stickiness", 1) @@ -1210,12 +1221,10 @@ socket=bin:10.0.2.10:5566 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) # enable the controller pat -h (global switch) -modparam("clusterer", "cluster_id", 1) # cluster 1 is controller- -managed -modparam("clusterer", "cluster_id", 2) # cluster 2 is controller- -managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1 +") loadmodule "clusterer_controller.so" # Cluster 1 — dialog replication group, LAN segment 10.0.1.0/24 @@ -1238,12 +1247,10 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) # enable the controller pat -h (global switch) -modparam("clusterer", "cluster_id", 1) # cluster 1 is controller- -managed -modparam("clusterer", "cluster_id", 2) # cluster 2 is controller- -managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1 +") loadmodule "clusterer_controller.so" modparam("clusterer_controller", "cluster", @@ -1268,10 +1275,11 @@ modparam("dispatcher", "cluster_id", 2) Which kind a cluster is follows from how it is declared: * Controller-managed — every cluster_id registered with the - clusterer module via modparam("clusterer", "cluster_id", N) - (and matched by a cluster entry in this module). Its - topology and this node's node_id are driven at runtime by - the controller; it never touches the database and always + clusterer module via modparam("clusterer", + "cluster_options", "cluster_id=N, use_controller=1") (and + matched by a cluster entry in this module). Its topology + and this node's node_id are driven at runtime by the + controller; it never touches the database and always behaves as db_mode=0, regardless of the global db_mode. * Native — every cluster loaded from the clusterer database (db_mode!=0) or provisioned statically. It behaves exactly @@ -1315,10 +1323,8 @@ s") modparam("clusterer", "my_node_id", 5) # this node's id in its nati ve/DB clusters (global) # controller side: cluster 1 is dynamic (no DB, no static rows) -modparam("clusterer", "use_controller", 1) # enable the controller path - (global switch) -modparam("clusterer", "cluster_id", 1) # register cluster 1 as con -troller-managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") loadmodule "clusterer_controller.so" modparam("clusterer_controller", "cluster", @@ -1350,10 +1356,8 @@ modparam("clusterer", "my_node_info", "cluster_id=7, url=bin:10.0. modparam("clusterer", "neighbor_node_info", "cluster_id=7, node_id=6, ur l=bin:10.0.0.11:5566") # controller side: cluster 1 is dynamic -modparam("clusterer", "use_controller", 1) # enable the contro -ller path (global switch) -modparam("clusterer", "cluster_id", 1) # register cluster - 1 as controller-managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") loadmodule "clusterer_controller.so" modparam("clusterer_controller", "cluster", @@ -1386,10 +1390,8 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) # enable the controller p -ath (global switch) -modparam("clusterer", "cluster_id", 1) # register cluster 1 as c -ontroller-managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") modparam("clusterer", "sharing_tag", "vip1/1=active") modparam("clusterer", "ping_interval", 4) modparam("clusterer", "ping_timeout", 1500) @@ -1667,9 +1669,10 @@ A.7. Test 6 — Multiple clusters over one BIN socket but both advertise the same BIN socket (bin:IP:3857). Each cluster must form independently and its replication must stay isolated over the shared socket. -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) -modparam("clusterer", "cluster_id", 2) +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1 +") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1 +") modparam("clusterer_controller", "cluster", "id=1,multicast=239.0.90.1:3333,bin_socket=bin:IP:3857") modparam("clusterer_controller", "cluster", diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index bd6e03f8fcb..31ca17589ea 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -5804,23 +5804,59 @@ static int mod_init(void) LM_INFO("clusterer_controller: clusterer API loaded - " "topology will be driven dynamically\n"); - /* The controller is only meaningful when the clusterer module is told - * to expect it (use_controller=1, a global switch): that is what - * pre-creates the controller-managed cluster stubs, sets each one's - * controller_managed flag (so they never touch the DB), and arms the - * guard that stops the controller from hijacking a native cluster of the - * same id. With use_controller=0 there is no controller-managed cluster - * at all and those safety mechanisms are off, so refuse to start rather - * than run the control plane against clusters clusterer never authorised - * us to drive. (Hybrid setups keep use_controller=1 - only the per- - * cluster kind differs - so this never trips them.) */ - if (!clctl.use_controller) { - LM_ERR("clusterer_controller: loaded but the clusterer module has " - "use_controller=0 - there is no controller-managed cluster and " - "the controller's safety guards are disabled. Set " - "modparam(\"clusterer\", \"use_controller\", 1), or remove the " - "clusterer_controller module.\n"); - return -1; + /* The clusterer marks a cluster controller-managed with + * cluster_options use_controller=1; that pre-creates its stub, sets the + * controller_managed flag (so it never touches the DB), and arms the + * guard against hijacking a native cluster of the same id. The set of + * clusterer-managed ids and the set of 'cluster' configs here must match + * exactly - either direction of mismatch is a hard error, since neither + * half is usable without the other (a managed id with no controller + * config has no bin socket or crypto params; a controller config for an + * unmanaged id has nothing legitimate to drive). Abort naming the id. */ + + /* (a) clusterer marks a cluster managed that we have no config for */ + { + int m, k, found; + for (m = 0; m < clctl.managed_count; m++) { + found = 0; + for (k = 0; k < cc_cluster_count; k++) + if (cc_clusters[k].cluster_id == clctl.managed_ids[m]) { + found = 1; + break; + } + if (!found) { + LM_ERR("clusterer_controller: the clusterer marks cluster %d " + "controller-managed (cluster_options use_controller=1) " + "but this module has no configuration for it - add " + "modparam(\"clusterer_controller\", \"cluster\", " + "\"id=%d, ...\"), or drop use_controller for that " + "cluster.\n", + clctl.managed_ids[m], clctl.managed_ids[m]); + return -1; + } + } + } + + /* (b) we have a config for a cluster the clusterer did not mark managed */ + { + int k, m, managed; + for (k = 0; k < cc_cluster_count; k++) { + managed = 0; + for (m = 0; m < clctl.managed_count; m++) + if (clctl.managed_ids[m] == cc_clusters[k].cluster_id) { + managed = 1; + break; + } + if (!managed) { + LM_ERR("clusterer_controller: cluster %d is configured here " + "but the clusterer did not mark it controller-managed " + "- add modparam(\"clusterer\", \"cluster_options\", " + "\"cluster_id=%d, use_controller=1\"), or remove this " + "module's 'cluster' config for it.\n", + cc_clusters[k].cluster_id, cc_clusters[k].cluster_id); + return -1; + } + } } /* When we manage sharing tags, start every tag as BACKUP and diff --git a/modules/clusterer_controller/doc/clusterer_controller_admin.xml b/modules/clusterer_controller/doc/clusterer_controller_admin.xml index b663231e511..5cb033b6e27 100644 --- a/modules/clusterer_controller/doc/clusterer_controller_admin.xml +++ b/modules/clusterer_controller/doc/clusterer_controller_admin.xml @@ -15,8 +15,9 @@ and the cluster topology is maintained dynamically at runtime. - When use_controller=1 is set in the - clusterer module, the + When a cluster is registered as controller-managed via + modparam("clusterer", "cluster_options", "cluster_id=N, + use_controller=1"), the clusterer_controller module takes over all topology management: it allocates unique node IDs, discovers peer BIN socket addresses, and calls the clusterer internal API to add @@ -503,16 +504,21 @@ clusterer — required, and it - must have - use_controller=1 set. That global switch - is what pre-creates the controller-managed cluster stubs, - marks them so they never touch the database, and arms the - guard that stops the controller from driving a native cluster - of the same id. If clusterer_controller - is loaded while clusterer has - use_controller=0, it refuses to start with - a clear error — there is no controller-managed cluster for it - to drive and those safety mechanisms would be off. + must register every controller-managed + cluster with + modparam("clusterer", "cluster_options", + "cluster_id=N, use_controller=1"). That per-cluster + parameter is what pre-creates the controller-managed cluster + stubs, marks them so they never touch the database, and arms + the guard that stops the controller from driving a native + cluster of the same id. The controller-managed ids declared here + and the cluster entries configured in + clusterer_controller must match + exactly: if either side names a cluster the + other does not, the controller refuses to start with an error + naming the offending id — a managed id with no controller config + has no BIN socket or crypto parameters, and a controller config + for an unmanaged id has nothing to drive. @@ -528,24 +534,29 @@ loadmodule order. - The reverse is also checked: if - use_controller=1 is set but the - clusterer_controller module is not loaded, - clusterer logs an error at startup, since the controller-managed + This is also checked from the other side: if a + cluster_options entry sets + use_controller=1 but the + clusterer_controller module is not loaded at + all, clusterer logs an error at startup, since the controller-managed cluster stubs would never obtain a node identity or form. clusterer itself keeps running, so any native or DB-backed clusters are unaffected. Hybrid deployments are unaffected. - use_controller is a single global switch; in a - hybrid instance (native and controller-managed clusters side by side) - it is always 1 and only the - per-cluster kind differs (native via DB/static - provisioning versus controller-managed via the - cluster_id list). Both consistency checks above - therefore only fire on a genuine module/config mismatch — never on a - hybrid or pure-controller setup. + use_controller is a per-cluster option carried in + cluster_options, defaulting to 0. + In a hybrid instance (native and controller-managed clusters side by + side), the controller-managed clusters each get a + cluster_options line with + use_controller=1, while the native ones are defined + the usual way (DB rows or static + my_node_info/neighbor_node_info) + and need no cluster_options line. The exact-match + check above compares only the controller-managed ids against the + controller's cluster entries, so it never fires on + a native cluster. All other modules that use the clusterer interface @@ -553,9 +564,10 @@ dispatcher, usrloc etc.) may be loaded in any order relative to clusterer_controller. The clusterer - module automatically creates a cluster stub when - use_controller=1 is set and a module - attempts to register a capability for an unknown cluster. + module automatically creates a cluster stub when a + cluster_options entry sets + use_controller=1 and a module attempts to register + a capability for that cluster.
@@ -985,9 +997,8 @@ modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") ... # clusters 1 and 2 registered as controller-managed on the clusterer side -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) -modparam("clusterer", "cluster_id", 2) +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1") # Enable automatic failover for every cluster (this is also the default) modparam("clusterer_controller", "manage_shtags", 1) @@ -1001,9 +1012,8 @@ modparam("clusterer_controller", "cluster", "id=2,multicast=239.0.90.2:3333") ... # clusters 1 and 2 registered as controller-managed on the clusterer side -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) -modparam("clusterer", "cluster_id", 2) +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1") # manage_shtags=1 globally, but cluster 2 uses its own MI/event-route scripts modparam("clusterer_controller", "manage_shtags", 1) @@ -1019,9 +1029,8 @@ modparam("clusterer_controller", "cluster", ... # clusters 1 and 2 registered as controller-managed on the clusterer side -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) -modparam("clusterer", "cluster_id", 2) +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1") # Disable automatic failover globally; enable it only for cluster 1 modparam("clusterer_controller", "manage_shtags", 0) @@ -1044,8 +1053,7 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) -modparam("clusterer", "cluster_id", 1) # register cluster 1 as controller-managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") modparam("clusterer", "sharing_tag", "vip1/1=active") modparam("clusterer", "ping_interval", 4) modparam("clusterer", "ping_timeout", 1500) @@ -1108,9 +1116,8 @@ modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") ... # both clusters registered as controller-managed on the clusterer side -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) -modparam("clusterer", "cluster_id", 2) +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1") # Global default (sticky) — omit entirely for the same effect modparam("clusterer_controller", "master_stickiness", 1) @@ -1494,9 +1501,8 @@ socket=bin:10.0.2.10:5566 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) -modparam("clusterer", "cluster_id", 1) # cluster 1 is controller-managed -modparam("clusterer", "cluster_id", 2) # cluster 2 is controller-managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1") loadmodule "clusterer_controller.so" # Cluster 1 — dialog replication group, LAN segment 10.0.1.0/24 @@ -1521,9 +1527,8 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) -modparam("clusterer", "cluster_id", 1) # cluster 1 is controller-managed -modparam("clusterer", "cluster_id", 2) # cluster 2 is controller-managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1") loadmodule "clusterer_controller.so" modparam("clusterer_controller", "cluster", @@ -1558,8 +1563,9 @@ modparam("dispatcher", "cluster_id", 2) Controller-managed — every cluster_id registered with the clusterer module - via modparam("clusterer", "cluster_id", N) - (and matched by a cluster entry in this module). + via modparam("clusterer", "cluster_options", + "cluster_id=N, use_controller=1") (and matched by a + cluster entry in this module). Its topology and this node's node_id are driven at runtime by the controller; it never touches the database and always behaves as db_mode=0, regardless of the @@ -1621,8 +1627,7 @@ modparam("clusterer", "db_mode", 1) modparam("clusterer", "db_url", "mysql://opensips:pass@localhost/opensips") modparam("clusterer", "my_node_id", 5) # this node's id in its native/DB clusters (global) # controller side: cluster 1 is dynamic (no DB, no static rows) -modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) -modparam("clusterer", "cluster_id", 1) # register cluster 1 as controller-managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") loadmodule "clusterer_controller.so" modparam("clusterer_controller", "cluster", @@ -1650,8 +1655,7 @@ modparam("clusterer", "my_node_id", 5) # this node's id in native modparam("clusterer", "my_node_info", "cluster_id=7, url=bin:10.0.0.10:5566") modparam("clusterer", "neighbor_node_info", "cluster_id=7, node_id=6, url=bin:10.0.0.11:5566") # controller side: cluster 1 is dynamic -modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) -modparam("clusterer", "cluster_id", 1) # register cluster 1 as controller-managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") loadmodule "clusterer_controller.so" modparam("clusterer_controller", "cluster", @@ -1689,8 +1693,7 @@ socket=bin:10.22.23.191:3857 loadmodule "proto_bin.so" loadmodule "clusterer.so" -modparam("clusterer", "use_controller", 1) # enable the controller path (global switch) -modparam("clusterer", "cluster_id", 1) # register cluster 1 as controller-managed +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") modparam("clusterer", "sharing_tag", "vip1/1=active") modparam("clusterer", "ping_interval", 4) modparam("clusterer", "ping_timeout", 1500) diff --git a/modules/clusterer_controller/doc/clusterer_controller_tests.xml b/modules/clusterer_controller/doc/clusterer_controller_tests.xml index 219306e537d..b09668c2c54 100644 --- a/modules/clusterer_controller/doc/clusterer_controller_tests.xml +++ b/modules/clusterer_controller/doc/clusterer_controller_tests.xml @@ -183,9 +183,8 @@ C svc=active tag=backup <-- joined as backup, did not steal and its replication must stay isolated over the shared socket. -modparam("clusterer", "use_controller", 1) -modparam("clusterer", "cluster_id", 1) -modparam("clusterer", "cluster_id", 2) +modparam("clusterer", "cluster_options", "cluster_id=1, use_controller=1") +modparam("clusterer", "cluster_options", "cluster_id=2, use_controller=1") modparam("clusterer_controller", "cluster", "id=1,multicast=239.0.90.1:3333,bin_socket=bin:IP:3857") modparam("clusterer_controller", "cluster", From 18c199ef68e7da36bf7c1eef20104b7ab749fdfd Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Wed, 15 Jul 2026 12:24:07 +1000 Subject: [PATCH 05/21] clusterer: gate controller integration behind a build-time flag; exclude clusterer_controller by default Make clusterer_controller an opt-in module and ensure the bundled clusterer module is completely unchanged when it is not built. Build wiring: - Add clusterer_controller to exclude_modules in Makefile.conf.template, so a stock build skips it (like the other modules with external-lib deps). Enable it via include_modules. - The top-level Makefile exports CLUSTERER_CTRL_SUPPORT=1 iff clusterer_controller is in the configured build (mirrors the module-selection rule, so it is stable across 'make all' and single-module rebuilds); clusterer/Makefile turns that into -DCLUSTERER_CTRL_SUPPORT. clusterer side, all under #ifdef CLUSTERER_CTRL_SUPPORT: - the clusterer_ctrl API (clusterer_ctrl.c) and the cluster_options / use_controller / cluster_id modparams + load_clusterer_ctrl_binds export; - the controller stub pre-create loop, the child_init guard, the shm current_id mirror, the on-demand stub, and the shtag_managed / controller_managed logic. - The per-cluster identity and hybrid-db_mode accessors (cluster_self_id, cl_db_mode, GET_CURRENT_ID, use_controller) get #else fallbacks to the stock globals (current_id / db_mode / 0), so their call sites compile to the exact upstream object code with no per-site #ifdef. add_node_info's internal self_id parameter is likewise gated (falls back to current_id). Result: a build without clusterer_controller produces the stock clusterer module - no cluster_options parameter (rejected as unknown), no behavioural change, no added exports. Verified with unifdef -UCLUSTERER_CTRL_SUPPORT against the base: no semantic difference from upstream. Enabling the controller rebuilds clusterer with the hooks; the two are a matched pair. Docs: new "Building the Module" section (admin guide + README). --- Makefile | 13 ++ Makefile.conf.template | 2 +- modules/clusterer/Makefile | 8 ++ modules/clusterer/api.h | 1 + modules/clusterer/clusterer.c | 56 ++++++++ modules/clusterer/clusterer_ctrl.c | 8 +- modules/clusterer/clusterer_ctrl.h | 4 - modules/clusterer/clusterer_mod.c | 38 +++++- modules/clusterer/node_info.c | 33 ++++- modules/clusterer/node_info.h | 44 +++++-- modules/clusterer/sharing_tags.c | 20 +++ modules/clusterer/sharing_tags.h | 2 + modules/clusterer/sync.c | 12 ++ modules/clusterer/topology.c | 8 +- modules/clusterer_controller/README | 123 +++++++++++------- .../doc/clusterer_controller_admin.xml | 37 ++++++ 16 files changed, 337 insertions(+), 72 deletions(-) diff --git a/Makefile b/Makefile index 085e6501e8a..75a93966157 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,19 @@ include Makefile.defs # always exclude the SVN dir override exclude_modules+= .svn $(skip_modules) +# The clusterer module exposes its controller-support API (and the extra +# cluster_options modparam) only when the clusterer_controller module is part of +# this build; otherwise it is a stock module. Mirror the module-selection rule +# below (built if force-included, else unless excluded) and export the result so +# clusterer/Makefile can define -DCLUSTERER_CTRL_SUPPORT. Derived from the +# configured include/exclude lists (not the current 'modules' subset) so it stays +# consistent whether you 'make all' or rebuild just modules/clusterer. +ifneq ($(filter clusterer_controller,$(include_modules)),) +export CLUSTERER_CTRL_SUPPORT:=1 +else ifeq ($(filter clusterer_controller,$(exclude_modules)),) +export CLUSTERER_CTRL_SUPPORT:=1 +endif + #always include this modules #include_modules?= diff --git a/Makefile.conf.template b/Makefile.conf.template index c6396cc5f49..971f35d277e 100644 --- a/Makefile.conf.template +++ b/Makefile.conf.template @@ -76,7 +76,7 @@ #uuid= UUID generator | uuid-dev # Modules omitted from the default build, generally due to external dependencies. -exclude_modules?= aaa_diameter aaa_radius auth_jwt auth_web3 b2b_logic_xml cachedb_cassandra cachedb_couchbase cachedb_dynamodb cachedb_memcached cachedb_mongodb cachedb_redis carrierroute cgrates compression cpl_c db_berkeley db_http db_mysql db_oracle db_perlvdb db_postgres db_sqlite db_unixodbc dialplan emergency event_rabbitmq event_kafka event_sqs h350 httpd http2d identity jabber json launch_darkly ldap lua mi_xmlrpc mmgeoip opentelemetry osp perl pi_http presence presence_dialoginfo presence_mwi presence_reginfo presence_xml presence_dfks proto_ipsec proto_sctp proto_tls proto_wss pua pua_bla pua_dialoginfo pua_mi pua_reginfo pua_usrloc pua_xmpp python regex rabbitmq_consumer rest_client rls rtp.io siprec sngtc snmpstats stir_shaken tls_mgm tls_openssl tls_wolfssl uuid xcap xcap_client xml xmpp +exclude_modules?= aaa_diameter aaa_radius auth_jwt auth_web3 b2b_logic_xml cachedb_cassandra cachedb_couchbase cachedb_dynamodb cachedb_memcached cachedb_mongodb cachedb_redis carrierroute cgrates clusterer_controller compression cpl_c db_berkeley db_http db_mysql db_oracle db_perlvdb db_postgres db_sqlite db_unixodbc dialplan emergency event_rabbitmq event_kafka event_sqs h350 httpd http2d identity jabber json launch_darkly ldap lua mi_xmlrpc mmgeoip opentelemetry osp perl pi_http presence presence_dialoginfo presence_mwi presence_reginfo presence_xml presence_dfks proto_ipsec proto_sctp proto_tls proto_wss pua pua_bla pua_dialoginfo pua_mi pua_reginfo pua_usrloc pua_xmpp python regex rabbitmq_consumer rest_client rls rtp.io siprec sngtc snmpstats stir_shaken tls_mgm tls_openssl tls_wolfssl uuid xcap xcap_client xml xmpp # Modules forced into the build, even when also listed in exclude_modules. include_modules?= diff --git a/modules/clusterer/Makefile b/modules/clusterer/Makefile index 3ca15b036d3..d5b014e8ea8 100644 --- a/modules/clusterer/Makefile +++ b/modules/clusterer/Makefile @@ -6,4 +6,12 @@ NAME=clusterer.so #DEFS+= -DCLUSTERER_DBG #DEFS+= -DCLUSTERER_EXTRA_BIN_DBG +# Controller support (the clusterer_ctrl API, the cluster_options modparam and +# the controller hooks) is compiled only when the clusterer_controller module is +# part of this build - the top-level Makefile exports CLUSTERER_CTRL_SUPPORT=1 in +# that case. A stock build produces the vanilla clusterer module. +ifeq ($(CLUSTERER_CTRL_SUPPORT),1) +DEFS+= -DCLUSTERER_CTRL_SUPPORT +endif + include ../../Makefile.modules diff --git a/modules/clusterer/api.h b/modules/clusterer/api.h index ad7ba8cd727..b6b8d3143c4 100644 --- a/modules/clusterer/api.h +++ b/modules/clusterer/api.h @@ -345,5 +345,6 @@ static inline module_dependency_t *get_deps_clusterer(const param_export_t *para return alloc_module_dep(MOD_TYPE_DEFAULT, "clusterer", DEP_ABORT); } + #endif /* CLUSTERER_API_H */ diff --git a/modules/clusterer/clusterer.c b/modules/clusterer/clusterer.c index b8b36c2885f..de610588d19 100644 --- a/modules/clusterer/clusterer.c +++ b/modules/clusterer/clusterer.c @@ -87,7 +87,9 @@ void sync_check_timer(utime_t ticks, void *param) lock_start_read(cl_list_lock); for (cl = *cluster_list; cl; cl = cl->next) { +#ifdef CLUSTERER_CTRL_SUPPORT if (!cl->current_node) continue; +#endif lock_get(cl->current_node->lock); if (!(cl->current_node->flags & NODE_STATE_ENABLED)) { lock_release(cl->current_node->lock); @@ -110,6 +112,7 @@ void sync_check_timer(utime_t ticks, void *param) cap->flags &= ~(CAP_SYNC_PENDING|CAP_SYNC_STARTUP); sr_set_status(cl_srg, STR2CI(cap->reg.sr_id), CAP_SR_SYNCED, STR2CI(CAP_SR_STATUS_STR(CAP_SR_SYNCED)), 0); +#ifdef CLUSTERER_CTRL_SUPPORT /* Seed-fallback for a still-PENDING sync (the transfer never * started - no donor responded to our request within * seed_fb_interval). This is exactly what the seed @@ -131,6 +134,13 @@ void sync_check_timer(utime_t ticks, void *param) cap->reg.name.len, cap->reg.name.s, cl->node_list == NULL ? "first/lone node" : "no donor sent data in due time"); +#else + sr_add_report_fmt(cl_srg, STR2CI(cap->reg.sr_id), 0, + "ERROR: Sync request aborted! (no donor found in due time)" + " => fallback to synced state"); + LM_ERR("Sync request aborted! (no donor found in due time)" + ", falling back to synced state\n"); +#endif /* send update about the state of this capability */ send_single_cap_update(cl, cap, 1); @@ -214,8 +224,10 @@ int cl_set_state(int cluster_id, int node_id, enum cl_node_state state) return 0; } +#ifdef CLUSTERER_CTRL_SUPPORT if (!cluster->current_node) return -1; +#endif lock_get(cluster->current_node->lock); @@ -648,6 +660,7 @@ clusterer_bridges_bcast_msg(bin_packet_t *packet, int src_cid) } +#ifdef CLUSTERER_CTRL_SUPPORT /* This node's id in @cluster_id, resolved without taking cl_list_lock: clusters * persist for the module's lifetime, and callers may already hold cl_list_lock * (read) or a different lock (shtags) whose order is cl_list_lock -> shtags_lock, @@ -663,13 +676,19 @@ static int trailer_self_id(int cluster_id) return cluster_self_id(cl); return -1; } +#endif int msg_add_trailer(bin_packet_t *packet, int cluster_id, int dst_id) { if (bin_push_int(packet, cluster_id) < 0) return -1; +#ifdef CLUSTERER_CTRL_SUPPORT if (bin_push_int(packet, trailer_self_id(cluster_id)) < 0) return -1; +#else + if (bin_push_int(packet, current_id) < 0) + return -1; +#endif if (bin_push_int(packet, dst_id) < 0) return -1; @@ -1170,8 +1189,10 @@ static void handle_remove_node(bin_packet_t *packet, cluster_info_t *cl) } if (target_node == cluster_self_id(cl)) { +#ifdef CLUSTERER_CTRL_SUPPORT if (!cl->current_node) return; +#endif lock_get(cl->current_node->lock); @@ -1219,6 +1240,13 @@ void bin_rcv_cl_extra_packets(bin_packet_t *packet, int packet_type, LM_DBG("received clusterer message from: %s:%hu with source id: %d and" " cluster id: %d\n", ip, port, source_id, cluster_id); +#ifndef CLUSTERER_CTRL_SUPPORT + if (source_id == current_id) { + LM_ERR("Received message with bad source - same node id as this instance\n"); + return; + } +#endif + gettimeofday(&now, NULL); if ((!db_mode || use_controller) && packet_type == CLUSTERER_REMOVE_NODE) @@ -1233,10 +1261,12 @@ void bin_rcv_cl_extra_packets(bin_packet_t *packet, int packet_type, goto exit; } +#ifdef CLUSTERER_CTRL_SUPPORT if (source_id == cluster_self_id(cl)) { LM_ERR("Received message with bad source - same node id as this instance\n"); goto exit; } +#endif lock_get(cl->current_node->lock); if (!(cl->current_node->flags & NODE_STATE_ENABLED)) { @@ -1357,6 +1387,13 @@ void bin_rcv_cl_packets(bin_packet_t *packet, int packet_type, LM_DBG("received clusterer message from: %s:%hu with source id: %d and " "cluster id: %d\n", ip, port, source_id, cl_id); +#ifndef CLUSTERER_CTRL_SUPPORT + if (source_id == current_id) { + LM_ERR("Received message with bad source - same node id as this instance\n"); + return; + } +#endif + if ((!db_mode || use_controller) && (packet_type == CLUSTERER_NODE_DESCRIPTION || packet_type == CLUSTERER_FULL_TOP_UPDATE)) lock_start_write(cl_list_lock); @@ -1369,6 +1406,7 @@ void bin_rcv_cl_packets(bin_packet_t *packet, int packet_type, goto exit; } +#ifdef CLUSTERER_CTRL_SUPPORT /* current_node is legitimately NULL while this node's identity is being * (re)established for a dynamically constructed cluster (clusterer_ctrl * update_identity: the cluster can already exist and receive BIN packets @@ -1384,6 +1422,7 @@ void bin_rcv_cl_packets(bin_packet_t *packet, int packet_type, LM_ERR("Received message with bad source - same node id as this instance\n"); goto exit; } +#endif lock_get(cl->current_node->lock); if (!(cl->current_node->flags & NODE_STATE_ENABLED)) { @@ -1522,6 +1561,13 @@ static void bin_rcv_mod_packets(bin_packet_t *packet, int packet_type, "cluster ids: %d->%d\n", ip, port, source_id, src_cluster_id, cluster_id); } +#ifndef CLUSTERER_CTRL_SUPPORT + if (source_id == current_id) { + LM_ERR("Received message with bad source - same node id as this instance\n"); + return; + } +#endif + cap = (struct capability_reg *)ptr; if (!cap) { LM_ERR("Failed to get bin callback parameter\n"); @@ -1538,10 +1584,12 @@ static void bin_rcv_mod_packets(bin_packet_t *packet, int packet_type, goto exit; } +#ifdef CLUSTERER_CTRL_SUPPORT if (source_id == cluster_self_id(cl)) { LM_ERR("Received message with bad source - same node id as this instance\n"); goto exit; } +#endif lock_get(cl->current_node->lock); if (!(cl->current_node->flags & NODE_STATE_ENABLED)) { @@ -1669,7 +1717,9 @@ int send_single_cap_update(cluster_info_t *cluster, struct local_cap *cap, timestamp = (int)(unsigned long)time(NULL); +#ifdef CLUSTERER_CTRL_SUPPORT if (!cluster->current_node) return -1; +#endif lock_get(cluster->current_node->lock); for (neigh = cluster->current_node->neighbour_list; neigh; @@ -1954,6 +2004,7 @@ int cl_register_cap(str *cap, cl_packet_cb_f packet_cb, cl_event_cb_f event_cb, cluster = get_cluster_by_id(cluster_id); if (!cluster) { +#ifdef CLUSTERER_CTRL_SUPPORT if (use_controller) { cluster = shm_malloc(sizeof *cluster); if (!cluster) { LM_ERR("no shm\n"); return -1; } @@ -1973,6 +2024,11 @@ int cl_register_cap(str *cap, cl_packet_cb_f packet_cb, cl_event_cb_f event_cb, db_mode ? "DB" : "script"); return -1; } +#else + LM_ERR("cluster id %d is not defined in the %s\n", cluster_id, + db_mode ? "DB" : "script"); + return -1; +#endif } new_cl_cap = shm_malloc(sizeof *new_cl_cap + cap->len + CAP_SR_ID_PREFIX_LEN); diff --git a/modules/clusterer/clusterer_ctrl.c b/modules/clusterer/clusterer_ctrl.c index 72ea99453ed..3616e94a3b4 100644 --- a/modules/clusterer/clusterer_ctrl.c +++ b/modules/clusterer/clusterer_ctrl.c @@ -25,6 +25,11 @@ #include "topology.h" /* delete_neighbour */ /* shtag_event_handler */ #include "clusterer_ctrl.h" +/* This whole API is compiled only when the clusterer_controller module is part + * of the build (see modules/clusterer/Makefile). In a stock clusterer build the + * translation unit is empty. */ +#ifdef CLUSTERER_CTRL_SUPPORT + /* declared in clusterer.c — raises E_CLUSTERER_NODE_STATE_CHANGED */ int report_node_state(enum clusterer_event event, int cluster_id, int node_id); @@ -382,7 +387,6 @@ int load_clusterer_ctrl_binds(clusterer_ctrl_binds_t *binds) return -1; } clusterer_ctrl_bound = 1; - binds->use_controller = use_controller; binds->managed_count = cc_stub_count; binds->managed_ids = cc_stub_ids; binds->set_my_identity = clusterer_ctrl_set_identity; @@ -396,3 +400,5 @@ int load_clusterer_ctrl_binds(clusterer_ctrl_binds_t *binds) binds->force_backup_shtags = clusterer_ctrl_force_backup_shtags; return 0; } + +#endif /* CLUSTERER_CTRL_SUPPORT */ diff --git a/modules/clusterer/clusterer_ctrl.h b/modules/clusterer/clusterer_ctrl.h index c7a41eb88ca..309570dce0b 100644 --- a/modules/clusterer/clusterer_ctrl.h +++ b/modules/clusterer/clusterer_ctrl.h @@ -158,10 +158,6 @@ typedef struct clusterer_ctrl_binds { */ int (*unset_shtag_managed)(int cluster_id); - /* Introspection: 1 if the clusterer has any controller-managed cluster - * (a cluster_options with use_controller=1), else 0. */ - int use_controller; - /* The set of cluster_ids the clusterer marked controller-managed * (cluster_options use_controller=1): count, and a pointer to the clusterer's * own array (valid for the process lifetime; read pre-fork in mod_init). The diff --git a/modules/clusterer/clusterer_mod.c b/modules/clusterer/clusterer_mod.c index 9a3d5695b6f..5e6cf8de421 100644 --- a/modules/clusterer/clusterer_mod.c +++ b/modules/clusterer/clusterer_mod.c @@ -39,7 +39,9 @@ #include "clusterer.h" #include "sync.h" #include "sharing_tags.h" +#ifdef CLUSTERER_CTRL_SUPPORT #include "clusterer_ctrl.h" +#endif #include "clusterer_evi.h" int ping_interval = DEFAULT_PING_INTERVAL; @@ -48,10 +50,16 @@ int ping_timeout = DEFAULT_PING_TIMEOUT; int seed_fb_interval = DEFAULT_SEED_FB_INTERVAL; int sync_timeout = DEFAULT_SYNC_TIMEOUT; int current_id = -1; -int *_current_id_shm = NULL; int db_mode = 1; int clusterer_enable_rerouting = 1; -int use_controller = 0; /* derived: 1 if any cluster_options sets use_controller=1 */ + +#ifdef CLUSTERER_CTRL_SUPPORT +/* shm-backed mirror of current_id so runtime identity changes are visible across + * forked workers; and the derived 'controller active' flag. Both exist only in a + * controller-enabled build - stock code reaches these through the GET_CURRENT_ID + * and use_controller fallbacks in node_info.h. */ +int *_current_id_shm = NULL; +int use_controller = 0; /* 1 if any cluster_options sets use_controller=1 */ /* cluster_ids marked controller-managed via 'cluster_options' (use_controller=1). * Not static: the controller loads a pointer to these through the ctrl binds so it @@ -156,6 +164,7 @@ static int cc_retired_param(modparam_t type, void *val) "\"cluster_id=N, use_controller=1\")\n"); return -1; } +#endif /* CLUSTERER_CTRL_SUPPORT */ str clusterer_db_url = {NULL, 0}; @@ -213,7 +222,9 @@ int cmd_check_addr(struct sip_msg *msg, int *cluster_id, str *ip_str, */ static const cmd_export_t cmds[] = { +#ifdef CLUSTERER_CTRL_SUPPORT {"load_clusterer_ctrl_binds", (cmd_function)load_clusterer_ctrl_binds, {{0,0,0}}, 0}, +#endif {"load_clusterer", (cmd_function)load_clusterer, {{0,0,0}}, 0}, {"cluster_broadcast_req", (cmd_function)cmd_broadcast_req, { {CMD_PARAM_INT,0,0}, @@ -266,9 +277,11 @@ static const param_export_t params[] = { {"flags_col", STR_PARAM, &flags_col.s }, {"description_col", STR_PARAM, &description_col.s }, {"db_mode", INT_PARAM, &db_mode }, +#ifdef CLUSTERER_CTRL_SUPPORT {"cluster_options", STR_PARAM|USE_FUNC_PARAM, (void*)cc_add_cluster_options}, {"use_controller", INT_PARAM|USE_FUNC_PARAM, (void*)cc_retired_param}, {"cluster_id", INT_PARAM|USE_FUNC_PARAM, (void*)cc_retired_param}, +#endif {"neighbor_node_info", STR_PARAM|USE_FUNC_PARAM, (void*)&provision_neighbor}, {"my_node_info", STR_PARAM|USE_FUNC_PARAM, @@ -500,6 +513,7 @@ static int mod_init(void) flags_col.len = strlen(flags_col.s); description_col.len = strlen(description_col.s); +#ifdef CLUSTERER_CTRL_SUPPORT /* db_mode defaults to 1 (DB-backed). If no DB URL is configured there are * no DB-backed native clusters, so drop to non-DB mode - this keeps * controller-only and purely static (my_node_info) setups working without @@ -507,11 +521,11 @@ static int mod_init(void) * keeps db_mode. */ if (!clusterer_db_url.s && !db_default_url) db_mode = 0; +#endif - /* The DB URL is required whenever native clusters are DB-backed - * (db_mode != 0), even alongside the controller. */ init_db_url(clusterer_db_url, db_mode == 0); +#ifdef CLUSTERER_CTRL_SUPPORT /* my_node_id identifies this node within its *native* clusters (DB or * static my_node_info). Controller clusters get their id assigned at * runtime, so my_node_id is only required when native clusters exist. */ @@ -519,6 +533,12 @@ static int mod_init(void) LM_CRIT("Invalid 'my_node_id' parameter (required for DB or static native clusters)\n"); return -1; } +#else + if (current_id < 1) { + LM_CRIT("Invalid 'my_node_id' parameter\n"); + return -1; + } +#endif if (ping_interval <= 0) { LM_WARN("Invalid ping_interval parameter, using default value\n"); ping_interval = DEFAULT_PING_INTERVAL; @@ -544,6 +564,7 @@ static int mod_init(void) LM_CRIT("Failed to init lock\n"); return -1; } +#ifdef CLUSTERER_CTRL_SUPPORT /* Move GET_CURRENT_ID to shared memory so all forked processes * see the same value when it is updated by the controller. */ { @@ -559,6 +580,7 @@ static int mod_init(void) *_shm_id = (current_id >= 1) ? current_id : -1; _current_id_shm = _shm_id; } +#endif /* if statistics are disabled, prevent their registration to core */ if (clusterer_enable_stats==0) @@ -625,6 +647,7 @@ static int mod_init(void) * cluster_id must be exclusively controller-managed OR native, never both - * the native clusters (DB loaded just above, or static from parse time) are * already in cluster_list, so an overlap is caught here. */ +#ifdef CLUSTERER_CTRL_SUPPORT if (use_controller) { int _ci; for (_ci = 0; _ci < cc_stub_count; _ci++) { @@ -663,6 +686,7 @@ static int mod_init(void) cc_stub_ids[_ci]); } } +#endif /* CLUSTERER_CTRL_SUPPORT */ /* register timer */ heartbeats_timer_interval = gcd(ping_interval*1000, ping_timeout); @@ -724,8 +748,12 @@ static int mod_init(void) /* check if the cluster IDs in the the sharing tag list are valid */ shtag_init_list(); shtag_init_reporting(); +#ifdef CLUSTERER_CTRL_SUPPORT if (!use_controller) shtag_validate_list(); +#else + shtag_validate_list(); +#endif return 0; error: @@ -749,6 +777,7 @@ static int child_init(int rank) } } +#ifdef CLUSTERER_CTRL_SUPPORT /* One-shot config sanity check (rank 1 fires once, and runs after every * module's mod_init, so a clusterer_controller would already have bound the * controller API): a controller-managed cluster is meaningless without that @@ -759,6 +788,7 @@ static int child_init(int rank) "they will never obtain a node identity or form. Load the " "clusterer_controller module, or drop use_controller from " "cluster_options.\n"); +#endif /* CLUSTERER_CTRL_SUPPORT */ return 0; } diff --git a/modules/clusterer/node_info.c b/modules/clusterer/node_info.c index 6025734a062..809c7299a5e 100644 --- a/modules/clusterer/node_info.c +++ b/modules/clusterer/node_info.c @@ -77,8 +77,16 @@ cluster_info_t **cluster_list; int load_cluster_bridges(cluster_info_t *cl_list); int add_node_info(node_info_t **new_info, cluster_info_t **cl_list, int *int_vals, +#ifdef CLUSTERER_CTRL_SUPPORT str *str_vals, int self_id) { +#else + str *str_vals) +{ +/* Without the controller a node has a single global identity; the caller passes + * no per-cluster id, so self_id is just current_id here. */ +#define self_id current_id +#endif char *host; int hlen, port; int proto; @@ -303,6 +311,9 @@ int add_node_info(node_info_t **new_info, cluster_info_t **cl_list, int *int_val } return -1; } +#ifndef CLUSTERER_CTRL_SUPPORT +#undef self_id +#endif #define check_val( _col, _val, _type, _not_null, _is_empty_check) \ do { \ @@ -499,7 +510,11 @@ int load_db_info(db_func_t *dr_dbf, db_con_t* db_hdl, cluster_info_t **cl_list) strlen(str_vals[STR_VALS_DESCRIPTION_COL].s) : 0; /* add info to backing list */ - if ((rc = add_node_info(&_, cl_list, int_vals, str_vals, GET_CURRENT_ID)) != 0) { + if ((rc = add_node_info(&_, cl_list, int_vals, str_vals +#ifdef CLUSTERER_CTRL_SUPPORT + , GET_CURRENT_ID +#endif + )) != 0) { LM_ERR("Unable to add node info to backing list\n"); if (rc < 0) { /* serious error happened, better give up */ @@ -770,7 +785,11 @@ int provision_neighbor(modparam_t type, void *val) *cluster_list = NULL; } - if (add_node_info(&new_info, cluster_list, int_vals, str_vals, GET_CURRENT_ID) < 0) { + if (add_node_info(&new_info, cluster_list, int_vals, str_vals +#ifdef CLUSTERER_CTRL_SUPPORT + , GET_CURRENT_ID +#endif + ) < 0) { LM_ERR("Unable to add node info to backing list\n"); return -1; } @@ -837,7 +856,11 @@ int provision_current(modparam_t type, void *val) *cluster_list = NULL; } - if (add_node_info(&new_info, cluster_list, int_vals, str_vals, GET_CURRENT_ID) != 0) { + if (add_node_info(&new_info, cluster_list, int_vals, str_vals +#ifdef CLUSTERER_CTRL_SUPPORT + , GET_CURRENT_ID +#endif + ) != 0) { LM_ERR("Unable to add node info to backing list\n"); return -1; } @@ -1149,7 +1172,11 @@ int cl_get_my_sip_addr(int cluster_id, str *out_addr) memset(out_addr, 0, sizeof *out_addr); rc = 0; } else { +#ifdef CLUSTERER_CTRL_SUPPORT if (cl->current_node && pkg_str_dup(out_addr, &cl->current_node->sip_addr) != 0) { +#else + if (pkg_str_dup(out_addr, &cl->current_node->sip_addr) != 0) { +#endif LM_ERR("oom\n"); memset(out_addr, 0, sizeof *out_addr); rc = -1; diff --git a/modules/clusterer/node_info.h b/modules/clusterer/node_info.h index b8ff8f7917f..42cfc78c472 100644 --- a/modules/clusterer/node_info.h +++ b/modules/clusterer/node_info.h @@ -137,16 +137,18 @@ struct cluster_info { cluster_bridge_t *bridges; /* replication links to other clusters */ +#ifdef CLUSTERER_CTRL_SUPPORT /* Set by clusterer_controller when manage_shtags=1 for this cluster. * Blocks MI and script-variable shtag activation to prevent conflicts * with controller-managed failover. */ int shtag_managed; /* 1 = this cluster's topology and identity are driven at runtime by - * clusterer_controller (registered via the 'cluster_id' modparam); it never - * touches the DB and behaves as db_mode=0 regardless of the global db_mode. + * clusterer_controller (registered via cluster_options use_controller=1); it + * never touches the DB and behaves as db_mode=0 regardless of global db_mode. * 0 = a native cluster defined via DB or static my_node_info/neighbor. */ int controller_managed; +#endif struct cluster_info *next; }; @@ -178,42 +180,62 @@ extern str clnk_shtag_col; extern str clnk_dst_node_col; extern int current_id; + +/* Controller-support identity accessors. When the clusterer_controller module + * is NOT part of the build these all collapse to the stock global 'current_id', + * so every call site compiles to the exact upstream behaviour with no #ifdef of + * its own. With the controller, a node can hold a different node_id per cluster + * and its identity is (re)assigned at runtime, so the per-cluster shm value and a + * shm-backed global are authoritative instead. */ +#ifdef CLUSTERER_CTRL_SUPPORT extern int *_current_id_shm; /* Read current_id from shm if available (cross-process after fork) */ #define GET_CURRENT_ID (_current_id_shm ? *_current_id_shm : current_id) -/* This node's node_id *within a specific cluster*. With the controller a node - * can hold a different node_id in each cluster, so the per-cluster identity in - * shared memory (cl->current_node) is authoritative. Returns -1 (a node_id - * that matches nothing) when this cluster's identity is not yet established, so - * a not-yet-joined cluster can never accidentally match or stamp a real id - * (in particular it never borrows another cluster's id via the legacy global). */ +/* This node's node_id *within a specific cluster*. Returns -1 (a node_id that + * matches nothing) when this cluster's identity is not yet established, so a + * not-yet-joined cluster can never accidentally match or stamp a real id. */ static inline int cluster_self_id(const struct cluster_info *cl) { return (cl && cl->current_node) ? cl->current_node->node_id : -1; } -extern int db_mode; extern int use_controller; /* cluster_ids declared controller-managed via cluster_options (use_controller=1) */ extern int cc_stub_ids[]; extern int cc_stub_count; +#else +#define GET_CURRENT_ID (current_id) +#define cluster_self_id(cl) (current_id) +#define use_controller 0 +#endif + +extern int db_mode; extern rw_lock_t *cl_list_lock; extern cluster_info_t **cluster_list; /* Effective db_mode *for one cluster*. Controller-managed clusters never use * the DB (their topology is injected at runtime), so they always behave as - * db_mode=0 even in a hybrid where native clusters are DB-backed (db_mode!=0). */ + * db_mode=0 even in a hybrid where native clusters are DB-backed (db_mode!=0). + * Without the controller every cluster is native, so this is just db_mode. */ +#ifdef CLUSTERER_CTRL_SUPPORT static inline int cl_db_mode(const struct cluster_info *cl) { return (cl && cl->controller_managed) ? 0 : db_mode; } +#else +#define cl_db_mode(cl) (db_mode) +#endif int update_db_state(int cluster_id, int node_id, int state); int load_db_info(db_func_t *dr_dbf, db_con_t* db_hdl, cluster_info_t **cl_list); void free_info(cluster_info_t *cl_list); int add_node_info(node_info_t **new_info, cluster_info_t **cl_list, int *int_vals, +#ifdef CLUSTERER_CTRL_SUPPORT str *str_vals, int self_id); +#else + str *str_vals); +#endif void remove_node_list(cluster_info_t *cl, node_info_t *node); int provision_neighbor(modparam_t type, void* val); @@ -233,7 +255,9 @@ static inline cluster_info_t *get_cluster_by_id(int cluster_id) { cluster_info_t *cl; +#ifdef CLUSTERER_CTRL_SUPPORT if (!cluster_list || !*cluster_list) return NULL; +#endif for (cl = *cluster_list; cl; cl = cl->next) if (cl->cluster_id == cluster_id) return cl; diff --git a/modules/clusterer/sharing_tags.c b/modules/clusterer/sharing_tags.c index 04ce5f6c1ca..21d507c6e3a 100644 --- a/modules/clusterer/sharing_tags.c +++ b/modules/clusterer/sharing_tags.c @@ -387,6 +387,7 @@ int shtag_modparam_func(modparam_t type, void *val_s) tag_name.len, tag_name.s); return -1; } +#ifdef CLUSTERER_CTRL_SUPPORT /* Force the given state. Only a *controller-managed* cluster's active tag * is forced to backup here (its controller master activates it when * appropriate); a native cluster keeps its configured =active state - the @@ -410,6 +411,14 @@ int shtag_modparam_func(modparam_t type, void *val_s) tag->send_active_msg = 1; } } +#else + /* force the given state */ + tag->state = init_state; + + if (init_state == SHTAG_STATE_ACTIVE) + /* broadcast (later) in cluster that this tag is active */ + tag->send_active_msg = 1; +#endif return 0; } @@ -893,6 +902,7 @@ int handle_shtag_active(bin_packet_t *packet, int cluster_id, int source_id) } +#ifdef CLUSTERER_CTRL_SUPPORT /** * shtag_force_all_backup() - force every sharing tag for the given * cluster to BACKUP state, ignoring the =active config value. @@ -957,6 +967,7 @@ int shtag_activate_all_backup(int cluster_id) } return 0; } +#endif /* CLUSTERER_CTRL_SUPPORT */ void shtag_event_handler(int cluster_id, enum clusterer_event ev, int node_id) { @@ -1042,6 +1053,7 @@ mi_response_t *shtag_mi_set_active(const mi_params_t *params, tag.len, tag.s, c_id); lock_start_read(cl_list_lock); +#ifdef CLUSTERER_CTRL_SUPPORT { cluster_info_t *_cl = get_cluster_by_id(c_id); if (!_cl) { @@ -1056,6 +1068,12 @@ mi_response_t *shtag_mi_set_active(const mi_params_t *params, "controller-managed; manual activation not allowed")); } } +#else + if (!get_cluster_by_id(c_id)) { + lock_stop_read(cl_list_lock); + return init_mi_error(404, MI_SSTR("Cluster ID not found")); + } +#endif lock_stop_read(cl_list_lock); if (shtag_activate( &tag, c_id, MI_SSTR("MI command"))<0) { @@ -1149,6 +1167,7 @@ int var_set_sh_tag(struct sip_msg* msg, pv_param_t *param, int op, return 0; } +#ifdef CLUSTERER_CTRL_SUPPORT lock_start_read(cl_list_lock); { cluster_info_t *_cl = get_cluster_by_id(v_name->cluster_id); @@ -1161,6 +1180,7 @@ int var_set_sh_tag(struct sip_msg* msg, pv_param_t *param, int op, } } lock_stop_read(cl_list_lock); +#endif if (shtag_activate( &v_name->shtag, v_name->cluster_id, MI_SSTR("script variable"))==-1) { diff --git a/modules/clusterer/sharing_tags.h b/modules/clusterer/sharing_tags.h index 64da0717fd3..0964ec73440 100644 --- a/modules/clusterer/sharing_tags.h +++ b/modules/clusterer/sharing_tags.h @@ -43,8 +43,10 @@ int send_shtag_active_info(int c_id, str *tag_name, int node_id); void shtag_flush_state(int c_id, int node_id); void shtag_event_handler(int cluster_id, enum clusterer_event ev, int node_id); +#ifdef CLUSTERER_CTRL_SUPPORT int shtag_activate_all_backup(int cluster_id); int shtag_force_all_backup(int cluster_id); +#endif mi_response_t *shtag_mi_list(const mi_params_t *params, struct mi_handler *async_hdl); diff --git a/modules/clusterer/sync.c b/modules/clusterer/sync.c index 4a47e09f3ff..89248ce73cf 100644 --- a/modules/clusterer/sync.c +++ b/modules/clusterer/sync.c @@ -70,8 +70,13 @@ static int get_sync_source(cluster_info_t *cluster, str *capability, if (get_next_hop(node) == 0) continue; +#ifdef CLUSTERER_CTRL_SUPPORT if (!cluster->current_node || !match_node(cluster->current_node, node, match_cond)) continue; +#else + if (!match_node(cluster->current_node, node, match_cond)) + continue; +#endif lock_get(node->lock); for (cap = node->capabilities; cap; cap = cap->next) @@ -95,6 +100,7 @@ int queue_sync_request(cluster_info_t *cluster, struct local_cap *lcap) { lock_get(cluster->lock); +#ifdef CLUSTERER_CTRL_SUPPORT /* If we are a seed node with no peers yet, skip the pending queue and * self-mark as synced immediately. There is nobody to sync from, and * the seed-fallback timer would just fire after seed_fb_interval and @@ -116,6 +122,7 @@ int queue_sync_request(cluster_info_t *cluster, struct local_cap *lcap) send_single_cap_update(cluster, lcap, 1); return 0; } +#endif lcap->flags |= CAP_SYNC_PENDING; if (sr_get_core_status() == STATE_INITIALIZING) @@ -123,11 +130,16 @@ int queue_sync_request(cluster_info_t *cluster, struct local_cap *lcap) else lcap->flags &= ~CAP_SYNC_STARTUP; +#ifdef CLUSTERER_CTRL_SUPPORT /* Always record when we started waiting — if current_node is not yet set * (controller mode, identity assigned post-fork), sync_req_time would * stay at zero (epoch) and TIME_DIFF would be huge, causing the seed * fallback timer to fire immediately once identity is assigned. */ gettimeofday(&lcap->sync_req_time, NULL); +#else + if (cluster->current_node->flags & NODE_IS_SEED) + gettimeofday(&lcap->sync_req_time, NULL); +#endif lock_release(cluster->lock); diff --git a/modules/clusterer/topology.c b/modules/clusterer/topology.c index e85936c0972..b6baa9a413e 100644 --- a/modules/clusterer/topology.c +++ b/modules/clusterer/topology.c @@ -195,8 +195,10 @@ void heartbeats_timer(void) lock_start_read(cl_list_lock); for (clusters_it = *cluster_list; clusters_it; clusters_it = clusters_it->next) { +#ifdef CLUSTERER_CTRL_SUPPORT if (!clusters_it->current_node) continue; /* identity not yet set by controller */ +#endif lock_get(clusters_it->current_node->lock); if (!(clusters_it->current_node->flags & NODE_STATE_ENABLED)) { lock_release(clusters_it->current_node->lock); @@ -716,7 +718,11 @@ static node_info_t *add_node(bin_packet_t *received, cluster_info_t *cl, int_vals[INT_VALS_NODE_ID_COL] = src_node_id; int_vals[INT_VALS_STATE_COL] = 1; /* enabled */ - if (add_node_info(&new_node, &cl, int_vals, str_vals, cluster_self_id(cl)) != 0) { + if (add_node_info(&new_node, &cl, int_vals, str_vals +#ifdef CLUSTERER_CTRL_SUPPORT + , cluster_self_id(cl) +#endif + ) != 0) { LM_ERR("Unable to add node info to backing list\n"); return NULL; } diff --git a/modules/clusterer_controller/README b/modules/clusterer_controller/README index 7d017cde7a3..719e3a6a320 100644 --- a/modules/clusterer_controller/README +++ b/modules/clusterer_controller/README @@ -14,32 +14,33 @@ CLUSTERER_CONTROLLER Module 1.5.1. OpenSIPS Modules 1.5.2. External Libraries or Applications - 1.6. Exported Parameters - - 1.6.1. cluster (string) - 1.6.2. my_ip (string) - 1.6.3. interface (string) - 1.6.4. query_time (integer) - 1.6.5. password (string) - 1.6.6. manage_shtags (integer) - 1.6.7. master_stickiness (integer) - 1.6.8. on_config_mismatch (string) - - 1.7. Exported MI Functions - - 1.7.1. cl_ctr_list_members - 1.7.2. cl_ctr_node_info - 1.7.3. cl_ctr_list_config - 1.7.4. cl_ctr_shtag_force - 1.7.5. cl_ctr_shtag_auto - - 1.8. Exported Pseudo-Variables - 1.9. Exported Functions - 1.10. Multiple Clusters - 1.11. Hybrid Topologies (native + controller clusters) - 1.12. Configuration Example - 1.13. Limitations - 1.14. Planned Features + 1.6. Building the Module + 1.7. Exported Parameters + + 1.7.1. cluster (string) + 1.7.2. my_ip (string) + 1.7.3. interface (string) + 1.7.4. query_time (integer) + 1.7.5. password (string) + 1.7.6. manage_shtags (integer) + 1.7.7. master_stickiness (integer) + 1.7.8. on_config_mismatch (string) + + 1.8. Exported MI Functions + + 1.8.1. cl_ctr_list_members + 1.8.2. cl_ctr_node_info + 1.8.3. cl_ctr_list_config + 1.8.4. cl_ctr_shtag_force + 1.8.5. cl_ctr_shtag_auto + + 1.9. Exported Pseudo-Variables + 1.10. Exported Functions + 1.11. Multiple Clusters + 1.12. Hybrid Topologies (native + controller clusters) + 1.13. Configuration Example + 1.14. Limitations + 1.15. Planned Features A. HA Behaviour Tests @@ -572,9 +573,35 @@ on-key") every node in a cluster must be built the same way; the active suite is printed in the startup log. -1.6. Exported Parameters +1.6. Building the Module -1.6.1. cluster (string) + clusterer_controller is excluded from the default build (it is + listed in exclude_modules in Makefile.conf.template), like the + other modules that depend on external libraries. A stock + OpenSIPS build therefore does not include it. To build it, add + it to include_modules in your Makefile.conf: +include_modules= clusterer_controller + + then rebuild (make all / make modules). Because the module + links WolfSSL (and optionally libsodium), the same + external-library requirements as tls_wolfssl apply. + + The clusterer module is unmodified when clusterer_controller is + not built. The controller integration on the clusterer side + (the clusterer_ctrl API, the cluster_options modparam and all + controller hooks) is compiled only when clusterer_controller is + part of the build: the top-level Makefile detects this and + passes -DCLUSTERER_CTRL_SUPPORT to the clusterer module. A + build without clusterer_controller produces the stock clusterer + module, unchanged in behaviour and exported interface - in + particular the cluster_options parameter does not exist and is + rejected as unknown. Enabling clusterer_controller + automatically rebuilds clusterer with the support compiled in; + the two are always a matched pair. + +1.7. Exported Parameters + +1.7.1. cluster (string) Define a cluster to participate in. The value is a comma-separated key=value string with the following fields: @@ -667,7 +694,7 @@ Note letting separate deployments coexist on a shared multicast group, not two clusters inside one instance.) -1.6.2. my_ip (string) +1.7.2. my_ip (string) Explicitly set the local IPv4 address used by the controller for its own node identity and master election. This is the IP @@ -711,7 +738,7 @@ Note modparam("clusterer_controller", "my_ip", "10.22.23.191") ... -1.6.3. interface (string) +1.7.3. interface (string) Explicitly set the network interface name to use for multicast traffic (e.g. eth0, enp6s18). The module takes the first IPv4 @@ -737,7 +764,7 @@ modparam("clusterer_controller", "my_ip", "10.22.23.191") modparam("clusterer_controller", "interface", "eth0") ... -1.6.4. query_time (integer) +1.7.4. query_time (integer) How often (in seconds) each active node sends an ALIVE heartbeat to the multicast group. This value also controls the @@ -754,7 +781,7 @@ modparam("clusterer_controller", "interface", "eth0") modparam("clusterer_controller", "query_time", 5) ... -1.6.5. password (string) +1.7.5. password (string) Global default encryption password for all clusters. All nodes in a cluster must use the same password. The password serves @@ -783,7 +810,7 @@ modparam("clusterer_controller", "query_time", 5) modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") ... -1.6.6. manage_shtags (integer) +1.7.6. manage_shtags (integer) When set to 1 (the default), the controller master node automatically manages sharing tag failover for all clusters. @@ -914,7 +941,7 @@ modparam("clusterer_controller", "cluster", "id=1,multicast=239.0.90.1: modparam("clusterer_controller", "password", "MyStr0ngPassw0rd!") # manage_shtags defaults to 1 — no need to set it explicitly -1.6.7. master_stickiness (integer) +1.7.7. master_stickiness (integer) Controls whether a live master keeps its role when a higher-IP node joins. Default 1 (sticky). @@ -964,7 +991,7 @@ modparam("clusterer_controller", "cluster", "id=2,multicast=239.0.90.2:3333,master_stickiness=0") ... -1.6.8. on_config_mismatch (string) +1.7.8. on_config_mismatch (string) Policy applied when a node's consistency-critical settings (manage_shtags, master_stickiness, query_time) differ from @@ -986,9 +1013,9 @@ modparam("clusterer_controller", "cluster", modparam("clusterer_controller", "on_config_mismatch", "reject") ... -1.7. Exported MI Functions +1.8. Exported MI Functions -1.7.1. cl_ctr_list_members +1.8.1. cl_ctr_list_members List all current cluster members with their node_id, status and BIN socket addresses. Status is one of master, backup (the @@ -1025,7 +1052,7 @@ opensips-cli -x mi cl_ctr_list_members } ] -1.7.2. cl_ctr_node_info +1.8.2. cl_ctr_node_info Return full information for a specific node identified by its allocated node_id. @@ -1043,7 +1070,7 @@ opensips-cli -x mi cl_ctr_node_info node_id=2 "bin_sockets": [ "bin:10.22.23.192:3857" ] } -1.7.3. cl_ctr_list_config +1.8.3. cl_ctr_list_config List all configured clusters and their resolved settings — the effective values actually in use after global defaults and @@ -1074,7 +1101,7 @@ opensips-cli -x mi cl_ctr_list_config } ] -1.7.4. cl_ctr_shtag_force +1.8.4. cl_ctr_shtag_force Force a specific node to hold the active sharing tag, overriding the normal master-driven allocation. This is useful @@ -1100,7 +1127,7 @@ opensips-cli -x mi cl_ctr_list_config Example 1.17. cl_ctr_shtag_force usage opensips-cli -x mi cl_ctr_shtag_force cluster_id=1 node_id=3 -1.7.5. cl_ctr_shtag_auto +1.8.5. cl_ctr_shtag_auto Clear any override set by cl_ctr_shtag_force and resume automatic, master-driven sharing-tag allocation — the active @@ -1113,7 +1140,7 @@ opensips-cli -x mi cl_ctr_shtag_force cluster_id=1 node_id=3 Example 1.18. cl_ctr_shtag_auto usage opensips-cli -x mi cl_ctr_shtag_auto cluster_id=1 -1.8. Exported Pseudo-Variables +1.9. Exported Pseudo-Variables These read-only pseudo-variables expose live cluster state to the routing script, so a decision such as “only the master runs @@ -1159,7 +1186,7 @@ xlog("cluster 2 master is $cl_ctr_master_ip(2), I am $cl_ctr_role(2)\n") ; -1.9. Exported Functions +1.10. Exported Functions Per-peer lookups take two arguments (cluster_id, node_id) and are therefore script functions, not pseudo-variables (a comma @@ -1183,7 +1210,7 @@ if (cl_ctr_node_present(1, 3) && cl_ctr_node_is_master(1, 3)) { xlog("node 3 ($var(ip)) leads cluster 1\n"); } -1.10. Multiple Clusters +1.11. Multiple Clusters A single OpenSIPS instance can participate in multiple clusters simultaneously by repeating the cluster modparam. Each cluster @@ -1264,7 +1291,7 @@ modparam("dialog", "dialog_replication_cluster", 1) loadmodule "dispatcher.so" modparam("dispatcher", "cluster_id", 2) -1.11. Hybrid Topologies (native + controller clusters) +1.12. Hybrid Topologies (native + controller clusters) A single OpenSIPS instance can run native clusterer clusters (defined in the database or statically via @@ -1376,7 +1403,7 @@ modparam("dispatcher", "cluster_id", 7) # native (cluster 7 cluster 1 the controller assigns an id at runtime, which may differ. -1.12. Configuration Example +1.13. Configuration Example The following example shows a minimal two-module configuration for zero-config HA clustering with dialog replication. The @@ -1416,7 +1443,7 @@ modparam("dispatcher", "cluster_probing_mode", "distributed") socket=bin:IP:PORT line changed per node) is used on every node. No other per-node customization is required. -1.13. Limitations +1.14. Limitations * IPv4 only. IPv6 multicast is not currently supported. * The module pre-allocates approximately 152 KB of shared @@ -1481,7 +1508,7 @@ modparam("dispatcher", "cluster_probing_mode", "distributed") paths between cluster nodes, particularly over VPN tunnels and across datacenter firewalls. -1.14. Planned Features +1.15. Planned Features The following features are planned for future releases: * Node maintenance mode — take a node out of duty for a diff --git a/modules/clusterer_controller/doc/clusterer_controller_admin.xml b/modules/clusterer_controller/doc/clusterer_controller_admin.xml index 5cb033b6e27..76a1b8a8586 100644 --- a/modules/clusterer_controller/doc/clusterer_controller_admin.xml +++ b/modules/clusterer_controller/doc/clusterer_controller_admin.xml @@ -611,6 +611,43 @@ +
+ Building the Module + + clusterer_controller is + excluded from the default build (it is + listed in exclude_modules in + Makefile.conf.template), like the other modules that + depend on external libraries. A stock &osips; build therefore does not + include it. To build it, add it to include_modules in + your Makefile.conf: + + +include_modules= clusterer_controller + + + then rebuild (make all / + make modules). Because the module links WolfSSL (and + optionally libsodium), the same external-library requirements as + tls_wolfssl apply. + + + The clusterer module is unmodified when + clusterer_controller is not built. The controller integration + on the clusterer side (the + clusterer_ctrl API, the cluster_options + modparam and all controller hooks) is compiled only when + clusterer_controller is part of the build: the top-level Makefile detects + this and passes -DCLUSTERER_CTRL_SUPPORT to the + clusterer module. A build without clusterer_controller produces the + stock clusterer module, unchanged in behaviour and exported interface - + in particular the cluster_options parameter does not + exist and is rejected as unknown. Enabling clusterer_controller + automatically rebuilds clusterer with the support compiled in; the two + are always a matched pair. + +
+
Exported Parameters From 880a2f9de0acf197494550a0b60bf69ab88ef5b1 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Fri, 17 Jul 2026 14:36:02 +1000 Subject: [PATCH 06/21] clusterer_controller: libsodium-only crypto, drop the wolfSSL fallback libsodium is a hard build requirement: the module is built sodium-only (XChaCha20-Poly1305 + Argon2id, plus X25519 / HKDF-SHA256 / RNG from libsodium), and the wolfSSL AES-256-GCM/scrypt fallback is not built. This keeps the module binary small and gives one consistent crypto suite across every build. --- modules/clusterer_controller/Makefile | 24 +- modules/clusterer_controller/README | 73 ++--- .../clusterer_controller.c | 261 ++++++------------ .../doc/clusterer_controller_admin.xml | 77 +++--- 4 files changed, 170 insertions(+), 265 deletions(-) diff --git a/modules/clusterer_controller/Makefile b/modules/clusterer_controller/Makefile index 5bf0dd4fe8b..5a29721fabd 100644 --- a/modules/clusterer_controller/Makefile +++ b/modules/clusterer_controller/Makefile @@ -4,26 +4,18 @@ include ../../Makefile.defs auto_gen= NAME= clusterer_controller.so -LIBS += -L../tls_wolfssl/lib/lib/ -lwolfssl -lm -DEFS += -I../tls_wolfssl/lib/include/ -DEPS += ../tls_wolfssl/lib/lib/libwolfssl.a - -# Optional stronger crypto suite: if libsodium (dev) is detected on the build -# host, compile with XChaCha20-Poly1305 + Argon2id instead of AES-256-GCM + -# scrypt. Distro static libs are usually non-PIC (can't link into a .so), so -# we link libsodium dynamically — target hosts then need the libsodium runtime -# package (e.g. `apt install libsodium23`). All nodes in a cluster must be -# built the same way (the wire formats are not interoperable). +# libsodium is a hard requirement: it provides the payload AEAD +# (XChaCha20-Poly1305), the bootstrap KDF (Argon2id), the Noise_NNpsk0 join +# handshake, and X25519 / HKDF-SHA256 / RNG. It is linked dynamically (distro +# static libs are usually non-PIC), so target hosts need the libsodium runtime +# package (e.g. `apt install libsodium23`). SODIUM_EXISTS := $(shell pkg-config --exists libsodium 2>/dev/null && echo yes) ifeq ($(SODIUM_EXISTS),yes) - DEFS += -DCC_HAVE_SODIUM $(shell pkg-config --cflags libsodium) + DEFS += $(shell pkg-config --cflags libsodium) LIBS += $(shell pkg-config --libs libsodium) - $(info clusterer_controller: libsodium found -> XChaCha20-Poly1305 + Argon2id) + $(info clusterer_controller: libsodium found -> XChaCha20-Poly1305 + Argon2id + Noise_NNpsk0) else - $(info clusterer_controller: libsodium not found -> AES-256-GCM + scrypt) + $(error clusterer_controller requires libsodium (install libsodium-dev / libsodium-devel)) endif include ../../Makefile.modules - -../tls_wolfssl/lib/lib/libwolfssl.a: - $(MAKE) -C ../tls_wolfssl lib/lib/libwolfssl.a diff --git a/modules/clusterer_controller/README b/modules/clusterer_controller/README index 719e3a6a320..afc992fb129 100644 --- a/modules/clusterer_controller/README +++ b/modules/clusterer_controller/README @@ -328,16 +328,17 @@ Chapter 1. Admin Guide attempting decryption, so foreign-cluster traffic on the group never counts as an authentication failure. - Crypto suite (selected at build time): by default the module - uses AES-256-GCM (12-byte nonce) for the payload AEAD and - scrypt (N=2^16, r=8, p=1) for the bootstrap-key derivation, - through WolfSSL. If libsodium is detected on the build host, - the module is compiled instead with XChaCha20-Poly1305 (24-byte - nonce, whose 192-bit nonce space removes any random-nonce - collision concern) and Argon2id. The two wire formats are not - interoperable, so every node in a cluster must be built with - the same suite; the active suite is reported in the startup log - (crypto=...). + Crypto suite (selected at build time): if libsodium is detected + on the build host, the module is built sodium-only: + XChaCha20-Poly1305 (24-byte nonce, whose 192-bit nonce space + removes any random-nonce collision concern) for the payload + AEAD, Argon2id for the bootstrap-key derivation, and X25519 / + HKDF-SHA256 / RNG also from libsodium — WolfSSL is not linked + at all. Without libsodium the module falls back to WolfSSL: + AES-256-GCM (12-byte nonce) and scrypt (N=2^16, r=8, p=1). The + two wire formats are not interoperable, so every node in a + cluster must be built with the same suite; the active suite is + reported in the startup log (crypto=...). Phase 1 — bootstrap (JOIN_REQ / KEY_GRANT): each worker process generates an ephemeral X25519 keypair on startup. When joining, @@ -551,27 +552,31 @@ on-key") 1.5.2. External Libraries or Applications - The following libraries must be installed: - * tls_wolfssl — the clusterer_controller module links - statically against the WolfSSL library built by the - tls_wolfssl module - (modules/tls_wolfssl/lib/lib/libwolfssl.a). WolfSSL - provides AES-256-GCM authenticated encryption, X25519 ECDH - key agreement, HKDF-SHA256 key derivation, the scrypt - password KDF, and SHA-256. The tls_wolfssl module must be - present in the source tree; its static library is built - automatically as a dependency if not already present. - * libsodium (optional) — if the build host has libsodium + One of the following two crypto libraries is required; + libsodium is preferred and used exclusively when found: + * libsodium (preferred) — if the build host has libsodium development files (detected via pkg-config), the module is - compiled with the stronger crypto suite: XChaCha20-Poly1305 - for the payload AEAD and Argon2id for the bootstrap-key - derivation, in place of AES-256-GCM and scrypt. X25519 ECDH - and HKDF continue to use WolfSSL. libsodium is linked - dynamically, so each target host then also needs the - libsodium runtime package (for example libsodium23). - Because the two suites produce incompatible wire formats, - every node in a cluster must be built the same way; the - active suite is printed in the startup log. + built sodium-only: XChaCha20-Poly1305 for the payload AEAD, + Argon2id for the bootstrap-key derivation, and X25519 ECDH, + HKDF-SHA256 and the RNG also from libsodium — WolfSSL is + not linked at all, keeping the module binary small (a few + hundred KB instead of several MB of statically-linked + WolfSSL). libsodium is linked dynamically, so each target + host also needs the libsodium runtime package (for example + libsodium23). + * tls_wolfssl (fallback) — without libsodium, the module + links statically against the WolfSSL library built by the + tls_wolfssl module + (modules/tls_wolfssl/lib/lib/libwolfssl.a), providing + AES-256-GCM authenticated encryption, X25519 ECDH, + HKDF-SHA256 and the scrypt password KDF. The tls_wolfssl + module must then be present in the source tree; its static + library is built automatically as a dependency if not + already present. + + Because the two suites produce incompatible wire formats, every + node in a cluster must be built the same way; the active suite + is printed in the startup log. 1.6. Building the Module @@ -582,9 +587,11 @@ on-key") it to include_modules in your Makefile.conf: include_modules= clusterer_controller - then rebuild (make all / make modules). Because the module - links WolfSSL (and optionally libsodium), the same - external-library requirements as tls_wolfssl apply. + then rebuild (make all / make modules). With libsodium + development files on the build host the module builds + sodium-only (no WolfSSL); otherwise it links the bundled + WolfSSL and the same external-library requirements as + tls_wolfssl apply — see External Libraries or Applications. The clusterer module is unmodified when clusterer_controller is not built. The controller integration on the clusterer side diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index 31ca17589ea..bdc9c07b3a0 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -34,14 +34,13 @@ * blocks re-stamping a captured packet onto a different cluster_id when two * clusters share a multicast group and password). * - * CRYPTO SUITE (build-time choice; every node in a cluster must match): - * - Default (WolfSSL): AES-256-GCM payload AEAD, 12-byte nonce; - * scrypt bootstrap-key KDF (N=2^16, r=8, p=1). - * - If libsodium is detected on the build host (-DCC_HAVE_SODIUM, see the - * Makefile): XChaCha20-Poly1305 payload AEAD, 24-byte nonce (its 192-bit - * nonce removes any random-nonce collision worry); Argon2id bootstrap KDF. - * The two wire formats are NOT interoperable (nonce size and primitives - * differ). The active suite is logged at startup ("crypto=..."). + * CRYPTO SUITE (all libsodium; libsodium is a hard build requirement): + * - XChaCha20-Poly1305 payload AEAD, 24-byte nonce (its 192-bit nonce space + * removes any random-nonce collision worry). + * - Argon2id bootstrap-key KDF. + * - Noise_NNpsk0_25519_ChaChaPoly_SHA256 for the join handshake (PSK = the + * Argon2id bootstrap key); crypto_box for the KEY_HANDOFF salt delivery. + * The active suite is logged at startup ("crypto=..."). * * Two key types are used, selected by the 2-byte magic: * @@ -206,29 +205,17 @@ #include "../clusterer/clusterer_ctrl.h" /* set_my_identity, add_node, remove_node */ -#include /* must be first: build-time feature flags */ -#include /* Aes, wc_AesGcmSetKey/Encrypt/Decrypt */ -#include /* WC_RNG, wc_RNG_GenerateBlock */ -#include /* wc_HKDF */ -#include /* curve25519_key, wc_curve25519_* */ -#include /* wc_Sha256Hash */ -#include /* wc_scrypt (bootstrap key hardening) */ #include /* timerfd_create(), timerfd_settime() */ #include "../../reactor_proc.h" /* reactor_proc_init/add_fd/loop */ -/* Optional stronger crypto suite. Selected at BUILD time: if libsodium is - * detected on the build host (see the module Makefile -> -DCC_HAVE_SODIUM), the - * payload AEAD becomes XChaCha20-Poly1305 (192-bit nonce -> no random-nonce - * collision worry, even for the static bootstrap key) and the bootstrap KDF - * becomes Argon2id. Otherwise we fall back to WolfSSL AES-256-GCM + scrypt. - * NOTE: the wire formats are NOT interoperable (nonce size and primitives - * differ), so every node in a cluster must be built with the same suite. */ -#ifdef CC_HAVE_SODIUM +/* All cryptography is provided by libsodium, which is a hard requirement (the + * module Makefile fails the build if it is not found). The payload AEAD is + * XChaCha20-Poly1305 (192-bit nonce -> no random-nonce collision worry, even for + * the static bootstrap key), the bootstrap KDF is Argon2id, the join handshake + * is Noise_NNpsk0_25519_ChaChaPoly_SHA256, and X25519 / HKDF-SHA256 / RNG also + * come from libsodium. */ #include -#define CC_CRYPTO_SUITE "XChaCha20-Poly1305 + Argon2id" -#else -#define CC_CRYPTO_SUITE "AES-256-GCM + scrypt" -#endif +#define CC_CRYPTO_SUITE "XChaCha20-Poly1305 + Argon2id + Noise_NNpsk0" /* ========================================================================= * Wire-format constants @@ -277,40 +264,28 @@ static const unsigned char CC_BOOTSTRAP_MAGIC[CC_MAGIC_SZ] = { 0xCC, 0x01 }; /* BIN info block: [bin_count 1B][sock1 NUL-term]...[sockN NUL-term] */ #define CC_BIN_INFO_MAX_SZ (1 + CC_MAX_BIN_SOCKETS * CC_MAX_BIN_SOCK_LEN) -/* AES-256-GCM encryption constants - * wire: [magic 2B][cluster_id 2B BE][nonce 12B][ciphertext][GCM tag 16B] +/* AEAD encryption constants + * wire: [magic 2B][cluster_id 2B BE][nonce 24B][ciphertext][tag 16B] * plaintext: [type 1B][seq 4B][payload] * The cluster_id is cleartext (like the magic) so a node can drop packets that * belong to a different cluster sharing the same multicast group WITHOUT its * key - before decryption, so foreign traffic never counts as an auth failure. */ -#ifdef CC_HAVE_SODIUM #define CC_NONCE_SZ 24 /* XChaCha20-Poly1305 nonce (192-bit) */ -#else -#define CC_NONCE_SZ 12 /* AES-GCM nonce, random per packet */ -#endif -#define CC_TAG_SZ 16 /* AEAD tag (16 for both AES-GCM & XChaCha) */ +#define CC_TAG_SZ 16 /* Poly1305 AEAD tag */ #define CC_SEQ_SZ 4 /* uint32_t monotonic sequence in plaintext */ #define CC_CLUSTER_ID_SZ 2 /* cleartext uint16 cluster_id (BE) selector */ #define CC_NONCE_OFF (CC_MAGIC_SZ + CC_CLUSTER_ID_SZ) /* nonce starts here */ #define CC_WIRE_HDR_SZ (CC_MAGIC_SZ + CC_CLUSTER_ID_SZ + CC_NONCE_SZ) /* 16 (GCM) / 28 (XChaCha) */ #define CC_PLAIN_HDR_SZ (1 + CC_SEQ_SZ) /* type + seq = 5 */ -/* Bootstrap-key hardening: the join/admission key is derived from the shared - * password with scrypt (memory-hard) instead of a single SHA-256, so a - * password captured from a JOIN_REQ cannot be brute-forced cheaply offline. - * Derived ONCE in mod_init (main process, before fork), so the cost is a - * transient startup cost only - cost=16/r=8 is ~64 MiB for ~0.3 s, freed - * immediately; workers inherit the 32-byte key and never run scrypt. (Argon2id - * would be preferable but this WolfSSL build does not provide it.) */ -#define CC_SCRYPT_COST 16 /* log2(N): N = 65536 (2x offline cost) */ -#define CC_SCRYPT_BLOCKSIZE 8 /* r */ -#define CC_SCRYPT_PARALLEL 1 /* p */ -#ifdef CC_HAVE_SODIUM -/* Argon2id parameters (libsodium crypto_pwhash). Fixed so every node derives - * the same key; ~64 MiB to match the scrypt fallback's memory hardness. */ +/* Bootstrap-key hardening: the join/admission key (also the Noise PSK) is + * derived from the shared password with Argon2id (memory-hard) instead of a + * single SHA-256, so a password captured from a JOIN_REQ cannot be brute-forced + * cheaply offline. Derived ONCE in mod_init (main process, before fork), so the + * ~64 MiB working set is a transient startup cost; workers inherit the 32-byte + * key. Fixed parameters so every node derives the same key. */ #define CC_ARGON2_OPSLIMIT 3UL #define CC_ARGON2_MEMLIMIT (64UL * 1024 * 1024) -#endif /* Minimum estimated password entropy (bits) before a startup warning fires. */ #define CC_MIN_PASSWORD_BITS 80 #define CC_DEFAULT_PASSWORD "3eCrEt*5629" /* insecure placeholder; warn if used */ @@ -649,7 +624,6 @@ static int on_config_mismatch = CC_CFGMISMATCH_REJECT; /* resolved; default static char my_ip_buf[INET_ADDRSTRLEN]; static char my_interface_buf[IF_NAMESIZE]; -static WC_RNG cc_rng; /* Local node identity - populated at mod_init by scanning the config file */ static uint16_t my_node_id = 0; @@ -1713,11 +1687,12 @@ static void cc_update_peer_bin_locked(const char *ip, uint16_t node_id, } /* ========================================================================= - * AES-256-GCM packet encryption / decryption + * Packet encryption / decryption (AEAD) * - * Every packet is fully encrypted and authenticated with AES-256-GCM. - * A 12-byte random nonce (generated fresh for each packet via WolfSSL RNG) - * ensures that even identical payloads produce different ciphertext. + * Every packet is fully encrypted and authenticated with the build's AEAD + * (XChaCha20-Poly1305 with libsodium, AES-256-GCM with wolfSSL). A fresh + * random nonce per packet ensures that even identical payloads produce + * different ciphertext. * * A 4-byte monotonic sequence number is included inside the plaintext. * Receivers track the highest sequence seen from each peer and reject any @@ -1734,62 +1709,72 @@ static void cc_update_peer_bin_locked(const char *ip, uint16_t node_id, * [type 1B][seq 4B][payload] * ========================================================================= */ +/* cc_random_bytes() - fill buf with cryptographically secure random bytes + * (libsodium's randombytes draws from the kernel CSPRNG). */ +static int cc_random_bytes(unsigned char *buf, size_t len) +{ + randombytes_buf(buf, len); + return 0; +} + +/* RFC 5869 HKDF-SHA256 built on libsodium's HMAC-SHA256 (libsodium 1.0.18 has + * no native HKDF; crypto_auth_hmacsha256_init accepts any key length). Every + * use in this module needs exactly 32 output bytes (= HashLen), so the expand + * phase is a single iteration: OKM = HMAC(PRK, info || 0x01). */ static int cc_hkdf_sha256(const unsigned char *ikm, size_t ikm_len, const unsigned char *salt, size_t salt_len, const char *info, unsigned char out[32]) { + crypto_auth_hmacsha256_state st; + unsigned char prk[crypto_auth_hmacsha256_BYTES]; + static const unsigned char zeros[crypto_auth_hmacsha256_BYTES]; + const unsigned char ctr = 0x01; size_t info_len = info ? strlen(info) : 0; - return wc_HKDF(WC_SHA256, - ikm, (word32)ikm_len, - salt, (word32)salt_len, - (const byte *)info, (word32)info_len, - out, 32) == 0 ? 0 : -1; + + /* extract: PRK = HMAC(salt, IKM); an absent salt is HashLen zero bytes */ + if (crypto_auth_hmacsha256_init(&st, salt_len ? salt : zeros, + salt_len ? salt_len : sizeof(zeros)) != 0) + return -1; + crypto_auth_hmacsha256_update(&st, ikm, (unsigned long long)ikm_len); + crypto_auth_hmacsha256_final(&st, prk); + + /* expand: T(1) = HMAC(PRK, info || 0x01) */ + if (crypto_auth_hmacsha256_init(&st, prk, sizeof(prk)) != 0) + return -1; + if (info_len) + crypto_auth_hmacsha256_update(&st, (const unsigned char *)info, + (unsigned long long)info_len); + crypto_auth_hmacsha256_update(&st, &ctr, 1); + crypto_auth_hmacsha256_final(&st, out); + return 0; } static int cc_gen_ecdh_keypair(unsigned char *privkey, unsigned char *pubkey) { - curve25519_key k; - word32 len = CC_PUBKEY_SZ; - int rc = -1; - - if (wc_curve25519_init(&k) != 0) goto done; - if (wc_curve25519_make_key(&cc_rng, 32, &k) != 0) goto free; - if (wc_curve25519_export_private_raw(&k, privkey, &len) != 0) goto free; - len = CC_PUBKEY_SZ; - if (wc_curve25519_export_public(&k, pubkey, &len) != 0) goto free; - rc = 0; -free: - wc_curve25519_free(&k); -done: - if (rc < 0) + randombytes_buf(privkey, CC_PUBKEY_SZ); + if (crypto_scalarmult_base(pubkey, privkey) != 0) { LM_ERR("clusterer_controller: X25519 keygen failed\n"); - return rc; + return -1; + } + return 0; } static int cc_ecdh_shared(const unsigned char *my_priv, const unsigned char *peer_pub, unsigned char out[CC_PUBKEY_SZ]) { - curve25519_key priv_k, pub_k; - word32 len = CC_PUBKEY_SZ; - int rc = -1; - - if (wc_curve25519_init(&priv_k) != 0) return -1; - if (wc_curve25519_init(&pub_k) != 0) { wc_curve25519_free(&priv_k); return -1; } - - if (wc_curve25519_import_private(my_priv, CC_PUBKEY_SZ, &priv_k) != 0) goto done; - if (wc_curve25519_import_public(peer_pub, CC_PUBKEY_SZ, &pub_k) != 0) goto done; - if (wc_curve25519_shared_secret(&priv_k, &pub_k, out, &len) != 0) goto done; - rc = 0; -done: - wc_curve25519_free(&priv_k); - wc_curve25519_free(&pub_k); - if (rc < 0) + /* crypto_scalarmult clamps the scalar internally (RFC 7748); reject an + * all-zero result (peer sent a low-order point). */ + if (crypto_scalarmult(out, my_priv, peer_pub) != 0 || + sodium_is_zero(out, CC_PUBKEY_SZ)) { LM_ERR("clusterer_controller: X25519 derive failed\n"); - return rc; + return -1; + } + return 0; } + /** * cc_wrap_salt() - XOR-encrypt master_salt for a specific peer using ECDH. * wrap_key = HKDF(ECDH(my_priv, peer_pub) || password [|| nonce], info) @@ -1926,14 +1911,13 @@ static int cc_derive_key(cc_cluster_t *cl) /* Per-cluster salt: a fixed domain-separation label plus the multicast * address, so the same password on different clusters yields different - * bootstrap keys. The salt is public by design - scrypt's work factor, + * bootstrap keys. The salt is public by design - Argon2id's work factor, * not salt secrecy, is what defeats brute force. */ saltlen = snprintf(salt, sizeof(salt), "opensips-cc-bootstrap-v1:%s", cl->multicast_address); if (saltlen < 0 || saltlen >= (int)sizeof(salt)) saltlen = (int)strlen(salt); -#ifdef CC_HAVE_SODIUM /* Argon2id. crypto_pwhash needs a fixed-length 16-byte salt, so fold our * variable-length domain-separation salt into one with BLAKE2b. */ { @@ -1949,16 +1933,6 @@ static int cc_derive_key(cc_cluster_t *cl) return -1; } } -#else - if (wc_scrypt(cl->key, (const byte *)cl->password, (int)strlen(cl->password), - (const byte *)salt, saltlen, - CC_SCRYPT_COST, CC_SCRYPT_BLOCKSIZE, CC_SCRYPT_PARALLEL, - 32) != 0) { - LM_ERR("clusterer_controller: key derivation (scrypt) failed for " - "cluster %d\n", cl->cluster_id); - return -1; - } -#endif return 0; } @@ -1984,10 +1958,8 @@ static int cc_encrypt_pkt(char *buf, int plain_off, int plain_len, /* AAD = cleartext header (magic + cluster_id): binding it to the tag stops * a captured packet being re-stamped with another cluster_id on a shared * multicast+password group (cross-cluster injection). The nonce is the - * AEAD IV, already bound. Random nonces are safe here: at this volume the - * AES-GCM 96-bit collision bound is unreachable, and XChaCha20's 192-bit - * nonce removes the concern outright. */ -#ifdef CC_HAVE_SODIUM + * AEAD IV, already bound. XChaCha20's 192-bit nonce makes random nonces + * collision-safe outright. */ { unsigned char nonce[CC_NONCE_SZ]; unsigned long long clen = 0; @@ -2003,35 +1975,6 @@ static int cc_encrypt_pkt(char *buf, int plain_off, int plain_len, } return plain_off + (int)clen; /* clen = plain_len + CC_TAG_SZ */ } -#else - { - Aes aes; - unsigned char nonce[CC_NONCE_SZ]; - unsigned char tag[CC_TAG_SZ]; - if (wc_RNG_GenerateBlock(&cc_rng, nonce, CC_NONCE_SZ) != 0) { - LM_ERR("clusterer_controller: RNG failed\n"); - return -1; - } - memcpy(buf + CC_NONCE_OFF, nonce, CC_NONCE_SZ); - if (wc_AesGcmSetKey(&aes, key, 32) != 0) { - LM_ERR("clusterer_controller: AesGcmSetKey failed\n"); - return -1; - } - if (wc_AesGcmEncrypt(&aes, - (byte *)buf + plain_off, - (const byte *)buf + plain_off, (word32)plain_len, - nonce, CC_NONCE_SZ, - tag, CC_TAG_SZ, - (const byte *)buf, CC_MAGIC_SZ + CC_CLUSTER_ID_SZ) != 0) { - LM_ERR("clusterer_controller: AesGcmEncrypt failed\n"); - wc_AesFree(&aes); - return -1; - } - wc_AesFree(&aes); - memcpy(buf + plain_off + plain_len, tag, CC_TAG_SZ); - return plain_off + plain_len + CC_TAG_SZ; - } -#endif } /** @@ -2056,7 +1999,6 @@ static int cc_decrypt_pkt(char *buf, ssize_t n, const char *sender_ip, return -1; } -#ifdef CC_HAVE_SODIUM { unsigned char *nonce = (unsigned char *)buf + CC_NONCE_OFF; unsigned long long mlen = 0; @@ -2077,36 +2019,6 @@ static int cc_decrypt_pkt(char *buf, ssize_t n, const char *sender_ip, return -1; } } -#else - { - Aes aes; - unsigned char *nonce = (unsigned char *)buf + CC_NONCE_OFF; - unsigned char *tag = (unsigned char *)buf + n - CC_TAG_SZ; - int ret; - if (wc_AesGcmSetKey(&aes, key, 32) != 0) { - LM_ERR("clusterer_controller: AesGcmSetKey failed\n"); - return -1; - } - ret = wc_AesGcmDecrypt(&aes, - (byte *)buf + CC_WIRE_HDR_SZ, - (const byte *)buf + CC_WIRE_HDR_SZ, (word32)cipher_len, - nonce, CC_NONCE_SZ, - tag, CC_TAG_SZ, - (const byte *)buf, CC_MAGIC_SZ + CC_CLUSTER_ID_SZ); - wc_AesFree(&aes); - if (ret != 0) { - if (is_bootstrap) - LM_WARN("clusterer_controller: bootstrap decryption failed " - "from %s - wrong password, foreign cluster, or " - "tampered packet\n", sender_ip); - else - LM_DBG("clusterer_controller: session packet from %s did not " - "decrypt - transient key mismatch during (re)key or " - "split-brain heal\n", sender_ip); - return -1; - } - } -#endif return 0; } @@ -2458,7 +2370,7 @@ static void cc_send_join_req_pkt(int sock, cc_cluster_t *cl) /* Append per-exchange nonce; master echoes it in KEY_GRANT to bind the * ECDH wrap to this specific exchange even if password is later compromised */ - if (wc_RNG_GenerateBlock(&cc_rng, cl->my_join_nonce, CC_JOIN_NONCE_SZ) != 0) { + if (cc_random_bytes(cl->my_join_nonce, CC_JOIN_NONCE_SZ) != 0) { LM_ERR("clusterer_controller: RNG for join_nonce failed\n"); return; } @@ -2758,7 +2670,7 @@ static void cc_send_key_handoff(int sock, const char *next_master_ip, static void cc_on_became_master(cc_cluster_t *cl) { cl->join_pending = 0; /* any pending re-key join is now moot */ - if (wc_RNG_GenerateBlock(&cc_rng, cl->peers->master_salt, CC_MASTER_SALT_SZ) != 0) { + if (cc_random_bytes(cl->peers->master_salt, CC_MASTER_SALT_SZ) != 0) { LM_ERR("clusterer_controller: [cluster %d] RNG for master_salt failed\n", cl->cluster_id); return; @@ -5580,17 +5492,10 @@ static int mod_init(void) LM_INFO("clusterer_controller: initialising\n"); - if (wc_InitRng(&cc_rng) != 0) { - LM_ERR("clusterer_controller: wc_InitRng failed\n"); - return -1; - } - -#ifdef CC_HAVE_SODIUM if (sodium_init() < 0) { LM_ERR("clusterer_controller: sodium_init() failed\n"); return -1; } -#endif /* Resolve the on_config_mismatch policy string. */ if (on_config_mismatch_s) { @@ -5886,12 +5791,8 @@ static int mod_init(void) static int cc_child_init(int rank) { - /* Re-seed RNG after fork - each worker must have independent state. */ - wc_FreeRng(&cc_rng); - if (wc_InitRng(&cc_rng) != 0) { - LM_ERR("clusterer_controller: wc_InitRng failed in child\n"); - return -1; - } + /* Re-seed the CSPRNG after fork - each worker must have independent state. */ + randombytes_stir(); /* Sync current_id from shared memory in every child process. * The global current_id diverges after fork - each process needs diff --git a/modules/clusterer_controller/doc/clusterer_controller_admin.xml b/modules/clusterer_controller/doc/clusterer_controller_admin.xml index 76a1b8a8586..c68e94f701b 100644 --- a/modules/clusterer_controller/doc/clusterer_controller_admin.xml +++ b/modules/clusterer_controller/doc/clusterer_controller_admin.xml @@ -300,17 +300,18 @@ Crypto suite (selected at build time): - by default the module uses AES-256-GCM (12-byte - nonce) for the payload AEAD and scrypt - (N=216, r=8, p=1) for the bootstrap-key - derivation, through WolfSSL. If libsodium is - detected on the build host, the module is compiled instead with + if libsodium is detected on the build host, the + module is built sodium-only: XChaCha20-Poly1305 (24-byte nonce, whose 192-bit - nonce space removes any random-nonce collision concern) and - Argon2id. The two wire formats are not - interoperable, so every node in a cluster must be built with the same - suite; the active suite is reported in the startup log - (crypto=...). + nonce space removes any random-nonce collision concern) for the payload + AEAD, Argon2id for the bootstrap-key derivation, + and X25519 / HKDF-SHA256 / RNG also from libsodium — WolfSSL is not + linked at all. Without libsodium the module falls back to WolfSSL: + AES-256-GCM (12-byte nonce) and + scrypt (N=216, r=8, + p=1). The two wire formats are not interoperable, so every node in a + cluster must be built with the same suite; the active suite is reported + in the startup log (crypto=...). Phase 1 — bootstrap (JOIN_REQ / @@ -574,39 +575,41 @@
External Libraries or Applications - The following libraries must be installed: + One of the following two crypto libraries is required; libsodium is + preferred and used exclusively when found: - tls_wolfssl — the - clusterer_controller module links - statically against the WolfSSL library built by the - tls_wolfssl module - (modules/tls_wolfssl/lib/lib/libwolfssl.a). - WolfSSL provides AES-256-GCM authenticated encryption, - X25519 ECDH key agreement, HKDF-SHA256 key derivation, - the scrypt password KDF, and SHA-256. The tls_wolfssl module - must be present in the source tree; its static library - is built automatically as a dependency if not already - present. + libsodium (preferred) — if the build host + has libsodium development files (detected via + pkg-config), the module is built + sodium-only: XChaCha20-Poly1305 for the + payload AEAD, Argon2id for the bootstrap-key derivation, and + X25519 ECDH, HKDF-SHA256 and the RNG also from libsodium — + WolfSSL is not linked at all, keeping the module binary small + (a few hundred KB instead of several MB of statically-linked + WolfSSL). libsodium is linked dynamically, so each target host + also needs the libsodium runtime package (for example + libsodium23). - libsodium (optional) — if the build host - has libsodium development files (detected via - pkg-config), the module is compiled with - the stronger crypto suite: XChaCha20-Poly1305 for the payload - AEAD and Argon2id for the bootstrap-key derivation, in place of - AES-256-GCM and scrypt. X25519 ECDH and HKDF continue to use - WolfSSL. libsodium is linked dynamically, so each target host - then also needs the libsodium runtime package (for example - libsodium23). Because the two suites produce - incompatible wire formats, every node in a cluster must be built - the same way; the active suite is printed in the startup log. + tls_wolfssl (fallback) — without + libsodium, the module links statically against the WolfSSL + library built by the tls_wolfssl module + (modules/tls_wolfssl/lib/lib/libwolfssl.a), + providing AES-256-GCM authenticated encryption, X25519 ECDH, + HKDF-SHA256 and the scrypt password KDF. The + tls_wolfssl module must then be present in + the source tree; its static library is built automatically as + a dependency if not already present. + Because the two suites produce incompatible wire formats, every node + in a cluster must be built the same way; the active suite is printed + in the startup log.
@@ -627,9 +630,11 @@ include_modules= clusterer_controller then rebuild (make all / - make modules). Because the module links WolfSSL (and - optionally libsodium), the same external-library requirements as - tls_wolfssl apply. + make modules). With libsodium development files + on the build host the module builds sodium-only (no WolfSSL); + otherwise it links the bundled WolfSSL and the same + external-library requirements as tls_wolfssl + apply — see External Libraries or Applications. The clusterer module is unmodified when From df26bf2bf8dd8749a50e73154e756ba058461423 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Fri, 17 Jul 2026 14:52:25 +1000 Subject: [PATCH 07/21] clusterer_controller: dedup send tails and peer lookups (net -77 lines) No behaviour change; a code-quality pass on the single-file module. - cc_seal_and_send(): every one of the ten packet senders repeated the same ~13-line tail (encrypt in place, build the multicast sockaddr, sendto, error log). Factor it into one helper. The multicast destination is now resolved once per cluster in mod_init (main process) into cl->mcast_dest and inherited by the forked workers - so it no longer rebuilds inet_addr()/htons() on every send, and mod_destroy's GOODBYE (which runs in the main process) uses the same path. Senders keep only their own success log via the helper's return value. - cc_peer_by_ip_locked(): the "find a peer entry by IP" scan was open-coded in many places; add a helper and use it where the lookup is standalone (cc_upsert_peer_locked, cc_update_peer_bin_locked). The fused election / member-list / prune loops are left as-is. - cc_recv_one(): reuse the is_bootstrap flag instead of re-memcmp'ing the packet magic three more times; drop the unreachable payload_len < 0 check (the minimum-length gate at the top already guarantees it is non-negative). Verified on a local two-node sodium cluster: join via KEY_GRANT + NODE_ASSIGN, sticky backup designation, master-death failover promotion, and graceful GOODBYE on shutdown - zero controller errors. Both build flavors (sodium-only and wolfSSL fallback) compile clean. --- .../clusterer_controller.c | 267 +++++++----------- 1 file changed, 95 insertions(+), 172 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index bdc9c07b3a0..04f879e0e9d 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -487,6 +487,7 @@ typedef struct cc_cluster_ { int cluster_id; char multicast_address[INET_ADDRSTRLEN]; int multicast_port; + struct sockaddr_in mcast_dest; /* resolved once in cc_setup_socket */ char password[1025]; unsigned char key[32]; /* bootstrap key = SHA256(password); JOIN only */ unsigned char session_key[32]; /* group key = HKDF(password, master_salt) */ @@ -1599,6 +1600,20 @@ static void cc_apply_master_from_list_locked(const char *master_ip, cc_cluster_t cl->peers->last_master[strnlen(master_ip, CC_MAX_IP_LEN)] = '\0'; } +/** + * cc_peer_by_ip_locked() - find a peer entry by IP string. + * Returns the entry pointer, or NULL if not present. + * Must be called with cl->peers->lock held (read or write). + */ +static cc_peer_t *cc_peer_by_ip_locked(cc_cluster_t *cl, const char *ip) +{ + int i; + for (i = 0; i < cl->peers->count; i++) + if (strcmp(cl->peers->entries[i].ip, ip) == 0) + return &cl->peers->entries[i]; + return NULL; +} + /** * cc_upsert_peer_locked() - insert or refresh a peer entry. * Does NOT call cc_elect_master(cl); callers do that explicitly. @@ -1606,20 +1621,19 @@ static void cc_apply_master_from_list_locked(const char *master_ip, cc_cluster_t */ static void cc_upsert_peer_locked(const char *src_ip, cc_cluster_t *cl) { - int i; unsigned int src_num = ip_to_num(src_ip); time_t now = time(NULL); + cc_peer_t *found; if (src_num == 0) { LM_WARN("clusterer_controller: ignoring invalid IP '%s'\n", src_ip); return; } - for (i = 0; i < cl->peers->count; i++) { - if (strcmp(cl->peers->entries[i].ip, src_ip) == 0) { - cl->peers->entries[i].last_seen = now; - return; /* updated */ - } + found = cc_peer_by_ip_locked(cl, src_ip); + if (found) { + found->last_seen = now; + return; /* updated */ } /* New entry */ @@ -1673,16 +1687,12 @@ static void cc_update_peer_bin_locked(const char *ip, uint16_t node_id, const char (*bin_sockets)[CC_MAX_BIN_SOCK_LEN], cc_cluster_t *cl) { - int i; - for (i = 0; i < cl->peers->count; i++) { - if (strcmp(cl->peers->entries[i].ip, ip) == 0) { - cl->peers->entries[i].node_id = node_id; - cl->peers->entries[i].bin_count = bin_count; - if (bin_count > 0) - memcpy(cl->peers->entries[i].bin_sockets, bin_sockets, - bin_count * CC_MAX_BIN_SOCK_LEN); - return; - } + cc_peer_t *e = cc_peer_by_ip_locked(cl, ip); + if (e) { + e->node_id = node_id; + e->bin_count = bin_count; + if (bin_count > 0) + memcpy(e->bin_sockets, bin_sockets, bin_count * CC_MAX_BIN_SOCK_LEN); } } @@ -2152,6 +2162,9 @@ static int cc_setup_socket(cc_cluster_t *cl) LM_INFO("clusterer_controller: [cluster %d] socket ready, joined %s:%d\n", cl->cluster_id, cl->multicast_address, cl->multicast_port); + /* cl->mcast_dest was resolved in mod_init (main process) and inherited + * across fork; no need to rebuild it here. */ + /* Set non-blocking so sendto() never hangs the worker if the kernel * UDP send buffer fills up. recvfrom() already relies on select() * for readiness, so O_NONBLOCK is safe and consistent for both. */ @@ -2168,6 +2181,39 @@ static int cc_setup_socket(cc_cluster_t *cl) * Packet senders * ========================================================================= */ +/** + * cc_seal_and_send() - encrypt a prepared packet in place and multicast it. + * + * On entry @pkt holds the cleartext framing (magic already stamped) plus the + * @plain_len-byte plaintext starting at CC_WIRE_HDR_SZ; cc_encrypt_pkt() fills + * the cluster_id + nonce and appends the AEAD tag. Every controller packet + * targets the cluster's multicast group (cached in cl->mcast_dest), so all + * senders share this tail. @type is only used for logging. + * + * @return 0 on success, -1 on encrypt or send failure. + */ +static int cc_seal_and_send(int sock, cc_cluster_t *cl, char *pkt, int plain_len, + const unsigned char *key, unsigned char type) +{ + int total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, key, + cl->cluster_id); + if (total_len < 0) + return -1; + + if (sendto(sock, pkt, total_len, 0, + (struct sockaddr *)&cl->mcast_dest, sizeof(cl->mcast_dest)) < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) + LM_DBG("clusterer_controller: [cluster %d] sendto (type=0x%02x) " + "would block\n", cl->cluster_id, type); + else + LM_ERR("clusterer_controller: [cluster %d] sendto (type=0x%02x): %s\n", + cl->cluster_id, type, strerror(errno)); + return -1; + } + LM_DBG("clusterer_controller: [cluster %d] sent 0x%02x\n", cl->cluster_id, type); + return 0; +} + /** * cc_send_pkt_with_ip() - build and multicast a small (ALIVE/GOODBYE) packet. * JOIN_REQ is handled by cc_send_join_req_pkt() which carries BIN socket info. @@ -2181,8 +2227,7 @@ static void cc_send_pkt_with_ip(int sock, unsigned char type, cc_cluster_t *cl) char pkt[CC_SMALL_PKT_SZ + CC_PUBKEY_SZ + CC_CONFIG_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); int ip_len = (int)strlen(my_ip); - int plain_len, total_len; - struct sockaddr_in dest; + int plain_len; if (ip_len > CC_MAX_IP_LEN) ip_len = CC_MAX_IP_LEN; @@ -2212,26 +2257,7 @@ static void cc_send_pkt_with_ip(int sock, unsigned char type, cc_cluster_t *cl) } } - total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->session_key, cl->cluster_id); - if (total_len < 0) - return; - - memset(&dest, 0, sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons((uint16_t)cl->multicast_port); - dest.sin_addr.s_addr = inet_addr(cl->multicast_address); - - if (sendto(sock, pkt, total_len, 0, - (struct sockaddr *)&dest, sizeof(dest)) < 0) { - if (errno == EAGAIN || errno == EWOULDBLOCK) - LM_DBG("clusterer_controller: [cluster %d] sendto (type=0x%02x) would block\n", - cl->cluster_id, type); - else - LM_ERR("clusterer_controller: [cluster %d] sendto (type=0x%02x): %s\n", - cl->cluster_id, type, strerror(errno)); - } else { - LM_DBG("clusterer_controller: [cluster %d] sent 0x%02x\n", cl->cluster_id, type); - } + cc_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, type); } #define cc_send_alive(sock, cl) cc_send_pkt_with_ip((sock), CC_PKT_ALIVE, (cl)) @@ -2250,8 +2276,7 @@ static void cc_send_list_pkt(int sock, unsigned char type, cc_cluster_t *cl) uint16_t count_be; char *p; time_t cutoff; - struct sockaddr_in dest; - int i, plain_len, total_len; + int i, plain_len; memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); /* nonce at [8..19] written by cc_encrypt_pkt */ @@ -2290,20 +2315,7 @@ static void cc_send_list_pkt(int sock, unsigned char type, cc_cluster_t *cl) plain_len = 1 + CC_SEQ_SZ + CC_LIST_COUNT_SZ + CC_NODE_ID_SZ + count * CC_IP_ENTRY_SZ; - total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->session_key, cl->cluster_id); - if (total_len < 0) - return; - - memset(&dest, 0, sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons((uint16_t)cl->multicast_port); - dest.sin_addr.s_addr = inet_addr(cl->multicast_address); - - if (sendto(sock, pkt, total_len, 0, - (struct sockaddr *)&dest, sizeof(dest)) < 0) - LM_ERR("clusterer_controller: [cluster %d] sendto MEMBER_LIST: %s\n", - cl->cluster_id, strerror(errno)); - else + if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, type) == 0) LM_INFO("clusterer_controller: [cluster %d] sent MEMBER_LIST (%d members)\n", cl->cluster_id, count); } @@ -2338,8 +2350,7 @@ static void cc_send_join_req_pkt(int sock, cc_cluster_t *cl) seq = htonl(++cl->peers->my_seq); int ip_len = (int)strlen(my_ip); char *p; - int plain_len, total_len; - struct sockaddr_in dest; + int plain_len; if (ip_len > CC_MAX_IP_LEN) ip_len = CC_MAX_IP_LEN; @@ -2388,20 +2399,7 @@ static void cc_send_join_req_pkt(int sock, cc_cluster_t *cl) } plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); - total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->key, cl->cluster_id); - if (total_len < 0) - return; - - memset(&dest, 0, sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons((uint16_t)cl->multicast_port); - dest.sin_addr.s_addr = inet_addr(cl->multicast_address); - - if (sendto(sock, pkt, total_len, 0, - (struct sockaddr *)&dest, sizeof(dest)) < 0) - LM_ERR("clusterer_controller: [cluster %d] sendto JOIN_REQ: %s\n", - cl->cluster_id, strerror(errno)); - else + if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->key, CC_PKT_JOIN_REQ) == 0) LM_DBG("clusterer_controller: [cluster %d] sent JOIN_REQ bin=%s\n", cl->cluster_id, cl->bin_socket); } @@ -2424,8 +2422,7 @@ static void cc_send_node_assign(int sock, const char *ip, uint16_t node_id, uint16_t nid_be = htons(node_id); int ip_len = (int)strnlen(ip, CC_MAX_IP_LEN); char *p; - int i, plain_len, total_len; - struct sockaddr_in dest; + int i, plain_len; memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); @@ -2458,20 +2455,7 @@ static void cc_send_node_assign(int sock, const char *ip, uint16_t node_id, * GCM auth on receipt ("session key mismatch"), driving a JOIN_REQ storm. * KEY_GRANT is sent before NODE_ASSIGN so the joiner already holds the * session key by the time this arrives. */ - total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->session_key, cl->cluster_id); - if (total_len < 0) - return; - - memset(&dest, 0, sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons((uint16_t)cl->multicast_port); - dest.sin_addr.s_addr = inet_addr(cl->multicast_address); - - if (sendto(sock, pkt, total_len, 0, - (struct sockaddr *)&dest, sizeof(dest)) < 0) - LM_ERR("clusterer_controller: [cluster %d] sendto NODE_ASSIGN: %s\n", - cl->cluster_id, strerror(errno)); - else + if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, CC_PKT_NODE_ASSIGN) == 0) LM_INFO("clusterer_controller: [cluster %d] NODE_ASSIGN node_id=%u ip=%s\n", cl->cluster_id, node_id, ip); } @@ -2485,27 +2469,14 @@ static void cc_send_master_alive(int sock, cc_cluster_t *cl) { char pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_TAG_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); - int total_len; - struct sockaddr_in dest; memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_MASTER_ALIVE; memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); /* no payload beyond type+seq */ - total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, CC_PLAIN_HDR_SZ, - cl->session_key, cl->cluster_id); - if (total_len < 0) return; - - memset(&dest, 0, sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons((uint16_t)cl->multicast_port); - dest.sin_addr.s_addr = inet_addr(cl->multicast_address); - - if (sendto(sock, pkt, total_len, 0, - (struct sockaddr *)&dest, sizeof(dest)) < 0) - LM_ERR("clusterer_controller: [cluster %d] sendto MASTER_ALIVE: %s\n", - cl->cluster_id, strerror(errno)); + cc_seal_and_send(sock, cl, pkt, CC_PLAIN_HDR_SZ, cl->session_key, + CC_PKT_MASTER_ALIVE); } /** @@ -2523,8 +2494,6 @@ static void cc_send_master_beacon(int sock, cc_cluster_t *cl) char pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + 2 + CC_TAG_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); uint16_t cnt_be; - int total_len; - struct sockaddr_in dest; lock_start_read(cl->peers->lock); cnt_be = htons((uint16_t)cl->peers->count); @@ -2535,19 +2504,8 @@ static void cc_send_master_beacon(int sock, cc_cluster_t *cl) memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); memcpy(pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ, &cnt_be, 2); - total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, CC_PLAIN_HDR_SZ + 2, - cl->key, cl->cluster_id); - if (total_len < 0) return; - - memset(&dest, 0, sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons((uint16_t)cl->multicast_port); - dest.sin_addr.s_addr = inet_addr(cl->multicast_address); - - if (sendto(sock, pkt, total_len, 0, - (struct sockaddr *)&dest, sizeof(dest)) < 0) - LM_ERR("clusterer_controller: [cluster %d] sendto MASTER_BEACON: %s\n", - cl->cluster_id, strerror(errno)); + cc_seal_and_send(sock, cl, pkt, CC_PLAIN_HDR_SZ + 2, cl->key, + CC_PKT_MASTER_BEACON); } /** @@ -2569,8 +2527,7 @@ static void cc_send_key_grant(int sock, const char *target_ip, cc_cluster_t *cl, uint32_t seq = htonl(++cl->peers->my_seq); unsigned char wrapped[CC_MASTER_SALT_SZ]; char *p; - int ip_len, plain_len, total_len; - struct sockaddr_in dest; + int ip_len, plain_len; if (cc_wrap_salt(cl->my_privkey, joiner_pubkey, cl->password, cl->peers->master_salt, "cc_key_grant", @@ -2590,19 +2547,7 @@ static void cc_send_key_grant(int sock, const char *target_ip, cc_cluster_t *cl, memcpy(p, wrapped, CC_MASTER_SALT_SZ); p += CC_MASTER_SALT_SZ; plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); - total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->key, cl->cluster_id); - if (total_len < 0) return; - - memset(&dest, 0, sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons((uint16_t)cl->multicast_port); - dest.sin_addr.s_addr = inet_addr(cl->multicast_address); - - if (sendto(sock, pkt, total_len, 0, - (struct sockaddr *)&dest, sizeof(dest)) < 0) - LM_ERR("clusterer_controller: [cluster %d] sendto KEY_GRANT: %s\n", - cl->cluster_id, strerror(errno)); - else + if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->key, CC_PKT_KEY_GRANT) == 0) LM_INFO("clusterer_controller: [cluster %d] sent KEY_GRANT to %s\n", cl->cluster_id, target_ip); } @@ -2621,8 +2566,7 @@ static void cc_send_key_handoff(int sock, const char *next_master_ip, uint32_t seq = htonl(++cl->peers->my_seq); unsigned char wrapped[CC_MASTER_SALT_SZ]; char *p; - int ip_len, plain_len, total_len; - struct sockaddr_in dest; + int ip_len, plain_len; if (cc_wrap_salt(cl->my_privkey, next_master_pubkey, cl->password, cl->peers->master_salt, "cc_key_handoff", @@ -2641,19 +2585,8 @@ static void cc_send_key_handoff(int sock, const char *next_master_ip, memcpy(p, wrapped, CC_MASTER_SALT_SZ); p += CC_MASTER_SALT_SZ; plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); - total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->session_key, cl->cluster_id); - if (total_len < 0) return; - - memset(&dest, 0, sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons((uint16_t)cl->multicast_port); - dest.sin_addr.s_addr = inet_addr(cl->multicast_address); - - if (sendto(sock, pkt, total_len, 0, - (struct sockaddr *)&dest, sizeof(dest)) < 0) - LM_ERR("clusterer_controller: [cluster %d] sendto KEY_HANDOFF: %s\n", - cl->cluster_id, strerror(errno)); - else + if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, + CC_PKT_KEY_HANDOFF) == 0) LM_INFO("clusterer_controller: [cluster %d] sent KEY_HANDOFF to %s\n", cl->cluster_id, next_master_ip); } @@ -4000,8 +3933,7 @@ static void cc_send_join_reject(int sock, const char *target_ip, cc_cluster_t *c { char pkt[CC_SMALL_PKT_SZ + 1]; /* +1 for the reason byte */ uint32_t seq = htonl(++cl->peers->my_seq); - int ip_len, plain_len, total_len; - struct sockaddr_in dest; + int ip_len, plain_len; ip_len = (int)strnlen(target_ip, CC_MAX_IP_LEN); @@ -4014,19 +3946,7 @@ static void cc_send_join_reject(int sock, const char *target_ip, cc_cluster_t *c pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + ip_len + 1] = (char)reason; plain_len = CC_PLAIN_HDR_SZ + ip_len + 1 + 1; - total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, cl->key, cl->cluster_id); - if (total_len < 0) return; - - memset(&dest, 0, sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons((uint16_t)cl->multicast_port); - dest.sin_addr.s_addr = inet_addr(cl->multicast_address); - - if (sendto(sock, pkt, total_len, 0, - (struct sockaddr *)&dest, sizeof(dest)) < 0) - LM_ERR("clusterer_controller: [cluster %d] sendto JOIN_REJECT failed: %s\n", - cl->cluster_id, strerror(errno)); - else + if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->key, CC_PKT_JOIN_REJECT) == 0) LM_WARN("clusterer_controller: [cluster %d] sent JOIN_REJECT to %s (%s)\n", cl->cluster_id, target_ip, reason == CC_REJECT_CONFIG ? "different cluster settings" @@ -4168,7 +4088,7 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) if (_new && strcmp(sender_ip_buf, my_ip) != 0) cl->auth_fail_pkts++; - if (memcmp(buf, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ) == 0) { + if (is_bootstrap) { /* Master: track per-IP bootstrap failures; send JOIN_REJECT on limit. * JOIN_REJECT is encrypted (BOOTSTRAP_MAGIC/GCM) so only nodes with @@ -4187,7 +4107,7 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) if (_new && _lm[0] != '\0' && strcmp(sender_ip_buf, _lm) == 0) cl->bootstrap_auth_fails++; } - if (memcmp(buf, CC_PACKET_MAGIC, CC_MAGIC_SZ) == 0 && !cl->join_pending) { + if (!is_bootstrap && !cl->join_pending) { /* Session key mismatch: request a re-key ONLY when the packet we * could not decrypt came from OUR current master (a legitimate key * rotation). Undecryptable session packets from any other source @@ -4210,7 +4130,7 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) * counter the worker uses - GOODBYE gets a valid monotonic seq number * without any special-casing. * Bootstrap packets (CC_BOOTSTRAP_MAGIC) use join_nonce instead. */ - if (memcmp(buf, CC_PACKET_MAGIC, CC_MAGIC_SZ) == 0) { + if (!is_bootstrap) { uint32_t pkt_seq; memcpy(&pkt_seq, buf + CC_WIRE_HDR_SZ + 1, CC_SEQ_SZ); pkt_seq = ntohl(pkt_seq); @@ -4220,15 +4140,10 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) pkt_type = (unsigned char)buf[CC_WIRE_HDR_SZ]; payload = buf + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + /* Non-negative: the minimum-length gate above guarantees + * n >= CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_TAG_SZ. */ payload_len = (int)(n - CC_WIRE_HDR_SZ - CC_PLAIN_HDR_SZ - CC_TAG_SZ); - if (payload_len < 0) { - LM_WARN("clusterer_controller: empty payload from %s, dropping\n", - sender_ip_buf); - return; - } - - switch (pkt_type) { case CC_PKT_ALIVE: { @@ -5534,8 +5449,16 @@ static int mod_init(void) /* ---- Parse and validate all cluster strings ------------------------ */ for (i = 0; i < cc_cluster_str_count; i++) { - if (cc_parse_cluster_str(cc_cluster_strs[i], &cc_clusters[i]) < 0) + cc_cluster_t *cl = &cc_clusters[i]; + if (cc_parse_cluster_str(cc_cluster_strs[i], cl) < 0) return -1; + /* Resolve the multicast destination once, in the main process, so both + * the forked workers (via fork) and mod_destroy's GOODBYE path (main + * process) can send without rebuilding it. */ + memset(&cl->mcast_dest, 0, sizeof(cl->mcast_dest)); + cl->mcast_dest.sin_family = AF_INET; + cl->mcast_dest.sin_port = htons((uint16_t)cl->multicast_port); + cl->mcast_dest.sin_addr.s_addr = inet_addr(cl->multicast_address); cc_cluster_count++; pkg_free(cc_cluster_strs[i]); cc_cluster_strs[i] = NULL; From ffca3dd1fb35456adb9c4fcd7bdb00059e66bafa Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Fri, 17 Jul 2026 15:41:51 +1000 Subject: [PATCH 08/21] clusterer_controller: replace the hand-rolled join handshake with Noise_NNpsk0 The join handshake used a bespoke construction: an ephemeral X25519 ECDH whose shared secret was fed, with the password and a per-exchange nonce, into HKDF to derive a key that XOR-wrapped the master_salt (cc_wrap_salt), plus a manual nonce echo to bind the exchange. Replace it with a standard, analysable handshake: Noise_NNpsk0_25519_ChaChaPoly_SHA256, PSK = the Argon2id bootstrap key. -> psk, e JOIN_REQ carries Noise message 1 (fresh ephemeral) <- e, ee KEY_GRANT carries Noise message 2; its AEAD payload = master_salt A compact Noise core (SymmetricState + CipherState + the NNpsk0 read/write) is added on libsodium primitives (X25519, ChaCha20-Poly1305, SHA-256, HMAC-SHA256); it was validated against the RFC 5869 HKDF vector and for self-consistency, tamper-detection and wrong-PSK rejection in a standalone harness before wiring in. The handshake hash binds the whole transcript, so the manual join_nonce echo, the cc_peer_t.join_nonce field, and CC_JOIN_NONCE_SZ are removed - a stale KEY_GRANT for a superseded JOIN_REQ now simply fails Noise msg-2 decryption. KEY_HANDOFF (a one-shot, which NNpsk0's 2-message shape does not fit) switches to an anonymous crypto_box_seal of the master_salt to the next master's long-lived X25519 key (learned via ALIVE); sender authenticity still comes from the session-key envelope. cc_wrap_salt and cc_ecdh_shared are deleted. The multicast transport, outer AEAD framing (bootstrap/session magic), sequence replay check and rate limiter are unchanged; wrong-password nodes are still rejected at the bootstrap envelope before the handshake runs. Verified on a local two-node cluster: join, sticky backup, master-death failover, GOODBYE, and wrong-password shutdown - all clean. --- .../clusterer_controller.c | 473 +++++++++++------- 1 file changed, 283 insertions(+), 190 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index 04f879e0e9d..567abf3bade 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -78,8 +78,8 @@ * * JOIN_REQ - bootstrap key, multicast * Sent by a new node on startup. - * Payload: IP(16B) + bin_info + pubkey(32B) + join_nonce(16B) - * join_nonce is random per-exchange; folded into the KEY_GRANT wrap key. + * Payload: IP(16B) + bin_info + noise_msg1(48B) + config(4B) + * noise_msg1 is Noise_NNpsk0 message 1 (fresh ephemeral + AEAD tag). * * MEMBER_LIST - session key, multicast * Master -> all: member count, the operator-forced sharing-tag holder @@ -98,15 +98,16 @@ * re-election. Two masters that share a session key resolve split-brain * here: the lower-IP one yields. * - * KEY_GRANT - bootstrap key, unicast to joiner + * KEY_GRANT - bootstrap key, multicast (addressed to joiner) * Master reply to JOIN_REQ. - * Payload: IP(16B) + master_pubkey(32B) + join_nonce(16B) + wrapped_salt(32B) - * wrapped_salt = master_salt XOR HKDF(ECDH(shared) || password || join_nonce) + * Payload: IP(16B) + noise_msg2(80B) + * noise_msg2 is Noise_NNpsk0 message 2; its AEAD payload is the master_salt. * - * KEY_HANDOFF - session key, unicast to next master + * KEY_HANDOFF - session key, multicast (addressed to next master) * Outgoing master on graceful shutdown -> next-highest-IP peer. Transfers * master_salt so the new master avoids a full re-join cycle. - * Payload: IP(16B) + sender_pubkey(32B) + wrapped_salt(32B) + * Payload: IP(16B) + crypto_box_seal(master_salt) sealed to the next + * master's long-lived X25519 key (learned from its ALIVE). * * JOIN_REJECT - bootstrap key, multicast * Master -> a source whose bootstrap packets repeatedly fail to decrypt @@ -251,7 +252,6 @@ static const unsigned char CC_BOOTSTRAP_MAGIC[CC_MAGIC_SZ] = { 0xCC, 0x01 }; #define CC_MAX_IP_LEN 15 /* "255.255.255.255" without NUL */ #define CC_PUBKEY_SZ 32 /* X25519 public key */ -#define CC_JOIN_NONCE_SZ 16 /* per-exchange nonce in JOIN_REQ/KEY_GRANT */ #define CC_MASTER_SALT_SZ 32 /* random salt generated by each new master */ /* MEMBER_LIST entry: IP (16B null-padded) + is_master (1B) = 17B. * Pubkeys are NOT carried here - nodes learn them from ALIVE packets, @@ -358,16 +358,21 @@ typedef struct { * accidental per-node config drift for the same cluster: * manage_shtags(1B) + master_stickiness(1B) + query_time(2B BE). */ #define CC_CONFIG_SZ 4 -/* JOIN_REQ: [ip NUL][bin_count 1B][sockets...][pubkey 32B][join_nonce 16B] */ +/* Noise handshake message sizes (NNpsk0, X25519, ChaChaPoly, SHA-256): + * msg 1 = e(32) + tag over empty payload(16) = 48 + * msg 2 = e(32) + AEAD(master_salt 32 + tag 16) = 80 */ +#define CC_NOISE_MSG1_SZ (32 + CC_TAG_SZ) +#define CC_NOISE_MSG2_SZ (32 + CC_MASTER_SALT_SZ + CC_TAG_SZ) +/* JOIN_REQ: [ip NUL][bin_count 1B][sockets...][noise_msg1 48B][config 4B] */ #define CC_JOIN_PKT_MAX_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 \ - + CC_BIN_INFO_MAX_SZ + CC_PUBKEY_SZ + CC_JOIN_NONCE_SZ \ + + CC_BIN_INFO_MAX_SZ + CC_NOISE_MSG1_SZ \ + CC_CONFIG_SZ + CC_TAG_SZ) -/* KEY_GRANT: [target_ip NUL][master_pubkey 32B][join_nonce 16B][wrapped_salt 32B] */ +/* KEY_GRANT: [target_ip NUL][noise_msg2 80B] */ #define CC_KEY_GRANT_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 \ - + CC_PUBKEY_SZ + CC_JOIN_NONCE_SZ + CC_MASTER_SALT_SZ + CC_TAG_SZ) -/* KEY_HANDOFF: [target_ip NUL][sender_pubkey 32B][wrapped_salt 32B] */ + + CC_NOISE_MSG2_SZ + CC_TAG_SZ) +/* KEY_HANDOFF: [target_ip NUL][crypto_box_seal(master_salt)] */ #define CC_KEY_HANDOFF_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 \ - + CC_PUBKEY_SZ + CC_MASTER_SALT_SZ + CC_TAG_SZ) + + crypto_box_SEALBYTES + CC_MASTER_SALT_SZ + CC_TAG_SZ) /* NODE_ASSIGN: [node_id 2B][ip NUL][bin_count 1B][sockets...] */ #define CC_NODE_ASSIGN_MAX_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_NODE_ID_SZ \ + CC_MAX_IP_LEN + 1 + CC_BIN_INFO_MAX_SZ + CC_TAG_SZ) @@ -479,6 +484,17 @@ typedef enum { /* Forward-declared so cc_cluster_t can embed a pointer */ typedef struct cc_peers_ cc_peers_t; +/* Noise handshake state (definitions with the Noise core further below). + * HASHLEN = 32 (SHA-256). cc_cluster_t embeds a cc_symstate_t for the + * initiator half of the join handshake. */ +#define CC_NOISE_HASHLEN 32 +typedef struct { unsigned char k[32]; uint64_t n; int has_key; } cc_cipherstate_t; +typedef struct { + unsigned char ck[CC_NOISE_HASHLEN]; + unsigned char h[CC_NOISE_HASHLEN]; + cc_cipherstate_t cs; +} cc_symstate_t; + /** * cc_cluster_t - per-cluster runtime state. * One instance per "cluster" modparam; one worker process per instance. @@ -505,11 +521,18 @@ typedef struct cc_cluster_ { int master_dead_tfd; /* non-master: fires on ka miss */ int identity_registered; /* 1 once update_identity called */ int shtag_bootstrapped; /* -1 = eligible, 1 = done */ - /* ECDH keypair - generated in cc_worker after fork, never leaves process */ + /* Long-lived X25519 keypair - generated in cc_worker after fork, never + * leaves the process. Advertised in ALIVE and used by crypto_box KEY_HANDOFF + * (NOT the Noise join handshake, which uses a fresh ephemeral per attempt). */ unsigned char my_privkey[CC_PUBKEY_SZ]; unsigned char my_pubkey[CC_PUBKEY_SZ]; - /* Per-exchange nonce sent in our JOIN_REQ; used to verify/unwrap KEY_GRANT */ - unsigned char my_join_nonce[CC_JOIN_NONCE_SZ]; + /* Noise (initiator) handshake state carried between our JOIN_REQ (msg 1) and + * the master's KEY_GRANT (msg 2): the SymmetricState and the fresh ephemeral + * private key. A new JOIN_REQ overwrites both, so a KEY_GRANT for a + * superseded attempt simply fails to decrypt and is dropped. Worker-local. */ + cc_symstate_t noise_hs; + unsigned char noise_e_priv[32]; + int noise_hs_valid; /* 1 after a JOIN_REQ, until KEY_GRANT/reset */ /* Set while a re-key JOIN_REQ is in flight; cleared on KEY_GRANT success * or master transition to prevent nonce stomping under packet flood. */ int join_pending; @@ -696,8 +719,8 @@ typedef struct cc_peer_ { uint16_t node_id; /* allocated by master; 0 = not yet assigned */ uint8_t bin_count; /* number of BIN listeners reported */ char bin_sockets[CC_MAX_BIN_SOCKETS][CC_MAX_BIN_SOCK_LEN]; - unsigned char pubkey[CC_PUBKEY_SZ]; /* X25519 public key; zero if unknown */ - unsigned char join_nonce[CC_JOIN_NONCE_SZ]; /* per-exchange nonce from JOIN_REQ */ + unsigned char pubkey[CC_PUBKEY_SZ]; /* long-lived X25519 pubkey (from ALIVE); + zero if unknown; used for KEY_HANDOFF */ uint32_t last_seq; /* highest seq accepted from this peer */ /* Peer's advertised consistency-critical config (from ALIVE), used to warn * on accidental per-node config drift. cfg_known=0 until first advertised; @@ -1770,68 +1793,174 @@ static int cc_gen_ecdh_keypair(unsigned char *privkey, unsigned char *pubkey) return 0; } -static int cc_ecdh_shared(const unsigned char *my_priv, - const unsigned char *peer_pub, - unsigned char out[CC_PUBKEY_SZ]) +/* ========================================================================= + * Noise_NNpsk0_25519_ChaChaPoly_SHA256 - the join handshake. + * + * NNpsk0: ephemeral-only, mutual authentication via a pre-shared key (the + * Argon2id bootstrap key), forward secrecy via the ephemeral-ephemeral DH. + * -> psk, e (JOIN_REQ carries msg 1) + * <- e, ee (KEY_GRANT carries msg 2, whose AEAD payload = master_salt) + * Both endpoints run this identical code, so byte-for-byte interop with other + * Noise libraries is not required; the module verifies self-consistency and the + * security properties in its own test harness. See RFC "The Noise Protocol + * Framework" rev 34. HASHLEN = 32 (SHA-256), DHLEN = 32 (X25519). + * ========================================================================= */ +#define CC_NOISE_PROTO "Noise_NNpsk0_25519_ChaChaPoly_SHA256" + +static void cc_hmac256(const unsigned char *key, size_t keylen, + const unsigned char *data, size_t datalen, + unsigned char out[CC_NOISE_HASHLEN]) { - /* crypto_scalarmult clamps the scalar internally (RFC 7748); reject an - * all-zero result (peer sent a low-order point). */ - if (crypto_scalarmult(out, my_priv, peer_pub) != 0 || - sodium_is_zero(out, CC_PUBKEY_SZ)) { - LM_ERR("clusterer_controller: X25519 derive failed\n"); - return -1; - } - return 0; + crypto_auth_hmacsha256_state st; + crypto_auth_hmacsha256_init(&st, key, keylen); + crypto_auth_hmacsha256_update(&st, data, datalen); + crypto_auth_hmacsha256_final(&st, out); } +/* Noise HKDF: chained HMAC outputs (num_outputs of 1..3, each HASHLEN bytes). */ +static void cc_noise_hkdf(const unsigned char ck[CC_NOISE_HASHLEN], + const unsigned char *ikm, size_t ikm_len, int num_outputs, + unsigned char o1[CC_NOISE_HASHLEN], + unsigned char o2[CC_NOISE_HASHLEN], + unsigned char o3[CC_NOISE_HASHLEN]) +{ + unsigned char tempkey[CC_NOISE_HASHLEN], buf[CC_NOISE_HASHLEN + 1]; + cc_hmac256(ck, CC_NOISE_HASHLEN, ikm, ikm_len, tempkey); /* extract */ + buf[0] = 0x01; + cc_hmac256(tempkey, CC_NOISE_HASHLEN, buf, 1, o1); + if (num_outputs >= 2) { + memcpy(buf, o1, CC_NOISE_HASHLEN); buf[CC_NOISE_HASHLEN] = 0x02; + cc_hmac256(tempkey, CC_NOISE_HASHLEN, buf, CC_NOISE_HASHLEN + 1, o2); + } + if (num_outputs >= 3) { + memcpy(buf, o2, CC_NOISE_HASHLEN); buf[CC_NOISE_HASHLEN] = 0x03; + cc_hmac256(tempkey, CC_NOISE_HASHLEN, buf, CC_NOISE_HASHLEN + 1, o3); + } + sodium_memzero(tempkey, sizeof tempkey); +} -/** - * cc_wrap_salt() - XOR-encrypt master_salt for a specific peer using ECDH. - * wrap_key = HKDF(ECDH(my_priv, peer_pub) || password [|| nonce], info) - * wrapped = master_salt XOR wrap_key - * - * nonce/nonce_len are optional (pass NULL/0 for KEY_HANDOFF). - * For KEY_GRANT, nonce is the per-exchange join_nonce from the JOIN_REQ, - * making every wrap_key unique even if the password is later compromised. - */ -static int cc_wrap_salt(const unsigned char *my_priv, - const unsigned char *peer_pub, - const char *password, - const unsigned char *master_salt, - const char *info, - const unsigned char *nonce, - size_t nonce_len, - unsigned char wrapped[CC_MASTER_SALT_SZ]) +static void cc_cs_nonce(uint64_t n, unsigned char out[12]) { - unsigned char ss[CC_PUBKEY_SZ]; - unsigned char ikm[CC_PUBKEY_SZ + 1024 + CC_JOIN_NONCE_SZ]; - size_t pass_len = strlen(password); - size_t ikm_len; - unsigned char wrap_key[32]; int i; - - if (cc_ecdh_shared(my_priv, peer_pub, ss) < 0) + memset(out, 0, 4); /* 32 bits of zeros */ + for (i = 0; i < 8; i++) out[4 + i] = (unsigned char)(n >> (8 * i)); /* LE64 */ +} +static int cc_cs_encrypt(cc_cipherstate_t *cs, const unsigned char *ad, size_t ad_len, + const unsigned char *pt, size_t pt_len, unsigned char *ct) +{ + unsigned char nonce[12]; unsigned long long clen = 0; + if (!cs->has_key) { memcpy(ct, pt, pt_len); return (int)pt_len; } + cc_cs_nonce(cs->n, nonce); + crypto_aead_chacha20poly1305_ietf_encrypt(ct, &clen, pt, pt_len, + ad, ad_len, NULL, nonce, cs->k); + cs->n++; + return (int)clen; +} +static int cc_cs_decrypt(cc_cipherstate_t *cs, const unsigned char *ad, size_t ad_len, + const unsigned char *ct, size_t ct_len, unsigned char *pt) +{ + unsigned char nonce[12]; unsigned long long plen = 0; + if (!cs->has_key) { memcpy(pt, ct, ct_len); return (int)ct_len; } + cc_cs_nonce(cs->n, nonce); + if (crypto_aead_chacha20poly1305_ietf_decrypt(pt, &plen, NULL, ct, ct_len, + ad, ad_len, nonce, cs->k) != 0) return -1; + cs->n++; + return (int)plen; +} - /* IKM = ss || password [|| nonce] */ - memcpy(ikm, ss, CC_PUBKEY_SZ); - if (pass_len > 1024) pass_len = 1024; - memcpy(ikm + CC_PUBKEY_SZ, password, pass_len); - ikm_len = CC_PUBKEY_SZ + pass_len; - if (nonce && nonce_len > 0) { - if (nonce_len > CC_JOIN_NONCE_SZ) nonce_len = CC_JOIN_NONCE_SZ; - memcpy(ikm + ikm_len, nonce, nonce_len); - ikm_len += nonce_len; - } - - if (cc_hkdf_sha256(ikm, ikm_len, - ss, CC_PUBKEY_SZ, /* use ss as salt too */ - info, wrap_key) < 0) - return -1; +static void cc_ss_mixhash(cc_symstate_t *s, const unsigned char *data, size_t len) +{ + crypto_hash_sha256_state st; + crypto_hash_sha256_init(&st); + crypto_hash_sha256_update(&st, s->h, CC_NOISE_HASHLEN); + crypto_hash_sha256_update(&st, data, len); + crypto_hash_sha256_final(&st, s->h); +} +static void cc_ss_init(cc_symstate_t *s) +{ + size_t nl = strlen(CC_NOISE_PROTO); /* <= HASHLEN, so pad it */ + memset(s->h, 0, CC_NOISE_HASHLEN); + memcpy(s->h, CC_NOISE_PROTO, nl); + memcpy(s->ck, s->h, CC_NOISE_HASHLEN); + s->cs.has_key = 0; s->cs.n = 0; +} +static void cc_ss_mixkey(cc_symstate_t *s, const unsigned char *ikm, size_t ikm_len) +{ + unsigned char o1[CC_NOISE_HASHLEN], o2[CC_NOISE_HASHLEN], o3[CC_NOISE_HASHLEN]; + cc_noise_hkdf(s->ck, ikm, ikm_len, 2, o1, o2, o3); + memcpy(s->ck, o1, CC_NOISE_HASHLEN); + memcpy(s->cs.k, o2, 32); s->cs.n = 0; s->cs.has_key = 1; +} +static void cc_ss_mixkeyhash(cc_symstate_t *s, const unsigned char *ikm, size_t ikm_len) +{ + unsigned char o1[CC_NOISE_HASHLEN], o2[CC_NOISE_HASHLEN], o3[CC_NOISE_HASHLEN]; + cc_noise_hkdf(s->ck, ikm, ikm_len, 3, o1, o2, o3); + memcpy(s->ck, o1, CC_NOISE_HASHLEN); + cc_ss_mixhash(s, o2, CC_NOISE_HASHLEN); + memcpy(s->cs.k, o3, 32); s->cs.n = 0; s->cs.has_key = 1; +} +static int cc_ss_encrypt_hash(cc_symstate_t *s, const unsigned char *pt, size_t pt_len, + unsigned char *ct) +{ + int cl = cc_cs_encrypt(&s->cs, s->h, CC_NOISE_HASHLEN, pt, pt_len, ct); + cc_ss_mixhash(s, ct, cl); + return cl; +} +static int cc_ss_decrypt_hash(cc_symstate_t *s, const unsigned char *ct, size_t ct_len, + unsigned char *pt) +{ + int pl = cc_cs_decrypt(&s->cs, s->h, CC_NOISE_HASHLEN, ct, ct_len, pt); + if (pl < 0) return -1; + cc_ss_mixhash(s, ct, ct_len); + return pl; +} - for (i = 0; i < CC_MASTER_SALT_SZ; i++) - wrapped[i] = master_salt[i] ^ wrap_key[i]; - return 0; +/* Initiator: write msg 1 (-> psk, e). Stores e keypair in @e_priv for msg 2. */ +static int cc_noise_write1(cc_symstate_t *s, unsigned char e_priv[32], + const unsigned char psk[32], + const unsigned char *payload, size_t pl_len, + unsigned char *out) +{ + unsigned char e_pub[32]; + cc_ss_init(s); + cc_ss_mixkeyhash(s, psk, 32); /* psk token (psk0) */ + randombytes_buf(e_priv, 32); crypto_scalarmult_base(e_pub, e_priv); /* e */ + memcpy(out, e_pub, 32); cc_ss_mixhash(s, e_pub, 32); cc_ss_mixkey(s, e_pub, 32); + return 32 + cc_ss_encrypt_hash(s, payload, pl_len, out + 32); +} +/* Responder: read msg 1, then write msg 2 (<- e, ee) with @payload. One-shot, + * so no responder state is retained afterwards. @out gets msg 2. */ +static int cc_noise_respond(const unsigned char psk[32], + const unsigned char *msg1, size_t msg1_len, + const unsigned char *payload, size_t pl_len, + unsigned char *out) +{ + cc_symstate_t s; + unsigned char re[32], e_priv[32], e_pub[32], dh[32], scratch[64]; + if (msg1_len < 32) return -1; + cc_ss_init(&s); + cc_ss_mixkeyhash(&s, psk, 32); + memcpy(re, msg1, 32); cc_ss_mixhash(&s, re, 32); cc_ss_mixkey(&s, re, 32); + if (cc_ss_decrypt_hash(&s, msg1 + 32, msg1_len - 32, scratch) < 0) + return -1; /* bad msg 1 (wrong psk/tamper) */ + randombytes_buf(e_priv, 32); crypto_scalarmult_base(e_pub, e_priv); + memcpy(out, e_pub, 32); cc_ss_mixhash(&s, e_pub, 32); cc_ss_mixkey(&s, e_pub, 32); + if (crypto_scalarmult(dh, e_priv, re) != 0) return -1; /* ee */ + cc_ss_mixkey(&s, dh, 32); + return 32 + cc_ss_encrypt_hash(&s, payload, pl_len, out + 32); +} +/* Initiator: read msg 2 (-> e, ee), recover @payload (master_salt). */ +static int cc_noise_read2(cc_symstate_t *s, const unsigned char e_priv[32], + const unsigned char *msg2, size_t msg2_len, + unsigned char *payload_out) +{ + unsigned char re2[32], dh[32]; + if (msg2_len < 32) return -1; + memcpy(re2, msg2, 32); cc_ss_mixhash(s, re2, 32); cc_ss_mixkey(s, re2, 32); + if (crypto_scalarmult(dh, e_priv, re2) != 0) return -1; + cc_ss_mixkey(s, dh, 32); + return cc_ss_decrypt_hash(s, msg2 + 32, msg2_len - 32, payload_out); } /** @@ -2375,18 +2504,17 @@ static void cc_send_join_req_pkt(int sock, cc_cluster_t *cl) p += slen + 1; } - /* Append our X25519 public key so master can wrap the session salt for us */ - memcpy(p, cl->my_pubkey, CC_PUBKEY_SZ); - p += CC_PUBKEY_SZ; - - /* Append per-exchange nonce; master echoes it in KEY_GRANT to bind the - * ECDH wrap to this specific exchange even if password is later compromised */ - if (cc_random_bytes(cl->my_join_nonce, CC_JOIN_NONCE_SZ) != 0) { - LM_ERR("clusterer_controller: RNG for join_nonce failed\n"); - return; + /* Noise msg 1 (-> psk, e): a fresh ephemeral public key plus an empty + * authenticated payload, PSK = the Argon2id bootstrap key (cl->key). Keep + * the initiator handshake state so we can read the master's KEY_GRANT + * (msg 2). A later JOIN_REQ overwrites it, so a stale KEY_GRANT just fails + * to decrypt and is dropped. */ + { + int m1 = cc_noise_write1(&cl->noise_hs, cl->noise_e_priv, cl->key, + NULL, 0, (unsigned char *)p); + cl->noise_hs_valid = 1; + p += m1; /* 48 bytes: 32 ephemeral + 16 tag */ } - memcpy(p, cl->my_join_nonce, CC_JOIN_NONCE_SZ); - p += CC_JOIN_NONCE_SZ; /* Advertise our consistency-critical config so the master can reject (or * warn about) a join with settings that differ from the running cluster. */ @@ -2510,29 +2638,32 @@ static void cc_send_master_beacon(int sock, cc_cluster_t *cl) /** * cc_send_key_grant() - send ECDH-wrapped master_salt to a joining node. - * Encrypted with bootstrap key (CC_BOOTSTRAP_MAGIC) so the joining node - * can read it before having the session key. + * Encrypted with bootstrap key (CC_BOOTSTRAP_MAGIC) so the joining node can + * read it before having the session key. * - * Payload: [target_ip NUL][my_pubkey 32B][join_nonce 16B][wrapped_salt 32B] + * Runs the Noise responder over the joiner's msg 1 and answers with msg 2, + * whose AEAD payload is the master_salt. The Noise handshake authenticates the + * exchange (PSK = bootstrap key) and binds it to the joiner's ephemeral, so a + * stale KEY_GRANT for a superseded JOIN_REQ fails on the joiner's side. * - * join_nonce is the per-exchange nonce from the JOIN_REQ. It is echoed back - * here (inside the authenticated GCM envelope) and folded into cc_wrap_salt's - * IKM so the wrap_key is unique per exchange even if the password leaks. + * Payload: [target_ip NUL][noise_msg2 64B] */ static void cc_send_key_grant(int sock, const char *target_ip, cc_cluster_t *cl, - const unsigned char *joiner_pubkey, - const unsigned char *joiner_nonce) + const unsigned char *msg1, int msg1_len) { char pkt[CC_KEY_GRANT_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); - unsigned char wrapped[CC_MASTER_SALT_SZ]; + unsigned char msg2[32 + CC_MASTER_SALT_SZ + CC_TAG_SZ]; char *p; - int ip_len, plain_len; + int ip_len, plain_len, m2; - if (cc_wrap_salt(cl->my_privkey, joiner_pubkey, cl->password, - cl->peers->master_salt, "cc_key_grant", - joiner_nonce, CC_JOIN_NONCE_SZ, wrapped) < 0) + m2 = cc_noise_respond(cl->key, msg1, (size_t)msg1_len, + cl->peers->master_salt, CC_MASTER_SALT_SZ, msg2); + if (m2 < 0) { + LM_DBG("clusterer_controller: [cluster %d] Noise msg1 from %s did not " + "verify - no KEY_GRANT sent\n", cl->cluster_id, target_ip); return; + } ip_len = (int)strnlen(target_ip, CC_MAX_IP_LEN); @@ -2542,9 +2673,7 @@ static void cc_send_key_grant(int sock, const char *target_ip, cc_cluster_t *cl, p = pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; memcpy(p, target_ip, ip_len); p[ip_len] = '\0'; p += ip_len + 1; - memcpy(p, cl->my_pubkey, CC_PUBKEY_SZ); p += CC_PUBKEY_SZ; - memcpy(p, joiner_nonce, CC_JOIN_NONCE_SZ); p += CC_JOIN_NONCE_SZ; - memcpy(p, wrapped, CC_MASTER_SALT_SZ); p += CC_MASTER_SALT_SZ; + memcpy(p, msg2, m2); p += m2; plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->key, CC_PKT_KEY_GRANT) == 0) @@ -2554,9 +2683,12 @@ static void cc_send_key_grant(int sock, const char *target_ip, cc_cluster_t *cl, /** * cc_send_key_handoff() - send master_salt to the next master before departing. - * Encrypted with session_key (CC_PACKET_MAGIC); only next master can unwrap. + * Outer envelope is the session key (CC_PACKET_MAGIC); the salt itself is sealed + * to the next master's long-lived X25519 key (learned via ALIVE) with an + * anonymous crypto_box - only that node can open it. Sender authenticity comes + * from the session-key envelope (only cluster members can send it). * - * Payload: [next_master_ip NUL][my_pubkey 32B][wrapped_salt 32B] + * Payload: [next_master_ip NUL][sealed_box 64B] */ static void cc_send_key_handoff(int sock, const char *next_master_ip, const unsigned char *next_master_pubkey, @@ -2564,14 +2696,16 @@ static void cc_send_key_handoff(int sock, const char *next_master_ip, { char pkt[CC_KEY_HANDOFF_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); - unsigned char wrapped[CC_MASTER_SALT_SZ]; + unsigned char sealed[crypto_box_SEALBYTES + CC_MASTER_SALT_SZ]; char *p; int ip_len, plain_len; - if (cc_wrap_salt(cl->my_privkey, next_master_pubkey, cl->password, - cl->peers->master_salt, "cc_key_handoff", - NULL, 0, wrapped) < 0) + if (crypto_box_seal(sealed, cl->peers->master_salt, CC_MASTER_SALT_SZ, + next_master_pubkey) != 0) { + LM_ERR("clusterer_controller: [cluster %d] crypto_box_seal failed\n", + cl->cluster_id); return; + } ip_len = (int)strnlen(next_master_ip, CC_MAX_IP_LEN); @@ -2581,8 +2715,7 @@ static void cc_send_key_handoff(int sock, const char *next_master_ip, p = pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; memcpy(p, next_master_ip, ip_len); p[ip_len] = '\0'; p += ip_len + 1; - memcpy(p, cl->my_pubkey, CC_PUBKEY_SZ); p += CC_PUBKEY_SZ; - memcpy(p, wrapped, CC_MASTER_SALT_SZ); p += CC_MASTER_SALT_SZ; + memcpy(p, sealed, sizeof(sealed)); p += sizeof(sealed); plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, @@ -2869,8 +3002,8 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, const char *end = payload + payload_len; char src_ip[CC_MAX_IP_LEN + 1]; char bin_socks[CC_MAX_BIN_SOCKETS][CC_MAX_BIN_SOCK_LEN]; - unsigned char joiner_pubkey[CC_PUBKEY_SZ]; - unsigned char joiner_nonce[CC_JOIN_NONCE_SZ]; + const unsigned char *noise_msg1 = NULL; /* points into payload */ + int noise_msg1_len = 0; uint8_t bin_cnt = 0; int ip_len, was_master, i; int j_cfg_present = 0, j_manage = 0, j_stick = 0, j_qt = 0; @@ -2904,18 +3037,11 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, } } - /* --- Parse X25519 public key (appended after BIN info) --- */ - memset(joiner_pubkey, 0, CC_PUBKEY_SZ); - if (p + CC_PUBKEY_SZ <= end) { - memcpy(joiner_pubkey, p, CC_PUBKEY_SZ); - p += CC_PUBKEY_SZ; - } - - /* --- Parse per-exchange join_nonce (appended after pubkey) --- */ - memset(joiner_nonce, 0, CC_JOIN_NONCE_SZ); - if (p + CC_JOIN_NONCE_SZ <= end) { - memcpy(joiner_nonce, p, CC_JOIN_NONCE_SZ); - p += CC_JOIN_NONCE_SZ; + /* --- Parse the Noise msg 1 (appended after BIN info) --- */ + if (p + CC_NOISE_MSG1_SZ <= end) { + noise_msg1 = (const unsigned char *)p; + noise_msg1_len = CC_NOISE_MSG1_SZ; + p += CC_NOISE_MSG1_SZ; } /* --- Parse the joiner's consistency-critical config (after join_nonce) --- */ @@ -3023,20 +3149,15 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, (const char (*)[CC_MAX_BIN_SOCK_LEN])bin_socks, cl); - /* Store joiner's public key, join_nonce, and reset last_seq in peer table. - * Resetting last_seq is essential: a restarted node begins its seq counter - * from 0, and without this reset peers would permanently reject its new - * packets (old last_seq > new seq) until the next key rotation. */ + /* Reset last_seq in the peer table. Essential: a restarted node begins its + * seq counter from 0, and without this reset peers would permanently reject + * its new packets (old last_seq > new seq) until the next key rotation. The + * joiner's long-lived pubkey (for a future KEY_HANDOFF) is learned from its + * ALIVE, not here - the JOIN_REQ now carries only an ephemeral Noise key. */ { - int _k; - for (_k = 0; _k < cl->peers->count; _k++) { - if (strcmp(cl->peers->entries[_k].ip, src_ip) == 0) { - memcpy(cl->peers->entries[_k].pubkey, joiner_pubkey, CC_PUBKEY_SZ); - memcpy(cl->peers->entries[_k].join_nonce, joiner_nonce, CC_JOIN_NONCE_SZ); - cl->peers->entries[_k].last_seq = 0; - break; - } - } + cc_peer_t *e = cc_peer_by_ip_locked(cl, src_ip); + if (e) + e->last_seq = 0; } lock_stop_write(cl->peers->lock); @@ -3056,8 +3177,8 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, /* Send KEY_GRANT first (bootstrap key) so joiner can derive session_key, * then NODE_ASSIGN + MEMBER_LIST (session key). */ - if (joiner_pubkey[0] || joiner_pubkey[1]) /* non-zero pubkey present */ - cc_send_key_grant(sock, src_ip, cl, joiner_pubkey, joiner_nonce); + if (noise_msg1) /* Noise msg 1 present -> run responder, answer msg 2 */ + cc_send_key_grant(sock, src_ip, cl, noise_msg1, noise_msg1_len); lock_start_write(cl->peers->lock); @@ -3684,15 +3805,8 @@ static void cc_handle_key_grant(const char *payload, int payload_len, const char *p = payload; const char *end = payload + payload_len; char target_ip[CC_MAX_IP_LEN + 1]; - unsigned char master_pubkey[CC_PUBKEY_SZ]; - unsigned char echoed_nonce[CC_JOIN_NONCE_SZ]; - unsigned char wrapped[CC_MASTER_SALT_SZ]; - unsigned char ss[CC_PUBKEY_SZ]; - unsigned char ikm[CC_PUBKEY_SZ + 1024 + CC_JOIN_NONCE_SZ]; - unsigned char wrap_key[32]; unsigned char new_salt[CC_MASTER_SALT_SZ]; - size_t pass_len, ikm_len; - int ip_len, i; + int ip_len; /* Parse target_ip */ ip_len = (int)strnlen(p, CC_MAX_IP_LEN); @@ -3720,37 +3834,26 @@ static void cc_handle_key_grant(const char *payload, int payload_len, } } - if (p + CC_PUBKEY_SZ + CC_JOIN_NONCE_SZ + CC_MASTER_SALT_SZ > end) { + if (p + CC_NOISE_MSG2_SZ > end) { LM_WARN("clusterer_controller: KEY_GRANT too short\n"); return; } - memcpy(master_pubkey, p, CC_PUBKEY_SZ); p += CC_PUBKEY_SZ; - memcpy(echoed_nonce, p, CC_JOIN_NONCE_SZ); p += CC_JOIN_NONCE_SZ; - memcpy(wrapped, p, CC_MASTER_SALT_SZ); - - /* Verify echoed nonce matches what we sent in JOIN_REQ */ - if (memcmp(echoed_nonce, cl->my_join_nonce, CC_JOIN_NONCE_SZ) != 0) { - LM_WARN("clusterer_controller: KEY_GRANT join_nonce mismatch - dropping\n"); - /* Clear join_pending so the next decryption failure or rejoin_tfd - * can issue a fresh JOIN_REQ. Without this the node stays blocked - * on a nonce that will never match (e.g. overwritten by rejoin - * timer or a master-election cycle between send and receipt). */ + + /* Complete the Noise handshake: read msg 2 with the initiator state stored + * when we sent the JOIN_REQ. Its AEAD payload is the master_salt. A stale + * KEY_GRANT (for a superseded JOIN_REQ) fails here and is dropped - this + * subsumes the old join_nonce echo check. */ + if (!cl->noise_hs_valid || + cc_noise_read2(&cl->noise_hs, cl->noise_e_priv, + (const unsigned char *)p, CC_NOISE_MSG2_SZ, new_salt) < 0) { + LM_DBG("clusterer_controller: KEY_GRANT from %s did not complete the " + "Noise handshake (stale or wrong key) - dropping\n", sender_ip); + /* Clear join_pending so the next decryption failure or rejoin_tfd can + * issue a fresh JOIN_REQ. */ cl->join_pending = 0; return; } - - /* Recover master_salt: XOR wrapped with HKDF(ECDH(my_priv, master_pub) || password || nonce) */ - if (cc_ecdh_shared(cl->my_privkey, master_pubkey, ss) < 0) return; - pass_len = strlen(cl->password); - if (pass_len > 1024) pass_len = 1024; - memcpy(ikm, ss, CC_PUBKEY_SZ); - memcpy(ikm + CC_PUBKEY_SZ, cl->password, pass_len); - memcpy(ikm + CC_PUBKEY_SZ + pass_len, echoed_nonce, CC_JOIN_NONCE_SZ); - ikm_len = CC_PUBKEY_SZ + pass_len + CC_JOIN_NONCE_SZ; - if (cc_hkdf_sha256(ikm, ikm_len, - ss, CC_PUBKEY_SZ, "cc_key_grant", wrap_key) < 0) return; - for (i = 0; i < CC_MASTER_SALT_SZ; i++) - new_salt[i] = wrapped[i] ^ wrap_key[i]; + cl->noise_hs_valid = 0; /* handshake consumed */ /* Apply: write to shm, derive session_key */ lock_start_write(cl->peers->lock); @@ -3770,7 +3873,7 @@ static void cc_handle_key_grant(const char *payload, int payload_len, /** * cc_handle_key_handoff() - process CC_PKT_KEY_HANDOFF. * Only acted on by the node that wins the next election (highest IP). - * Payload: [next_master_ip NUL][sender_pubkey 32B][wrapped_salt 32B] + * Payload: [next_master_ip NUL][crypto_box_seal(master_salt)] */ static void cc_handle_key_handoff(const char *payload, int payload_len, const char *sender_ip, cc_cluster_t *cl) @@ -3778,14 +3881,8 @@ static void cc_handle_key_handoff(const char *payload, int payload_len, const char *p = payload; const char *end = payload + payload_len; char target_ip[CC_MAX_IP_LEN + 1]; - unsigned char sender_pubkey[CC_PUBKEY_SZ]; - unsigned char wrapped[CC_MASTER_SALT_SZ]; - unsigned char ss[CC_PUBKEY_SZ]; - unsigned char ikm[CC_PUBKEY_SZ + 1024]; - unsigned char wrap_key[32]; unsigned char new_salt[CC_MASTER_SALT_SZ]; - size_t pass_len; - int ip_len, i; + int ip_len; ip_len = (int)strnlen(p, CC_MAX_IP_LEN); if (p + ip_len >= end) return; @@ -3795,22 +3892,18 @@ static void cc_handle_key_handoff(const char *payload, int payload_len, /* Only the intended next master unwraps this */ if (strcmp(target_ip, my_ip) != 0) return; - if (p + CC_PUBKEY_SZ + CC_MASTER_SALT_SZ > end) { + if (p + (int)(crypto_box_SEALBYTES + CC_MASTER_SALT_SZ) > end) { LM_WARN("clusterer_controller: KEY_HANDOFF too short\n"); return; } - memcpy(sender_pubkey, p, CC_PUBKEY_SZ); p += CC_PUBKEY_SZ; - memcpy(wrapped, p, CC_MASTER_SALT_SZ); - - if (cc_ecdh_shared(cl->my_privkey, sender_pubkey, ss) < 0) return; - pass_len = strlen(cl->password); - if (pass_len > 1024) pass_len = 1024; - memcpy(ikm, ss, CC_PUBKEY_SZ); - memcpy(ikm + CC_PUBKEY_SZ, cl->password, pass_len); - if (cc_hkdf_sha256(ikm, CC_PUBKEY_SZ + pass_len, - ss, CC_PUBKEY_SZ, "cc_key_handoff", wrap_key) < 0) return; - for (i = 0; i < CC_MASTER_SALT_SZ; i++) - new_salt[i] = wrapped[i] ^ wrap_key[i]; + /* Open the anonymous sealed box with our own long-lived keypair. */ + if (crypto_box_seal_open(new_salt, (const unsigned char *)p, + crypto_box_SEALBYTES + CC_MASTER_SALT_SZ, + cl->my_pubkey, cl->my_privkey) != 0) { + LM_WARN("clusterer_controller: [cluster %d] KEY_HANDOFF from %s did not " + "open - dropping\n", cl->cluster_id, sender_ip); + return; + } /* Store salt and re-derive so session_key is guaranteed consistent with * the adopted salt (the incoming master's salt may differ from ours). */ From d1ee636b9c1274bca793d9f77025a32627be5e66 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Fri, 17 Jul 2026 15:45:27 +1000 Subject: [PATCH 09/21] clusterer_controller: docs - Noise join handshake, libsodium-only crypto Update the admin guide (and regenerated README) for the crypto changes: - crypto is now libsodium-only (XChaCha20-Poly1305 + Argon2id); the wolfSSL fallback and its AES-256-GCM/scrypt description are removed, and the build now requires libsodium; - the Phase 1 join handshake is described as Noise_NNpsk0_25519_ChaChaPoly_ SHA256 (PSK = bootstrap key) instead of the hand-rolled ECDH-and-XOR wrap; - JOIN_REQ / KEY_GRANT carry Noise messages 1 / 2; KEY_HANDOFF uses a crypto_box sealed to the next master's key; - password / bootstrap-key wording updated (Argon2id, Noise PSK). --- modules/clusterer_controller/README | 184 ++++++++---------- .../doc/clusterer_controller_admin.xml | 163 +++++++--------- 2 files changed, 148 insertions(+), 199 deletions(-) diff --git a/modules/clusterer_controller/README b/modules/clusterer_controller/README index afc992fb129..af2c121ec25 100644 --- a/modules/clusterer_controller/README +++ b/modules/clusterer_controller/README @@ -147,20 +147,19 @@ Chapter 1. Admin Guide the password — i.e. they are configured identically, which is the operator's responsibility to avoid. - Every packet is encrypted and authenticated with an AEAD — - AES-256-GCM by default, or XChaCha20-Poly1305 when the module - is built against libsodium (see the Security Architecture + Every packet is encrypted and authenticated with the + XChaCha20-Poly1305 AEAD (see the Security Architecture section). Two distinct encryption keys are used depending on the communication phase: - * Bootstrap key — derived from the configured password with a - memory-hard KDF (scrypt N=2^16, r=8, p=1, or Argon2id in - the libsodium build, with a per-cluster salt), so a + * Bootstrap key — derived from the configured password with + the Argon2id memory-hard KDF (per-cluster salt), so a password captured from a bootstrap packet cannot be - brute-forced cheaply offline. Derived once at startup. Used - for the admission handshake (JOIN_REQ, KEY_GRANT, - JOIN_REJECT) and the split-brain MASTER_BEACON — traffic - that must be readable before a session key exists, or by - masters holding different session keys. + brute-forced cheaply offline. Derived once at startup. It + is both the outer AEAD key for the admission handshake + (JOIN_REQ, KEY_GRANT, JOIN_REJECT) and the split-brain + MASTER_BEACON — traffic that must be readable before a + session key exists, or by masters holding different session + keys — and the pre-shared key for the Noise join handshake. * Session key — derived via HKDF-SHA256 from the password and a 32-byte master salt generated once when the cluster first bootstraps. All normal cluster traffic uses this key. It is @@ -172,16 +171,16 @@ Chapter 1. Admin Guide identifying the key type and a 2-byte cluster_id, both in cleartext (the magic and cluster_id must be readable before decryption to select the key and to filter foreign clusters), - followed by a random AEAD nonce (12 bytes for AES-256-GCM, 24 - for XChaCha20-Poly1305) and the ciphertext. The magic and - cluster_id are additionally bound into the authentication tag - as AAD, so they cannot be altered undetected. Only the payload - is encrypted; the authenticated plaintext begins with a 1-byte - packet type and a 4-byte monotonic sequence number used for - replay protection. A 16-byte authentication tag follows the - ciphertext. Packets for a different cluster_id are dropped - before decryption; packets encrypted with a different password - fail authentication and are silently discarded. + followed by a random 24-byte AEAD nonce and the ciphertext. The + magic and cluster_id are additionally bound into the + authentication tag as AAD, so they cannot be altered + undetected. Only the payload is encrypted; the authenticated + plaintext begins with a 1-byte packet type and a 4-byte + monotonic sequence number used for replay protection. A 16-byte + authentication tag follows the ciphertext. Packets for a + different cluster_id are dropped before decryption; packets + encrypted with a different password fail authentication and are + silently discarded. The following packet types are defined: * ALIVE — periodic heartbeat sent by every active node every @@ -192,9 +191,9 @@ Chapter 1. Admin Guide used for configuration-drift detection (see Security Architecture). Encrypted with the session key. * JOIN_REQ — sent at startup by a new node, carrying its IP, - BIN socket list, X25519 ephemeral public key, and a 16-byte - random join nonce. Encrypted with the bootstrap key so it - can be sent before a session key exists. + BIN socket list, and Noise message 1 (the initiator's fresh + ephemeral public key). Encrypted with the bootstrap key so + it can be sent before a session key exists. * MEMBER_LIST — sent by the master in response to a JOIN_REQ, carrying the member count, the operator-forced sharing-tag holder node_id (0 = automatic), and the full peer IP list @@ -211,17 +210,17 @@ Chapter 1. Admin Guide (independent of query_time). Used by all peers to detect master failure quickly (3-second timeout). Encrypted with the session key. - * KEY_GRANT — unicast response to a JOIN_REQ, sent by the - master directly to the joining node. Contains the master's - X25519 public key, the original join nonce, and the master - salt wrapped with a per-exchange key derived from - ECDH(master_priv, joiner_pub) + password + join_nonce via - HKDF-SHA256. Encrypted with the bootstrap key. - * KEY_HANDOFF — unicast packet sent by the outgoing master on - graceful shutdown to the next-highest-IP peer, delivering - the master salt so that peer can become the new master - without a full re-join cycle. Encrypted with the session + * KEY_GRANT — the master's response to a JOIN_REQ, addressed + to the joining node. Carries Noise message 2 (the master's + ephemeral plus the AEAD-encrypted master salt), completing + the Noise_NNpsk0 handshake. Encrypted with the bootstrap key. + * KEY_HANDOFF — sent by the outgoing master on graceful + shutdown to the next-highest-IP peer, delivering the master + salt (as an anonymous crypto_box sealed to that peer's + long-lived X25519 key, learned from its ALIVE) so it can + become the new master without a full re-join cycle. + Encrypted with the session key. * JOIN_REJECT — sent by the master to a joining node whose JOIN_REQ repeatedly fails authentication (wrong password). After CC_JOIN_FAIL_LIMIT (3) consecutive bootstrap-key @@ -328,32 +327,28 @@ Chapter 1. Admin Guide attempting decryption, so foreign-cluster traffic on the group never counts as an authentication failure. - Crypto suite (selected at build time): if libsodium is detected - on the build host, the module is built sodium-only: + Crypto (all libsodium; a hard requirement): the payload AEAD is XChaCha20-Poly1305 (24-byte nonce, whose 192-bit nonce space - removes any random-nonce collision concern) for the payload - AEAD, Argon2id for the bootstrap-key derivation, and X25519 / - HKDF-SHA256 / RNG also from libsodium — WolfSSL is not linked - at all. Without libsodium the module falls back to WolfSSL: - AES-256-GCM (12-byte nonce) and scrypt (N=2^16, r=8, p=1). The - two wire formats are not interoperable, so every node in a - cluster must be built with the same suite; the active suite is - reported in the startup log (crypto=...). - - Phase 1 — bootstrap (JOIN_REQ / KEY_GRANT): each worker process - generates an ephemeral X25519 keypair on startup. When joining, - the node sends its public key and a 16-byte random join nonce - in the JOIN_REQ, encrypted with the bootstrap key (a - memory-hard KDF of the password — scrypt, or Argon2id in the - libsodium build). The master responds with a KEY_GRANT - encrypted with the same bootstrap key, containing its own - public key, the echoed join nonce, and the master salt wrapped - with a per-exchange key derived via HKDF-SHA256 from: -IKM = ECDH(master_priv, joiner_pub) || password || join_nonce - - This ensures that even if the password is compromised, - previously recorded exchanges cannot be decrypted without both - the ephemeral private key and the per-exchange join nonce. + removes any random-nonce collision concern); the bootstrap-key + KDF is Argon2id; and X25519 / HKDF-SHA256 / RNG also come from + libsodium. The active suite is reported in the startup log + (crypto=...). + + Phase 1 — join handshake (JOIN_REQ / KEY_GRANT): the join is a + Noise_NNpsk0_25519_ChaChaPoly_SHA256 handshake with the + pre-shared key set to the Argon2id bootstrap key: +-> psk, e JOIN_REQ carries Noise message 1 (a fresh ephemeral) +<- e, ee KEY_GRANT carries Noise message 2; its AEAD payload = mast +er_salt + + Authentication comes from the shared PSK, forward secrecy from + the ephemeral-ephemeral DH, and the Noise handshake hash binds + the whole transcript — so a stale KEY_GRANT for a superseded + JOIN_REQ simply fails to decrypt. Both handshake messages also + travel inside the bootstrap-key AEAD envelope, so a + wrong-password node is rejected at the envelope before the + handshake is even reached. This replaces the earlier + hand-rolled ECDH-and-XOR salt wrap. Phase 2 — session (all other packets): once the master salt is known, all nodes derive the session key as: @@ -552,31 +547,17 @@ on-key") 1.5.2. External Libraries or Applications - One of the following two crypto libraries is required; - libsodium is preferred and used exclusively when found: - * libsodium (preferred) — if the build host has libsodium - development files (detected via pkg-config), the module is - built sodium-only: XChaCha20-Poly1305 for the payload AEAD, - Argon2id for the bootstrap-key derivation, and X25519 ECDH, - HKDF-SHA256 and the RNG also from libsodium — WolfSSL is - not linked at all, keeping the module binary small (a few - hundred KB instead of several MB of statically-linked - WolfSSL). libsodium is linked dynamically, so each target - host also needs the libsodium runtime package (for example - libsodium23). - * tls_wolfssl (fallback) — without libsodium, the module - links statically against the WolfSSL library built by the - tls_wolfssl module - (modules/tls_wolfssl/lib/lib/libwolfssl.a), providing - AES-256-GCM authenticated encryption, X25519 ECDH, - HKDF-SHA256 and the scrypt password KDF. The tls_wolfssl - module must then be present in the source tree; its static - library is built automatically as a dependency if not - already present. - - Because the two suites produce incompatible wire formats, every - node in a cluster must be built the same way; the active suite - is printed in the startup log. + libsodium is required: + * libsodium (required) — provides the payload AEAD + (XChaCha20-Poly1305), the bootstrap-key KDF (Argon2id), the + Noise_NNpsk0 join handshake, and X25519 / HKDF-SHA256 / + RNG. The build fails with a clear error if libsodium + development files are not found. It is linked dynamically, + so each target host also needs the libsodium runtime + package (for example libsodium23). No other crypto library + is used or linked. + + The active suite is printed in the startup log (crypto=...). 1.6. Building the Module @@ -587,11 +568,9 @@ on-key") it to include_modules in your Makefile.conf: include_modules= clusterer_controller - then rebuild (make all / make modules). With libsodium - development files on the build host the module builds - sodium-only (no WolfSSL); otherwise it links the bundled - WolfSSL and the same external-library requirements as - tls_wolfssl apply — see External Libraries or Applications. + then rebuild (make all / make modules). libsodium development + files must be present on the build host or the build fails — + see External Libraries or Applications. The clusterer module is unmodified when clusterer_controller is not built. The controller integration on the clusterer side @@ -618,9 +597,10 @@ include_modules= clusterer_controller * multicast (required) — IPv4 multicast address and UDP port in the form A.B.C.D:PORT. The address must be in the 224.0.0.0/4 range. - * password (optional) — AES-256 encryption key material. All - nodes in the same cluster must use the same password. Falls - back to the global password modparam if not set. + * password (optional) — XChaCha20-Poly1305 encryption key + material. All nodes in the same cluster must use the same + password. Falls back to the global password modparam if not + set. * bin_socket (optional) — BIN socket to advertise for this cluster, in the form bin:IP:PORT. Required when multiple clusters are defined (so the controller knows which @@ -793,10 +773,10 @@ modparam("clusterer_controller", "query_time", 5) Global default encryption password for all clusters. All nodes in a cluster must use the same password. The password serves two purposes: - * Bootstrap key — the password is stretched with scrypt - (memory-hard, per-cluster salt) to encrypt JOIN_REQ and - KEY_GRANT packets during the initial key exchange phase - before a session key is established. + * Bootstrap key — the password is stretched with Argon2id + (memory-hard, per-cluster salt); it is the pre-shared key + for the Noise join handshake (JOIN_REQ / KEY_GRANT) and the + AEAD key for bootstrap traffic before a session key exists. * Session key material — the password is fed into HKDF-SHA256 together with the master salt to derive the session key used for all normal cluster traffic. @@ -806,7 +786,7 @@ modparam("clusterer_controller", "query_time", 5) Default value is “3eCrEt*5629”. Change this in production. Use a long, high-entropy secret rather than a memorable phrase — - scrypt raises the cost of an offline guess, but only a strong + Argon2id raises the cost of an offline guess, but only a strong secret removes the risk. A generated key is ideal, e.g. openssl rand -base64 32. The module logs a startup warning if the configured password is the default or has an estimated entropy @@ -1491,11 +1471,11 @@ modparam("dispatcher", "cluster_probing_mode", "distributed") tunnel (GRE-over-IPsec, Geneve-over-IPsec) that presents a multicast-capable interface. Running the controller over such a setup results in double encryption - (application-layer AES-256-GCM plus IPsec ESP), which is - harmless but adds minor CPU overhead. IPsec ESP tunnel mode - also reduces the effective MTU by approximately 50 bytes, - which compounds the MEMBER_LIST fragmentation issue - described below. + (application-layer XChaCha20-Poly1305 plus IPsec ESP), + which is harmless but adds minor CPU overhead. IPsec ESP + tunnel mode also reduces the effective MTU by approximately + 50 bytes, which compounds the MEMBER_LIST fragmentation + issue described below. * MEMBER_LIST fragmentation: the MEMBER_LIST packet grows with cluster size and reaches approximately 4395 bytes at the maximum of 256 nodes. This exceeds the 1472-byte UDP diff --git a/modules/clusterer_controller/doc/clusterer_controller_admin.xml b/modules/clusterer_controller/doc/clusterer_controller_admin.xml index c68e94f701b..fbc519a8101 100644 --- a/modules/clusterer_controller/doc/clusterer_controller_admin.xml +++ b/modules/clusterer_controller/doc/clusterer_controller_admin.xml @@ -63,23 +63,21 @@ which is the operator's responsibility to avoid. - Every packet is encrypted and authenticated with an AEAD — - AES-256-GCM by default, or XChaCha20-Poly1305 when the module is built - against libsodium (see the Security Architecture - section). Two distinct encryption keys are used depending on the - communication phase: + Every packet is encrypted and authenticated with the XChaCha20-Poly1305 + AEAD (see the Security Architecture section). Two + distinct encryption keys are used depending on the communication phase: Bootstrap key — derived from - the configured password with a memory-hard KDF - (scrypt N=216, r=8, - p=1, or Argon2id in the libsodium build, with a - per-cluster salt), so a password captured from a bootstrap packet - cannot be brute-forced cheaply offline. Derived once at startup. - Used for the admission handshake (JOIN_REQ, KEY_GRANT, JOIN_REJECT) - and the split-brain MASTER_BEACON — traffic that must be readable - before a session key exists, or by masters holding different - session keys. + the configured password with the + Argon2id memory-hard KDF (per-cluster salt), + so a password captured from a bootstrap packet cannot be + brute-forced cheaply offline. Derived once at startup. It is both + the outer AEAD key for the admission handshake (JOIN_REQ, + KEY_GRANT, JOIN_REJECT) and the split-brain MASTER_BEACON — traffic + that must be readable before a session key exists, or by masters + holding different session keys — and the pre-shared key for the + Noise join handshake. Session key — derived via @@ -96,8 +94,7 @@ identifying the key type and a 2-byte cluster_id, both in cleartext (the magic and cluster_id must be readable before decryption to select the key and to filter foreign clusters), followed by a random - AEAD nonce (12 bytes for AES-256-GCM, 24 for XChaCha20-Poly1305) and - the ciphertext. The magic and cluster_id are additionally bound into + 24-byte AEAD nonce and the ciphertext. The magic and cluster_id are additionally bound into the authentication tag as AAD, so they cannot be altered undetected. Only the payload is encrypted; the authenticated plaintext begins with a 1-byte packet type and a 4-byte monotonic sequence number used for @@ -123,8 +120,8 @@ JOIN_REQ — sent at startup - by a new node, carrying its IP, BIN socket list, X25519 - ephemeral public key, and a 16-byte random join nonce. + by a new node, carrying its IP, BIN socket list, and Noise + message 1 (the initiator's fresh ephemeral public key). Encrypted with the bootstrap key so it can be sent before a session key exists. @@ -156,19 +153,19 @@ the session key. - KEY_GRANT — unicast response - to a JOIN_REQ, sent by the master directly to the joining - node. Contains the master's X25519 public key, the original - join nonce, and the master salt wrapped with a per-exchange - key derived from ECDH(master_priv, joiner_pub) + password + - join_nonce via HKDF-SHA256. Encrypted with the bootstrap key. + KEY_GRANT — the master's + response to a JOIN_REQ, addressed to the joining node. Carries + Noise message 2 (the master's ephemeral plus the AEAD-encrypted + master salt), completing the Noise_NNpsk0 handshake. Encrypted + with the bootstrap key. - KEY_HANDOFF — unicast packet - sent by the outgoing master on graceful shutdown to the - next-highest-IP peer, delivering the master salt so that peer - can become the new master without a full re-join cycle. - Encrypted with the session key. + KEY_HANDOFF — sent by the + outgoing master on graceful shutdown to the next-highest-IP peer, + delivering the master salt (as an anonymous crypto_box sealed to + that peer's long-lived X25519 key, learned from its ALIVE) so it + can become the new master without a full re-join cycle. Encrypted + with the session key. JOIN_REJECT — sent by the @@ -299,35 +296,27 @@ failure. - Crypto suite (selected at build time): - if libsodium is detected on the build host, the - module is built sodium-only: - XChaCha20-Poly1305 (24-byte nonce, whose 192-bit - nonce space removes any random-nonce collision concern) for the payload - AEAD, Argon2id for the bootstrap-key derivation, - and X25519 / HKDF-SHA256 / RNG also from libsodium — WolfSSL is not - linked at all. Without libsodium the module falls back to WolfSSL: - AES-256-GCM (12-byte nonce) and - scrypt (N=216, r=8, - p=1). The two wire formats are not interoperable, so every node in a - cluster must be built with the same suite; the active suite is reported - in the startup log (crypto=...). + Crypto (all libsodium; a hard requirement): + the payload AEAD is XChaCha20-Poly1305 (24-byte + nonce, whose 192-bit nonce space removes any random-nonce collision + concern); the bootstrap-key KDF is Argon2id; and + X25519 / HKDF-SHA256 / RNG also come from libsodium. The active suite is + reported in the startup log (crypto=...). - Phase 1 — bootstrap (JOIN_REQ / - KEY_GRANT): each worker process generates an ephemeral - X25519 keypair on startup. When joining, the node sends its - public key and a 16-byte random join nonce in the JOIN_REQ, - encrypted with the bootstrap key (a memory-hard KDF of the password — - scrypt, or Argon2id in the libsodium build). The - master responds with a KEY_GRANT encrypted with the same bootstrap - key, containing its own public key, the echoed join nonce, and - the master salt wrapped with a per-exchange key derived via - HKDF-SHA256 from: - IKM = ECDH(master_priv, joiner_pub) || password || join_nonce - This ensures that even if the password is compromised, previously - recorded exchanges cannot be decrypted without both the ephemeral - private key and the per-exchange join nonce. + Phase 1 — join handshake (JOIN_REQ / + KEY_GRANT): the join is a + Noise_NNpsk0_25519_ChaChaPoly_SHA256 handshake with + the pre-shared key set to the Argon2id bootstrap key: + -> psk, e JOIN_REQ carries Noise message 1 (a fresh ephemeral) +<- e, ee KEY_GRANT carries Noise message 2; its AEAD payload = master_salt + Authentication comes from the shared PSK, forward secrecy from the + ephemeral-ephemeral DH, and the Noise handshake hash binds the whole + transcript — so a stale KEY_GRANT for a superseded JOIN_REQ simply fails + to decrypt. Both handshake messages also travel inside the bootstrap-key + AEAD envelope, so a wrong-password node is rejected at the envelope + before the handshake is even reached. This replaces the earlier + hand-rolled ECDH-and-XOR salt wrap. Phase 2 — session (all other @@ -575,41 +564,23 @@
External Libraries or Applications - One of the following two crypto libraries is required; libsodium is - preferred and used exclusively when found: + libsodium is required: - libsodium (preferred) — if the build host - has libsodium development files (detected via - pkg-config), the module is built - sodium-only: XChaCha20-Poly1305 for the - payload AEAD, Argon2id for the bootstrap-key derivation, and - X25519 ECDH, HKDF-SHA256 and the RNG also from libsodium — - WolfSSL is not linked at all, keeping the module binary small - (a few hundred KB instead of several MB of statically-linked - WolfSSL). libsodium is linked dynamically, so each target host - also needs the libsodium runtime package (for example - libsodium23). - - - - - tls_wolfssl (fallback) — without - libsodium, the module links statically against the WolfSSL - library built by the tls_wolfssl module - (modules/tls_wolfssl/lib/lib/libwolfssl.a), - providing AES-256-GCM authenticated encryption, X25519 ECDH, - HKDF-SHA256 and the scrypt password KDF. The - tls_wolfssl module must then be present in - the source tree; its static library is built automatically as - a dependency if not already present. + libsodium (required) — provides the + payload AEAD (XChaCha20-Poly1305), the bootstrap-key KDF + (Argon2id), the Noise_NNpsk0 join handshake, and X25519 / + HKDF-SHA256 / RNG. The build fails with a clear error if + libsodium development files are not found. It is linked + dynamically, so each target host also needs the libsodium + runtime package (for example libsodium23). + No other crypto library is used or linked. - Because the two suites produce incompatible wire formats, every node - in a cluster must be built the same way; the active suite is printed - in the startup log. + The active suite is printed in the startup log + (crypto=...).
@@ -630,11 +601,9 @@ include_modules= clusterer_controller then rebuild (make all / - make modules). With libsodium development files - on the build host the module builds sodium-only (no WolfSSL); - otherwise it links the bundled WolfSSL and the same - external-library requirements as tls_wolfssl - apply — see External Libraries or Applications. + make modules). libsodium development files must be + present on the build host or the build fails — see + External Libraries or Applications. The clusterer module is unmodified when @@ -676,7 +645,7 @@ include_modules= clusterer_controller password (optional) - — AES-256 encryption key material. All nodes in the + — XChaCha20-Poly1305 encryption key material. All nodes in the same cluster must use the same password. Falls back to the global password modparam if not set. @@ -927,10 +896,10 @@ modparam("clusterer_controller", "query_time", 5) Bootstrap key — - the password is stretched with scrypt - (memory-hard, per-cluster salt) to encrypt JOIN_REQ and - KEY_GRANT packets during the initial key exchange phase - before a session key is established. + the password is stretched with Argon2id + (memory-hard, per-cluster salt); it is the pre-shared key for + the Noise join handshake (JOIN_REQ / KEY_GRANT) and the AEAD + key for bootstrap traffic before a session key exists. Session key material — @@ -946,7 +915,7 @@ modparam("clusterer_controller", "query_time", 5) Default value is 3eCrEt*5629. Change this in production. Use a long, high-entropy - secret rather than a memorable phrase — scrypt raises the cost of + secret rather than a memorable phrase — Argon2id raises the cost of an offline guess, but only a strong secret removes the risk. A generated key is ideal, e.g. openssl rand -base64 32. The module logs a startup warning if the configured password is the @@ -1825,7 +1794,7 @@ modparam("dispatcher", "cluster_probing_mode", "distributed") run multicast inside an inner tunnel (GRE-over-IPsec, Geneve-over-IPsec) that presents a multicast-capable interface. Running the controller over such a setup results - in double encryption (application-layer AES-256-GCM plus + in double encryption (application-layer XChaCha20-Poly1305 plus IPsec ESP), which is harmless but adds minor CPU overhead. IPsec ESP tunnel mode also reduces the effective MTU by approximately 50 bytes, which compounds the MEMBER_LIST From 6980446cedef4742a6156350a369d8e6c960e773 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Sat, 18 Jul 2026 12:59:36 +1000 Subject: [PATCH 10/21] clusterer_controller: log this node role transitions on every election The election result was only visible as a cluster-wide roles list; a node could not see its OWN transition on its own line - e.g. a rejoining higher-IP peer would silently demote this backup to a plain member with nothing marking the change. Track the last-known self role (enum cc_role) per cluster and, at the end of cc_elect_master(), emit a single "my role changed: X -> Y" line whenever it differs. This covers every election trigger (goodbye, join, timer) in one place. Zero-initialised to member, matching a not-yet-joined node. The post-goodbye "re-election complete" line drops its now-redundant role clause and just names the elected master. --- .../clusterer_controller.c | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index 567abf3bade..f7b1f1c6cc4 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -495,6 +495,20 @@ typedef struct { cc_cipherstate_t cs; } cc_symstate_t; +/* This node's own role in a cluster. Tracked across elections so + * cc_elect_master() can log an explicit transition (member->backup, + * backup->master, master->member, ...) whenever it changes. */ +enum cc_role { CC_ROLE_MEMBER = 0, CC_ROLE_BACKUP, CC_ROLE_MASTER }; + +static inline const char *cc_role_name(int r) +{ + switch (r) { + case CC_ROLE_MASTER: return "master"; + case CC_ROLE_BACKUP: return "backup"; + default: return "member"; + } +} + /** * cc_cluster_t - per-cluster runtime state. * One instance per "cluster" modparam; one worker process per instance. @@ -608,6 +622,11 @@ typedef struct cc_cluster_ { * otherwise a demoted node keeps broadcasting MASTER_ALIVE and lower-IP * peers oscillate between two masters. Worker-local. */ int master_ka_armed; + /* This node's last-known role in the cluster (enum cc_role). Compared in + * cc_elect_master() to emit a one-line transition log on every change. + * Zero-initialised to CC_ROLE_MEMBER, which matches a not-yet-joined node. + * Worker-local. */ + int my_role; } cc_cluster_t; static cc_cluster_t cc_clusters[CC_MAX_CLUSTERS]; @@ -1418,8 +1437,28 @@ static void cc_elect_master(cc_cluster_t *cl) (b_ip && strcmp(b_ip, my_ip) == 0) ? " [me]" : "", n_in); } + + /* Log this node's own role transition (member/backup/master) whenever + * it changes, so a failover or a peer (re)joining that demotes/promotes + * us gets its own line rather than being implied by the roles list. */ + { + int my_new_role = i_am_elected ? CC_ROLE_MASTER : + ((b_ip && strcmp(b_ip, my_ip) == 0) ? CC_ROLE_BACKUP : + CC_ROLE_MEMBER); + if (my_new_role != cl->my_role) { + LM_INFO("clusterer_controller: [cluster %d] my role changed: " + "%s -> %s\n", cl->cluster_id, + cc_role_name(cl->my_role), cc_role_name(my_new_role)); + cl->my_role = my_new_role; + } + } } else { - /* No eligible peer - cluster has no master */ + /* No eligible peer - cluster has no master; we are a plain member. */ + if (cl->my_role != CC_ROLE_MEMBER) { + LM_INFO("clusterer_controller: [cluster %d] my role changed: %s -> " + "member\n", cl->cluster_id, cc_role_name(cl->my_role)); + cl->my_role = CC_ROLE_MEMBER; + } if (cl->peers->last_master[0] != '\0') { LM_INFO("clusterer_controller: [cluster %d] master lost (%s), " "no eligible peers in election window\n", @@ -3546,8 +3585,10 @@ static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) cc_send_member_list(sock, cl); } } else { + /* This node's own role change (if any) is logged separately by + * cc_elect_master() as a "my role changed" transition line. */ LM_INFO("clusterer_controller: re-election complete - " - "master is %s, my role is member (%d node(s) remaining)\n", + "master is %s (%d node(s) remaining)\n", new_master[0] ? new_master : "(none)", remaining); } From 016747495becfc7db6a1f143200be8306568152f Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Sat, 18 Jul 2026 19:12:38 +1000 Subject: [PATCH 11/21] tm: render the anycast Via cid from the live node id, not a frozen one tm_init_cluster() read this node cluster id once, at mod_init, and baked it into the ";cid=" Via parameter (and into a cached tm_node_id used to decide whether an anycast reply is ours). That assumes the id is known and stable by mod_init, which holds for a statically configured clusterer but not for a controller-managed one: there the id is assigned at runtime, after the node joins, and can change on re-election. So mod_init froze the still-unassigned id (-1, printed as its unsigned form 18446744073709551615) into every outgoing Via, and every node compared incoming cids against -1 - t_anycast_replicate() could never route a reply to the owning node. Read the id live instead: pre-build only the fixed ";=" prefix at init, and have tm_via_cid() append the current get_my_id() per request (returning no parameter while the node still has no id); compare incoming cids against the live get_my_id() too. cl_get_my_id() already returns the runtime id, so this needs nothing from the caller and self-corrects across re-elections. --- modules/clusterer_controller/README | 14 ++++++ .../doc/clusterer_controller_admin.xml | 16 +++++++ modules/tm/README | 6 +++ modules/tm/cluster.c | 45 ++++++++++++++----- modules/tm/cluster.h | 7 ++- modules/tm/doc/tm_admin.xml | 7 +++ 6 files changed, 82 insertions(+), 13 deletions(-) diff --git a/modules/clusterer_controller/README b/modules/clusterer_controller/README index af2c121ec25..ead30db6638 100644 --- a/modules/clusterer_controller/README +++ b/modules/clusterer_controller/README @@ -545,6 +545,20 @@ on-key") use_controller=1 and a module attempts to register a capability for that cluster. + Because a controller-managed node receives its node_id at + runtime (after it joins) rather than from static configuration + at startup, any module that stamps this node's id into + on-the-wire data must read it live per message instead of + caching it once at initialisation — a value read at startup + would be the not-yet-assigned placeholder, and it may also + change on a re-election. In particular tm's anycast support + (tm_replication_cluster / t_anycast_replicate()) stamps this + node's id into the cid Via parameter so that a reply landing on + a different anycast member can be relayed to the node holding + the transaction; it renders that parameter from the current id + on each request, so anycast reply routing works unchanged under + a controller-managed cluster. + 1.5.2. External Libraries or Applications libsodium is required: diff --git a/modules/clusterer_controller/doc/clusterer_controller_admin.xml b/modules/clusterer_controller/doc/clusterer_controller_admin.xml index fbc519a8101..16c889e89cb 100644 --- a/modules/clusterer_controller/doc/clusterer_controller_admin.xml +++ b/modules/clusterer_controller/doc/clusterer_controller_admin.xml @@ -559,6 +559,22 @@ use_controller=1 and a module attempts to register a capability for that cluster. + + Because a controller-managed node receives its + node_id at runtime (after it joins) rather than + from static configuration at startup, any module that stamps this + node's id into on-the-wire data must read it live per message + instead of caching it once at initialisation — a value read at + startup would be the not-yet-assigned placeholder, and it may also + change on a re-election. In particular tm's + anycast support (tm_replication_cluster / + t_anycast_replicate()) stamps this node's id + into the cid Via parameter so that a reply landing + on a different anycast member can be relayed to the node holding the + transaction; it renders that parameter from the current id on each + request, so anycast reply routing works unchanged under a + controller-managed cluster. +
diff --git a/modules/tm/README b/modules/tm/README index e3927aea6f1..5377b35ef55 100644 --- a/modules/tm/README +++ b/modules/tm/README @@ -720,6 +720,12 @@ modparam("tm", "auto_100trying", 0) Check out the tm_anycast section for more details. + This node's clusterer id is read at the moment each request is + sent (and each reply examined), not cached at startup, so + anycast replication also works when the id is assigned at + runtime and may change — as with a clusterer_controller-managed + cluster. + Anycast replication is disabled by default. Example 1.20. Set tm_replication_cluster parameter diff --git a/modules/tm/cluster.c b/modules/tm/cluster.c index 467eb8429c5..67db7f85d5d 100644 --- a/modules/tm/cluster.c +++ b/modules/tm/cluster.c @@ -32,7 +32,9 @@ str tm_cid; int tm_repl_cluster = 0; int tm_repl_auto_cancel = 1; str tm_cluster_param = str_init(TM_CLUSTER_DEFAULT_PARAM); -static int tm_node_id = 0; +/* length of the fixed ";=" prefix of tm_cid; the node id is appended + * live per-request by tm_via_cid() */ +static int tm_cid_prefix_len; static str tm_repl_cap = str_init("tm-repl"); struct clusterer_binds cluster_api; @@ -234,8 +236,6 @@ static void receive_tm_repl(bin_packet_t *packet) int tm_init_cluster(void) { - str cid; - if (tm_repl_cluster == 0) { LM_DBG("tm_replication_cluster not set - not engaging!\n"); return 0; @@ -260,11 +260,13 @@ int tm_init_cluster(void) /* overwrite structure to disable clusterer */ goto cluster_error; } - tm_node_id = cluster_api.get_my_id(); - - /* build the via param */ - cid.s = int2str(tm_node_id, &cid.len); - tm_cid.s = pkg_malloc(1/*;*/ + tm_cluster_param.len + 1/*=*/ + cid.len); + /* Pre-build only the fixed ";=" prefix. The node id is NOT read + * here: under a controller-managed cluster it is assigned at runtime + * (after this mod_init) and may change on re-election, so freezing it + * now would stamp a stale (typically -1) id into every Via. tm_via_cid() + * appends the current id per-request instead. Room is left for the + * largest id an int2str() can render. */ + tm_cid.s = pkg_malloc(1/*;*/ + tm_cluster_param.len + 1/*=*/ + INT2STR_MAX_LEN); if (!tm_cid.s) { LM_ERR("out of pkg memory!\n"); goto cluster_error; @@ -274,8 +276,7 @@ int tm_init_cluster(void) memcpy(tm_cid.s + tm_cid.len, tm_cluster_param.s, tm_cluster_param.len); tm_cid.len += tm_cluster_param.len; tm_cid.s[tm_cid.len++] = '='; - memcpy(tm_cid.s + tm_cid.len, cid.s, cid.len); - tm_cid.len += cid.len; + tm_cid_prefix_len = tm_cid.len; return 0; @@ -284,6 +285,28 @@ int tm_init_cluster(void) return -1; } +str *tm_via_cid(void) +{ + char *id_s; + int id, id_len; + + if (!tm_cluster_enabled()) + return NULL; + + /* read this node's id live - it is only known once the node has been + * assigned one in the cluster (immediately for a static clusterer, + * after the runtime join for a controller-managed one) and can change + * on re-election; advertise no cid until we actually have one */ + id = cluster_api.get_my_id(); + if (id < 0) + return NULL; + + id_s = int2str((unsigned long)id, &id_len); + memcpy(tm_cid.s + tm_cid_prefix_len, id_s, id_len); + tm_cid.len = tm_cid_prefix_len + id_len; + return &tm_cid; +} + #define TM_BIN_PUSH(_t, _f, _d) \ do { \ if (bin_push_##_t(&packet, _f) < 0) { \ @@ -500,7 +523,7 @@ int tm_reply_replicate(struct sip_msg *msg) /* if there was no parameter, or it was, but it was ours, handle it */ if (cid < 0) return 0; - if (cid == tm_node_id) { + if (cid == cluster_api.get_my_id()) { LM_DBG("reply should be processed by us (%d)\n", cid); return 0; } diff --git a/modules/tm/cluster.h b/modules/tm/cluster.h index 39783fe8810..9fad793ea57 100644 --- a/modules/tm/cluster.h +++ b/modules/tm/cluster.h @@ -51,7 +51,10 @@ int tm_anycast_cancel(struct sip_msg *msg); /* returns true if clusterer is enabled */ #define tm_cluster_enabled() (cluster_api.register_capability != 0) -/* returns the via parameter for the cluster */ -#define tm_via_cid() (tm_cluster_enabled()?&tm_cid:0) +/* Returns the ";=" Via parameter for the cluster, rendered + * with this node's CURRENT clusterer id (read live, not frozen at init - the + * id is assigned at runtime under a controller-managed cluster and may change + * on re-election). NULL when clustering is off or no id is assigned yet. */ +str *tm_via_cid(void); #endif /* _TM_CLUSTER_H_ */ diff --git a/modules/tm/doc/tm_admin.xml b/modules/tm/doc/tm_admin.xml index 5eca76d0fa5..42be4661978 100644 --- a/modules/tm/doc/tm_admin.xml +++ b/modules/tm/doc/tm_admin.xml @@ -791,6 +791,13 @@ modparam("tm", "auto_100trying", 0) Check out the section for more details. + This node's clusterer id is read at the moment each request is sent + (and each reply examined), not cached at startup, so anycast + replication also works when the id is assigned at runtime and may + change — as with a clusterer_controller-managed + cluster. + + Anycast replication is disabled by default. From 5dc976db0bb682f39207f142cc082941f0312b15 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Sat, 18 Jul 2026 21:26:43 +1000 Subject: [PATCH 12/21] clusterer_controller: trim the maintenance-mode roadmap comment The block flagged the sharing-tag override, its clearing, and the per-node shtag status as future work, but all three shipped: cl_ctr_shtag_force, cl_ctr_shtag_auto, and the shtag_mode field of cl_ctr_list_config. Reduce it to the one item still outstanding - node maintenance mode - and note that it would build on the shtag commands already in place, so the source no longer reads as unfinished where it is not. --- .../clusterer_controller.c | 76 +++---------------- 1 file changed, 12 insertions(+), 64 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index f7b1f1c6cc4..0f320bbcc99 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -407,72 +407,20 @@ typedef enum { CC_NODE_ACTIVE = 1 /* fully participating, sends ALIVE */ } cc_node_state_t; -/* TODO: maintenance mode +/* Planned: node maintenance mode (see the module documentation roadmap). * - * Add CC_NODE_MAINTENANCE = 2 to cc_node_state_t. A node in maintenance - * must never become master, must not participate in elections (treated as - * absent from cc_elect_master), and must not become active shtag holder for - * any cluster even when manage_shtags=1. + * A CC_NODE_MAINTENANCE state would let a node be drained without leaving the + * cluster: never elected master, excluded from cc_elect_master (advertised in + * the ALIVE payload so peers drop it from the election window without waiting + * for a MEMBER_LIST refresh), and never the active sharing-tag holder even + * with manage_shtags=1. It is local policy, so it would live in cc_cluster_t + * (not the shared peer table) and survive MEMBER_LIST resets, toggled by a + * local MI command. * - * Entry / exit: new MI command cc_maintenance {on|off} sets the flag on the - * local node. Advertise the state in the ALIVE payload so all peers know to - * exclude this node from elections without waiting for a MEMBER_LIST refresh. - * The maintenance flag should survive MEMBER_LIST resets (it is local policy, - * not part of the cluster-wide peer table - store it in cc_cluster_t, not - * cc_peer_t). - * - * In cc_elect_master: skip any peer whose ALIVE-advertised maintenance flag - * is set (treat it as not in the election window even if last_seen is fresh). - * - * In cc_transition_to_active and shtag activation paths: gate all - * activate_backup_shtags / set_active_shtag calls behind - * if (cl->peers->node_state != CC_NODE_MAINTENANCE) - * - * TODO: shtag override mode - * - * Add MI command cc_set_shtag_holder to force a - * specific non-maintenance node to be the active shtag holder for a cluster, - * overriding the normal master-drives-shtag logic. Persist the override in - * a new field cc_peers_t.shtag_override_ip[CC_MAX_IP_LEN+1] (in shm so - * it is visible across processes). - * - * When shtag_override_ip is set for a cluster: - * - the designated node calls set_active_shtag regardless of mastership - * - all other nodes stay in backup shtag state - * - the override is NOT cleared on MEMBER_LIST or key rotation - it is - * explicit operator intent and must be cleared only by - * cc_clear_shtag_override (see below) or automatically when the - * overridden node enters maintenance mode - * - if the overridden node is put into maintenance mode, the override is - * automatically cleared and normal master-driven shtag logic resumes - * - * TODO: shtag override clear - * - * Add MI command cc_clear_shtag_override to cancel a - * previously set shtag override and return the cluster to normal mode where - * the elected master drives shtag activation. - * - * Implementation: - * - zero cc_peers_t.shtag_override_ip for the cluster - * - the current master immediately calls set_active_shtag on itself and - * activate_backup_shtags on all other nodes, restoring normal state - * - non-master nodes that were held in backup shtag state due to the - * override need no explicit action - the next election cycle reapplies - * correct shtag assignments automatically - * - log at INFO: "shtag override cleared for cluster , resuming - * normal master-driven shtag assignment (master: )" - * - return error if no override is active for the given cluster_id - * - * TODO: MI status commands - * - * cc_list_shtags - list all clusters with their active shtag holder, - * whether the holder was elected normally or overridden, and which nodes - * are in maintenance mode. Output columns: cluster_id, shtag, holder_ip, - * status in {elected | overridden | maintenance | no_holder}. - * - * cl_ctr_list_members should be extended to include a 'mode' field per node: - * active | maintenance, and an 'shtag_status' field: holder | backup | - * overridden | n/a (when manage_shtags=0 for that cluster). + * It would build on what is already here: the sharing-tag override + * (cl_ctr_shtag_force / cl_ctr_shtag_auto) would auto-clear when the forced + * node enters maintenance, and cl_ctr_list_members / cl_ctr_list_config would + * gain a per-node maintenance indicator alongside the existing shtag_mode. */ /* ========================================================================= From 9913e5c9d63ecbb288495b61c11d4bceacdf93f6 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Sun, 19 Jul 2026 17:56:14 +1000 Subject: [PATCH 13/21] clusterer_controller: rename the cc_/CC_ prefix to cl_ctr_/CL_CTR_ Rename every internal identifier from the cc_ prefix to cl_ctr_, and the CC_ macros to CL_CTR_, so the whole module reads consistently with the public cl_ctr_* MI commands, pseudo-variables and script functions rather than carrying the older cc_ shorthand alongside them. This covers the static functions, the types (cl_ctr_cluster_t, cl_ctr_peer_t, ...), the module variables and the CL_CTR_ constants, across clusterer_controller.c and the controller-side hooks in clusterer (clusterer_ctrl.c/.h, clusterer_mod.c, node_info.h), plus the docs and the join-reject test tool. The wire magic bytes (the 0xCC prefix) are unchanged. Two key-derivation inputs are renamed along with everything else - the HKDF label "cc_session" -> "cl_ctr_session" and the bootstrap salt "opensips-cc-bootstrap-v1:" -> "opensips-cl-ctr-bootstrap-v1:" - which changes the derived session and bootstrap keys, so every node of a cluster must run this build; a mixed old/new cluster cannot interoperate. --- modules/clusterer/clusterer_ctrl.c | 4 +- modules/clusterer/clusterer_ctrl.h | 4 +- modules/clusterer/clusterer_mod.c | 42 +- modules/clusterer/node_info.h | 4 +- modules/clusterer_controller/README | 16 +- .../clusterer_controller.c | 2218 ++++++++--------- .../doc/clusterer_controller_admin.xml | 16 +- ...ect_test.py => cl_ctr_join_reject_test.py} | 4 +- 8 files changed, 1154 insertions(+), 1154 deletions(-) rename modules/clusterer_controller/test/{cc_join_reject_test.py => cl_ctr_join_reject_test.py} (97%) diff --git a/modules/clusterer/clusterer_ctrl.c b/modules/clusterer/clusterer_ctrl.c index 3616e94a3b4..de9d5052107 100644 --- a/modules/clusterer/clusterer_ctrl.c +++ b/modules/clusterer/clusterer_ctrl.c @@ -387,8 +387,8 @@ int load_clusterer_ctrl_binds(clusterer_ctrl_binds_t *binds) return -1; } clusterer_ctrl_bound = 1; - binds->managed_count = cc_stub_count; - binds->managed_ids = cc_stub_ids; + binds->managed_count = cl_ctr_stub_count; + binds->managed_ids = cl_ctr_stub_ids; binds->set_my_identity = clusterer_ctrl_set_identity; binds->add_node = clusterer_ctrl_add_node; binds->remove_node = clusterer_ctrl_remove_node; diff --git a/modules/clusterer/clusterer_ctrl.h b/modules/clusterer/clusterer_ctrl.h index 309570dce0b..d5ba9f15289 100644 --- a/modules/clusterer/clusterer_ctrl.h +++ b/modules/clusterer/clusterer_ctrl.h @@ -64,7 +64,7 @@ typedef struct clusterer_ctrl_binds { * The clusterer ping timer picks it up within one ping interval * and establishes the BIN link automatically. * - * Called on every CC_PKT_NODE_ASSIGN received for a peer. + * Called on every CL_CTR_PKT_NODE_ASSIGN received for a peer. * Safe to call if the node already exists — returns 0 (no-op). * * @cluster_id must match a cluster initialised by set_my_identity() @@ -80,7 +80,7 @@ typedef struct clusterer_ctrl_binds { * Removes from the node_list and cleans up routing state. * The BIN connection is closed by the clusterer's own cleanup path. * - * Called on CC_PKT_GOODBYE or when election-window expiry removes + * Called on CL_CTR_PKT_GOODBYE or when election-window expiry removes * the peer from the controller's own peer table. * * @cluster_id cluster the node belongs to diff --git a/modules/clusterer/clusterer_mod.c b/modules/clusterer/clusterer_mod.c index 5e6cf8de421..21181a31d41 100644 --- a/modules/clusterer/clusterer_mod.c +++ b/modules/clusterer/clusterer_mod.c @@ -64,12 +64,12 @@ int use_controller = 0; /* 1 if any cluster_options sets use_controller=1 */ /* cluster_ids marked controller-managed via 'cluster_options' (use_controller=1). * Not static: the controller loads a pointer to these through the ctrl binds so it * can verify it has a matching 'cluster' config for each. */ -int cc_stub_ids[64]; -int cc_stub_count = 0; +int cl_ctr_stub_ids[64]; +int cl_ctr_stub_count = 0; /* extract "=" from a "key=value, key=value" cluster_options string. * Returns 0 and sets *out on success, 1 if the key is absent, -1 if malformed. */ -static int cc_opt_int(str *descr, str *name, int *out) +static int cl_ctr_opt_int(str *descr, str *name, int *out) { char *p, *pe; str aux; @@ -103,7 +103,7 @@ static int cc_opt_int(str *descr, str *name, int *out) * use_controller=1 registers the cluster as controller-managed - its identity * and peer list are then driven at runtime by clusterer_controller. Every other * clusterer option stays a global modparam. */ -static int cc_add_cluster_options(modparam_t type, void *val) +static int cl_ctr_add_cluster_options(modparam_t type, void *val) { static str cid_prop = str_init("cluster_id"); static str uc_prop = str_init("use_controller"); @@ -117,7 +117,7 @@ static int cc_add_cluster_options(modparam_t type, void *val) descr.s = (char *)val; descr.len = strlen(descr.s); - rc = cc_opt_int(&descr, &cid_prop, &cid); + rc = cl_ctr_opt_int(&descr, &cid_prop, &cid); if (rc < 0) return -1; if (rc == 1 || cid < 1) { @@ -126,7 +126,7 @@ static int cc_add_cluster_options(modparam_t type, void *val) return -1; } - rc = cc_opt_int(&descr, &uc_prop, &uc); /* absent (rc==1) -> uc stays 0 */ + rc = cl_ctr_opt_int(&descr, &uc_prop, &uc); /* absent (rc==1) -> uc stays 0 */ if (rc < 0) return -1; if (uc != 0 && uc != 1) { @@ -138,17 +138,17 @@ static int cc_add_cluster_options(modparam_t type, void *val) if (uc == 0) return 0; /* native (the default): nothing to register */ - for (i = 0; i < cc_stub_count; i++) - if (cc_stub_ids[i] == cid) { + for (i = 0; i < cl_ctr_stub_count; i++) + if (cl_ctr_stub_ids[i] == cid) { LM_ERR("clusterer: cluster_options declares cluster_id %d as " "controller-managed more than once\n", cid); return -1; } - if (cc_stub_count >= (int)(sizeof cc_stub_ids / sizeof cc_stub_ids[0])) { + if (cl_ctr_stub_count >= (int)(sizeof cl_ctr_stub_ids / sizeof cl_ctr_stub_ids[0])) { LM_ERR("clusterer: too many controller-managed clusters\n"); return -1; } - cc_stub_ids[cc_stub_count++] = cid; + cl_ctr_stub_ids[cl_ctr_stub_count++] = cid; use_controller = 1; /* at least one controller-managed cluster exists */ return 0; } @@ -156,7 +156,7 @@ static int cc_add_cluster_options(modparam_t type, void *val) /* Retired interim modparams (never released): controller-managed clusters are now * declared via 'cluster_options'. Kept registered only to fail with a clear * migration hint instead of a generic "unknown parameter". */ -static int cc_retired_param(modparam_t type, void *val) +static int cl_ctr_retired_param(modparam_t type, void *val) { LM_ERR("clusterer: the 'use_controller'/'cluster_id' modparams have been " "replaced; declare each controller-managed cluster with " @@ -278,9 +278,9 @@ static const param_export_t params[] = { {"description_col", STR_PARAM, &description_col.s }, {"db_mode", INT_PARAM, &db_mode }, #ifdef CLUSTERER_CTRL_SUPPORT - {"cluster_options", STR_PARAM|USE_FUNC_PARAM, (void*)cc_add_cluster_options}, - {"use_controller", INT_PARAM|USE_FUNC_PARAM, (void*)cc_retired_param}, - {"cluster_id", INT_PARAM|USE_FUNC_PARAM, (void*)cc_retired_param}, + {"cluster_options", STR_PARAM|USE_FUNC_PARAM, (void*)cl_ctr_add_cluster_options}, + {"use_controller", INT_PARAM|USE_FUNC_PARAM, (void*)cl_ctr_retired_param}, + {"cluster_id", INT_PARAM|USE_FUNC_PARAM, (void*)cl_ctr_retired_param}, #endif {"neighbor_node_info", STR_PARAM|USE_FUNC_PARAM, (void*)&provision_neighbor}, @@ -650,40 +650,40 @@ static int mod_init(void) #ifdef CLUSTERER_CTRL_SUPPORT if (use_controller) { int _ci; - for (_ci = 0; _ci < cc_stub_count; _ci++) { + for (_ci = 0; _ci < cl_ctr_stub_count; _ci++) { cluster_info_t *_ex; for (_ex = *cluster_list; _ex; _ex = _ex->next) - if (_ex->cluster_id == cc_stub_ids[_ci]) { + if (_ex->cluster_id == cl_ctr_stub_ids[_ci]) { LM_ERR("clusterer: cluster_id %d is registered as " "controller-managed (cluster_options use_controller=1) but is " "already defined via native config " "(my_node_info/neighbor_node_info/DB) or listed " "twice - a cluster_id must be unique and either " "controller-managed or native, not both\n", - cc_stub_ids[_ci]); + cl_ctr_stub_ids[_ci]); return -1; } cluster_info_t *_cl = shm_malloc(sizeof *_cl); if (!_cl) { LM_ERR("clusterer: no shm for cluster %d stub\n", - cc_stub_ids[_ci]); + cl_ctr_stub_ids[_ci]); return -1; } memset(_cl, 0, sizeof *_cl); - _cl->cluster_id = cc_stub_ids[_ci]; + _cl->cluster_id = cl_ctr_stub_ids[_ci]; _cl->controller_managed = 1; _cl->lock = lock_alloc(); if (!_cl->lock || lock_init(_cl->lock) == NULL) { LM_ERR("clusterer: lock_alloc failed for cluster %d\n", - cc_stub_ids[_ci]); + cl_ctr_stub_ids[_ci]); shm_free(_cl); return -1; } _cl->next = *cluster_list; *cluster_list = _cl; LM_INFO("clusterer: pre-created controller-managed cluster %d stub\n", - cc_stub_ids[_ci]); + cl_ctr_stub_ids[_ci]); } } #endif /* CLUSTERER_CTRL_SUPPORT */ diff --git a/modules/clusterer/node_info.h b/modules/clusterer/node_info.h index 42cfc78c472..953456bc790 100644 --- a/modules/clusterer/node_info.h +++ b/modules/clusterer/node_info.h @@ -201,8 +201,8 @@ static inline int cluster_self_id(const struct cluster_info *cl) } extern int use_controller; /* cluster_ids declared controller-managed via cluster_options (use_controller=1) */ -extern int cc_stub_ids[]; -extern int cc_stub_count; +extern int cl_ctr_stub_ids[]; +extern int cl_ctr_stub_count; #else #define GET_CURRENT_ID (current_id) #define cluster_self_id(cl) (current_id) diff --git a/modules/clusterer_controller/README b/modules/clusterer_controller/README index ead30db6638..a0fe9651bdc 100644 --- a/modules/clusterer_controller/README +++ b/modules/clusterer_controller/README @@ -223,7 +223,7 @@ Chapter 1. Admin Guide Encrypted with the session key. * JOIN_REJECT — sent by the master to a joining node whose JOIN_REQ repeatedly fails authentication (wrong password). - After CC_JOIN_FAIL_LIMIT (3) consecutive bootstrap-key + After CL_CTR_JOIN_FAIL_LIMIT (3) consecutive bootstrap-key decryption failures from the same source IP the master sends a JOIN_REJECT to that IP. Encrypted with the bootstrap key so it cannot be forged by a node that does @@ -382,15 +382,15 @@ on-key") Join authentication and rejection: the master tracks consecutive bootstrap-key decryption failures per source IP in - a small worker-local table (CC_JOIN_FAIL_TABLE_SZ = 8 slots). - When any source IP accumulates CC_JOIN_FAIL_LIMIT (3) + a small worker-local table (CL_CTR_JOIN_FAIL_TABLE_SZ = 8 slots). + When any source IP accumulates CL_CTR_JOIN_FAIL_LIMIT (3) consecutive failures — indicating a node attempting to join with the wrong password — the master sends an encrypted JOIN_REJECT packet and stops responding to further JOIN_REQs from that IP. On the joining side, a received JOIN_REJECT is only acted on - while the node is still in the initial join phase (CC_NODE_NEW + while the node is still in the initial join phase (CL_CTR_NODE_NEW state) and is addressed to this node; an already-active cluster member ignores any JOIN_REJECT unconditionally, so a node with the correct password can never be evicted by a peer. @@ -400,7 +400,7 @@ on-key") so it relies on a self-contained signal instead: while joining it counts packets received from other peers that it cannot decrypt. If, at the join deadline, the node is still unjoined - and has seen CC_JOIN_FAIL_LIMIT or more such undecryptable + and has seen CL_CTR_JOIN_FAIL_LIMIT or more such undecryptable packets, it concludes that a cluster it cannot authenticate to exists on the group and shuts down OpenSIPS with a critical log message — rather than promoting itself into a lone, split-brain @@ -419,7 +419,7 @@ on-key") cluster into a re-JOIN churn. Peer table exhaustion defence: the peer table is bounded at - CC_MAX_PEERS (256) entries. When the table is full, the master + CL_CTR_MAX_PEERS (256) entries. When the table is full, the master rejects JOIN_REQ packets from unknown IPs with a JOIN_REJECT response. Known peers that are reconnecting after a restart continue to be admitted regardless of the table count, since @@ -1467,7 +1467,7 @@ modparam("dispatcher", "cluster_probing_mode", "distributed") re-flood/prune cycle could cause a gap in MASTER_ALIVE delivery and trigger a spurious master re-election. If this occurs, increase the effective timeout by raising - CC_MASTER_KA_MISSED in the source from 3 to 5 or higher. + CL_CTR_MASTER_KA_MISSED in the source from 3 to 5 or higher. PIM Sparse Mode (PIM-SM) or networks with IGMP snooping do not have this issue. * L2 overlay tunnels (Geneve, VXLAN, GRE): if the overlay @@ -1478,7 +1478,7 @@ modparam("dispatcher", "cluster_probing_mode", "distributed") controller will not function, as it has no unicast fallback. Encapsulation overhead also adds latency and jitter; on high-latency overlays consider raising - CC_MASTER_KA_MISSED to avoid spurious re-elections. + CL_CTR_MASTER_KA_MISSED to avoid spurious re-elections. * IPsec-protected links: native IPsec multicast requires GDOI/GET VPN (RFC 6407), which is rarely deployed. The recommended approach is to run multicast inside an inner diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index 0f320bbcc99..cc5e9cb5816 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -84,7 +84,7 @@ * MEMBER_LIST - session key, multicast * Master -> all: member count, the operator-forced sharing-tag holder * node_id (0 = automatic), and the full peer IP list, so all nodes elect - * identically. Only accepted from the current master (CC_NODE_NEW aside). + * identically. Only accepted from the current master (CL_CTR_NODE_NEW aside). * * GOODBYE - session key, multicast * Graceful shutdown. Peers remove sender immediately without timeout. @@ -93,8 +93,8 @@ * Master -> all: allocate node_id + BIN socket for a joining node. * * MASTER_ALIVE - session key, multicast - * Master-only keepalive every CC_MASTER_KA_INTERVAL seconds. Peers declare - * the master dead after CC_MASTER_KA_TIMEOUT seconds of silence and trigger + * Master-only keepalive every CL_CTR_MASTER_KA_INTERVAL seconds. Peers declare + * the master dead after CL_CTR_MASTER_KA_TIMEOUT seconds of silence and trigger * re-election. Two masters that share a session key resolve split-brain * here: the lower-IP one yields. * @@ -115,19 +115,19 @@ * read it; a wrong-password joiner also self-terminates at its deadline. * * MASTER_BEACON - bootstrap key, multicast - * Master-only, every CC_MASTER_BEACON_EVERY keepalive ticks. Because it + * Master-only, every CL_CTR_MASTER_BEACON_EVERY keepalive ticks. Because it * uses the bootstrap key it is readable even by a master holding a * DIFFERENT session key, which is how a split brain between independently * bootstrapped partitions is detected and merged. Payload: member count. * * NODE STATE MACHINE * ------------------ - * CC_NODE_NEW --> (MEMBER_LIST or KEY_GRANT received) --> CC_NODE_ACTIVE - * --> (join_deadline expired) --> CC_NODE_ACTIVE + * CL_CTR_NODE_NEW --> (MEMBER_LIST or KEY_GRANT received) --> CL_CTR_NODE_ACTIVE + * --> (join_deadline expired) --> CL_CTR_NODE_ACTIVE * - * CC_NODE_NEW: receive only; do NOT send ALIVE or MASTER_ALIVE. - * CC_NODE_ACTIVE: send ALIVE every query_time seconds. - * Master also sends MASTER_ALIVE every CC_MASTER_KA_INTERVAL s. + * CL_CTR_NODE_NEW: receive only; do NOT send ALIVE or MASTER_ALIVE. + * CL_CTR_NODE_ACTIVE: send ALIVE every query_time seconds. + * Master also sends MASTER_ALIVE every CL_CTR_MASTER_KA_INTERVAL s. * * MASTER ELECTION * --------------- @@ -216,7 +216,7 @@ * is Noise_NNpsk0_25519_ChaChaPoly_SHA256, and X25519 / HKDF-SHA256 / RNG also * come from libsodium. */ #include -#define CC_CRYPTO_SUITE "XChaCha20-Poly1305 + Argon2id + Noise_NNpsk0" +#define CL_CTR_CRYPTO_SUITE "XChaCha20-Poly1305 + Argon2id + Noise_NNpsk0" /* ========================================================================= * Wire-format constants @@ -227,42 +227,42 @@ * Both share the 0xCC prefix; the second byte distinguishes the key tier. * Only a sanity/routing tag on a dedicated multicast group:port - the real * confidentiality and integrity come from the AES-256-GCM payload. */ -#define CC_MAGIC_SZ 2 -static const unsigned char CC_PACKET_MAGIC[CC_MAGIC_SZ] = { 0xCC, 0x00 }; -static const unsigned char CC_BOOTSTRAP_MAGIC[CC_MAGIC_SZ] = { 0xCC, 0x01 }; +#define CL_CTR_MAGIC_SZ 2 +static const unsigned char CL_CTR_PACKET_MAGIC[CL_CTR_MAGIC_SZ] = { 0xCC, 0x00 }; +static const unsigned char CL_CTR_BOOTSTRAP_MAGIC[CL_CTR_MAGIC_SZ] = { 0xCC, 0x01 }; /* Packet type bytes */ -#define CC_PKT_ALIVE 0x01 -#define CC_PKT_JOIN_REQ 0x02 -#define CC_PKT_MEMBER_LIST 0x03 /* master -> joining node: here is the cluster */ -#define CC_PKT_GOODBYE 0x04 /* graceful shutdown notification */ -#define CC_PKT_NODE_ASSIGN 0x05 /* master -> multicast: here is your node_id */ -#define CC_PKT_MASTER_ALIVE 0x06 /* master-only keepalive; ~1 s interval */ -#define CC_PKT_KEY_GRANT 0x07 /* master -> joiner: ECDH-wrapped master_salt */ -#define CC_PKT_KEY_HANDOFF 0x08 /* outgoing master -> next master: salt handoff */ -#define CC_PKT_JOIN_REJECT 0x09 /* master -> joiner: authentication rejected */ -#define CC_PKT_MASTER_BEACON 0x0A /* master-only announce (BOOTSTRAP key) so +#define CL_CTR_PKT_ALIVE 0x01 +#define CL_CTR_PKT_JOIN_REQ 0x02 +#define CL_CTR_PKT_MEMBER_LIST 0x03 /* master -> joining node: here is the cluster */ +#define CL_CTR_PKT_GOODBYE 0x04 /* graceful shutdown notification */ +#define CL_CTR_PKT_NODE_ASSIGN 0x05 /* master -> multicast: here is your node_id */ +#define CL_CTR_PKT_MASTER_ALIVE 0x06 /* master-only keepalive; ~1 s interval */ +#define CL_CTR_PKT_KEY_GRANT 0x07 /* master -> joiner: ECDH-wrapped master_salt */ +#define CL_CTR_PKT_KEY_HANDOFF 0x08 /* outgoing master -> next master: salt handoff */ +#define CL_CTR_PKT_JOIN_REJECT 0x09 /* master -> joiner: authentication rejected */ +#define CL_CTR_PKT_MASTER_BEACON 0x0A /* master-only announce (BOOTSTRAP key) so * masters with divergent session keys can * still discover each other and merge a * split brain; payload = member count 2B BE */ /* Number of consecutive bootstrap-decrypt failures before a JOIN_REJECT is sent */ -#define CC_JOIN_FAIL_LIMIT 3 -#define CC_JOIN_FAIL_TABLE_SZ 8 /* max simultaneous rejected IPs tracked by master */ +#define CL_CTR_JOIN_FAIL_LIMIT 3 +#define CL_CTR_JOIN_FAIL_TABLE_SZ 8 /* max simultaneous rejected IPs tracked by master */ -#define CC_MAX_IP_LEN 15 /* "255.255.255.255" without NUL */ -#define CC_PUBKEY_SZ 32 /* X25519 public key */ -#define CC_MASTER_SALT_SZ 32 /* random salt generated by each new master */ +#define CL_CTR_MAX_IP_LEN 15 /* "255.255.255.255" without NUL */ +#define CL_CTR_PUBKEY_SZ 32 /* X25519 public key */ +#define CL_CTR_MASTER_SALT_SZ 32 /* random salt generated by each new master */ /* MEMBER_LIST entry: IP (16B null-padded) + is_master (1B) = 17B. * Pubkeys are NOT carried here - nodes learn them from ALIVE packets, * keeping MEMBER_LIST small enough to avoid excessive IP fragmentation. */ -#define CC_IP_ENTRY_SZ 17 -#define CC_LIST_COUNT_SZ 2 /* MEMBER_LIST count field: uint16_t BE */ -#define CC_NODE_ID_SZ 2 /* uint16_t node_id, big-endian */ -#define CC_MAX_BIN_SOCKETS 8 /* max BIN listeners per node */ -#define CC_MAX_BIN_SOCK_LEN 64 /* "bin:255.255.255.255:65535" = 26 chars */ +#define CL_CTR_IP_ENTRY_SZ 17 +#define CL_CTR_LIST_COUNT_SZ 2 /* MEMBER_LIST count field: uint16_t BE */ +#define CL_CTR_NODE_ID_SZ 2 /* uint16_t node_id, big-endian */ +#define CL_CTR_MAX_BIN_SOCKETS 8 /* max BIN listeners per node */ +#define CL_CTR_MAX_BIN_SOCK_LEN 64 /* "bin:255.255.255.255:65535" = 26 chars */ /* BIN info block: [bin_count 1B][sock1 NUL-term]...[sockN NUL-term] */ -#define CC_BIN_INFO_MAX_SZ (1 + CC_MAX_BIN_SOCKETS * CC_MAX_BIN_SOCK_LEN) +#define CL_CTR_BIN_INFO_MAX_SZ (1 + CL_CTR_MAX_BIN_SOCKETS * CL_CTR_MAX_BIN_SOCK_LEN) /* AEAD encryption constants * wire: [magic 2B][cluster_id 2B BE][nonce 24B][ciphertext][tag 16B] @@ -270,13 +270,13 @@ static const unsigned char CC_BOOTSTRAP_MAGIC[CC_MAGIC_SZ] = { 0xCC, 0x01 }; * The cluster_id is cleartext (like the magic) so a node can drop packets that * belong to a different cluster sharing the same multicast group WITHOUT its * key - before decryption, so foreign traffic never counts as an auth failure. */ -#define CC_NONCE_SZ 24 /* XChaCha20-Poly1305 nonce (192-bit) */ -#define CC_TAG_SZ 16 /* Poly1305 AEAD tag */ -#define CC_SEQ_SZ 4 /* uint32_t monotonic sequence in plaintext */ -#define CC_CLUSTER_ID_SZ 2 /* cleartext uint16 cluster_id (BE) selector */ -#define CC_NONCE_OFF (CC_MAGIC_SZ + CC_CLUSTER_ID_SZ) /* nonce starts here */ -#define CC_WIRE_HDR_SZ (CC_MAGIC_SZ + CC_CLUSTER_ID_SZ + CC_NONCE_SZ) /* 16 (GCM) / 28 (XChaCha) */ -#define CC_PLAIN_HDR_SZ (1 + CC_SEQ_SZ) /* type + seq = 5 */ +#define CL_CTR_NONCE_SZ 24 /* XChaCha20-Poly1305 nonce (192-bit) */ +#define CL_CTR_TAG_SZ 16 /* Poly1305 AEAD tag */ +#define CL_CTR_SEQ_SZ 4 /* uint32_t monotonic sequence in plaintext */ +#define CL_CTR_CLUSTER_ID_SZ 2 /* cleartext uint16 cluster_id (BE) selector */ +#define CL_CTR_NONCE_OFF (CL_CTR_MAGIC_SZ + CL_CTR_CLUSTER_ID_SZ) /* nonce starts here */ +#define CL_CTR_WIRE_HDR_SZ (CL_CTR_MAGIC_SZ + CL_CTR_CLUSTER_ID_SZ + CL_CTR_NONCE_SZ) /* 16 (GCM) / 28 (XChaCha) */ +#define CL_CTR_PLAIN_HDR_SZ (1 + CL_CTR_SEQ_SZ) /* type + seq = 5 */ /* Bootstrap-key hardening: the join/admission key (also the Noise PSK) is * derived from the shared password with Argon2id (memory-hard) instead of a @@ -284,24 +284,24 @@ static const unsigned char CC_BOOTSTRAP_MAGIC[CC_MAGIC_SZ] = { 0xCC, 0x01 }; * cheaply offline. Derived ONCE in mod_init (main process, before fork), so the * ~64 MiB working set is a transient startup cost; workers inherit the 32-byte * key. Fixed parameters so every node derives the same key. */ -#define CC_ARGON2_OPSLIMIT 3UL -#define CC_ARGON2_MEMLIMIT (64UL * 1024 * 1024) +#define CL_CTR_ARGON2_OPSLIMIT 3UL +#define CL_CTR_ARGON2_MEMLIMIT (64UL * 1024 * 1024) /* Minimum estimated password entropy (bits) before a startup warning fires. */ -#define CC_MIN_PASSWORD_BITS 80 -#define CC_DEFAULT_PASSWORD "3eCrEt*5629" /* insecure placeholder; warn if used */ +#define CL_CTR_MIN_PASSWORD_BITS 80 +#define CL_CTR_DEFAULT_PASSWORD "3eCrEt*5629" /* insecure placeholder; warn if used */ -/* Master keepalive: master sends CC_PKT_MASTER_ALIVE every CC_MASTER_KA_INTERVAL - * seconds. Peers declare master dead after CC_MASTER_KA_MISSED missed packets. */ -#define CC_MASTER_KA_INTERVAL 1 /* seconds between MASTER_ALIVE sends */ -#define CC_MASTER_KA_MISSED 3 /* missed keepalives before re-election */ -#define CC_MASTER_KA_TIMEOUT (CC_MASTER_KA_INTERVAL * CC_MASTER_KA_MISSED) +/* Master keepalive: master sends CL_CTR_PKT_MASTER_ALIVE every CL_CTR_MASTER_KA_INTERVAL + * seconds. Peers declare master dead after CL_CTR_MASTER_KA_MISSED missed packets. */ +#define CL_CTR_MASTER_KA_INTERVAL 1 /* seconds between MASTER_ALIVE sends */ +#define CL_CTR_MASTER_KA_MISSED 3 /* missed keepalives before re-election */ +#define CL_CTR_MASTER_KA_TIMEOUT (CL_CTR_MASTER_KA_INTERVAL * CL_CTR_MASTER_KA_MISSED) -/* Split-brain merge: a master emits a CC_PKT_MASTER_BEACON (bootstrap key) once - * every CC_MASTER_BEACON_EVERY MASTER_ALIVE ticks. This is the only traffic two +/* Split-brain merge: a master emits a CL_CTR_PKT_MASTER_BEACON (bootstrap key) once + * every CL_CTR_MASTER_BEACON_EVERY MASTER_ALIVE ticks. This is the only traffic two * masters with divergent session keys can both read, so it bounds split-brain - * convergence to ~CC_MASTER_BEACON_EVERY seconds while keeping bootstrap-key use + * convergence to ~CL_CTR_MASTER_BEACON_EVERY seconds while keeping bootstrap-key use * (and thus exposure) rare compared with the 1 s session keepalive. */ -#define CC_MASTER_BEACON_EVERY 5 /* MASTER_ALIVE ticks between beacons (~5 s) */ +#define CL_CTR_MASTER_BEACON_EVERY 5 /* MASTER_ALIVE ticks between beacons (~5 s) */ /* Split-brain PREVENTION at join time. When several nodes cold-start together * they all exchange (bootstrap-decryptable) JOIN_REQs, so each learns the other @@ -309,23 +309,23 @@ static const unsigned char CC_BOOTSTRAP_MAGIC[CC_MAGIC_SZ] = { 0xCC, 0x01 }; * NOT self-promote; it defers (re-sending JOIN_REQ) so the highest-IP starter * becomes the single master and everyone joins it - no divergent keys ever form. * Bounded so a higher-IP node that heard-then-died cannot stall us forever. */ -#define CC_JOIN_DEFER_SECS 1 /* seconds per deferral round */ -#define CC_JOIN_DEFER_MAX 4 /* consecutive deferrals for a *silent* */ +#define CL_CTR_JOIN_DEFER_SECS 1 /* seconds per deferral round */ +#define CL_CTR_JOIN_DEFER_MAX 4 /* consecutive deferrals for a *silent* */ /* higher-IP peer before we promote anyway */ -#define CC_JOIN_DEFER_HARDMAX 20 /* absolute cap on deferrals incl. resets - */ +#define CL_CTR_JOIN_DEFER_HARDMAX 20 /* absolute cap on deferrals incl. resets - */ /* a peer stuck joining can't stall forever */ -#define CC_JOIN_REQ_MIN_US 500000 /* min microseconds between JOIN_REQ sends */ +#define CL_CTR_JOIN_REQ_MIN_US 500000 /* min microseconds between JOIN_REQ sends */ /* Per-source-IP rate limiter: checked before decryption to shed floods cheaply. - * Tracks up to CC_RATE_TBL_SZ source IPs with a 1-second sliding window. */ -#define CC_RATE_TBL_SZ 256 /* one slot per peer; matches max cluster size */ -#define CC_RATE_LIMIT 20 /* max packets per second per source IP */ + * Tracks up to CL_CTR_RATE_TBL_SZ source IPs with a 1-second sliding window. */ +#define CL_CTR_RATE_TBL_SZ 256 /* one slot per peer; matches max cluster size */ +#define CL_CTR_RATE_LIMIT 20 /* max packets per second per source IP */ typedef struct { uint32_t ip; /* network byte order; 0 = empty slot */ time_t window_start; int count; -} cc_rate_entry_t; +} cl_ctr_rate_entry_t; /* Max packet sizes: wire(20) = magic(8) + nonce(12); plain(5) = type(1) + seq(4) * MASTER_ALIVE : wire(20) + plain(5) + tag(16) = 41 bytes @@ -353,67 +353,67 @@ typedef struct { * Firewalls that block fragmented UDP packets will silently drop MEMBER_LIST, * preventing new nodes from joining. The DF bit is not set so fragmentation * occurs transparently where the network allows it. */ -#define CC_SMALL_PKT_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 + CC_TAG_SZ) +#define CL_CTR_SMALL_PKT_SZ (CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_MAX_IP_LEN + 1 + CL_CTR_TAG_SZ) /* Consistency-critical settings advertised in ALIVE so peers can detect * accidental per-node config drift for the same cluster: * manage_shtags(1B) + master_stickiness(1B) + query_time(2B BE). */ -#define CC_CONFIG_SZ 4 +#define CL_CTR_CONFIG_SZ 4 /* Noise handshake message sizes (NNpsk0, X25519, ChaChaPoly, SHA-256): * msg 1 = e(32) + tag over empty payload(16) = 48 * msg 2 = e(32) + AEAD(master_salt 32 + tag 16) = 80 */ -#define CC_NOISE_MSG1_SZ (32 + CC_TAG_SZ) -#define CC_NOISE_MSG2_SZ (32 + CC_MASTER_SALT_SZ + CC_TAG_SZ) +#define CL_CTR_NOISE_MSG1_SZ (32 + CL_CTR_TAG_SZ) +#define CL_CTR_NOISE_MSG2_SZ (32 + CL_CTR_MASTER_SALT_SZ + CL_CTR_TAG_SZ) /* JOIN_REQ: [ip NUL][bin_count 1B][sockets...][noise_msg1 48B][config 4B] */ -#define CC_JOIN_PKT_MAX_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 \ - + CC_BIN_INFO_MAX_SZ + CC_NOISE_MSG1_SZ \ - + CC_CONFIG_SZ + CC_TAG_SZ) +#define CL_CTR_JOIN_PKT_MAX_SZ (CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_MAX_IP_LEN + 1 \ + + CL_CTR_BIN_INFO_MAX_SZ + CL_CTR_NOISE_MSG1_SZ \ + + CL_CTR_CONFIG_SZ + CL_CTR_TAG_SZ) /* KEY_GRANT: [target_ip NUL][noise_msg2 80B] */ -#define CC_KEY_GRANT_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 \ - + CC_NOISE_MSG2_SZ + CC_TAG_SZ) +#define CL_CTR_KEY_GRANT_SZ (CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_MAX_IP_LEN + 1 \ + + CL_CTR_NOISE_MSG2_SZ + CL_CTR_TAG_SZ) /* KEY_HANDOFF: [target_ip NUL][crypto_box_seal(master_salt)] */ -#define CC_KEY_HANDOFF_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_MAX_IP_LEN + 1 \ - + crypto_box_SEALBYTES + CC_MASTER_SALT_SZ + CC_TAG_SZ) +#define CL_CTR_KEY_HANDOFF_SZ (CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_MAX_IP_LEN + 1 \ + + crypto_box_SEALBYTES + CL_CTR_MASTER_SALT_SZ + CL_CTR_TAG_SZ) /* NODE_ASSIGN: [node_id 2B][ip NUL][bin_count 1B][sockets...] */ -#define CC_NODE_ASSIGN_MAX_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_NODE_ID_SZ \ - + CC_MAX_IP_LEN + 1 + CC_BIN_INFO_MAX_SZ + CC_TAG_SZ) -#define CC_LIST_PKT_MAX_SZ (CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_LIST_COUNT_SZ \ - + CC_NODE_ID_SZ /* forced-shtag node_id */ \ - + CC_MAX_PEERS * CC_IP_ENTRY_SZ + CC_TAG_SZ) +#define CL_CTR_NODE_ASSIGN_MAX_SZ (CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_NODE_ID_SZ \ + + CL_CTR_MAX_IP_LEN + 1 + CL_CTR_BIN_INFO_MAX_SZ + CL_CTR_TAG_SZ) +#define CL_CTR_LIST_PKT_MAX_SZ (CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_LIST_COUNT_SZ \ + + CL_CTR_NODE_ID_SZ /* forced-shtag node_id */ \ + + CL_CTR_MAX_PEERS * CL_CTR_IP_ENTRY_SZ + CL_CTR_TAG_SZ) /* Large enough to receive a fully reassembled UDP datagram (max 65507 bytes) */ -#define CC_RECV_BUF_SZ 65536 +#define CL_CTR_RECV_BUF_SZ 65536 /* ========================================================================= * Peer-table constants * ========================================================================= */ -#define CC_MAX_PEERS 256 +#define CL_CTR_MAX_PEERS 256 /* - * CC_ELECT_FACTOR - election window = query_time x CC_ELECT_FACTOR. + * CL_CTR_ELECT_FACTOR - election window = query_time x CL_CTR_ELECT_FACTOR. * QUANTIZED: all nodes evaluate the same cutoff simultaneously. * - * CC_PURGE_FACTOR - memory-cleanup window = query_time x CC_PURGE_FACTOR. + * CL_CTR_PURGE_FACTOR - memory-cleanup window = query_time x CL_CTR_PURGE_FACTOR. * Not quantized; only affects when entries are freed, not who is elected. */ -#define CC_ELECT_FACTOR 3 -#define CC_PURGE_FACTOR 6 +#define CL_CTR_ELECT_FACTOR 3 +#define CL_CTR_PURGE_FACTOR 6 /* ========================================================================= * Node-join state machine * ========================================================================= */ typedef enum { - CC_NODE_NEW = 0, /* sent JOIN_REQ, awaiting MEMBER_LIST or timeout */ - CC_NODE_ACTIVE = 1 /* fully participating, sends ALIVE */ -} cc_node_state_t; + CL_CTR_NODE_NEW = 0, /* sent JOIN_REQ, awaiting MEMBER_LIST or timeout */ + CL_CTR_NODE_ACTIVE = 1 /* fully participating, sends ALIVE */ +} cl_ctr_node_state_t; /* Planned: node maintenance mode (see the module documentation roadmap). * - * A CC_NODE_MAINTENANCE state would let a node be drained without leaving the - * cluster: never elected master, excluded from cc_elect_master (advertised in + * A CL_CTR_NODE_MAINTENANCE state would let a node be drained without leaving the + * cluster: never elected master, excluded from cl_ctr_elect_master (advertised in * the ALIVE payload so peers drop it from the election window without waiting * for a MEMBER_LIST refresh), and never the active sharing-tag holder even - * with manage_shtags=1. It is local policy, so it would live in cc_cluster_t + * with manage_shtags=1. It is local policy, so it would live in cl_ctr_cluster_t * (not the shared peer table) and survive MEMBER_LIST resets, toggled by a * local MI command. * @@ -427,54 +427,54 @@ typedef enum { * Cluster table * ========================================================================= */ -#define CC_MAX_CLUSTERS 16 /* max cluster= entries */ +#define CL_CTR_MAX_CLUSTERS 16 /* max cluster= entries */ -/* Forward-declared so cc_cluster_t can embed a pointer */ -typedef struct cc_peers_ cc_peers_t; +/* Forward-declared so cl_ctr_cluster_t can embed a pointer */ +typedef struct cl_ctr_peers_ cl_ctr_peers_t; /* Noise handshake state (definitions with the Noise core further below). - * HASHLEN = 32 (SHA-256). cc_cluster_t embeds a cc_symstate_t for the + * HASHLEN = 32 (SHA-256). cl_ctr_cluster_t embeds a cl_ctr_symstate_t for the * initiator half of the join handshake. */ -#define CC_NOISE_HASHLEN 32 -typedef struct { unsigned char k[32]; uint64_t n; int has_key; } cc_cipherstate_t; +#define CL_CTR_NOISE_HASHLEN 32 +typedef struct { unsigned char k[32]; uint64_t n; int has_key; } cl_ctr_cipherstate_t; typedef struct { - unsigned char ck[CC_NOISE_HASHLEN]; - unsigned char h[CC_NOISE_HASHLEN]; - cc_cipherstate_t cs; -} cc_symstate_t; + unsigned char ck[CL_CTR_NOISE_HASHLEN]; + unsigned char h[CL_CTR_NOISE_HASHLEN]; + cl_ctr_cipherstate_t cs; +} cl_ctr_symstate_t; /* This node's own role in a cluster. Tracked across elections so - * cc_elect_master() can log an explicit transition (member->backup, + * cl_ctr_elect_master() can log an explicit transition (member->backup, * backup->master, master->member, ...) whenever it changes. */ -enum cc_role { CC_ROLE_MEMBER = 0, CC_ROLE_BACKUP, CC_ROLE_MASTER }; +enum cl_ctr_role { CL_CTR_ROLE_MEMBER = 0, CL_CTR_ROLE_BACKUP, CL_CTR_ROLE_MASTER }; -static inline const char *cc_role_name(int r) +static inline const char *cl_ctr_role_name(int r) { switch (r) { - case CC_ROLE_MASTER: return "master"; - case CC_ROLE_BACKUP: return "backup"; + case CL_CTR_ROLE_MASTER: return "master"; + case CL_CTR_ROLE_BACKUP: return "backup"; default: return "member"; } } /** - * cc_cluster_t - per-cluster runtime state. + * cl_ctr_cluster_t - per-cluster runtime state. * One instance per "cluster" modparam; one worker process per instance. */ -typedef struct cc_cluster_ { +typedef struct cl_ctr_cluster_ { int cluster_id; char multicast_address[INET_ADDRSTRLEN]; int multicast_port; - struct sockaddr_in mcast_dest; /* resolved once in cc_setup_socket */ + struct sockaddr_in mcast_dest; /* resolved once in cl_ctr_setup_socket */ char password[1025]; unsigned char key[32]; /* bootstrap key = SHA256(password); JOIN only */ unsigned char session_key[32]; /* group key = HKDF(password, master_salt) */ int manage_shtags; /* per-cluster override; defaults to global manage_shtags */ int master_stickiness; /* per-cluster override; -1 = inherit global */ - cc_peers_t *peers; /* per-cluster peer table in shm */ + cl_ctr_peers_t *peers; /* per-cluster peer table in shm */ /* BIN socket resolved at mod_init - advertised in JOIN_REQ/NODE_ASSIGN */ - char bin_socket[CC_MAX_BIN_SOCK_LEN]; /* "bin:IP:PORT" */ - /* Worker-process fds and state - valid only inside cc_worker after fork */ + char bin_socket[CL_CTR_MAX_BIN_SOCK_LEN]; /* "bin:IP:PORT" */ + /* Worker-process fds and state - valid only inside cl_ctr_worker after fork */ int sock; /* multicast UDP socket */ int alive_tfd; /* periodic ALIVE timer */ int join_tfd; /* one-shot join deadline */ @@ -483,44 +483,44 @@ typedef struct cc_cluster_ { int master_dead_tfd; /* non-master: fires on ka miss */ int identity_registered; /* 1 once update_identity called */ int shtag_bootstrapped; /* -1 = eligible, 1 = done */ - /* Long-lived X25519 keypair - generated in cc_worker after fork, never + /* Long-lived X25519 keypair - generated in cl_ctr_worker after fork, never * leaves the process. Advertised in ALIVE and used by crypto_box KEY_HANDOFF * (NOT the Noise join handshake, which uses a fresh ephemeral per attempt). */ - unsigned char my_privkey[CC_PUBKEY_SZ]; - unsigned char my_pubkey[CC_PUBKEY_SZ]; + unsigned char my_privkey[CL_CTR_PUBKEY_SZ]; + unsigned char my_pubkey[CL_CTR_PUBKEY_SZ]; /* Noise (initiator) handshake state carried between our JOIN_REQ (msg 1) and * the master's KEY_GRANT (msg 2): the SymmetricState and the fresh ephemeral * private key. A new JOIN_REQ overwrites both, so a KEY_GRANT for a * superseded attempt simply fails to decrypt and is dropped. Worker-local. */ - cc_symstate_t noise_hs; + cl_ctr_symstate_t noise_hs; unsigned char noise_e_priv[32]; int noise_hs_valid; /* 1 after a JOIN_REQ, until KEY_GRANT/reset */ /* Set while a re-key JOIN_REQ is in flight; cleared on KEY_GRANT success * or master transition to prevent nonce stomping under packet flood. */ int join_pending; /* 1 once a valid session_key has been established - either generated at - * cluster bootstrap (cc_on_became_master) or adopted from the current master + * cluster bootstrap (cl_ctr_on_became_master) or adopted from the current master * master via KEY_GRANT / KEY_HANDOFF. A node must NOT act as master * (broadcast MASTER_ALIVE) while this is 0, or it would encrypt with an * underived key that no member can decrypt. */ int have_session_key; /* Master-side per-IP table tracking bootstrap-decrypt failures. - * Worker-local (no shm, no lock needed). After CC_JOIN_FAIL_LIMIT + * Worker-local (no shm, no lock needed). After CL_CTR_JOIN_FAIL_LIMIT * failures from the same source IP the master sends JOIN_REJECT. */ struct { uint32_t ip_num; - char ip[CC_MAX_IP_LEN + 1]; + char ip[CL_CTR_MAX_IP_LEN + 1]; int count; int rejected; /* 1 = JOIN_REJECT already sent; suppress repeats */ - } join_fail_tbl[CC_JOIN_FAIL_TABLE_SZ]; + } join_fail_tbl[CL_CTR_JOIN_FAIL_TABLE_SZ]; /* Joiner-side auth-failure detection - no lock needed (worker-local). */ int bootstrap_auth_fails; /* consecutive bootstrap decrypt failures - during CC_NODE_NEW; reset on KEY_GRANT */ + during CL_CTR_NODE_NEW; reset on KEY_GRANT */ int join_attempt_count; /* rejoin_tfd fires since last KEY_GRANT */ - /* Count of packets from OTHER peers during CC_NODE_NEW that we could not + /* Count of packets from OTHER peers during CL_CTR_NODE_NEW that we could not * decrypt (any magic). Non-zero means a cluster (or rogue) whose key we do * not share exists on the group - evidence we may have the wrong password. - * This is only *evidence*: cc_on_join_tfd never self-terminates on it + * This is only *evidence*: cl_ctr_on_join_tfd never self-terminates on it * immediately (that would let start-up noise or a flood kill a healthy * node); it defers and re-joins, and a KEY_GRANT resets this counter. Only * a correct-password joiner ever receives a KEY_GRANT, so persistence of @@ -528,16 +528,16 @@ typedef struct cc_cluster_ { * wrong-password / foreign-cluster condition. */ int auth_fail_pkts; /* Number of join rounds we have already deferred because we still could not - * authenticate. We only give up (shut down) after CC_JOIN_DEFER_MAX such + * authenticate. We only give up (shut down) after CL_CTR_JOIN_DEFER_MAX such * rounds, so a KEY_GRANT that is merely slow, or a brief burst of start-up * noise or crafted garbage, never self-terminates a correctly-configured * node. Reset on successful authentication. */ int auth_defer_count; /* master_salt lives in cl->peers->master_salt (shm) so mod_destroy can * read it. session_key is the worker-local derived key cache. */ - /* Per-source-IP rate limiter table - pkg_malloc'd in cc_worker after fork */ - cc_rate_entry_t *rate_tbl; - /* Last shtag decision this worker applied, so cc_apply_shtags_decision() + /* Per-source-IP rate limiter table - pkg_malloc'd in cl_ctr_worker after fork */ + cl_ctr_rate_entry_t *rate_tbl; + /* Last shtag decision this worker applied, so cl_ctr_apply_shtags_decision() * logs the *reason* only when the decision (or its cause) actually * changes - not on every idempotent re-apply. Worker-local. * shtag_last_active: -1 unknown, 0 backup, 1 active. @@ -545,7 +545,7 @@ typedef struct cc_cluster_ { int shtag_last_active; uint16_t shtag_last_forced; /* Counts MASTER_ALIVE ticks so a beacon is emitted every - * CC_MASTER_BEACON_EVERY of them. Worker-local (master path only). */ + * CL_CTR_MASTER_BEACON_EVERY of them. Worker-local (master path only). */ unsigned int beacon_tick; /* How many times we have deferred self-promotion at the join deadline * because a higher-IP node was also still joining (split-brain @@ -554,7 +554,7 @@ typedef struct cc_cluster_ { * for it rather than self-promoting into a divergent-key split brain). * Worker-local; reset once we leave the NEW state. */ int join_defer_count; - /* Total deferrals across resets - an absolute cap (CC_JOIN_DEFER_HARDMAX) + /* Total deferrals across resets - an absolute cap (CL_CTR_JOIN_DEFER_HARDMAX) * so a peer that keeps sending JOIN_REQ yet never becomes master cannot * defer us forever. Worker-local; reset once we leave the NEW state. */ int join_defer_total; @@ -563,26 +563,26 @@ typedef struct cc_cluster_ { * flood the group with JOIN_REQs. 0 = never sent. Worker-local. */ utime_t last_join_req_utime; /* 1 while our MASTER_ALIVE keepalive timer is armed (i.e. we are acting as - * master and broadcasting). Set by cc_arm_master_timers(). Lets - * cc_elect_master() enforce the invariant "keepalive armed <=> I am the + * master and broadcasting). Set by cl_ctr_arm_master_timers(). Lets + * cl_ctr_elect_master() enforce the invariant "keepalive armed <=> I am the * elected master": an election that demotes us (clears is_master) without * going through a yield/member-list path must still stop the keepalive, * otherwise a demoted node keeps broadcasting MASTER_ALIVE and lower-IP * peers oscillate between two masters. Worker-local. */ int master_ka_armed; - /* This node's last-known role in the cluster (enum cc_role). Compared in - * cc_elect_master() to emit a one-line transition log on every change. - * Zero-initialised to CC_ROLE_MEMBER, which matches a not-yet-joined node. + /* This node's last-known role in the cluster (enum cl_ctr_role). Compared in + * cl_ctr_elect_master() to emit a one-line transition log on every change. + * Zero-initialised to CL_CTR_ROLE_MEMBER, which matches a not-yet-joined node. * Worker-local. */ int my_role; -} cc_cluster_t; +} cl_ctr_cluster_t; -static cc_cluster_t cc_clusters[CC_MAX_CLUSTERS]; -static int cc_cluster_count = 0; +static cl_ctr_cluster_t cl_ctr_clusters[CL_CTR_MAX_CLUSTERS]; +static int cl_ctr_cluster_count = 0; /* Raw "cluster" strings collected during modparam parsing */ -static char *cc_cluster_strs[CC_MAX_CLUSTERS]; -static int cc_cluster_str_count = 0; +static char *cl_ctr_cluster_strs[CL_CTR_MAX_CLUSTERS]; +static int cl_ctr_cluster_str_count = 0; /* ========================================================================= * Module parameters @@ -592,7 +592,7 @@ static int cc_cluster_str_count = 0; static char *my_ip = NULL; /* explicit IP, or NULL for auto-detect */ static char *my_interface = NULL; /* explicit interface name, or NULL */ static int query_time = 5; -static char *password = CC_DEFAULT_PASSWORD; /* default; falls back per cluster */ +static char *password = CL_CTR_DEFAULT_PASSWORD; /* default; falls back per cluster */ /* Policy when a node's consistency-critical settings (manage_shtags/ * master_stickiness/query_time) differ from the running cluster (a master is @@ -602,16 +602,16 @@ static char *password = CC_DEFAULT_PASSWORD; /* default; falls back per * down with a clear message (default); * "adopt" - the node adopts the master's (authoritative) settings at * runtime and continues. */ -#define CC_CFGMISMATCH_WARN 0 -#define CC_CFGMISMATCH_REJECT 1 -#define CC_CFGMISMATCH_ADOPT 2 +#define CL_CTR_CFGMISMATCH_WARN 0 +#define CL_CTR_CFGMISMATCH_REJECT 1 +#define CL_CTR_CFGMISMATCH_ADOPT 2 /* JOIN_REJECT reason codes (1 byte after the target IP in the payload). */ -#define CC_REJECT_GENERIC 0 /* wrong password / unauthorized / table full */ -#define CC_REJECT_CONFIG 1 /* different cluster settings (reject policy) */ +#define CL_CTR_REJECT_GENERIC 0 /* wrong password / unauthorized / table full */ +#define CL_CTR_REJECT_CONFIG 1 /* different cluster settings (reject policy) */ static char *on_config_mismatch_s = NULL; /* raw modparam string */ -static int on_config_mismatch = CC_CFGMISMATCH_REJECT; /* resolved; default reject */ +static int on_config_mismatch = CL_CTR_CFGMISMATCH_REJECT; /* resolved; default reject */ -/* Resolved at mod_init time - always valid after cc_resolve_local_identity() */ +/* Resolved at mod_init time - always valid after cl_ctr_resolve_local_identity() */ static char my_ip_buf[INET_ADDRSTRLEN]; static char my_interface_buf[IF_NAMESIZE]; @@ -632,36 +632,36 @@ static int manage_shtags = 1; * 0 = not sticky - pure highest-IP election, so a higher-IP node * takes over as master as soon as it appears (more handovers). */ static int master_stickiness = 1; -static char my_bin_sockets[CC_MAX_BIN_SOCKETS][CC_MAX_BIN_SOCK_LEN]; +static char my_bin_sockets[CL_CTR_MAX_BIN_SOCKETS][CL_CTR_MAX_BIN_SOCK_LEN]; static int my_bin_count = 0; /** - * cc_add_cluster_param() - collect "cluster" modparam strings. + * cl_ctr_add_cluster_param() - collect "cluster" modparam strings. * Actual parsing happens in mod_init() after all params are set. */ -static int cc_add_cluster_param(modparam_t type, void *val) +static int cl_ctr_add_cluster_param(modparam_t type, void *val) { - if (cc_cluster_str_count >= CC_MAX_CLUSTERS) { + if (cl_ctr_cluster_str_count >= CL_CTR_MAX_CLUSTERS) { LM_ERR("clusterer_controller: too many clusters (max %d)\n", - CC_MAX_CLUSTERS); + CL_CTR_MAX_CLUSTERS); return -1; } { size_t _len = strlen((char *)val) + 1; - cc_cluster_strs[cc_cluster_str_count] = pkg_malloc(_len); - if (!cc_cluster_strs[cc_cluster_str_count]) { + cl_ctr_cluster_strs[cl_ctr_cluster_str_count] = pkg_malloc(_len); + if (!cl_ctr_cluster_strs[cl_ctr_cluster_str_count]) { LM_ERR("clusterer_controller: pkg_malloc failed\n"); return -1; } - memcpy(cc_cluster_strs[cc_cluster_str_count], (char *)val, _len); + memcpy(cl_ctr_cluster_strs[cl_ctr_cluster_str_count], (char *)val, _len); } - cc_cluster_str_count++; + cl_ctr_cluster_str_count++; return 0; } static const param_export_t params[] = { - {"cluster", STR_PARAM | USE_FUNC_PARAM, (void *)cc_add_cluster_param}, + {"cluster", STR_PARAM | USE_FUNC_PARAM, (void *)cl_ctr_add_cluster_param}, {"my_ip", STR_PARAM, &my_ip}, {"interface", STR_PARAM, &my_interface}, {"query_time", INT_PARAM, &query_time}, @@ -676,8 +676,8 @@ static const param_export_t params[] = { * Peer table (shared memory) * ========================================================================= */ -typedef struct cc_peer_ { - char ip[CC_MAX_IP_LEN + 1]; +typedef struct cl_ctr_peer_ { + char ip[CL_CTR_MAX_IP_LEN + 1]; unsigned int ip_num; time_t last_seen; int is_master; @@ -685,8 +685,8 @@ typedef struct cc_peer_ { int in_election; /* 1 = currently inside the election window */ uint16_t node_id; /* allocated by master; 0 = not yet assigned */ uint8_t bin_count; /* number of BIN listeners reported */ - char bin_sockets[CC_MAX_BIN_SOCKETS][CC_MAX_BIN_SOCK_LEN]; - unsigned char pubkey[CC_PUBKEY_SZ]; /* long-lived X25519 pubkey (from ALIVE); + char bin_sockets[CL_CTR_MAX_BIN_SOCKETS][CL_CTR_MAX_BIN_SOCK_LEN]; + unsigned char pubkey[CL_CTR_PUBKEY_SZ]; /* long-lived X25519 pubkey (from ALIVE); zero if unknown; used for KEY_HANDOFF */ uint32_t last_seq; /* highest seq accepted from this peer */ /* Peer's advertised consistency-critical config (from ALIVE), used to warn @@ -697,21 +697,21 @@ typedef struct cc_peer_ { int cfg_master_stickiness; int cfg_query_time; int cfg_warned; -} cc_peer_t; +} cl_ctr_peer_t; -struct cc_peers_ { - cc_peer_t entries[CC_MAX_PEERS]; +struct cl_ctr_peers_ { + cl_ctr_peer_t entries[CL_CTR_MAX_PEERS]; int count; /* rw_lock_t allows concurrent readers (MI, future script functions) - * while still serialising the single writer (cc_worker). */ + * while still serialising the single writer (cl_ctr_worker). */ rw_lock_t *lock; - cc_node_state_t node_state; + cl_ctr_node_state_t node_state; time_t join_deadline; /* last elected master IP - used to detect and log master changes */ - char last_master[CC_MAX_IP_LEN + 1]; + char last_master[CL_CTR_MAX_IP_LEN + 1]; /* master_salt: generated by each new master, shared here so mod_destroy * (running in main process) can derive session_key for GOODBYE. */ - unsigned char master_salt[CC_MASTER_SALT_SZ]; + unsigned char master_salt[CL_CTR_MASTER_SALT_SZ]; /* my_seq: monotonic send counter; in shm so mod_destroy can use it for * GOODBYE without needing the worker's private state. Reset to 0 on * every session key rotation so last_seq counters reset cleanly. */ @@ -721,7 +721,7 @@ struct cc_peers_ { * cluster (cl_ctr_shtag_force MI), suspending automatic allocation until * cl_ctr_shtag_auto clears it. Propagated to all nodes in the MEMBER_LIST. */ uint16_t shtag_forced_node_id; - /* worker_proc_no: OpenSIPS process index of this cluster's cc_worker, + /* worker_proc_no: OpenSIPS process index of this cluster's cl_ctr_worker, * published here (shm) after fork so MI handlers running in a different * process can target the worker with ipc_send_rpc(). -1 until set. */ int worker_proc_no; @@ -741,7 +741,7 @@ struct cc_peers_ { * ========================================================================= */ /* Drain the expiration counter so the fd stops being readable. */ -static void cc_drain_tfd(int tfd) +static void cl_ctr_drain_tfd(int tfd) { uint64_t exp; if (read(tfd, &exp, sizeof(exp)) < 0 && errno != EAGAIN) @@ -749,7 +749,7 @@ static void cc_drain_tfd(int tfd) } /* Arm a timerfd. Pass sec_value=0 to disarm. */ -static void cc_arm_tfd(int tfd, time_t sec_value, time_t sec_interval) +static void cl_ctr_arm_tfd(int tfd, time_t sec_value, time_t sec_interval) { struct itimerspec its; memset(&its, 0, sizeof(its)); @@ -764,33 +764,33 @@ static void cc_arm_tfd(int tfd, time_t sec_value, time_t sec_interval) * ========================================================================= */ static int mod_init(void); -static int cc_child_init(int rank); +static int cl_ctr_child_init(int rank); static void mod_destroy(void); -static void cc_worker(int rank); -static int cc_on_sock(int fd, void *param, int was_timeout); -static int cc_on_alive_tfd(int fd, void *param, int was_timeout); -static void cc_arm_master_timers(cc_cluster_t *cl, int i_am_master); -static int cc_on_join_tfd(int fd, void *param, int was_timeout); -static int cc_on_rejoin_tfd(int fd, void *param, int was_timeout); -static int cc_on_master_alive_tfd(int fd, void *param, int was_timeout); -static int cc_on_master_dead_tfd(int fd, void *param, int was_timeout); +static void cl_ctr_worker(int rank); +static int cl_ctr_on_sock(int fd, void *param, int was_timeout); +static int cl_ctr_on_alive_tfd(int fd, void *param, int was_timeout); +static void cl_ctr_arm_master_timers(cl_ctr_cluster_t *cl, int i_am_master); +static int cl_ctr_on_join_tfd(int fd, void *param, int was_timeout); +static int cl_ctr_on_rejoin_tfd(int fd, void *param, int was_timeout); +static int cl_ctr_on_master_alive_tfd(int fd, void *param, int was_timeout); +static int cl_ctr_on_master_dead_tfd(int fd, void *param, int was_timeout); static mi_response_t *mi_cl_ctr_members(const mi_params_t *params, struct mi_handler *hdl); -static void cc_handle_member_list(const char *payload, int payload_len, - const char *sender_ip, cc_cluster_t *cl); -static void cc_handle_join_req(int sock, const char *payload, int payload_len, - cc_cluster_t *cl); -static void cc_handle_node_assign(const char *payload, int payload_len, - const char *sender_ip, cc_cluster_t *cl); -static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl); -static void cc_handle_master_alive(const char *sender_ip, cc_cluster_t *cl); -static void cc_handle_key_grant(const char *payload, int payload_len, - const char *sender_ip, cc_cluster_t *cl); -static void cc_handle_key_handoff(const char *payload, int payload_len, - const char *sender_ip, cc_cluster_t *cl); -static void cc_handle_join_reject(const char *payload, int payload_len, - const char *sender_ip, cc_cluster_t *cl); -static void cc_send_join_reject(int sock, const char *target_ip, cc_cluster_t *cl, +static void cl_ctr_handle_member_list(const char *payload, int payload_len, + const char *sender_ip, cl_ctr_cluster_t *cl); +static void cl_ctr_handle_join_req(int sock, const char *payload, int payload_len, + cl_ctr_cluster_t *cl); +static void cl_ctr_handle_node_assign(const char *payload, int payload_len, + const char *sender_ip, cl_ctr_cluster_t *cl); +static void cl_ctr_handle_goodbye(int sock, const char *src_ip, cl_ctr_cluster_t *cl); +static void cl_ctr_handle_master_alive(const char *sender_ip, cl_ctr_cluster_t *cl); +static void cl_ctr_handle_key_grant(const char *payload, int payload_len, + const char *sender_ip, cl_ctr_cluster_t *cl); +static void cl_ctr_handle_key_handoff(const char *payload, int payload_len, + const char *sender_ip, cl_ctr_cluster_t *cl); +static void cl_ctr_handle_join_reject(const char *payload, int payload_len, + const char *sender_ip, cl_ctr_cluster_t *cl); +static void cl_ctr_send_join_reject(int sock, const char *target_ip, cl_ctr_cluster_t *cl, int reason); static mi_response_t *mi_cl_ctr_node_info(const mi_params_t *params, struct mi_handler *hdl); @@ -806,7 +806,7 @@ static mi_response_t *mi_cl_ctr_shtag_auto(const mi_params_t *params, * ========================================================================= */ static proc_export_t procs[] = { - {"clusterer_controller worker", 0, 0, cc_worker, 1, + {"clusterer_controller worker", 0, 0, cl_ctr_worker, 1, PROC_FLAG_INITCHILD | PROC_FLAG_HAS_IPC}, {0, 0, 0, 0, 0, 0} }; @@ -891,20 +891,20 @@ static const mi_export_t mi_cmds[] = { * no setter is exported. * ========================================================================= */ -enum cc_pv_field { - CC_PV_ROLE, /* master | backup | member | joining */ - CC_PV_IS_MASTER, /* 1 / 0 */ - CC_PV_MASTER_IP, /* current master's IP, NULL if none */ - CC_PV_BACKUP_IP, /* current backup's IP, NULL if none */ - CC_PV_NODE_ID, /* this node's id in the cluster, NULL if unassigned */ - CC_PV_MY_IP, /* controller identity IP */ - CC_PV_MEMBERS, /* live member count */ - CC_PV_SHTAG_MODE, /* auto | forced */ - CC_PV_FORCED_NODE, /* node pinned by cl_ctr_shtag_force, NULL if auto */ +enum cl_ctr_pv_field { + CL_CTR_PV_ROLE, /* master | backup | member | joining */ + CL_CTR_PV_IS_MASTER, /* 1 / 0 */ + CL_CTR_PV_MASTER_IP, /* current master's IP, NULL if none */ + CL_CTR_PV_BACKUP_IP, /* current backup's IP, NULL if none */ + CL_CTR_PV_NODE_ID, /* this node's id in the cluster, NULL if unassigned */ + CL_CTR_PV_MY_IP, /* controller identity IP */ + CL_CTR_PV_MEMBERS, /* live member count */ + CL_CTR_PV_SHTAG_MODE, /* auto | forced */ + CL_CTR_PV_FORCED_NODE, /* node pinned by cl_ctr_shtag_force, NULL if auto */ }; /* Optional (cluster_id) argument; bare form leaves the spec zeroed (cid 0). */ -static int cc_pv_parse_cluster(pv_spec_p sp, const str *in) +static int cl_ctr_pv_parse_cluster(pv_spec_p sp, const str *in) { unsigned int cid; str s; @@ -934,34 +934,34 @@ static int cc_pv_parse_cluster(pv_spec_p sp, const str *in) return 0; } -static int cc_pv_get(struct sip_msg *msg, pv_param_t *param, pv_value_t *res, - enum cc_pv_field field) +static int cl_ctr_pv_get(struct sip_msg *msg, pv_param_t *param, pv_value_t *res, + enum cl_ctr_pv_field field) { - static char cc_pv_ipbuf[INET_ADDRSTRLEN]; - static int cc_pv_warned_ambiguous; - cc_cluster_t *cl = NULL; - cc_peer_t *me = NULL, *master = NULL, *backup = NULL; + static char cl_ctr_pv_ipbuf[INET_ADDRSTRLEN]; + static int cl_ctr_pv_warned_ambiguous; + cl_ctr_cluster_t *cl = NULL; + cl_ctr_peer_t *me = NULL, *master = NULL, *backup = NULL; const char *sval = NULL; int cid, i, have_int = 0, ival = 0, joining; cid = param->pvn.u.isname.name.n; if (cid == 0) { - if (cc_cluster_count == 1) { - cl = &cc_clusters[0]; + if (cl_ctr_cluster_count == 1) { + cl = &cl_ctr_clusters[0]; } else { - if (!cc_pv_warned_ambiguous) { + if (!cl_ctr_pv_warned_ambiguous) { LM_WARN("clusterer_controller: bare $cl_ctr_* used with %d " "clusters configured - specify the cluster id, e.g. " - "$cl_ctr_role(%d)\n", cc_cluster_count, - cc_cluster_count ? cc_clusters[0].cluster_id : 1); - cc_pv_warned_ambiguous = 1; + "$cl_ctr_role(%d)\n", cl_ctr_cluster_count, + cl_ctr_cluster_count ? cl_ctr_clusters[0].cluster_id : 1); + cl_ctr_pv_warned_ambiguous = 1; } return pv_get_null(msg, param, res); } } else { - for (i = 0; i < cc_cluster_count; i++) - if (cc_clusters[i].cluster_id == cid) { - cl = &cc_clusters[i]; + for (i = 0; i < cl_ctr_cluster_count; i++) + if (cl_ctr_clusters[i].cluster_id == cid) { + cl = &cl_ctr_clusters[i]; break; } } @@ -970,9 +970,9 @@ static int cc_pv_get(struct sip_msg *msg, pv_param_t *param, pv_value_t *res, lock_start_read(cl->peers->lock); - joining = (cl->peers->node_state == CC_NODE_NEW); + joining = (cl->peers->node_state == CL_CTR_NODE_NEW); for (i = 0; i < cl->peers->count; i++) { - cc_peer_t *e = &cl->peers->entries[i]; + cl_ctr_peer_t *e = &cl->peers->entries[i]; if (e->is_master) master = e; if (e->is_backup) @@ -982,7 +982,7 @@ static int cc_pv_get(struct sip_msg *msg, pv_param_t *param, pv_value_t *res, } switch (field) { - case CC_PV_ROLE: + case CL_CTR_PV_ROLE: if (joining) sval = "joining"; else if (me && me->is_master) @@ -992,41 +992,41 @@ static int cc_pv_get(struct sip_msg *msg, pv_param_t *param, pv_value_t *res, else sval = "member"; break; - case CC_PV_IS_MASTER: + case CL_CTR_PV_IS_MASTER: have_int = 1; ival = (!joining && me && me->is_master) ? 1 : 0; break; - case CC_PV_MASTER_IP: + case CL_CTR_PV_MASTER_IP: if (master) { - strncpy(cc_pv_ipbuf, master->ip, sizeof(cc_pv_ipbuf) - 1); - cc_pv_ipbuf[sizeof(cc_pv_ipbuf) - 1] = '\0'; - sval = cc_pv_ipbuf; + strncpy(cl_ctr_pv_ipbuf, master->ip, sizeof(cl_ctr_pv_ipbuf) - 1); + cl_ctr_pv_ipbuf[sizeof(cl_ctr_pv_ipbuf) - 1] = '\0'; + sval = cl_ctr_pv_ipbuf; } break; - case CC_PV_BACKUP_IP: + case CL_CTR_PV_BACKUP_IP: if (backup) { - strncpy(cc_pv_ipbuf, backup->ip, sizeof(cc_pv_ipbuf) - 1); - cc_pv_ipbuf[sizeof(cc_pv_ipbuf) - 1] = '\0'; - sval = cc_pv_ipbuf; + strncpy(cl_ctr_pv_ipbuf, backup->ip, sizeof(cl_ctr_pv_ipbuf) - 1); + cl_ctr_pv_ipbuf[sizeof(cl_ctr_pv_ipbuf) - 1] = '\0'; + sval = cl_ctr_pv_ipbuf; } break; - case CC_PV_NODE_ID: + case CL_CTR_PV_NODE_ID: if (me && me->node_id > 0) { have_int = 1; ival = me->node_id; } break; - case CC_PV_MY_IP: + case CL_CTR_PV_MY_IP: sval = my_ip; /* resolved in mod_init, constant afterwards */ break; - case CC_PV_MEMBERS: + case CL_CTR_PV_MEMBERS: have_int = 1; ival = cl->peers->count; break; - case CC_PV_SHTAG_MODE: + case CL_CTR_PV_SHTAG_MODE: sval = cl->peers->shtag_forced_node_id ? "forced" : "auto"; break; - case CC_PV_FORCED_NODE: + case CL_CTR_PV_FORCED_NODE: if (cl->peers->shtag_forced_node_id) { have_int = 1; ival = cl->peers->shtag_forced_node_id; @@ -1045,9 +1045,9 @@ static int cc_pv_get(struct sip_msg *msg, pv_param_t *param, pv_value_t *res, return pv_get_null(msg, param, res); } -#define CC_PV_WRAP(_fn, _field) \ +#define CL_CTR_PV_WRAP(_fn, _field) \ static int _fn(struct sip_msg *msg, pv_param_t *param, pv_value_t *res) \ -{ return cc_pv_get(msg, param, res, _field); } +{ return cl_ctr_pv_get(msg, param, res, _field); } /* ------------------------------------------------------------------------- * Per-peer lookups: query a *specific* node in a cluster. These are script @@ -1060,19 +1060,19 @@ static int _fn(struct sip_msg *msg, pv_param_t *param, pv_value_t *res) \ /* Find peer (cluster_id, node_id); cid<=0 => the sole configured cluster. * Returns 0 and fills the requested out params on success, -1 if the cluster * or node is unknown. Caller must not hold the peers lock. */ -static int cc_find_peer(int cid, int nid, int *is_master, int *is_backup, +static int cl_ctr_find_peer(int cid, int nid, int *is_master, int *is_backup, char *ipbuf, int ipbuf_sz) { - cc_cluster_t *cl = NULL; + cl_ctr_cluster_t *cl = NULL; int i, rc = -1; if (cid <= 0) { - if (cc_cluster_count == 1) - cl = &cc_clusters[0]; + if (cl_ctr_cluster_count == 1) + cl = &cl_ctr_clusters[0]; } else { - for (i = 0; i < cc_cluster_count; i++) - if (cc_clusters[i].cluster_id == cid) { - cl = &cc_clusters[i]; + for (i = 0; i < cl_ctr_cluster_count; i++) + if (cl_ctr_clusters[i].cluster_id == cid) { + cl = &cl_ctr_clusters[i]; break; } } @@ -1081,7 +1081,7 @@ static int cc_find_peer(int cid, int nid, int *is_master, int *is_backup, lock_start_read(cl->peers->lock); for (i = 0; i < cl->peers->count; i++) { - cc_peer_t *e = &cl->peers->entries[i]; + cl_ctr_peer_t *e = &cl->peers->entries[i]; if (e->node_id != nid) continue; if (is_master) *is_master = e->is_master; @@ -1097,7 +1097,7 @@ static int cc_find_peer(int cid, int nid, int *is_master, int *is_backup, return rc; } -static int cc_out_str(struct sip_msg *msg, pv_spec_t *out, const char *str_s) +static int cl_ctr_out_str(struct sip_msg *msg, pv_spec_t *out, const char *str_s) { pv_value_t val; memset(&val, 0, sizeof val); @@ -1111,7 +1111,7 @@ static int cc_out_str(struct sip_msg *msg, pv_spec_t *out, const char *str_s) static int w_cl_ctr_node_is_master(struct sip_msg *msg, int *cid, int *nid) { int im = 0; - if (cc_find_peer(cid ? *cid : 0, nid ? *nid : 0, &im, NULL, NULL, 0) < 0) + if (cl_ctr_find_peer(cid ? *cid : 0, nid ? *nid : 0, &im, NULL, NULL, 0) < 0) return -1; return im ? 1 : -1; } @@ -1119,7 +1119,7 @@ static int w_cl_ctr_node_is_master(struct sip_msg *msg, int *cid, int *nid) /* cl_ctr_node_present(cluster_id, node_id) -> true if node_id is a live member */ static int w_cl_ctr_node_present(struct sip_msg *msg, int *cid, int *nid) { - return cc_find_peer(cid ? *cid : 0, nid ? *nid : 0, NULL, NULL, NULL, 0) == 0 + return cl_ctr_find_peer(cid ? *cid : 0, nid ? *nid : 0, NULL, NULL, NULL, 0) == 0 ? 1 : -1; } @@ -1128,9 +1128,9 @@ static int w_cl_ctr_get_node_role(struct sip_msg *msg, int *cid, int *nid, pv_spec_t *out) { int im = 0, ib = 0; - if (cc_find_peer(cid ? *cid : 0, nid ? *nid : 0, &im, &ib, NULL, 0) < 0) + if (cl_ctr_find_peer(cid ? *cid : 0, nid ? *nid : 0, &im, &ib, NULL, 0) < 0) return -1; - return cc_out_str(msg, out, im ? "master" : (ib ? "backup" : "member")) == 0 + return cl_ctr_out_str(msg, out, im ? "master" : (ib ? "backup" : "member")) == 0 ? 1 : -1; } @@ -1139,46 +1139,46 @@ static int w_cl_ctr_get_node_ip(struct sip_msg *msg, int *cid, int *nid, pv_spec_t *out) { char ipbuf[INET_ADDRSTRLEN]; - if (cc_find_peer(cid ? *cid : 0, nid ? *nid : 0, NULL, NULL, + if (cl_ctr_find_peer(cid ? *cid : 0, nid ? *nid : 0, NULL, NULL, ipbuf, sizeof ipbuf) < 0) return -1; - return cc_out_str(msg, out, ipbuf) == 0 ? 1 : -1; + return cl_ctr_out_str(msg, out, ipbuf) == 0 ? 1 : -1; } -CC_PV_WRAP(cc_pv_role, CC_PV_ROLE) -CC_PV_WRAP(cc_pv_is_master, CC_PV_IS_MASTER) -CC_PV_WRAP(cc_pv_master_ip, CC_PV_MASTER_IP) -CC_PV_WRAP(cc_pv_backup_ip, CC_PV_BACKUP_IP) -CC_PV_WRAP(cc_pv_node_id, CC_PV_NODE_ID) -CC_PV_WRAP(cc_pv_my_ip, CC_PV_MY_IP) -CC_PV_WRAP(cc_pv_members, CC_PV_MEMBERS) -CC_PV_WRAP(cc_pv_shtag_mode, CC_PV_SHTAG_MODE) -CC_PV_WRAP(cc_pv_forced_node, CC_PV_FORCED_NODE) - -static const pv_export_t cc_mod_vars[] = { +CL_CTR_PV_WRAP(cl_ctr_pv_role, CL_CTR_PV_ROLE) +CL_CTR_PV_WRAP(cl_ctr_pv_is_master, CL_CTR_PV_IS_MASTER) +CL_CTR_PV_WRAP(cl_ctr_pv_master_ip, CL_CTR_PV_MASTER_IP) +CL_CTR_PV_WRAP(cl_ctr_pv_backup_ip, CL_CTR_PV_BACKUP_IP) +CL_CTR_PV_WRAP(cl_ctr_pv_node_id, CL_CTR_PV_NODE_ID) +CL_CTR_PV_WRAP(cl_ctr_pv_my_ip, CL_CTR_PV_MY_IP) +CL_CTR_PV_WRAP(cl_ctr_pv_members, CL_CTR_PV_MEMBERS) +CL_CTR_PV_WRAP(cl_ctr_pv_shtag_mode, CL_CTR_PV_SHTAG_MODE) +CL_CTR_PV_WRAP(cl_ctr_pv_forced_node, CL_CTR_PV_FORCED_NODE) + +static const pv_export_t cl_ctr_mod_vars[] = { { {"cl_ctr_role", sizeof("cl_ctr_role")-1}, 1100, - cc_pv_role, 0, cc_pv_parse_cluster, 0, 0, 0 }, + cl_ctr_pv_role, 0, cl_ctr_pv_parse_cluster, 0, 0, 0 }, { {"cl_ctr_is_master", sizeof("cl_ctr_is_master")-1}, 1101, - cc_pv_is_master, 0, cc_pv_parse_cluster, 0, 0, 0 }, + cl_ctr_pv_is_master, 0, cl_ctr_pv_parse_cluster, 0, 0, 0 }, { {"cl_ctr_master_ip", sizeof("cl_ctr_master_ip")-1}, 1102, - cc_pv_master_ip, 0, cc_pv_parse_cluster, 0, 0, 0 }, + cl_ctr_pv_master_ip, 0, cl_ctr_pv_parse_cluster, 0, 0, 0 }, { {"cl_ctr_backup_ip", sizeof("cl_ctr_backup_ip")-1}, 1103, - cc_pv_backup_ip, 0, cc_pv_parse_cluster, 0, 0, 0 }, + cl_ctr_pv_backup_ip, 0, cl_ctr_pv_parse_cluster, 0, 0, 0 }, { {"cl_ctr_node_id", sizeof("cl_ctr_node_id")-1}, 1104, - cc_pv_node_id, 0, cc_pv_parse_cluster, 0, 0, 0 }, + cl_ctr_pv_node_id, 0, cl_ctr_pv_parse_cluster, 0, 0, 0 }, { {"cl_ctr_my_ip", sizeof("cl_ctr_my_ip")-1}, 1105, - cc_pv_my_ip, 0, cc_pv_parse_cluster, 0, 0, 0 }, + cl_ctr_pv_my_ip, 0, cl_ctr_pv_parse_cluster, 0, 0, 0 }, { {"cl_ctr_members", sizeof("cl_ctr_members")-1}, 1106, - cc_pv_members, 0, cc_pv_parse_cluster, 0, 0, 0 }, + cl_ctr_pv_members, 0, cl_ctr_pv_parse_cluster, 0, 0, 0 }, { {"cl_ctr_shtag_mode", sizeof("cl_ctr_shtag_mode")-1}, 1107, - cc_pv_shtag_mode, 0, cc_pv_parse_cluster, 0, 0, 0 }, + cl_ctr_pv_shtag_mode, 0, cl_ctr_pv_parse_cluster, 0, 0, 0 }, { {"cl_ctr_forced_node", sizeof("cl_ctr_forced_node")-1}, 1108, - cc_pv_forced_node, 0, cc_pv_parse_cluster, 0, 0, 0 }, + cl_ctr_pv_forced_node, 0, cl_ctr_pv_parse_cluster, 0, 0, 0 }, { {0, 0}, 0, 0, 0, 0, 0, 0, 0 } }; /* Script functions: per-peer lookups (two args -> functions, not variables). */ -static const cmd_export_t cc_cmds[] = { +static const cmd_export_t cl_ctr_cmds[] = { {"cl_ctr_node_is_master", (cmd_function)w_cl_ctr_node_is_master, { {CMD_PARAM_INT, 0, 0}, {CMD_PARAM_INT, 0, 0}, {0, 0, 0}}, ALL_ROUTES}, {"cl_ctr_node_present", (cmd_function)w_cl_ctr_node_present, { @@ -1201,19 +1201,19 @@ struct module_exports exports = { DEFAULT_DLFLAGS, 0, &deps, - cc_cmds, /* cmds - per-peer lookup functions */ + cl_ctr_cmds, /* cmds - per-peer lookup functions */ 0, /* acmds */ params, 0, /* stats */ mi_cmds, - cc_mod_vars, /* pvs - read-only $cl_ctr_* variables */ + cl_ctr_mod_vars, /* pvs - read-only $cl_ctr_* variables */ 0, /* transforms */ procs, 0, /* pre_init_f */ mod_init, 0, /* response_f */ mod_destroy, - cc_child_init, /* child_init_f */ + cl_ctr_child_init, /* child_init_f */ 0 /* reload_confirm_f */ }; @@ -1230,21 +1230,21 @@ static unsigned int ip_to_num(const char *ip) } /** - * cc_election_cutoff() - quantized stale cutoff for master election. + * cl_ctr_election_cutoff() - quantized stale cutoff for master election. * * All NTP-synchronized nodes compute the same value at the same second, * so they always evaluate the identical eligible-peer set and elect the * same master. */ -static time_t cc_election_cutoff(void) +static time_t cl_ctr_election_cutoff(void) { time_t now = time(NULL); return (now / (time_t)query_time) * (time_t)query_time - - (time_t)(query_time * CC_ELECT_FACTOR); + - (time_t)(query_time * CL_CTR_ELECT_FACTOR); } /** - * cc_elect_master(cl) - mark the peer with the highest IP as master. + * cl_ctr_elect_master(cl) - mark the peer with the highest IP as master. * * Uses the quantized election window so all NTP-synchronized nodes evaluate * the same eligible set and elect the same master. @@ -1253,33 +1253,33 @@ static time_t cc_election_cutoff(void) * * in_election 1->0 A peer's last_seen fell outside the election window - * the node is considered down. Logged immediately so the - * operator sees the event without waiting for cc_prune_stale(cl) - * (which only fires at CC_PURGE_FACTOR x query_time). + * operator sees the event without waiting for cl_ctr_prune_stale(cl) + * (which only fires at CL_CTR_PURGE_FACTOR x query_time). * * last_master The elected master IP changed - either because the * previous master went down, or a higher-IP node joined. * * Must be called with cl->peers->lock held. */ -static void cc_elect_master(cc_cluster_t *cl) +static void cl_ctr_elect_master(cl_ctr_cluster_t *cl) { - time_t cutoff = cc_election_cutoff(); + time_t cutoff = cl_ctr_election_cutoff(); unsigned int top_num = 0; int i, n_in = 0, top_idx = -1, cur_master_idx = -1, master_idx = -1; int i_am_elected = 0; - char prev_master[CC_MAX_IP_LEN + 1]; - char prev_backup[CC_MAX_IP_LEN + 1]; + char prev_master[CL_CTR_MAX_IP_LEN + 1]; + char prev_backup[CL_CTR_MAX_IP_LEN + 1]; prev_backup[0] = '\0'; /* Snapshot the master we had before this election, for change reporting. */ { - size_t _l = strnlen(cl->peers->last_master, CC_MAX_IP_LEN); + size_t _l = strnlen(cl->peers->last_master, CL_CTR_MAX_IP_LEN); memcpy(prev_master, cl->peers->last_master, _l); prev_master[_l] = '\0'; } for (i = 0; i < cl->peers->count; i++) { - cc_peer_t *e = &cl->peers->entries[i]; + cl_ctr_peer_t *e = &cl->peers->entries[i]; int now_in = (e->last_seen >= cutoff); /* Detect peer dropping out of the election window */ @@ -1295,7 +1295,7 @@ static void cc_elect_master(cc_cluster_t *cl) if (e->is_master && now_in) cur_master_idx = i; if (e->is_backup) { - size_t _l = strnlen(e->ip, CC_MAX_IP_LEN); + size_t _l = strnlen(e->ip, CL_CTR_MAX_IP_LEN); memcpy(prev_backup, e->ip, _l); prev_backup[_l] = '\0'; } @@ -1319,7 +1319,7 @@ static void cc_elect_master(cc_cluster_t *cl) * master_stickiness == 0: pure highest-IP election - the highest-IP peer * always wins, preempting any lower-IP current master. * Split-brain (two live masters, e.g. after a partition heal) is resolved - * separately in cc_handle_master_alive by yielding to the highest IP. */ + * separately in cl_ctr_handle_master_alive by yielding to the highest IP. */ if (cl->master_stickiness == 1 && cur_master_idx >= 0) master_idx = cur_master_idx; else @@ -1337,10 +1337,10 @@ static void cc_elect_master(cc_cluster_t *cl) /* Designate the BACKUP: the highest-IP in-window peer that is not the * master. Deterministic across all nodes, so everyone agrees who takes - * over next; on master failure cc_elect_master (no live current master) + * over next; on master failure cl_ctr_elect_master (no live current master) * promotes exactly this node. */ for (j = 0; j < cl->peers->count; j++) { - cc_peer_t *e = &cl->peers->entries[j]; + cl_ctr_peer_t *e = &cl->peers->entries[j]; if (j == master_idx || !e->in_election) continue; if (e->ip_num > b_num) { @@ -1357,7 +1357,7 @@ static void cc_elect_master(cc_cluster_t *cl) /* Persist the elected master for the next round / other handlers. */ { - size_t _l = strnlen(m_ip, CC_MAX_IP_LEN); + size_t _l = strnlen(m_ip, CL_CTR_MAX_IP_LEN); memcpy(cl->peers->last_master, m_ip, _l); cl->peers->last_master[_l] = '\0'; } @@ -1390,22 +1390,22 @@ static void cc_elect_master(cc_cluster_t *cl) * it changes, so a failover or a peer (re)joining that demotes/promotes * us gets its own line rather than being implied by the roles list. */ { - int my_new_role = i_am_elected ? CC_ROLE_MASTER : - ((b_ip && strcmp(b_ip, my_ip) == 0) ? CC_ROLE_BACKUP : - CC_ROLE_MEMBER); + int my_new_role = i_am_elected ? CL_CTR_ROLE_MASTER : + ((b_ip && strcmp(b_ip, my_ip) == 0) ? CL_CTR_ROLE_BACKUP : + CL_CTR_ROLE_MEMBER); if (my_new_role != cl->my_role) { LM_INFO("clusterer_controller: [cluster %d] my role changed: " "%s -> %s\n", cl->cluster_id, - cc_role_name(cl->my_role), cc_role_name(my_new_role)); + cl_ctr_role_name(cl->my_role), cl_ctr_role_name(my_new_role)); cl->my_role = my_new_role; } } } else { /* No eligible peer - cluster has no master; we are a plain member. */ - if (cl->my_role != CC_ROLE_MEMBER) { + if (cl->my_role != CL_CTR_ROLE_MEMBER) { LM_INFO("clusterer_controller: [cluster %d] my role changed: %s -> " - "member\n", cl->cluster_id, cc_role_name(cl->my_role)); - cl->my_role = CC_ROLE_MEMBER; + "member\n", cl->cluster_id, cl_ctr_role_name(cl->my_role)); + cl->my_role = CL_CTR_ROLE_MEMBER; } if (cl->peers->last_master[0] != '\0') { LM_INFO("clusterer_controller: [cluster %d] master lost (%s), " @@ -1421,19 +1421,19 @@ static void cc_elect_master(cc_cluster_t *cl) * broadcasting MASTER_ALIVE as a phantom master and lower-IP peers * oscillate between two masters. Only disarm here: promoting a new master * (arming) is done by the became-master paths, which also establish the - * session key. cc_arm_master_timers() only issues timerfd syscalls, so it + * session key. cl_ctr_arm_master_timers() only issues timerfd syscalls, so it * is safe under cl->peers->lock. */ if (!i_am_elected && cl->master_ka_armed) - cc_arm_master_timers(cl, 0); + cl_ctr_arm_master_timers(cl, 0); } /** - * cc_i_am_master_locked(cl) - return 1 if my_ip is currently elected master. + * cl_ctr_i_am_master_locked(cl) - return 1 if my_ip is currently elected master. * Must be called with cl->peers->lock held. */ -static int cc_i_am_master_locked(cc_cluster_t *cl) +static int cl_ctr_i_am_master_locked(cl_ctr_cluster_t *cl) { - time_t cutoff = cc_election_cutoff(); + time_t cutoff = cl_ctr_election_cutoff(); int i; for (i = 0; i < cl->peers->count; i++) { @@ -1446,7 +1446,7 @@ static int cc_i_am_master_locked(cc_cluster_t *cl) } /** - * cc_ip_beats_master_locked() - check whether a candidate IP would displace + * cl_ctr_ip_beats_master_locked() - check whether a candidate IP would displace * the current master in an election. * * Returns 1 (re-election is worth running) when: @@ -1458,9 +1458,9 @@ static int cc_i_am_master_locked(cc_cluster_t *cl) * * Must be called with cl->peers->lock held. */ -static int cc_ip_beats_master_locked(unsigned int ip_num, cc_cluster_t *cl) +static int cl_ctr_ip_beats_master_locked(unsigned int ip_num, cl_ctr_cluster_t *cl) { - time_t cutoff = cc_election_cutoff(); + time_t cutoff = cl_ctr_election_cutoff(); int i; for (i = 0; i < cl->peers->count; i++) { @@ -1472,7 +1472,7 @@ static int cc_ip_beats_master_locked(unsigned int ip_num, cc_cluster_t *cl) } /** - * cc_apply_shtags_decision() - (de)activate this node's sharing tags per policy. + * cl_ctr_apply_shtags_decision() - (de)activate this node's sharing tags per policy. * * Decides whether THIS node should be the active sharing-tag holder for the * cluster and calls clusterer accordingly. Idempotent (activate/force-backup @@ -1485,7 +1485,7 @@ static int cc_ip_beats_master_locked(unsigned int ip_num, cc_cluster_t *cl) * to backup. Automatic allocation is suspended. * forced == 0 : automatic mode - the current master is the active holder. */ -static void cc_apply_shtags_decision(cc_cluster_t *cl, int i_am_master, +static void cl_ctr_apply_shtags_decision(cl_ctr_cluster_t *cl, int i_am_master, uint16_t forced) { int activate; @@ -1499,7 +1499,7 @@ static void cc_apply_shtags_decision(cc_cluster_t *cl, int i_am_master, activate = i_am_master; /* Log the reason on this node, but only when the decision or its cause - * changes - cc_apply_shtags_decision() is called on every relevant event + * changes - cl_ctr_apply_shtags_decision() is called on every relevant event * and clusterer's own activate/force calls are idempotent, so logging * unconditionally would flood. This makes the "why" visible on EVERY * node (master, forced holder, and passive backups alike). */ @@ -1530,30 +1530,30 @@ static void cc_apply_shtags_decision(cc_cluster_t *cl, int i_am_master, } /** - * cc_apply_shtags() - convenience wrapper that reads the current state under a + * cl_ctr_apply_shtags() - convenience wrapper that reads the current state under a * read lock and applies the shtag decision. Call WITHOUT cl->peers->lock held. */ -static void cc_apply_shtags(cc_cluster_t *cl) +static void cl_ctr_apply_shtags(cl_ctr_cluster_t *cl) { int i_am_master; uint16_t forced; lock_start_read(cl->peers->lock); - i_am_master = cc_i_am_master_locked(cl); + i_am_master = cl_ctr_i_am_master_locked(cl); forced = cl->peers->shtag_forced_node_id; lock_stop_read(cl->peers->lock); - cc_apply_shtags_decision(cl, i_am_master, forced); + cl_ctr_apply_shtags_decision(cl, i_am_master, forced); } /** - * cc_prune_stale(cl) - free entries far outside the election window. + * cl_ctr_prune_stale(cl) - free entries far outside the election window. * Memory management only - does not affect election outcomes. * Must be called with cl->peers->lock held. */ -static void cc_prune_stale(cc_cluster_t *cl) +static void cl_ctr_prune_stale(cl_ctr_cluster_t *cl) { - time_t cutoff = time(NULL) - (time_t)(query_time * CC_PURGE_FACTOR); + time_t cutoff = time(NULL) - (time_t)(query_time * CL_CTR_PURGE_FACTOR); int i; for (i = 0; i < cl->peers->count; i++) { @@ -1564,7 +1564,7 @@ static void cc_prune_stale(cc_cluster_t *cl) cl->peers->count--; if (i < cl->peers->count) cl->peers->entries[i] = cl->peers->entries[cl->peers->count]; - memset(&cl->peers->entries[cl->peers->count], 0, sizeof(cc_peer_t)); + memset(&cl->peers->entries[cl->peers->count], 0, sizeof(cl_ctr_peer_t)); i--; /* cl_list_lock and cl->peers->lock are independent - no deadlock */ if (clctl_loaded && pruned_id > 0) @@ -1579,21 +1579,21 @@ static void cc_prune_stale(cc_cluster_t *cl) } /* Re-apply shtag policy (override-aware); inside the lock, so pass * the state directly rather than calling the locking wrapper. */ - cc_apply_shtags_decision(cl, cc_i_am_master_locked(cl), + cl_ctr_apply_shtags_decision(cl, cl_ctr_i_am_master_locked(cl), cl->peers->shtag_forced_node_id); } } } /** - * cc_apply_master_from_list_locked() - apply master designation from a + * cl_ctr_apply_master_from_list_locked() - apply master designation from a * received MEMBER_LIST/MEMBER_LIST packet. * * Zeros all is_master flags in the peer table, then sets is_master=1 for * the entry matching master_ip. Updates last_master accordingly. * Must be called with cl->peers->lock held. */ -static void cc_apply_master_from_list_locked(const char *master_ip, cc_cluster_t *cl) +static void cl_ctr_apply_master_from_list_locked(const char *master_ip, cl_ctr_cluster_t *cl) { int i; @@ -1606,16 +1606,16 @@ static void cc_apply_master_from_list_locked(const char *master_ip, cc_cluster_t } memcpy(cl->peers->last_master, master_ip, - strnlen(master_ip, CC_MAX_IP_LEN)); - cl->peers->last_master[strnlen(master_ip, CC_MAX_IP_LEN)] = '\0'; + strnlen(master_ip, CL_CTR_MAX_IP_LEN)); + cl->peers->last_master[strnlen(master_ip, CL_CTR_MAX_IP_LEN)] = '\0'; } /** - * cc_peer_by_ip_locked() - find a peer entry by IP string. + * cl_ctr_peer_by_ip_locked() - find a peer entry by IP string. * Returns the entry pointer, or NULL if not present. * Must be called with cl->peers->lock held (read or write). */ -static cc_peer_t *cc_peer_by_ip_locked(cc_cluster_t *cl, const char *ip) +static cl_ctr_peer_t *cl_ctr_peer_by_ip_locked(cl_ctr_cluster_t *cl, const char *ip) { int i; for (i = 0; i < cl->peers->count; i++) @@ -1625,37 +1625,37 @@ static cc_peer_t *cc_peer_by_ip_locked(cc_cluster_t *cl, const char *ip) } /** - * cc_upsert_peer_locked() - insert or refresh a peer entry. - * Does NOT call cc_elect_master(cl); callers do that explicitly. + * cl_ctr_upsert_peer_locked() - insert or refresh a peer entry. + * Does NOT call cl_ctr_elect_master(cl); callers do that explicitly. * Must be called with cl->peers->lock held. */ -static void cc_upsert_peer_locked(const char *src_ip, cc_cluster_t *cl) +static void cl_ctr_upsert_peer_locked(const char *src_ip, cl_ctr_cluster_t *cl) { unsigned int src_num = ip_to_num(src_ip); time_t now = time(NULL); - cc_peer_t *found; + cl_ctr_peer_t *found; if (src_num == 0) { LM_WARN("clusterer_controller: ignoring invalid IP '%s'\n", src_ip); return; } - found = cc_peer_by_ip_locked(cl, src_ip); + found = cl_ctr_peer_by_ip_locked(cl, src_ip); if (found) { found->last_seen = now; return; /* updated */ } /* New entry */ - if (cl->peers->count >= CC_MAX_PEERS) { + if (cl->peers->count >= CL_CTR_MAX_PEERS) { LM_WARN("clusterer_controller: peer table full, ignoring %s\n", src_ip); return; } { - cc_peer_t *e = &cl->peers->entries[cl->peers->count]; - memcpy(e->ip, src_ip, strnlen(src_ip, CC_MAX_IP_LEN)); - e->ip[strnlen(src_ip, CC_MAX_IP_LEN)] = '\0'; + cl_ctr_peer_t *e = &cl->peers->entries[cl->peers->count]; + memcpy(e->ip, src_ip, strnlen(src_ip, CL_CTR_MAX_IP_LEN)); + e->ip[strnlen(src_ip, CL_CTR_MAX_IP_LEN)] = '\0'; e->ip_num = src_num; e->last_seen = now; e->is_master = 0; @@ -1666,10 +1666,10 @@ static void cc_upsert_peer_locked(const char *src_ip, cc_cluster_t *cl) } /** - * cc_alloc_node_id_locked() - find the lowest unused node_id >= 1. + * cl_ctr_alloc_node_id_locked() - find the lowest unused node_id >= 1. * Must be called with cl->peers->lock held for write. */ -static uint16_t cc_alloc_node_id_locked(cc_cluster_t *cl) +static uint16_t cl_ctr_alloc_node_id_locked(cl_ctr_cluster_t *cl) { uint16_t id; int i, used; @@ -1685,24 +1685,24 @@ static uint16_t cc_alloc_node_id_locked(cc_cluster_t *cl) if (!used) return id; } - return 0; /* table full (shouldn't happen with CC_MAX_PEERS=256) */ + return 0; /* table full (shouldn't happen with CL_CTR_MAX_PEERS=256) */ } /** - * cc_update_peer_bin_locked() - store node_id and BIN sockets for a peer. + * cl_ctr_update_peer_bin_locked() - store node_id and BIN sockets for a peer. * Must be called with cl->peers->lock held. */ -static void cc_update_peer_bin_locked(const char *ip, uint16_t node_id, +static void cl_ctr_update_peer_bin_locked(const char *ip, uint16_t node_id, uint8_t bin_count, - const char (*bin_sockets)[CC_MAX_BIN_SOCK_LEN], - cc_cluster_t *cl) + const char (*bin_sockets)[CL_CTR_MAX_BIN_SOCK_LEN], + cl_ctr_cluster_t *cl) { - cc_peer_t *e = cc_peer_by_ip_locked(cl, ip); + cl_ctr_peer_t *e = cl_ctr_peer_by_ip_locked(cl, ip); if (e) { e->node_id = node_id; e->bin_count = bin_count; if (bin_count > 0) - memcpy(e->bin_sockets, bin_sockets, bin_count * CC_MAX_BIN_SOCK_LEN); + memcpy(e->bin_sockets, bin_sockets, bin_count * CL_CTR_MAX_BIN_SOCK_LEN); } } @@ -1720,7 +1720,7 @@ static void cc_update_peer_bin_locked(const char *ip, uint16_t node_id, * without requiring NTP-synchronised clocks or a finite nonce cache. * The counter resets to 0 on every session key rotation; old packets * encrypted with the previous key fail AES-GCM authentication anyway. - * Bootstrap-key packets (CC_BOOTSTRAP_MAGIC) skip the sequence check - + * Bootstrap-key packets (CL_CTR_BOOTSTRAP_MAGIC) skip the sequence check - * their per-exchange join_nonce provides equivalent replay protection. * * Wire layout: @@ -1729,9 +1729,9 @@ static void cc_update_peer_bin_locked(const char *ip, uint16_t node_id, * [type 1B][seq 4B][payload] * ========================================================================= */ -/* cc_random_bytes() - fill buf with cryptographically secure random bytes +/* cl_ctr_random_bytes() - fill buf with cryptographically secure random bytes * (libsodium's randombytes draws from the kernel CSPRNG). */ -static int cc_random_bytes(unsigned char *buf, size_t len) +static int cl_ctr_random_bytes(unsigned char *buf, size_t len) { randombytes_buf(buf, len); return 0; @@ -1741,7 +1741,7 @@ static int cc_random_bytes(unsigned char *buf, size_t len) * no native HKDF; crypto_auth_hmacsha256_init accepts any key length). Every * use in this module needs exactly 32 output bytes (= HashLen), so the expand * phase is a single iteration: OKM = HMAC(PRK, info || 0x01). */ -static int cc_hkdf_sha256(const unsigned char *ikm, size_t ikm_len, +static int cl_ctr_hkdf_sha256(const unsigned char *ikm, size_t ikm_len, const unsigned char *salt, size_t salt_len, const char *info, unsigned char out[32]) @@ -1770,9 +1770,9 @@ static int cc_hkdf_sha256(const unsigned char *ikm, size_t ikm_len, return 0; } -static int cc_gen_ecdh_keypair(unsigned char *privkey, unsigned char *pubkey) +static int cl_ctr_gen_ecdh_keypair(unsigned char *privkey, unsigned char *pubkey) { - randombytes_buf(privkey, CC_PUBKEY_SZ); + randombytes_buf(privkey, CL_CTR_PUBKEY_SZ); if (crypto_scalarmult_base(pubkey, privkey) != 0) { LM_ERR("clusterer_controller: X25519 keygen failed\n"); return -1; @@ -1792,11 +1792,11 @@ static int cc_gen_ecdh_keypair(unsigned char *privkey, unsigned char *pubkey) * security properties in its own test harness. See RFC "The Noise Protocol * Framework" rev 34. HASHLEN = 32 (SHA-256), DHLEN = 32 (X25519). * ========================================================================= */ -#define CC_NOISE_PROTO "Noise_NNpsk0_25519_ChaChaPoly_SHA256" +#define CL_CTR_NOISE_PROTO "Noise_NNpsk0_25519_ChaChaPoly_SHA256" -static void cc_hmac256(const unsigned char *key, size_t keylen, +static void cl_ctr_hmac256(const unsigned char *key, size_t keylen, const unsigned char *data, size_t datalen, - unsigned char out[CC_NOISE_HASHLEN]) + unsigned char out[CL_CTR_NOISE_HASHLEN]) { crypto_auth_hmacsha256_state st; crypto_auth_hmacsha256_init(&st, key, keylen); @@ -1805,50 +1805,50 @@ static void cc_hmac256(const unsigned char *key, size_t keylen, } /* Noise HKDF: chained HMAC outputs (num_outputs of 1..3, each HASHLEN bytes). */ -static void cc_noise_hkdf(const unsigned char ck[CC_NOISE_HASHLEN], +static void cl_ctr_noise_hkdf(const unsigned char ck[CL_CTR_NOISE_HASHLEN], const unsigned char *ikm, size_t ikm_len, int num_outputs, - unsigned char o1[CC_NOISE_HASHLEN], - unsigned char o2[CC_NOISE_HASHLEN], - unsigned char o3[CC_NOISE_HASHLEN]) + unsigned char o1[CL_CTR_NOISE_HASHLEN], + unsigned char o2[CL_CTR_NOISE_HASHLEN], + unsigned char o3[CL_CTR_NOISE_HASHLEN]) { - unsigned char tempkey[CC_NOISE_HASHLEN], buf[CC_NOISE_HASHLEN + 1]; - cc_hmac256(ck, CC_NOISE_HASHLEN, ikm, ikm_len, tempkey); /* extract */ + unsigned char tempkey[CL_CTR_NOISE_HASHLEN], buf[CL_CTR_NOISE_HASHLEN + 1]; + cl_ctr_hmac256(ck, CL_CTR_NOISE_HASHLEN, ikm, ikm_len, tempkey); /* extract */ buf[0] = 0x01; - cc_hmac256(tempkey, CC_NOISE_HASHLEN, buf, 1, o1); + cl_ctr_hmac256(tempkey, CL_CTR_NOISE_HASHLEN, buf, 1, o1); if (num_outputs >= 2) { - memcpy(buf, o1, CC_NOISE_HASHLEN); buf[CC_NOISE_HASHLEN] = 0x02; - cc_hmac256(tempkey, CC_NOISE_HASHLEN, buf, CC_NOISE_HASHLEN + 1, o2); + memcpy(buf, o1, CL_CTR_NOISE_HASHLEN); buf[CL_CTR_NOISE_HASHLEN] = 0x02; + cl_ctr_hmac256(tempkey, CL_CTR_NOISE_HASHLEN, buf, CL_CTR_NOISE_HASHLEN + 1, o2); } if (num_outputs >= 3) { - memcpy(buf, o2, CC_NOISE_HASHLEN); buf[CC_NOISE_HASHLEN] = 0x03; - cc_hmac256(tempkey, CC_NOISE_HASHLEN, buf, CC_NOISE_HASHLEN + 1, o3); + memcpy(buf, o2, CL_CTR_NOISE_HASHLEN); buf[CL_CTR_NOISE_HASHLEN] = 0x03; + cl_ctr_hmac256(tempkey, CL_CTR_NOISE_HASHLEN, buf, CL_CTR_NOISE_HASHLEN + 1, o3); } sodium_memzero(tempkey, sizeof tempkey); } -static void cc_cs_nonce(uint64_t n, unsigned char out[12]) +static void cl_ctr_cs_nonce(uint64_t n, unsigned char out[12]) { int i; memset(out, 0, 4); /* 32 bits of zeros */ for (i = 0; i < 8; i++) out[4 + i] = (unsigned char)(n >> (8 * i)); /* LE64 */ } -static int cc_cs_encrypt(cc_cipherstate_t *cs, const unsigned char *ad, size_t ad_len, +static int cl_ctr_cs_encrypt(cl_ctr_cipherstate_t *cs, const unsigned char *ad, size_t ad_len, const unsigned char *pt, size_t pt_len, unsigned char *ct) { unsigned char nonce[12]; unsigned long long clen = 0; if (!cs->has_key) { memcpy(ct, pt, pt_len); return (int)pt_len; } - cc_cs_nonce(cs->n, nonce); + cl_ctr_cs_nonce(cs->n, nonce); crypto_aead_chacha20poly1305_ietf_encrypt(ct, &clen, pt, pt_len, ad, ad_len, NULL, nonce, cs->k); cs->n++; return (int)clen; } -static int cc_cs_decrypt(cc_cipherstate_t *cs, const unsigned char *ad, size_t ad_len, +static int cl_ctr_cs_decrypt(cl_ctr_cipherstate_t *cs, const unsigned char *ad, size_t ad_len, const unsigned char *ct, size_t ct_len, unsigned char *pt) { unsigned char nonce[12]; unsigned long long plen = 0; if (!cs->has_key) { memcpy(pt, ct, ct_len); return (int)ct_len; } - cc_cs_nonce(cs->n, nonce); + cl_ctr_cs_nonce(cs->n, nonce); if (crypto_aead_chacha20poly1305_ietf_decrypt(pt, &plen, NULL, ct, ct_len, ad, ad_len, nonce, cs->k) != 0) return -1; @@ -1856,113 +1856,113 @@ static int cc_cs_decrypt(cc_cipherstate_t *cs, const unsigned char *ad, size_t a return (int)plen; } -static void cc_ss_mixhash(cc_symstate_t *s, const unsigned char *data, size_t len) +static void cl_ctr_ss_mixhash(cl_ctr_symstate_t *s, const unsigned char *data, size_t len) { crypto_hash_sha256_state st; crypto_hash_sha256_init(&st); - crypto_hash_sha256_update(&st, s->h, CC_NOISE_HASHLEN); + crypto_hash_sha256_update(&st, s->h, CL_CTR_NOISE_HASHLEN); crypto_hash_sha256_update(&st, data, len); crypto_hash_sha256_final(&st, s->h); } -static void cc_ss_init(cc_symstate_t *s) +static void cl_ctr_ss_init(cl_ctr_symstate_t *s) { - size_t nl = strlen(CC_NOISE_PROTO); /* <= HASHLEN, so pad it */ - memset(s->h, 0, CC_NOISE_HASHLEN); - memcpy(s->h, CC_NOISE_PROTO, nl); - memcpy(s->ck, s->h, CC_NOISE_HASHLEN); + size_t nl = strlen(CL_CTR_NOISE_PROTO); /* <= HASHLEN, so pad it */ + memset(s->h, 0, CL_CTR_NOISE_HASHLEN); + memcpy(s->h, CL_CTR_NOISE_PROTO, nl); + memcpy(s->ck, s->h, CL_CTR_NOISE_HASHLEN); s->cs.has_key = 0; s->cs.n = 0; } -static void cc_ss_mixkey(cc_symstate_t *s, const unsigned char *ikm, size_t ikm_len) +static void cl_ctr_ss_mixkey(cl_ctr_symstate_t *s, const unsigned char *ikm, size_t ikm_len) { - unsigned char o1[CC_NOISE_HASHLEN], o2[CC_NOISE_HASHLEN], o3[CC_NOISE_HASHLEN]; - cc_noise_hkdf(s->ck, ikm, ikm_len, 2, o1, o2, o3); - memcpy(s->ck, o1, CC_NOISE_HASHLEN); + unsigned char o1[CL_CTR_NOISE_HASHLEN], o2[CL_CTR_NOISE_HASHLEN], o3[CL_CTR_NOISE_HASHLEN]; + cl_ctr_noise_hkdf(s->ck, ikm, ikm_len, 2, o1, o2, o3); + memcpy(s->ck, o1, CL_CTR_NOISE_HASHLEN); memcpy(s->cs.k, o2, 32); s->cs.n = 0; s->cs.has_key = 1; } -static void cc_ss_mixkeyhash(cc_symstate_t *s, const unsigned char *ikm, size_t ikm_len) +static void cl_ctr_ss_mixkeyhash(cl_ctr_symstate_t *s, const unsigned char *ikm, size_t ikm_len) { - unsigned char o1[CC_NOISE_HASHLEN], o2[CC_NOISE_HASHLEN], o3[CC_NOISE_HASHLEN]; - cc_noise_hkdf(s->ck, ikm, ikm_len, 3, o1, o2, o3); - memcpy(s->ck, o1, CC_NOISE_HASHLEN); - cc_ss_mixhash(s, o2, CC_NOISE_HASHLEN); + unsigned char o1[CL_CTR_NOISE_HASHLEN], o2[CL_CTR_NOISE_HASHLEN], o3[CL_CTR_NOISE_HASHLEN]; + cl_ctr_noise_hkdf(s->ck, ikm, ikm_len, 3, o1, o2, o3); + memcpy(s->ck, o1, CL_CTR_NOISE_HASHLEN); + cl_ctr_ss_mixhash(s, o2, CL_CTR_NOISE_HASHLEN); memcpy(s->cs.k, o3, 32); s->cs.n = 0; s->cs.has_key = 1; } -static int cc_ss_encrypt_hash(cc_symstate_t *s, const unsigned char *pt, size_t pt_len, +static int cl_ctr_ss_encrypt_hash(cl_ctr_symstate_t *s, const unsigned char *pt, size_t pt_len, unsigned char *ct) { - int cl = cc_cs_encrypt(&s->cs, s->h, CC_NOISE_HASHLEN, pt, pt_len, ct); - cc_ss_mixhash(s, ct, cl); + int cl = cl_ctr_cs_encrypt(&s->cs, s->h, CL_CTR_NOISE_HASHLEN, pt, pt_len, ct); + cl_ctr_ss_mixhash(s, ct, cl); return cl; } -static int cc_ss_decrypt_hash(cc_symstate_t *s, const unsigned char *ct, size_t ct_len, +static int cl_ctr_ss_decrypt_hash(cl_ctr_symstate_t *s, const unsigned char *ct, size_t ct_len, unsigned char *pt) { - int pl = cc_cs_decrypt(&s->cs, s->h, CC_NOISE_HASHLEN, ct, ct_len, pt); + int pl = cl_ctr_cs_decrypt(&s->cs, s->h, CL_CTR_NOISE_HASHLEN, ct, ct_len, pt); if (pl < 0) return -1; - cc_ss_mixhash(s, ct, ct_len); + cl_ctr_ss_mixhash(s, ct, ct_len); return pl; } /* Initiator: write msg 1 (-> psk, e). Stores e keypair in @e_priv for msg 2. */ -static int cc_noise_write1(cc_symstate_t *s, unsigned char e_priv[32], +static int cl_ctr_noise_write1(cl_ctr_symstate_t *s, unsigned char e_priv[32], const unsigned char psk[32], const unsigned char *payload, size_t pl_len, unsigned char *out) { unsigned char e_pub[32]; - cc_ss_init(s); - cc_ss_mixkeyhash(s, psk, 32); /* psk token (psk0) */ + cl_ctr_ss_init(s); + cl_ctr_ss_mixkeyhash(s, psk, 32); /* psk token (psk0) */ randombytes_buf(e_priv, 32); crypto_scalarmult_base(e_pub, e_priv); /* e */ - memcpy(out, e_pub, 32); cc_ss_mixhash(s, e_pub, 32); cc_ss_mixkey(s, e_pub, 32); - return 32 + cc_ss_encrypt_hash(s, payload, pl_len, out + 32); + memcpy(out, e_pub, 32); cl_ctr_ss_mixhash(s, e_pub, 32); cl_ctr_ss_mixkey(s, e_pub, 32); + return 32 + cl_ctr_ss_encrypt_hash(s, payload, pl_len, out + 32); } /* Responder: read msg 1, then write msg 2 (<- e, ee) with @payload. One-shot, * so no responder state is retained afterwards. @out gets msg 2. */ -static int cc_noise_respond(const unsigned char psk[32], +static int cl_ctr_noise_respond(const unsigned char psk[32], const unsigned char *msg1, size_t msg1_len, const unsigned char *payload, size_t pl_len, unsigned char *out) { - cc_symstate_t s; + cl_ctr_symstate_t s; unsigned char re[32], e_priv[32], e_pub[32], dh[32], scratch[64]; if (msg1_len < 32) return -1; - cc_ss_init(&s); - cc_ss_mixkeyhash(&s, psk, 32); - memcpy(re, msg1, 32); cc_ss_mixhash(&s, re, 32); cc_ss_mixkey(&s, re, 32); - if (cc_ss_decrypt_hash(&s, msg1 + 32, msg1_len - 32, scratch) < 0) + cl_ctr_ss_init(&s); + cl_ctr_ss_mixkeyhash(&s, psk, 32); + memcpy(re, msg1, 32); cl_ctr_ss_mixhash(&s, re, 32); cl_ctr_ss_mixkey(&s, re, 32); + if (cl_ctr_ss_decrypt_hash(&s, msg1 + 32, msg1_len - 32, scratch) < 0) return -1; /* bad msg 1 (wrong psk/tamper) */ randombytes_buf(e_priv, 32); crypto_scalarmult_base(e_pub, e_priv); - memcpy(out, e_pub, 32); cc_ss_mixhash(&s, e_pub, 32); cc_ss_mixkey(&s, e_pub, 32); + memcpy(out, e_pub, 32); cl_ctr_ss_mixhash(&s, e_pub, 32); cl_ctr_ss_mixkey(&s, e_pub, 32); if (crypto_scalarmult(dh, e_priv, re) != 0) return -1; /* ee */ - cc_ss_mixkey(&s, dh, 32); - return 32 + cc_ss_encrypt_hash(&s, payload, pl_len, out + 32); + cl_ctr_ss_mixkey(&s, dh, 32); + return 32 + cl_ctr_ss_encrypt_hash(&s, payload, pl_len, out + 32); } /* Initiator: read msg 2 (-> e, ee), recover @payload (master_salt). */ -static int cc_noise_read2(cc_symstate_t *s, const unsigned char e_priv[32], +static int cl_ctr_noise_read2(cl_ctr_symstate_t *s, const unsigned char e_priv[32], const unsigned char *msg2, size_t msg2_len, unsigned char *payload_out) { unsigned char re2[32], dh[32]; if (msg2_len < 32) return -1; - memcpy(re2, msg2, 32); cc_ss_mixhash(s, re2, 32); cc_ss_mixkey(s, re2, 32); + memcpy(re2, msg2, 32); cl_ctr_ss_mixhash(s, re2, 32); cl_ctr_ss_mixkey(s, re2, 32); if (crypto_scalarmult(dh, e_priv, re2) != 0) return -1; - cc_ss_mixkey(s, dh, 32); - return cc_ss_decrypt_hash(s, msg2 + 32, msg2_len - 32, payload_out); + cl_ctr_ss_mixkey(s, dh, 32); + return cl_ctr_ss_decrypt_hash(s, msg2 + 32, msg2_len - 32, payload_out); } /** - * cc_derive_session_key() - derive group key from password + master_salt. + * cl_ctr_derive_session_key() - derive group key from password + master_salt. * Reads master_salt from cl->peers->master_salt (shm, caller holds write lock). * Stores result in cl->session_key (worker-local cache). * Must be called with cl->peers->lock held for WRITE. */ -static int cc_derive_session_key(cc_cluster_t *cl) +static int cl_ctr_derive_session_key(cl_ctr_cluster_t *cl) { int i; size_t pass_len = strlen(cl->password); - if (cc_hkdf_sha256((unsigned char *)cl->password, pass_len, - cl->peers->master_salt, CC_MASTER_SALT_SZ, - "cc_session", cl->session_key) < 0) { + if (cl_ctr_hkdf_sha256((unsigned char *)cl->password, pass_len, + cl->peers->master_salt, CL_CTR_MASTER_SALT_SZ, + "cl_ctr_session", cl->session_key) < 0) { LM_ERR("clusterer_controller: [cluster %d] session key derivation failed\n", cl->cluster_id); return -1; @@ -1978,12 +1978,12 @@ static int cc_derive_session_key(cc_cluster_t *cl) /** - * cc_password_entropy_bits() - conservative estimate of a password's entropy. + * cl_ctr_password_entropy_bits() - conservative estimate of a password's entropy. * bits ~= length * floor(log2(charset)), where charset is the union of the * character classes present. floor() makes it a slight under-estimate, so the * weak-password warning errs toward firing. No libm dependency. */ -static int cc_password_entropy_bits(const char *p) +static int cl_ctr_password_entropy_bits(const char *p) { int have_lower = 0, have_upper = 0, have_digit = 0, have_sym = 0; int charset, bits_per_char = 0, tmp; @@ -2005,7 +2005,7 @@ static int cc_password_entropy_bits(const char *p) } /** - * cc_derive_key() - derive the 32-byte bootstrap (admission) key from the + * cl_ctr_derive_key() - derive the 32-byte bootstrap (admission) key from the * shared password. Used only for the join handshake (JOIN_REQ / KEY_GRANT / * JOIN_REJECT); normal traffic uses the ECDH-agreed session key. * @@ -2015,20 +2015,20 @@ static int cc_password_entropy_bits(const char *p) * ~64 MiB scrypt working set is a transient startup cost only; workers inherit * the derived key and never run scrypt. */ -static int cc_derive_key(cc_cluster_t *cl) +static int cl_ctr_derive_key(cl_ctr_cluster_t *cl) { char salt[64]; int saltlen, bits; /* Warn on a weak or default admission password. scrypt raises the cost of * each offline guess, but only a high-entropy secret removes the risk. */ - if (strcmp(cl->password, CC_DEFAULT_PASSWORD) == 0) { + if (strcmp(cl->password, CL_CTR_DEFAULT_PASSWORD) == 0) { LM_WARN("clusterer_controller: [cluster %d] using the built-in default " "password - set a strong 'password' (e.g. `openssl rand -base64 32`)\n", cl->cluster_id); } else { - bits = cc_password_entropy_bits(cl->password); - if (bits < CC_MIN_PASSWORD_BITS) + bits = cl_ctr_password_entropy_bits(cl->password); + if (bits < CL_CTR_MIN_PASSWORD_BITS) LM_WARN("clusterer_controller: [cluster %d] weak password (~%d bits " "of entropy) - an attacker who captures a JOIN packet can " "brute-force it offline; use a long random string, e.g. " @@ -2039,7 +2039,7 @@ static int cc_derive_key(cc_cluster_t *cl) * address, so the same password on different clusters yields different * bootstrap keys. The salt is public by design - Argon2id's work factor, * not salt secrecy, is what defeats brute force. */ - saltlen = snprintf(salt, sizeof(salt), "opensips-cc-bootstrap-v1:%s", + saltlen = snprintf(salt, sizeof(salt), "opensips-cl-ctr-bootstrap-v1:%s", cl->multicast_address); if (saltlen < 0 || saltlen >= (int)sizeof(salt)) saltlen = (int)strlen(salt); @@ -2052,7 +2052,7 @@ static int cc_derive_key(cc_cluster_t *cl) (const unsigned char *)salt, (size_t)saltlen, NULL, 0); if (crypto_pwhash(cl->key, 32, cl->password, strlen(cl->password), - salt16, CC_ARGON2_OPSLIMIT, CC_ARGON2_MEMLIMIT, + salt16, CL_CTR_ARGON2_OPSLIMIT, CL_CTR_ARGON2_MEMLIMIT, crypto_pwhash_ALG_ARGON2ID13) != 0) { LM_ERR("clusterer_controller: key derivation (Argon2id) failed for " "cluster %d (out of memory?)\n", cl->cluster_id); @@ -2063,23 +2063,23 @@ static int cc_derive_key(cc_cluster_t *cl) } /** - * cc_encrypt_pkt() - encrypt plaintext in-place and append the GCM tag. + * cl_ctr_encrypt_pkt() - encrypt plaintext in-place and append the GCM tag. * - * On entry: buf[0..CC_MAGIC_SZ-1] = magic (set by caller) + * On entry: buf[0..CL_CTR_MAGIC_SZ-1] = magic (set by caller) * buf[plain_off..] = plaintext to encrypt - * On return: buf[CC_MAGIC_SZ..] = cleartext cluster_id (BE) - * buf[CC_NONCE_OFF..] = random nonce + * On return: buf[CL_CTR_MAGIC_SZ..] = cleartext cluster_id (BE) + * buf[CL_CTR_NONCE_OFF..] = random nonce * buf[plain_off..] = ciphertext (same length) - * buf[plain_off+plain_len..+CC_TAG_SZ-1] = GCM tag + * buf[plain_off+plain_len..+CL_CTR_TAG_SZ-1] = GCM tag * * @return total packet length, or -1 on error */ -static int cc_encrypt_pkt(char *buf, int plain_off, int plain_len, +static int cl_ctr_encrypt_pkt(char *buf, int plain_off, int plain_len, const unsigned char *key, int cluster_id) { uint16_t cid_be = htons((uint16_t)cluster_id); - memcpy(buf + CC_MAGIC_SZ, &cid_be, CC_CLUSTER_ID_SZ); /* cleartext selector */ + memcpy(buf + CL_CTR_MAGIC_SZ, &cid_be, CL_CTR_CLUSTER_ID_SZ); /* cleartext selector */ /* AAD = cleartext header (magic + cluster_id): binding it to the tag stops * a captured packet being re-stamped with another cluster_id on a shared @@ -2087,37 +2087,37 @@ static int cc_encrypt_pkt(char *buf, int plain_off, int plain_len, * AEAD IV, already bound. XChaCha20's 192-bit nonce makes random nonces * collision-safe outright. */ { - unsigned char nonce[CC_NONCE_SZ]; + unsigned char nonce[CL_CTR_NONCE_SZ]; unsigned long long clen = 0; - randombytes_buf(nonce, CC_NONCE_SZ); - memcpy(buf + CC_NONCE_OFF, nonce, CC_NONCE_SZ); + randombytes_buf(nonce, CL_CTR_NONCE_SZ); + memcpy(buf + CL_CTR_NONCE_OFF, nonce, CL_CTR_NONCE_SZ); if (crypto_aead_xchacha20poly1305_ietf_encrypt( (unsigned char *)buf + plain_off, &clen, (const unsigned char *)buf + plain_off, (unsigned long long)plain_len, - (const unsigned char *)buf, CC_MAGIC_SZ + CC_CLUSTER_ID_SZ, + (const unsigned char *)buf, CL_CTR_MAGIC_SZ + CL_CTR_CLUSTER_ID_SZ, NULL, nonce, key) != 0) { LM_ERR("clusterer_controller: XChaCha20-Poly1305 encrypt failed\n"); return -1; } - return plain_off + (int)clen; /* clen = plain_len + CC_TAG_SZ */ + return plain_off + (int)clen; /* clen = plain_len + CL_CTR_TAG_SZ */ } } /** - * cc_decrypt_pkt() - authenticate and decrypt a received packet in-place. + * cl_ctr_decrypt_pkt() - authenticate and decrypt a received packet in-place. * * On entry: buf = [magic 2B][cluster_id 2B][nonce][ciphertext][tag 16B] - * On return: buf[CC_WIRE_HDR_SZ..] = plaintext (type + seq + payload) + * On return: buf[CL_CTR_WIRE_HDR_SZ..] = plaintext (type + seq + payload) * - * AAD must match cc_encrypt_pkt(): the cleartext header (magic+cluster_id), so + * AAD must match cl_ctr_encrypt_pkt(): the cleartext header (magic+cluster_id), so * a packet re-stamped with another cluster_id fails authentication. * * @return 0 on success, -1 to drop (wrong key, tampered, or too short) */ -static int cc_decrypt_pkt(char *buf, ssize_t n, const char *sender_ip, +static int cl_ctr_decrypt_pkt(char *buf, ssize_t n, const char *sender_ip, const unsigned char *key, int is_bootstrap) { - ssize_t cipher_len = n - CC_WIRE_HDR_SZ - CC_TAG_SZ; /* plaintext length */ + ssize_t cipher_len = n - CL_CTR_WIRE_HDR_SZ - CL_CTR_TAG_SZ; /* plaintext length */ if (cipher_len <= 0) { LM_INFO("clusterer_controller: packet from %s too short to decrypt " @@ -2126,13 +2126,13 @@ static int cc_decrypt_pkt(char *buf, ssize_t n, const char *sender_ip, } { - unsigned char *nonce = (unsigned char *)buf + CC_NONCE_OFF; + unsigned char *nonce = (unsigned char *)buf + CL_CTR_NONCE_OFF; unsigned long long mlen = 0; if (crypto_aead_xchacha20poly1305_ietf_decrypt( - (unsigned char *)buf + CC_WIRE_HDR_SZ, &mlen, NULL, - (const unsigned char *)buf + CC_WIRE_HDR_SZ, - (unsigned long long)(cipher_len + CC_TAG_SZ), - (const unsigned char *)buf, CC_MAGIC_SZ + CC_CLUSTER_ID_SZ, + (unsigned char *)buf + CL_CTR_WIRE_HDR_SZ, &mlen, NULL, + (const unsigned char *)buf + CL_CTR_WIRE_HDR_SZ, + (unsigned long long)(cipher_len + CL_CTR_TAG_SZ), + (const unsigned char *)buf, CL_CTR_MAGIC_SZ + CL_CTR_CLUSTER_ID_SZ, nonce, key) != 0) { if (is_bootstrap) LM_WARN("clusterer_controller: bootstrap decryption failed " @@ -2149,17 +2149,17 @@ static int cc_decrypt_pkt(char *buf, ssize_t n, const char *sender_ip, } /** - * cc_check_and_update_seq() - reject replayed or reordered packets. + * cl_ctr_check_and_update_seq() - reject replayed or reordered packets. * Looks up sender_ip in the peer table; requires pkt_seq > last_seq. * Updates last_seq on accept. Unknown senders (new nodes not yet in * the peer table) are accepted so their first packet (ALIVE/JOIN_REQ) * can populate the table. - * Only called for CC_PACKET_MAGIC packets; bootstrap packets use join_nonce. - * Single-threaded caller (cc_worker reactor); no lock needed for the check. + * Only called for CL_CTR_PACKET_MAGIC packets; bootstrap packets use join_nonce. + * Single-threaded caller (cl_ctr_worker reactor); no lock needed for the check. * @return 0 to accept, -1 to drop. */ -static int cc_check_and_update_seq(const char *sender_ip, uint32_t pkt_seq, - cc_cluster_t *cl) +static int cl_ctr_check_and_update_seq(const char *sender_ip, uint32_t pkt_seq, + cl_ctr_cluster_t *cl) { int i; for (i = 0; i < cl->peers->count; i++) { @@ -2180,7 +2180,7 @@ static int cc_check_and_update_seq(const char *sender_ip, uint32_t pkt_seq, * Socket setup * ========================================================================= */ -static int cc_setup_socket(cc_cluster_t *cl) +static int cl_ctr_setup_socket(cl_ctr_cluster_t *cl) { int sock; int yes = 1; @@ -2246,7 +2246,7 @@ static int cc_setup_socket(cc_cluster_t *cl) /* Pin the sending interface to my_ip so that loopback packets carry * my_ip as source address - this makes self-loopback detection in - * cc_handle_member_list() reliable on multi-homed hosts. */ + * cl_ctr_handle_member_list() reliable on multi-homed hosts. */ { struct in_addr local_if; local_if.s_addr = inet_addr(my_ip); @@ -2298,20 +2298,20 @@ static int cc_setup_socket(cc_cluster_t *cl) * ========================================================================= */ /** - * cc_seal_and_send() - encrypt a prepared packet in place and multicast it. + * cl_ctr_seal_and_send() - encrypt a prepared packet in place and multicast it. * * On entry @pkt holds the cleartext framing (magic already stamped) plus the - * @plain_len-byte plaintext starting at CC_WIRE_HDR_SZ; cc_encrypt_pkt() fills + * @plain_len-byte plaintext starting at CL_CTR_WIRE_HDR_SZ; cl_ctr_encrypt_pkt() fills * the cluster_id + nonce and appends the AEAD tag. Every controller packet * targets the cluster's multicast group (cached in cl->mcast_dest), so all * senders share this tail. @type is only used for logging. * * @return 0 on success, -1 on encrypt or send failure. */ -static int cc_seal_and_send(int sock, cc_cluster_t *cl, char *pkt, int plain_len, +static int cl_ctr_seal_and_send(int sock, cl_ctr_cluster_t *cl, char *pkt, int plain_len, const unsigned char *key, unsigned char type) { - int total_len = cc_encrypt_pkt(pkt, CC_WIRE_HDR_SZ, plain_len, key, + int total_len = cl_ctr_encrypt_pkt(pkt, CL_CTR_WIRE_HDR_SZ, plain_len, key, cl->cluster_id); if (total_len < 0) return -1; @@ -2331,62 +2331,62 @@ static int cc_seal_and_send(int sock, cc_cluster_t *cl, char *pkt, int plain_len } /** - * cc_send_pkt_with_ip() - build and multicast a small (ALIVE/GOODBYE) packet. - * JOIN_REQ is handled by cc_send_join_req_pkt() which carries BIN socket info. + * cl_ctr_send_pkt_with_ip() - build and multicast a small (ALIVE/GOODBYE) packet. + * JOIN_REQ is handled by cl_ctr_send_join_req_pkt() which carries BIN socket info. * * ALIVE: [type 1B][seq 4B][ip NUL][pubkey 32B] - peers learn our pubkey here * GOODBYE: [type 1B][seq 4B][ip NUL] - no pubkey needed */ -static void cc_send_pkt_with_ip(int sock, unsigned char type, cc_cluster_t *cl) +static void cl_ctr_send_pkt_with_ip(int sock, unsigned char type, cl_ctr_cluster_t *cl) { /* Sized for ALIVE which carries an extra pubkey + config descriptor */ - char pkt[CC_SMALL_PKT_SZ + CC_PUBKEY_SZ + CC_CONFIG_SZ]; + char pkt[CL_CTR_SMALL_PKT_SZ + CL_CTR_PUBKEY_SZ + CL_CTR_CONFIG_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); int ip_len = (int)strlen(my_ip); int plain_len; - if (ip_len > CC_MAX_IP_LEN) - ip_len = CC_MAX_IP_LEN; + if (ip_len > CL_CTR_MAX_IP_LEN) + ip_len = CL_CTR_MAX_IP_LEN; - memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); + memcpy(pkt, CL_CTR_PACKET_MAGIC, CL_CTR_MAGIC_SZ); - pkt[CC_WIRE_HDR_SZ] = (char)type; - memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); - memcpy(pkt + CC_WIRE_HDR_SZ + 1 + CC_SEQ_SZ, my_ip, ip_len); - pkt[CC_WIRE_HDR_SZ + 1 + CC_SEQ_SZ + ip_len] = '\0'; - plain_len = 1 + CC_SEQ_SZ + ip_len + 1; + pkt[CL_CTR_WIRE_HDR_SZ] = (char)type; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1 + CL_CTR_SEQ_SZ, my_ip, ip_len); + pkt[CL_CTR_WIRE_HDR_SZ + 1 + CL_CTR_SEQ_SZ + ip_len] = '\0'; + plain_len = 1 + CL_CTR_SEQ_SZ + ip_len + 1; /* ALIVE appends our X25519 public key so peers accumulate pubkeys * without bloating MEMBER_LIST (avoids excessive IP fragmentation). */ - if (type == CC_PKT_ALIVE) { - memcpy(pkt + CC_WIRE_HDR_SZ + plain_len, cl->my_pubkey, CC_PUBKEY_SZ); - plain_len += CC_PUBKEY_SZ; + if (type == CL_CTR_PKT_ALIVE) { + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + plain_len, cl->my_pubkey, CL_CTR_PUBKEY_SZ); + plain_len += CL_CTR_PUBKEY_SZ; /* Advertise our effective consistency-critical config so peers can * detect accidental per-node config drift for the same cluster. */ { - char *c = pkt + CC_WIRE_HDR_SZ + plain_len; + char *c = pkt + CL_CTR_WIRE_HDR_SZ + plain_len; uint16_t qt = htons((uint16_t)(query_time & 0xFFFF)); c[0] = (char)(cl->manage_shtags ? 1 : 0); c[1] = (char)(cl->master_stickiness ? 1 : 0); memcpy(c + 2, &qt, 2); - plain_len += CC_CONFIG_SZ; + plain_len += CL_CTR_CONFIG_SZ; } } - cc_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, type); + cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, type); } -#define cc_send_alive(sock, cl) cc_send_pkt_with_ip((sock), CC_PKT_ALIVE, (cl)) -#define cc_send_join_req(sock, cl) cc_send_join_req_pkt((sock), (cl)) +#define cl_ctr_send_alive(sock, cl) cl_ctr_send_pkt_with_ip((sock), CL_CTR_PKT_ALIVE, (cl)) +#define cl_ctr_send_join_req(sock, cl) cl_ctr_send_join_req_pkt((sock), (cl)) /** - * cc_send_list_pkt() - encrypt and multicast the active peer table. + * cl_ctr_send_list_pkt() - encrypt and multicast the active peer table. * * Wire: [magic 2B][cluster_id 2B][nonce 12B][AES-256-GCM([type 1B][seq 4B][count 2B][entries...])][tag 16B] */ -static void cc_send_list_pkt(int sock, unsigned char type, cc_cluster_t *cl) +static void cl_ctr_send_list_pkt(int sock, unsigned char type, cl_ctr_cluster_t *cl) { - char pkt[CC_LIST_PKT_MAX_SZ]; + char pkt[CL_CTR_LIST_PKT_MAX_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); uint16_t count = 0; uint16_t count_be; @@ -2394,71 +2394,71 @@ static void cc_send_list_pkt(int sock, unsigned char type, cc_cluster_t *cl) time_t cutoff; int i, plain_len; - memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); - /* nonce at [8..19] written by cc_encrypt_pkt */ + memcpy(pkt, CL_CTR_PACKET_MAGIC, CL_CTR_MAGIC_SZ); + /* nonce at [8..19] written by cl_ctr_encrypt_pkt */ /* Plaintext: [type][seq][count BE][forced_shtag_node_id BE][entries...] */ - pkt[CC_WIRE_HDR_SZ] = (char)type; - memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + pkt[CL_CTR_WIRE_HDR_SZ] = (char)type; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); /* count filled after iteration; forced_shtag_node_id filled below */ - p = pkt + CC_WIRE_HDR_SZ + 1 + CC_SEQ_SZ + CC_LIST_COUNT_SZ + CC_NODE_ID_SZ; + p = pkt + CL_CTR_WIRE_HDR_SZ + 1 + CL_CTR_SEQ_SZ + CL_CTR_LIST_COUNT_SZ + CL_CTR_NODE_ID_SZ; - cutoff = time(NULL) - (time_t)(query_time * CC_ELECT_FACTOR); + cutoff = time(NULL) - (time_t)(query_time * CL_CTR_ELECT_FACTOR); lock_start_read(cl->peers->lock); { uint16_t forced_be = htons(cl->peers->shtag_forced_node_id); - memcpy(pkt + CC_WIRE_HDR_SZ + 1 + CC_SEQ_SZ + CC_LIST_COUNT_SZ, - &forced_be, CC_NODE_ID_SZ); + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1 + CL_CTR_SEQ_SZ + CL_CTR_LIST_COUNT_SZ, + &forced_be, CL_CTR_NODE_ID_SZ); } - for (i = 0; i < cl->peers->count && count < CC_MAX_PEERS; i++) { - cc_peer_t *e = &cl->peers->entries[i]; + for (i = 0; i < cl->peers->count && count < CL_CTR_MAX_PEERS; i++) { + cl_ctr_peer_t *e = &cl->peers->entries[i]; if (e->last_seen < cutoff) continue; /* Entry layout: [ip 16B null-padded][is_master 1B] = 17B */ - memset(p, 0, CC_IP_ENTRY_SZ); - memcpy(p, e->ip, strnlen(e->ip, CC_MAX_IP_LEN)); - p[CC_IP_ENTRY_SZ - 1] = (char)(e->is_master ? 1 : 0); - p += CC_IP_ENTRY_SZ; + memset(p, 0, CL_CTR_IP_ENTRY_SZ); + memcpy(p, e->ip, strnlen(e->ip, CL_CTR_MAX_IP_LEN)); + p[CL_CTR_IP_ENTRY_SZ - 1] = (char)(e->is_master ? 1 : 0); + p += CL_CTR_IP_ENTRY_SZ; count++; } lock_stop_read(cl->peers->lock); count_be = htons(count); - memcpy(pkt + CC_WIRE_HDR_SZ + 1 + CC_SEQ_SZ, &count_be, CC_LIST_COUNT_SZ); + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1 + CL_CTR_SEQ_SZ, &count_be, CL_CTR_LIST_COUNT_SZ); - plain_len = 1 + CC_SEQ_SZ + CC_LIST_COUNT_SZ + CC_NODE_ID_SZ - + count * CC_IP_ENTRY_SZ; - if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, type) == 0) + plain_len = 1 + CL_CTR_SEQ_SZ + CL_CTR_LIST_COUNT_SZ + CL_CTR_NODE_ID_SZ + + count * CL_CTR_IP_ENTRY_SZ; + if (cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, type) == 0) LM_INFO("clusterer_controller: [cluster %d] sent MEMBER_LIST (%d members)\n", cl->cluster_id, count); } -#define cc_send_member_list(sock, cl) cc_send_list_pkt((sock), CC_PKT_MEMBER_LIST, (cl)) +#define cl_ctr_send_member_list(sock, cl) cl_ctr_send_list_pkt((sock), CL_CTR_PKT_MEMBER_LIST, (cl)) /** - * cc_send_join_req_pkt() - send CC_PKT_JOIN_REQ with BIN socket info. + * cl_ctr_send_join_req_pkt() - send CL_CTR_PKT_JOIN_REQ with BIN socket info. * * Payload: [ip NUL][bin_count 1B][sock1 NUL]...[sockN NUL] */ -static void cc_send_join_req_pkt(int sock, cc_cluster_t *cl) +static void cl_ctr_send_join_req_pkt(int sock, cl_ctr_cluster_t *cl) { - char pkt[CC_JOIN_PKT_MAX_SZ]; + char pkt[CL_CTR_JOIN_PKT_MAX_SZ]; uint32_t seq; /* Rate-limit JOIN_REQ transmissions. During a key-mismatch / split-brain * heal several code paths (defer timer, session-mismatch re-key, rejoin * timer) can each ask to (re)send a JOIN_REQ within the same second. A * JOIN_REQ is idempotent - the master answers whichever one arrives - so - * drop any that lands within CC_JOIN_REQ_MIN_US of the previous send; the + * drop any that lands within CL_CTR_JOIN_REQ_MIN_US of the previous send; the * next timer tick resends if the join is still pending. The very first * send (last_join_req_utime == 0) is never throttled. */ { utime_t now_us = get_uticks(); if (cl->last_join_req_utime != 0 && - (utime_t)(now_us - cl->last_join_req_utime) < CC_JOIN_REQ_MIN_US) + (utime_t)(now_us - cl->last_join_req_utime) < CL_CTR_JOIN_REQ_MIN_US) return; cl->last_join_req_utime = now_us; } @@ -2468,15 +2468,15 @@ static void cc_send_join_req_pkt(int sock, cc_cluster_t *cl) char *p; int plain_len; - if (ip_len > CC_MAX_IP_LEN) - ip_len = CC_MAX_IP_LEN; + if (ip_len > CL_CTR_MAX_IP_LEN) + ip_len = CL_CTR_MAX_IP_LEN; - memcpy(pkt, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ); /* JOIN_REQ uses bootstrap key */ + memcpy(pkt, CL_CTR_BOOTSTRAP_MAGIC, CL_CTR_MAGIC_SZ); /* JOIN_REQ uses bootstrap key */ /* Plaintext: [type][seq][ip NUL][bin_count][sock1 NUL]...[sockN NUL][pubkey 32B] */ - pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_JOIN_REQ; - memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); - p = pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + pkt[CL_CTR_WIRE_HDR_SZ] = (char)CL_CTR_PKT_JOIN_REQ; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); + p = pkt + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ; memcpy(p, my_ip, ip_len); p[ip_len] = '\0'; @@ -2484,7 +2484,7 @@ static void cc_send_join_req_pkt(int sock, cc_cluster_t *cl) /* Advertise only the BIN socket resolved for this specific cluster */ { - int slen = (int)strnlen(cl->bin_socket, CC_MAX_BIN_SOCK_LEN - 1); + int slen = (int)strnlen(cl->bin_socket, CL_CTR_MAX_BIN_SOCK_LEN - 1); *p++ = 1; /* bin_count */ memcpy(p, cl->bin_socket, slen); p[slen] = '\0'; @@ -2497,7 +2497,7 @@ static void cc_send_join_req_pkt(int sock, cc_cluster_t *cl) * (msg 2). A later JOIN_REQ overwrites it, so a stale KEY_GRANT just fails * to decrypt and is dropped. */ { - int m1 = cc_noise_write1(&cl->noise_hs, cl->noise_e_priv, cl->key, + int m1 = cl_ctr_noise_write1(&cl->noise_hs, cl->noise_e_priv, cl->key, NULL, 0, (unsigned char *)p); cl->noise_hs_valid = 1; p += m1; /* 48 bytes: 32 ephemeral + 16 tag */ @@ -2513,41 +2513,41 @@ static void cc_send_join_req_pkt(int sock, cc_cluster_t *cl) p += 2; } - plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); - if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->key, CC_PKT_JOIN_REQ) == 0) + plain_len = (int)(p - (pkt + CL_CTR_WIRE_HDR_SZ)); + if (cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->key, CL_CTR_PKT_JOIN_REQ) == 0) LM_DBG("clusterer_controller: [cluster %d] sent JOIN_REQ bin=%s\n", cl->cluster_id, cl->bin_socket); } /** - * cc_send_node_assign() - send CC_PKT_NODE_ASSIGN to multicast. + * cl_ctr_send_node_assign() - send CL_CTR_PKT_NODE_ASSIGN to multicast. * * Payload: [node_id 2B BE][ip NUL][bin_count 1B][sock1 NUL]...[sockN NUL] * * Sent by master after allocating a node_id. All cluster members receive * it and update their peer tables accordingly. */ -static void cc_send_node_assign(int sock, const char *ip, uint16_t node_id, +static void cl_ctr_send_node_assign(int sock, const char *ip, uint16_t node_id, uint8_t bin_count, - const char (*bin_sockets)[CC_MAX_BIN_SOCK_LEN], - cc_cluster_t *cl) + const char (*bin_sockets)[CL_CTR_MAX_BIN_SOCK_LEN], + cl_ctr_cluster_t *cl) { - char pkt[CC_NODE_ASSIGN_MAX_SZ]; + char pkt[CL_CTR_NODE_ASSIGN_MAX_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); uint16_t nid_be = htons(node_id); - int ip_len = (int)strnlen(ip, CC_MAX_IP_LEN); + int ip_len = (int)strnlen(ip, CL_CTR_MAX_IP_LEN); char *p; int i, plain_len; - memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); + memcpy(pkt, CL_CTR_PACKET_MAGIC, CL_CTR_MAGIC_SZ); - pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_NODE_ASSIGN; - memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); - p = pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + pkt[CL_CTR_WIRE_HDR_SZ] = (char)CL_CTR_PKT_NODE_ASSIGN; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); + p = pkt + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ; /* node_id (2B BE) */ - memcpy(p, &nid_be, CC_NODE_ID_SZ); - p += CC_NODE_ID_SZ; + memcpy(p, &nid_be, CL_CTR_NODE_ID_SZ); + p += CL_CTR_NODE_ID_SZ; /* IP NUL */ memcpy(p, ip, ip_len); @@ -2557,56 +2557,56 @@ static void cc_send_node_assign(int sock, const char *ip, uint16_t node_id, /* BIN sockets */ *p++ = (char)bin_count; for (i = 0; i < bin_count; i++) { - int slen = (int)strnlen(bin_sockets[i], CC_MAX_BIN_SOCK_LEN - 1); + int slen = (int)strnlen(bin_sockets[i], CL_CTR_MAX_BIN_SOCK_LEN - 1); memcpy(p, bin_sockets[i], slen); p[slen] = '\0'; p += slen + 1; } - plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); - /* NODE_ASSIGN carries the CC_PACKET_MAGIC session magic, so the receiver + plain_len = (int)(p - (pkt + CL_CTR_WIRE_HDR_SZ)); + /* NODE_ASSIGN carries the CL_CTR_PACKET_MAGIC session magic, so the receiver * decrypts it with session_key. It MUST therefore be encrypted with * session_key, not the bootstrap key - otherwise every NODE_ASSIGN fails * GCM auth on receipt ("session key mismatch"), driving a JOIN_REQ storm. * KEY_GRANT is sent before NODE_ASSIGN so the joiner already holds the * session key by the time this arrives. */ - if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, CC_PKT_NODE_ASSIGN) == 0) + if (cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, CL_CTR_PKT_NODE_ASSIGN) == 0) LM_INFO("clusterer_controller: [cluster %d] NODE_ASSIGN node_id=%u ip=%s\n", cl->cluster_id, node_id, ip); } /** - * cc_send_master_alive() - master-only keepalive, no payload beyond the header. - * Encrypted with session_key (CC_PACKET_MAGIC). + * cl_ctr_send_master_alive() - master-only keepalive, no payload beyond the header. + * Encrypted with session_key (CL_CTR_PACKET_MAGIC). */ -static void cc_send_master_alive(int sock, cc_cluster_t *cl) +static void cl_ctr_send_master_alive(int sock, cl_ctr_cluster_t *cl) { - char pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_TAG_SZ]; + char pkt[CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_TAG_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); - memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); - pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_MASTER_ALIVE; - memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); + memcpy(pkt, CL_CTR_PACKET_MAGIC, CL_CTR_MAGIC_SZ); + pkt[CL_CTR_WIRE_HDR_SZ] = (char)CL_CTR_PKT_MASTER_ALIVE; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); /* no payload beyond type+seq */ - cc_seal_and_send(sock, cl, pkt, CC_PLAIN_HDR_SZ, cl->session_key, - CC_PKT_MASTER_ALIVE); + cl_ctr_seal_and_send(sock, cl, pkt, CL_CTR_PLAIN_HDR_SZ, cl->session_key, + CL_CTR_PKT_MASTER_ALIVE); } /** - * cc_send_master_beacon() - master-only split-brain merge announcement. + * cl_ctr_send_master_beacon() - master-only split-brain merge announcement. * - * Encrypted with the BOOTSTRAP key (CC_BOOTSTRAP_MAGIC) - unlike MASTER_ALIVE, + * Encrypted with the BOOTSTRAP key (CL_CTR_BOOTSTRAP_MAGIC) - unlike MASTER_ALIVE, * which uses the per-cluster session key. Every correctly-configured node * shares the bootstrap key, so two masters that hold *different* session keys * (e.g. after an all-node simultaneous cold start) can still read each other's * beacon and reconcile. Payload is this partition's member count (2B BE); the - * sender IP comes from the datagram source. See cc_handle_master_beacon(). + * sender IP comes from the datagram source. See cl_ctr_handle_master_beacon(). */ -static void cc_send_master_beacon(int sock, cc_cluster_t *cl) +static void cl_ctr_send_master_beacon(int sock, cl_ctr_cluster_t *cl) { - char pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + 2 + CC_TAG_SZ]; + char pkt[CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + 2 + CL_CTR_TAG_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); uint16_t cnt_be; @@ -2614,18 +2614,18 @@ static void cc_send_master_beacon(int sock, cc_cluster_t *cl) cnt_be = htons((uint16_t)cl->peers->count); lock_stop_read(cl->peers->lock); - memcpy(pkt, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ); - pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_MASTER_BEACON; - memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); - memcpy(pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ, &cnt_be, 2); + memcpy(pkt, CL_CTR_BOOTSTRAP_MAGIC, CL_CTR_MAGIC_SZ); + pkt[CL_CTR_WIRE_HDR_SZ] = (char)CL_CTR_PKT_MASTER_BEACON; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ, &cnt_be, 2); - cc_seal_and_send(sock, cl, pkt, CC_PLAIN_HDR_SZ + 2, cl->key, - CC_PKT_MASTER_BEACON); + cl_ctr_seal_and_send(sock, cl, pkt, CL_CTR_PLAIN_HDR_SZ + 2, cl->key, + CL_CTR_PKT_MASTER_BEACON); } /** - * cc_send_key_grant() - send ECDH-wrapped master_salt to a joining node. - * Encrypted with bootstrap key (CC_BOOTSTRAP_MAGIC) so the joining node can + * cl_ctr_send_key_grant() - send ECDH-wrapped master_salt to a joining node. + * Encrypted with bootstrap key (CL_CTR_BOOTSTRAP_MAGIC) so the joining node can * read it before having the session key. * * Runs the Noise responder over the joiner's msg 1 and answers with msg 2, @@ -2635,78 +2635,78 @@ static void cc_send_master_beacon(int sock, cc_cluster_t *cl) * * Payload: [target_ip NUL][noise_msg2 64B] */ -static void cc_send_key_grant(int sock, const char *target_ip, cc_cluster_t *cl, +static void cl_ctr_send_key_grant(int sock, const char *target_ip, cl_ctr_cluster_t *cl, const unsigned char *msg1, int msg1_len) { - char pkt[CC_KEY_GRANT_SZ]; + char pkt[CL_CTR_KEY_GRANT_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); - unsigned char msg2[32 + CC_MASTER_SALT_SZ + CC_TAG_SZ]; + unsigned char msg2[32 + CL_CTR_MASTER_SALT_SZ + CL_CTR_TAG_SZ]; char *p; int ip_len, plain_len, m2; - m2 = cc_noise_respond(cl->key, msg1, (size_t)msg1_len, - cl->peers->master_salt, CC_MASTER_SALT_SZ, msg2); + m2 = cl_ctr_noise_respond(cl->key, msg1, (size_t)msg1_len, + cl->peers->master_salt, CL_CTR_MASTER_SALT_SZ, msg2); if (m2 < 0) { LM_DBG("clusterer_controller: [cluster %d] Noise msg1 from %s did not " "verify - no KEY_GRANT sent\n", cl->cluster_id, target_ip); return; } - ip_len = (int)strnlen(target_ip, CC_MAX_IP_LEN); + ip_len = (int)strnlen(target_ip, CL_CTR_MAX_IP_LEN); - memcpy(pkt, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ); - pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_KEY_GRANT; - memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); - p = pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + memcpy(pkt, CL_CTR_BOOTSTRAP_MAGIC, CL_CTR_MAGIC_SZ); + pkt[CL_CTR_WIRE_HDR_SZ] = (char)CL_CTR_PKT_KEY_GRANT; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); + p = pkt + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ; memcpy(p, target_ip, ip_len); p[ip_len] = '\0'; p += ip_len + 1; memcpy(p, msg2, m2); p += m2; - plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); - if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->key, CC_PKT_KEY_GRANT) == 0) + plain_len = (int)(p - (pkt + CL_CTR_WIRE_HDR_SZ)); + if (cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->key, CL_CTR_PKT_KEY_GRANT) == 0) LM_INFO("clusterer_controller: [cluster %d] sent KEY_GRANT to %s\n", cl->cluster_id, target_ip); } /** - * cc_send_key_handoff() - send master_salt to the next master before departing. - * Outer envelope is the session key (CC_PACKET_MAGIC); the salt itself is sealed + * cl_ctr_send_key_handoff() - send master_salt to the next master before departing. + * Outer envelope is the session key (CL_CTR_PACKET_MAGIC); the salt itself is sealed * to the next master's long-lived X25519 key (learned via ALIVE) with an * anonymous crypto_box - only that node can open it. Sender authenticity comes * from the session-key envelope (only cluster members can send it). * * Payload: [next_master_ip NUL][sealed_box 64B] */ -static void cc_send_key_handoff(int sock, const char *next_master_ip, +static void cl_ctr_send_key_handoff(int sock, const char *next_master_ip, const unsigned char *next_master_pubkey, - cc_cluster_t *cl) + cl_ctr_cluster_t *cl) { - char pkt[CC_KEY_HANDOFF_SZ]; + char pkt[CL_CTR_KEY_HANDOFF_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); - unsigned char sealed[crypto_box_SEALBYTES + CC_MASTER_SALT_SZ]; + unsigned char sealed[crypto_box_SEALBYTES + CL_CTR_MASTER_SALT_SZ]; char *p; int ip_len, plain_len; - if (crypto_box_seal(sealed, cl->peers->master_salt, CC_MASTER_SALT_SZ, + if (crypto_box_seal(sealed, cl->peers->master_salt, CL_CTR_MASTER_SALT_SZ, next_master_pubkey) != 0) { LM_ERR("clusterer_controller: [cluster %d] crypto_box_seal failed\n", cl->cluster_id); return; } - ip_len = (int)strnlen(next_master_ip, CC_MAX_IP_LEN); + ip_len = (int)strnlen(next_master_ip, CL_CTR_MAX_IP_LEN); - memcpy(pkt, CC_PACKET_MAGIC, CC_MAGIC_SZ); - pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_KEY_HANDOFF; - memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); - p = pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + memcpy(pkt, CL_CTR_PACKET_MAGIC, CL_CTR_MAGIC_SZ); + pkt[CL_CTR_WIRE_HDR_SZ] = (char)CL_CTR_PKT_KEY_HANDOFF; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); + p = pkt + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ; memcpy(p, next_master_ip, ip_len); p[ip_len] = '\0'; p += ip_len + 1; memcpy(p, sealed, sizeof(sealed)); p += sizeof(sealed); - plain_len = (int)(p - (pkt + CC_WIRE_HDR_SZ)); - if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, - CC_PKT_KEY_HANDOFF) == 0) + plain_len = (int)(p - (pkt + CL_CTR_WIRE_HDR_SZ)); + if (cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, + CL_CTR_PKT_KEY_HANDOFF) == 0) LM_INFO("clusterer_controller: [cluster %d] sent KEY_HANDOFF to %s\n", cl->cluster_id, next_master_ip); } @@ -2716,72 +2716,72 @@ static void cc_send_key_handoff(int sock, const char *next_master_ip, * ========================================================================= */ /** - * cc_on_became_master() - side effects when this node wins an election. + * cl_ctr_on_became_master() - side effects when this node wins an election. * Generates fresh master_salt, derives session_key, arms MASTER_ALIVE timer, * disarms master_dead watchdog. Must be called with cl->peers->lock held WRITE. */ -static void cc_on_became_master(cc_cluster_t *cl) +static void cl_ctr_on_became_master(cl_ctr_cluster_t *cl) { cl->join_pending = 0; /* any pending re-key join is now moot */ - if (cc_random_bytes(cl->peers->master_salt, CC_MASTER_SALT_SZ) != 0) { + if (cl_ctr_random_bytes(cl->peers->master_salt, CL_CTR_MASTER_SALT_SZ) != 0) { LM_ERR("clusterer_controller: [cluster %d] RNG for master_salt failed\n", cl->cluster_id); return; } - cc_derive_session_key(cl); /* reads cl->peers->master_salt */ + cl_ctr_derive_session_key(cl); /* reads cl->peers->master_salt */ LM_INFO("clusterer_controller: [cluster %d] became master - " "new master_salt generated, session key rotated\n", cl->cluster_id); /* Timer ops outside lock (timerfd is worker-local, no shm concern) */ } /** - * cc_arm_master_timers() - arm/disarm keepalive timers after election. + * cl_ctr_arm_master_timers() - arm/disarm keepalive timers after election. * Call WITHOUT cl->peers->lock held. * i_am_master: 1 = we won, arm MASTER_ALIVE, disarm dead-watchdog. * 0 = we lost, arm dead-watchdog, disarm MASTER_ALIVE. */ -static void cc_arm_master_timers(cc_cluster_t *cl, int i_am_master) +static void cl_ctr_arm_master_timers(cl_ctr_cluster_t *cl, int i_am_master) { cl->master_ka_armed = i_am_master ? 1 : 0; if (i_am_master) { - cc_arm_tfd(cl->master_alive_tfd, CC_MASTER_KA_INTERVAL, CC_MASTER_KA_INTERVAL); - cc_arm_tfd(cl->master_dead_tfd, 0, 0); /* disarm watchdog */ + cl_ctr_arm_tfd(cl->master_alive_tfd, CL_CTR_MASTER_KA_INTERVAL, CL_CTR_MASTER_KA_INTERVAL); + cl_ctr_arm_tfd(cl->master_dead_tfd, 0, 0); /* disarm watchdog */ } else { - cc_arm_tfd(cl->master_alive_tfd, 0, 0); /* disarm sender */ - cc_arm_tfd(cl->master_dead_tfd, CC_MASTER_KA_TIMEOUT, 0); + cl_ctr_arm_tfd(cl->master_alive_tfd, 0, 0); /* disarm sender */ + cl_ctr_arm_tfd(cl->master_dead_tfd, CL_CTR_MASTER_KA_TIMEOUT, 0); } } /** - * cc_request_rekey() - ask the current key-holder for the session key. + * cl_ctr_request_rekey() - ask the current key-holder for the session key. * Sends a single JOIN_REQ (bootstrap key), guarded by join_pending so a fresh * nonce is not stomped while one exchange is in flight. Used when a node is * elected master but has not yet adopted the cluster key (KEY_GRANT lost). * Call WITHOUT cl->peers->lock held. */ -static void cc_request_rekey(cc_cluster_t *cl) +static void cl_ctr_request_rekey(cl_ctr_cluster_t *cl) { if (cl->join_pending) return; cl->join_pending = 1; - cc_send_join_req(cl->sock, cl); + cl_ctr_send_join_req(cl->sock, cl); } /** - * cc_transition_to_active() - switch from CC_NODE_NEW to CC_NODE_ACTIVE. + * cl_ctr_transition_to_active() - switch from CL_CTR_NODE_NEW to CL_CTR_NODE_ACTIVE. * * Disarms the join-phase timers, sends the first ALIVE immediately, then * arms the periodic ALIVE timer. Called from both: * - the join_tfd handler (deadline expired, fresh cluster), and - * - cc_handle_member_list (master responded before deadline). + * - cl_ctr_handle_member_list (master responded before deadline). * Must be called with cl->peers->lock NOT held. */ -static void cc_transition_to_active(cc_cluster_t *cl) +static void cl_ctr_transition_to_active(cl_ctr_cluster_t *cl) { - cc_arm_tfd(cl->join_tfd, 0, 0); /* disarm one-shot deadline */ - cc_arm_tfd(cl->rejoin_tfd, 0, 0); /* disarm JOIN_REQ retry */ - cc_send_alive(cl->sock, cl); /* first heartbeat now */ - cc_arm_tfd(cl->alive_tfd, query_time, query_time); /* periodic from here */ + cl_ctr_arm_tfd(cl->join_tfd, 0, 0); /* disarm one-shot deadline */ + cl_ctr_arm_tfd(cl->rejoin_tfd, 0, 0); /* disarm JOIN_REQ retry */ + cl_ctr_send_alive(cl->sock, cl); /* first heartbeat now */ + cl_ctr_arm_tfd(cl->alive_tfd, query_time, query_time); /* periodic from here */ } /* ========================================================================= @@ -2789,19 +2789,19 @@ static void cc_transition_to_active(cc_cluster_t *cl) * ========================================================================= */ /** - * cc_fmt_cfg_diff() - render only the consistency-critical settings that + * cl_ctr_fmt_cfg_diff() - render only the consistency-critical settings that * actually DIFFER into 'out' (e.g. "manage_shtags cluster=1 node=0"), so a * mismatch log names just the offending setting(s) rather than all three. * 'la'/'lb' are the labels for the local/peer sides ("cluster"/"node" or * "local"/"peer"). */ -static void cc_fmt_cfg_diff(char *out, int outsz, const char *la, const char *lb, +static void cl_ctr_fmt_cfg_diff(char *out, int outsz, const char *la, const char *lb, int a_manage, int b_manage, int a_stick, int b_stick, int a_qt, int b_qt) { int n = 0; out[0] = '\0'; -#define CC_DIFF_APPEND(cond, name, av, bv) \ +#define CL_CTR_DIFF_APPEND(cond, name, av, bv) \ do { \ if (cond) { \ int _w = snprintf(out + n, (n < outsz) ? (outsz - n) : 0, \ @@ -2810,18 +2810,18 @@ static void cc_fmt_cfg_diff(char *out, int outsz, const char *la, const char *lb if (_w > 0) { n += _w; if (n > outsz) n = outsz; } \ } \ } while (0) - CC_DIFF_APPEND(a_manage != b_manage, "manage_shtags", a_manage, b_manage); - CC_DIFF_APPEND(a_stick != b_stick, "master_stickiness", a_stick, b_stick); - CC_DIFF_APPEND(a_qt != b_qt, "query_time", a_qt, b_qt); -#undef CC_DIFF_APPEND + CL_CTR_DIFF_APPEND(a_manage != b_manage, "manage_shtags", a_manage, b_manage); + CL_CTR_DIFF_APPEND(a_stick != b_stick, "master_stickiness", a_stick, b_stick); + CL_CTR_DIFF_APPEND(a_qt != b_qt, "query_time", a_qt, b_qt); +#undef CL_CTR_DIFF_APPEND } /** - * cc_adopt_config() - adopt the running cluster's consistency-critical settings + * cl_ctr_adopt_config() - adopt the running cluster's consistency-critical settings * (on_config_mismatch=adopt). Called on a non-master node when the master's * advertised config differs from ours. Call WITHOUT cl->peers->lock held. */ -static void cc_adopt_config(cc_cluster_t *cl, int new_manage, int new_stick, +static void cl_ctr_adopt_config(cl_ctr_cluster_t *cl, int new_manage, int new_stick, int new_qt, int is_active) { int old_manage = cl->manage_shtags ? 1 : 0; @@ -2842,7 +2842,7 @@ static void cc_adopt_config(cc_cluster_t *cl, int new_manage, int new_stick, if (new_qt >= 1 && new_qt != query_time) { query_time = new_qt; if (is_active) - cc_arm_tfd(cl->alive_tfd, query_time, query_time); + cl_ctr_arm_tfd(cl->alive_tfd, query_time, query_time); } /* Mirror the effective values into shm so cl_ctr_list_config (MI process) @@ -2862,44 +2862,44 @@ static void cc_adopt_config(cc_cluster_t *cl, int new_manage, int new_stick, } } /* Reconcile tag state under the new policy (no-op when now unmanaged). */ - cc_apply_shtags(cl); + cl_ctr_apply_shtags(cl); } } /** - * cc_handle_alive() - process a CC_PKT_ALIVE packet. + * cl_ctr_handle_alive() - process a CL_CTR_PKT_ALIVE packet. * * Regular heartbeat path: upsert the sender, re-elect. - * Only called in CC_NODE_ACTIVE state; ignored while joining (cc_recv_one + * Only called in CL_CTR_NODE_ACTIVE state; ignored while joining (cl_ctr_recv_one * still dispatches them so the peer table builds up before the timeout). */ -static void cc_handle_alive(const char *src_ip, +static void cl_ctr_handle_alive(const char *src_ip, const unsigned char *pubkey, /* may be NULL */ int cfg_present, int peer_manage, int peer_stick, int peer_qt, - cc_cluster_t *cl) + cl_ctr_cluster_t *cl) { int prev_master, now_master; int warn = 0, adopt = 0, is_active = 0, ent = -1; - char warn_ip[CC_MAX_IP_LEN + 1] = ""; + char warn_ip[CL_CTR_MAX_IP_LEN + 1] = ""; int loc_manage = cl->manage_shtags ? 1 : 0; int loc_stick = cl->master_stickiness ? 1 : 0; int loc_qt = query_time; int mism = 0, changed = 0; lock_start_write(cl->peers->lock); - prev_master = cc_i_am_master_locked(cl); - cc_upsert_peer_locked(src_ip, cl); + prev_master = cl_ctr_i_am_master_locked(cl); + cl_ctr_upsert_peer_locked(src_ip, cl); { int _i; for (_i = 0; _i < cl->peers->count; _i++) { - cc_peer_t *e = &cl->peers->entries[_i]; + cl_ctr_peer_t *e = &cl->peers->entries[_i]; if (strcmp(e->ip, src_ip) != 0) continue; ent = _i; /* Store pubkey so master can use it for KEY_HANDOFF / KEY_GRANT */ if (pubkey) - memcpy(e->pubkey, pubkey, CC_PUBKEY_SZ); + memcpy(e->pubkey, pubkey, CL_CTR_PUBKEY_SZ); if (cfg_present) { changed = (!e->cfg_known || e->cfg_manage_shtags != peer_manage @@ -2916,21 +2916,21 @@ static void cc_handle_alive(const char *src_ip, break; } } - cc_elect_master(cl); - now_master = cc_i_am_master_locked(cl); - is_active = (cl->peers->node_state == CC_NODE_ACTIVE); + cl_ctr_elect_master(cl); + now_master = cl_ctr_i_am_master_locked(cl); + is_active = (cl->peers->node_state == CL_CTR_NODE_ACTIVE); /* Config-consistency handling, decided once the master is known. In * 'adopt' mode a non-master node takes the master's settings; otherwise a * mismatch is logged once per peer (re-logged only if the peer's advertised - * config changes, cleared when it matches). cc_elect_master does not + * config changes, cleared when it matches). cl_ctr_elect_master does not * reorder entries, so the captured index stays valid. */ if (cfg_present && ent >= 0) { - cc_peer_t *e = &cl->peers->entries[ent]; + cl_ctr_peer_t *e = &cl->peers->entries[ent]; int sender_is_master = (cl->peers->last_master[0] != '\0' && strcmp(src_ip, cl->peers->last_master) == 0); if (!mism) { e->cfg_warned = 0; - } else if (on_config_mismatch == CC_CFGMISMATCH_ADOPT + } else if (on_config_mismatch == CL_CTR_CFGMISMATCH_ADOPT && sender_is_master && !now_master) { adopt = 1; e->cfg_warned = 0; @@ -2944,7 +2944,7 @@ static void cc_handle_alive(const char *src_ip, if (warn) { char diff[160]; - cc_fmt_cfg_diff(diff, sizeof(diff), "local", "peer", + cl_ctr_fmt_cfg_diff(diff, sizeof(diff), "local", "peer", loc_manage, peer_manage, loc_stick, peer_stick, loc_qt, peer_qt); LM_WARN("clusterer_controller: [cluster %d] CONFIG MISMATCH with peer %s " @@ -2953,22 +2953,22 @@ static void cc_handle_alive(const char *src_ip, cl->cluster_id, warn_ip, diff); } if (adopt) - cc_adopt_config(cl, peer_manage, peer_stick, peer_qt, is_active); + cl_ctr_adopt_config(cl, peer_manage, peer_stick, peer_qt, is_active); /* Defer acting as master until we hold the cluster key. In normal * operation a higher-IP joiner has already adopted the key via KEY_GRANT * before winning here, so this only guards the pathological case where the * KEY_GRANT was lost during the join. Request a re-key instead of * broadcasting an undecryptable MASTER_ALIVE. */ if (now_master && !cl->have_session_key) { - cc_request_rekey(cl); + cl_ctr_request_rekey(cl); return; } if (prev_master != now_master) - cc_arm_master_timers(cl, now_master); + cl_ctr_arm_master_timers(cl, now_master); } /** - * cc_handle_join_req() - process a CC_PKT_JOIN_REQ packet. + * cl_ctr_handle_join_req() - process a CL_CTR_PKT_JOIN_REQ packet. * * Payload: [ip NUL][bin_count 1B][sock1 NUL]...[sockN NUL] * @@ -2982,13 +2982,13 @@ static void cc_handle_alive(const char *src_ip, * 4. If joining IP > own IP: run election, send MEMBER_LIST. * 5. If joining IP <= own IP: send MEMBER_LIST (self still master). */ -static void cc_handle_join_req(int sock, const char *payload, int payload_len, - cc_cluster_t *cl) +static void cl_ctr_handle_join_req(int sock, const char *payload, int payload_len, + cl_ctr_cluster_t *cl) { const char *p = payload; const char *end = payload + payload_len; - char src_ip[CC_MAX_IP_LEN + 1]; - char bin_socks[CC_MAX_BIN_SOCKETS][CC_MAX_BIN_SOCK_LEN]; + char src_ip[CL_CTR_MAX_IP_LEN + 1]; + char bin_socks[CL_CTR_MAX_BIN_SOCKETS][CL_CTR_MAX_BIN_SOCK_LEN]; const unsigned char *noise_msg1 = NULL; /* points into payload */ int noise_msg1_len = 0; uint8_t bin_cnt = 0; @@ -2997,7 +2997,7 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, uint16_t new_id; /* --- Parse IP --- */ - ip_len = (int)strnlen(p, CC_MAX_IP_LEN); + ip_len = (int)strnlen(p, CL_CTR_MAX_IP_LEN); if (p + ip_len >= end) { LM_WARN("clusterer_controller: JOIN_REQ payload truncated\n"); return; @@ -3014,10 +3014,10 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, memset(bin_socks, 0, sizeof(bin_socks)); if (p < end) { bin_cnt = (uint8_t)*p++; - if (bin_cnt > CC_MAX_BIN_SOCKETS) - bin_cnt = CC_MAX_BIN_SOCKETS; + if (bin_cnt > CL_CTR_MAX_BIN_SOCKETS) + bin_cnt = CL_CTR_MAX_BIN_SOCKETS; for (i = 0; i < (int)bin_cnt && p < end; i++) { - int slen = (int)strnlen(p, CC_MAX_BIN_SOCK_LEN - 1); + int slen = (int)strnlen(p, CL_CTR_MAX_BIN_SOCK_LEN - 1); memcpy(bin_socks[i], p, slen); bin_socks[i][slen] = '\0'; p += slen + 1; @@ -3025,21 +3025,21 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, } /* --- Parse the Noise msg 1 (appended after BIN info) --- */ - if (p + CC_NOISE_MSG1_SZ <= end) { + if (p + CL_CTR_NOISE_MSG1_SZ <= end) { noise_msg1 = (const unsigned char *)p; - noise_msg1_len = CC_NOISE_MSG1_SZ; - p += CC_NOISE_MSG1_SZ; + noise_msg1_len = CL_CTR_NOISE_MSG1_SZ; + p += CL_CTR_NOISE_MSG1_SZ; } /* --- Parse the joiner's consistency-critical config (after join_nonce) --- */ - if (p + CC_CONFIG_SZ <= end) { + if (p + CL_CTR_CONFIG_SZ <= end) { uint16_t qt_be; j_manage = (unsigned char)p[0]; j_stick = (unsigned char)p[1]; memcpy(&qt_be, p + 2, 2); j_qt = ntohs(qt_be); j_cfg_present = 1; - p += CC_CONFIG_SZ; + p += CL_CTR_CONFIG_SZ; } LM_INFO("clusterer_controller: [cluster %d] JOIN_REQ from %s " @@ -3047,8 +3047,8 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, lock_start_write(cl->peers->lock); - was_master = (cl->peers->node_state == CC_NODE_ACTIVE) && - cc_i_am_master_locked(cl); + was_master = (cl->peers->node_state == CL_CTR_NODE_ACTIVE) && + cl_ctr_i_am_master_locked(cl); if (!was_master) { /* Split-brain prevention: while we are ourselves still joining, record @@ -3056,8 +3056,8 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, * highest-IP starter instead of every node self-promoting into a * divergent-key lone master. (An active non-master simply ignores it; * the master drives discovery.) */ - if (cl->peers->node_state == CC_NODE_NEW) { - cc_upsert_peer_locked(src_ip, cl); + if (cl->peers->node_state == CL_CTR_NODE_NEW) { + cl_ctr_upsert_peer_locked(src_ip, cl); /* A fresh JOIN_REQ from a *higher-IP* peer proves it is still alive * and joining, so reset our split-brain defer budget: keep waiting * for it to become master instead of prematurely self-promoting into @@ -3077,19 +3077,19 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, * the policy is 'reject', refuse the join so the node shuts down rather * than joining and behaving inconsistently. (warn/adopt admit the node; * the joiner then warns or adopts on the master's ALIVE.) */ - if (j_cfg_present && on_config_mismatch == CC_CFGMISMATCH_REJECT) { + if (j_cfg_present && on_config_mismatch == CL_CTR_CFGMISMATCH_REJECT) { int loc_manage = cl->manage_shtags ? 1 : 0; int loc_stick = cl->master_stickiness ? 1 : 0; if (j_manage != loc_manage || j_stick != loc_stick || j_qt != query_time) { char diff[160]; lock_stop_write(cl->peers->lock); - cc_fmt_cfg_diff(diff, sizeof(diff), "cluster", "node", + cl_ctr_fmt_cfg_diff(diff, sizeof(diff), "cluster", "node", loc_manage, j_manage, loc_stick, j_stick, query_time, j_qt); LM_WARN("clusterer_controller: [cluster %d] rejecting JOIN_REQ from %s: " "different settings than the running cluster (%s)\n", cl->cluster_id, src_ip, diff); - cc_send_join_reject(sock, src_ip, cl, CC_REJECT_CONFIG); + cl_ctr_send_join_reject(sock, src_ip, cl, CL_CTR_REJECT_CONFIG); return; } } @@ -3097,7 +3097,7 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, /* Reject JOIN_REQ from an unknown IP when the peer table is full. * Known peers (reconnecting after restart) are still allowed through * since they already occupy a slot. */ - if (cl->peers->count >= CC_MAX_PEERS) { + if (cl->peers->count >= CL_CTR_MAX_PEERS) { int _found = 0, _fi; for (_fi = 0; _fi < cl->peers->count; _fi++) { if (strcmp(cl->peers->entries[_fi].ip, src_ip) == 0) { @@ -3109,8 +3109,8 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, lock_stop_write(cl->peers->lock); LM_WARN("clusterer_controller: [cluster %d] peer table full " "(%d/%d), rejecting JOIN_REQ from %s\n", - cl->cluster_id, cl->peers->count, CC_MAX_PEERS, src_ip); - cc_send_join_reject(sock, src_ip, cl, CC_REJECT_GENERIC); + cl->cluster_id, cl->peers->count, CL_CTR_MAX_PEERS, src_ip); + cl_ctr_send_join_reject(sock, src_ip, cl, CL_CTR_REJECT_GENERIC); return; } } @@ -3118,7 +3118,7 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, /* Upsert peer and store BIN info + node_id. * If this IP already has a node_id (rejoining after crash/restart), * reuse it so the id stays stable and clusterer stays in sync. */ - cc_upsert_peer_locked(src_ip, cl); + cl_ctr_upsert_peer_locked(src_ip, cl); { int _i; new_id = 0; @@ -3129,11 +3129,11 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, } } if (new_id == 0) - new_id = cc_alloc_node_id_locked(cl); + new_id = cl_ctr_alloc_node_id_locked(cl); } - cc_update_peer_bin_locked(src_ip, new_id, + cl_ctr_update_peer_bin_locked(src_ip, new_id, bin_cnt, - (const char (*)[CC_MAX_BIN_SOCK_LEN])bin_socks, + (const char (*)[CL_CTR_MAX_BIN_SOCK_LEN])bin_socks, cl); /* Reset last_seq in the peer table. Essential: a restarted node begins its @@ -3142,7 +3142,7 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, * joiner's long-lived pubkey (for a future KEY_HANDOFF) is learned from its * ALIVE, not here - the JOIN_REQ now carries only an ephemeral Noise key. */ { - cc_peer_t *e = cc_peer_by_ip_locked(cl, src_ip); + cl_ctr_peer_t *e = cl_ctr_peer_by_ip_locked(cl, src_ip); if (e) e->last_seq = 0; } @@ -3154,7 +3154,7 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, { uint32_t _ip_num = ip_to_num(src_ip); int _fi; - for (_fi = 0; _fi < CC_JOIN_FAIL_TABLE_SZ; _fi++) { + for (_fi = 0; _fi < CL_CTR_JOIN_FAIL_TABLE_SZ; _fi++) { if (cl->join_fail_tbl[_fi].ip_num == _ip_num) { memset(&cl->join_fail_tbl[_fi], 0, sizeof(cl->join_fail_tbl[_fi])); break; @@ -3165,22 +3165,22 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, /* Send KEY_GRANT first (bootstrap key) so joiner can derive session_key, * then NODE_ASSIGN + MEMBER_LIST (session key). */ if (noise_msg1) /* Noise msg 1 present -> run responder, answer msg 2 */ - cc_send_key_grant(sock, src_ip, cl, noise_msg1, noise_msg1_len); + cl_ctr_send_key_grant(sock, src_ip, cl, noise_msg1, noise_msg1_len); lock_start_write(cl->peers->lock); /* Send NODE_ASSIGN for joining node */ - cc_send_node_assign(sock, src_ip, new_id, bin_cnt, - (const char (*)[CC_MAX_BIN_SOCK_LEN])bin_socks, cl); + cl_ctr_send_node_assign(sock, src_ip, new_id, bin_cnt, + (const char (*)[CL_CTR_MAX_BIN_SOCK_LEN])bin_socks, cl); /* Send NODE_ASSIGN for each existing peer so joining node learns * all current node_ids and BIN sockets */ for (i = 0; i < cl->peers->count; i++) { - cc_peer_t *e = &cl->peers->entries[i]; + cl_ctr_peer_t *e = &cl->peers->entries[i]; if (strcmp(e->ip, src_ip) == 0 || e->node_id == 0) continue; - cc_send_node_assign(sock, e->ip, e->node_id, e->bin_count, - (const char (*)[CC_MAX_BIN_SOCK_LEN])e->bin_sockets, + cl_ctr_send_node_assign(sock, e->ip, e->node_id, e->bin_count, + (const char (*)[CL_CTR_MAX_BIN_SOCK_LEN])e->bin_sockets, cl); } @@ -3191,7 +3191,7 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, * Instead we stay master and designate ourselves in the MEMBER_LIST. The * joiner adopts our session key via the KEY_GRANT sent above and joins as * a member. If it has a higher IP it will win the very next ALIVE-driven - * election (cc_handle_alive) - but by then it holds the key, so when it + * election (cl_ctr_handle_alive) - but by then it holds the key, so when it * takes over it broadcasts with a key every member already has. This * keeps the deterministic "highest IP is master" outcome while deferring * the actual takeover until after the key has been transferred. */ @@ -3199,36 +3199,36 @@ static void cc_handle_join_req(int sock, const char *payload, int payload_len, LM_INFO("clusterer_controller: [cluster %d] I am master, " "new node %s assigned node_id=%u\n", cl->cluster_id, src_ip, new_id); - cc_send_member_list(sock, cl); + cl_ctr_send_member_list(sock, cl); } /** - * cc_handle_member_list() - process a CC_PKT_MEMBER_LIST from the master. + * cl_ctr_handle_member_list() - process a CL_CTR_PKT_MEMBER_LIST from the master. * * This is THE authoritative packet for cluster state. All nodes - both * the joining node and existing active members - update their peer tables * and master designation directly from this list. No independent election * is run; the master's word is final. * - * Joining node (CC_NODE_NEW): + * Joining node (CL_CTR_NODE_NEW): * - Replaces its empty peer table with the master's list. - * - Transitions to CC_NODE_ACTIVE. + * - Transitions to CL_CTR_NODE_ACTIVE. * - If the list designates us as master (our IP has is_master=1): we * accept mastership immediately and log accordingly. * - * Active member / old master (CC_NODE_ACTIVE): + * Active member / old master (CL_CTR_NODE_ACTIVE): * - Upserts any new peers from the list. * - Applies master designation from the list (may demote old master). * - Logs who is now master. */ -static void cc_handle_member_list(const char *payload, int payload_len, - const char *sender_ip, cc_cluster_t *cl) +static void cl_ctr_handle_member_list(const char *payload, int payload_len, + const char *sender_ip, cl_ctr_cluster_t *cl) { uint16_t count; uint16_t forced_node_id; int i; const char *p; - char designated_master[CC_MAX_IP_LEN + 1]; + char designated_master[CL_CTR_MAX_IP_LEN + 1]; /* MEMBER_LIST is authoritative cluster state and must only come from the * current master. Any cluster member holding the session key could forge @@ -3246,11 +3246,11 @@ static void cc_handle_member_list(const char *payload, int payload_len, strcmp(sender_ip, cl->peers->last_master) == 0); lock_stop_read(cl->peers->lock); if (!_from_master) { - /* Also allow during CC_NODE_NEW: we have no master yet, so any + /* Also allow during CL_CTR_NODE_NEW: we have no master yet, so any * MEMBER_LIST is our first authoritative view of the cluster. */ int _is_new; lock_start_read(cl->peers->lock); - _is_new = (cl->peers->node_state == CC_NODE_NEW); + _is_new = (cl->peers->node_state == CL_CTR_NODE_NEW); lock_stop_read(cl->peers->lock); if (!_is_new) { LM_WARN("clusterer_controller: MEMBER_LIST from non-master %s " @@ -3264,42 +3264,42 @@ static void cc_handle_member_list(const char *payload, int payload_len, designated_master[0] = '\0'; - if (payload_len < CC_LIST_COUNT_SZ + CC_NODE_ID_SZ) { + if (payload_len < CL_CTR_LIST_COUNT_SZ + CL_CTR_NODE_ID_SZ) { LM_WARN("clusterer_controller: MEMBER_LIST too short\n"); return; } { uint16_t count_be, forced_be; - memcpy(&count_be, payload, CC_LIST_COUNT_SZ); + memcpy(&count_be, payload, CL_CTR_LIST_COUNT_SZ); count = ntohs(count_be); /* forced-shtag node_id follows the count (0 = automatic allocation) */ - memcpy(&forced_be, payload + CC_LIST_COUNT_SZ, CC_NODE_ID_SZ); + memcpy(&forced_be, payload + CL_CTR_LIST_COUNT_SZ, CL_CTR_NODE_ID_SZ); forced_node_id = ntohs(forced_be); } - if (count > CC_MAX_PEERS) { + if (count > CL_CTR_MAX_PEERS) { LM_WARN("clusterer_controller: MEMBER_LIST count %u exceeds " - "max peers %d, dropping\n", count, CC_MAX_PEERS); + "max peers %d, dropping\n", count, CL_CTR_MAX_PEERS); return; } - if (payload_len < CC_LIST_COUNT_SZ + CC_NODE_ID_SZ - + (int)count * CC_IP_ENTRY_SZ) { + if (payload_len < CL_CTR_LIST_COUNT_SZ + CL_CTR_NODE_ID_SZ + + (int)count * CL_CTR_IP_ENTRY_SZ) { LM_WARN("clusterer_controller: MEMBER_LIST truncated " "(count=%u, got %d bytes)\n", count, payload_len); return; } - p = payload + CC_LIST_COUNT_SZ + CC_NODE_ID_SZ; + p = payload + CL_CTR_LIST_COUNT_SZ + CL_CTR_NODE_ID_SZ; /* First pass: collect designated master IP */ for (i = 0; i < (int)count; i++) { - const char *entry = p + i * CC_IP_ENTRY_SZ; - unsigned char is_master = (unsigned char)entry[CC_IP_ENTRY_SZ - 1]; + const char *entry = p + i * CL_CTR_IP_ENTRY_SZ; + unsigned char is_master = (unsigned char)entry[CL_CTR_IP_ENTRY_SZ - 1]; if (is_master) { - memcpy(designated_master, entry, CC_MAX_IP_LEN); - designated_master[CC_MAX_IP_LEN] = '\0'; + memcpy(designated_master, entry, CL_CTR_MAX_IP_LEN); + designated_master[CL_CTR_MAX_IP_LEN] = '\0'; break; } } @@ -3311,14 +3311,14 @@ static void cc_handle_member_list(const char *payload, int payload_len, * sent JOIN_REQ: the MEMBER_LIST is the broadcast announcement that a * join event occurred. Without the reset, non-master peers would reject * the restarted node's new packets (old last_seq > new low seq). */ - for (i = 0; i < (int)count; i++, p += CC_IP_ENTRY_SZ) { - char ip_buf[CC_MAX_IP_LEN + 1]; + for (i = 0; i < (int)count; i++, p += CL_CTR_IP_ENTRY_SZ) { + char ip_buf[CL_CTR_MAX_IP_LEN + 1]; int _j; - memcpy(ip_buf, p, CC_MAX_IP_LEN); - ip_buf[CC_MAX_IP_LEN] = '\0'; + memcpy(ip_buf, p, CL_CTR_MAX_IP_LEN); + ip_buf[CL_CTR_MAX_IP_LEN] = '\0'; if (ip_buf[0] == '\0') continue; - cc_upsert_peer_locked(ip_buf, cl); + cl_ctr_upsert_peer_locked(ip_buf, cl); for (_j = 0; _j < cl->peers->count; _j++) { if (strcmp(cl->peers->entries[_j].ip, ip_buf) == 0) { cl->peers->entries[_j].last_seq = 0; @@ -3327,17 +3327,17 @@ static void cc_handle_member_list(const char *payload, int payload_len, } } - /* Snapshot master status BEFORE cc_apply_master_from_list_locked clears it */ - int was_master_before = cc_i_am_master_locked(cl); + /* Snapshot master status BEFORE cl_ctr_apply_master_from_list_locked clears it */ + int was_master_before = cl_ctr_i_am_master_locked(cl); /* Apply the master designation from the list - no local election */ if (designated_master[0] != '\0') - cc_apply_master_from_list_locked(designated_master, cl); + cl_ctr_apply_master_from_list_locked(designated_master, cl); /* The master is authoritative for the shtag override too. */ cl->peers->shtag_forced_node_id = forced_node_id; - int was_new = (cl->peers->node_state == CC_NODE_NEW); + int was_new = (cl->peers->node_state == CL_CTR_NODE_NEW); if (was_new) { /* Only act as master if the list designates us AND we already hold the * cluster key. Incumbent masters no longer hand over during a join, so @@ -3348,7 +3348,7 @@ static void cc_handle_member_list(const char *payload, int payload_len, int _taking_over = (designated_master[0] != '\0' && strcmp(designated_master, my_ip) == 0 && cl->have_session_key); - cl->peers->node_state = CC_NODE_ACTIVE; + cl->peers->node_state = CL_CTR_NODE_ACTIVE; if (_taking_over) { lock_stop_write(cl->peers->lock); LM_INFO("clusterer_controller: received MEMBER_LIST (%u members) " @@ -3362,9 +3362,9 @@ static void cc_handle_member_list(const char *payload, int payload_len, count, sender_ip, designated_master[0] ? designated_master : "(none)"); } - cc_transition_to_active(cl); + cl_ctr_transition_to_active(cl); if (_taking_over) - cc_arm_master_timers(cl, 1); /* start MASTER_ALIVE, disarm dead watchdog */ + cl_ctr_arm_master_timers(cl, 1); /* start MASTER_ALIVE, disarm dead watchdog */ } else { /* Active node - log the master update (may be self-demotion) */ int i_am_master = (designated_master[0] != '\0' && @@ -3380,7 +3380,7 @@ static void cc_handle_member_list(const char *payload, int payload_len, designated_master[0] ? designated_master : "(none)", count); /* Fix up keepalive timers: stop sending MASTER_ALIVE, arm watchdog. */ - cc_arm_master_timers(cl, 0); + cl_ctr_arm_master_timers(cl, 0); } else { /* Already a member - this is just a routine MEMBER_LIST refresh (e.g. * a periodic re-broadcast or a shtag-override update). No role change, @@ -3393,11 +3393,11 @@ static void cc_handle_member_list(const char *payload, int payload_len, /* (Re)apply the shtag decision now that the forced-node override and the * master designation from this MEMBER_LIST have both been stored. */ - cc_apply_shtags(cl); + cl_ctr_apply_shtags(cl); } /** - * cc_handle_goodbye() - process a CC_PKT_GOODBYE packet. + * cl_ctr_handle_goodbye() - process a CL_CTR_PKT_GOODBYE packet. * * Remove the departing node from the peer table immediately. * @@ -3405,19 +3405,19 @@ static void cc_handle_member_list(const char *payload, int payload_len, * 1. Only one node remains - we are alone and must assume mastership. * 2. Our IP is higher than the current master's IP, or the master entry * no longer exists because the departing node was the master. - * cc_ip_beats_master_locked() covers both cases: it returns 1 when + * cl_ctr_ip_beats_master_locked() covers both cases: it returns 1 when * no is_master entry is present (departed master) or when our IP * numerically exceeds the current master's. * * All other departures (a member leaves while a higher-IP master is alive) * require no immediate action - the next periodic ALIVE cycle runs - * cc_elect_master(cl) within query_time seconds and self-corrects if needed. + * cl_ctr_elect_master(cl) within query_time seconds and self-corrects if needed. */ -static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) +static void cl_ctr_handle_goodbye(int sock, const char *src_ip, cl_ctr_cluster_t *cl) { int i, i_am_master, master_unchanged, remaining; - char prev_master[CC_MAX_IP_LEN + 1]; - char new_master[CC_MAX_IP_LEN + 1]; + char prev_master[CL_CTR_MAX_IP_LEN + 1]; + char new_master[CL_CTR_MAX_IP_LEN + 1]; uint16_t departed_node_id = 0; LM_INFO("clusterer_controller: GOODBYE from %s\n", src_ip); @@ -3430,7 +3430,7 @@ static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) cl->peers->count--; if (i < cl->peers->count) cl->peers->entries[i] = cl->peers->entries[cl->peers->count]; - memset(&cl->peers->entries[cl->peers->count], 0, sizeof(cc_peer_t)); + memset(&cl->peers->entries[cl->peers->count], 0, sizeof(cl_ctr_peer_t)); break; } } @@ -3450,8 +3450,8 @@ static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) /* --- Decide whether re-election is warranted --- */ if (remaining <= 1) { /* We are the only node left - no election needed, promote directly. */ - int was_master = cc_i_am_master_locked(cl); - cc_apply_master_from_list_locked(my_ip, cl); + int was_master = cl_ctr_i_am_master_locked(cl); + cl_ctr_apply_master_from_list_locked(my_ip, cl); lock_stop_write(cl->peers->lock); if (was_master) { LM_INFO("clusterer_controller: %s departed - I am the only " @@ -3462,12 +3462,12 @@ static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) } if (clctl_loaded && departed_node_id > 0) clctl.remove_node(cl->cluster_id, departed_node_id); - cc_apply_shtags(cl); + cl_ctr_apply_shtags(cl); return; } - if (cc_ip_beats_master_locked(ip_to_num(my_ip), cl)) { - /* Two sub-cases both return 1 from cc_ip_beats_master_locked: + if (cl_ctr_ip_beats_master_locked(ip_to_num(my_ip), cl)) { + /* Two sub-cases both return 1 from cl_ctr_ip_beats_master_locked: * a) departing node was the master (no master entry remains) * b) our IP is genuinely higher than the current master (anomaly) */ if (strcmp(src_ip, cl->peers->last_master) == 0) { @@ -3482,7 +3482,7 @@ static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) } else { /* Snapshot before releasing - must not read last_master after unlock */ { - size_t _l = strnlen(cl->peers->last_master, CC_MAX_IP_LEN); + size_t _l = strnlen(cl->peers->last_master, CL_CTR_MAX_IP_LEN); memcpy(prev_master, cl->peers->last_master, _l); prev_master[_l] = '\0'; } @@ -3496,24 +3496,24 @@ static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) clctl.remove_node(cl->cluster_id, departed_node_id); /* Re-apply shtag policy: the still-active master takes over the * departed node's tags, unless an operator override is in effect. */ - cc_apply_shtags(cl); + cl_ctr_apply_shtags(cl); return; } memcpy(prev_master, cl->peers->last_master, - strnlen(cl->peers->last_master, CC_MAX_IP_LEN)); - prev_master[strnlen(cl->peers->last_master, CC_MAX_IP_LEN)] = '\0'; + strnlen(cl->peers->last_master, CL_CTR_MAX_IP_LEN)); + prev_master[strnlen(cl->peers->last_master, CL_CTR_MAX_IP_LEN)] = '\0'; { - cc_elect_master(cl); - i_am_master = cc_i_am_master_locked(cl); + cl_ctr_elect_master(cl); + i_am_master = cl_ctr_i_am_master_locked(cl); } master_unchanged = (strcmp(prev_master, cl->peers->last_master) == 0); - i_am_master = cc_i_am_master_locked(cl); + i_am_master = cl_ctr_i_am_master_locked(cl); /* Snapshot post-election master before releasing - used in member log */ { - size_t _l = strnlen(cl->peers->last_master, CC_MAX_IP_LEN); + size_t _l = strnlen(cl->peers->last_master, CL_CTR_MAX_IP_LEN); memcpy(new_master, cl->peers->last_master, _l); new_master[_l] = '\0'; } @@ -3529,12 +3529,12 @@ static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) "I reclaimed mastership after %s departed " "(%d node(s) remaining) - sending MEMBER_LIST\n", src_ip, remaining); - cc_arm_master_timers(cl, 1); - cc_send_member_list(sock, cl); + cl_ctr_arm_master_timers(cl, 1); + cl_ctr_send_member_list(sock, cl); } } else { /* This node's own role change (if any) is logged separately by - * cc_elect_master() as a "my role changed" transition line. */ + * cl_ctr_elect_master() as a "my role changed" transition line. */ LM_INFO("clusterer_controller: re-election complete - " "master is %s (%d node(s) remaining)\n", new_master[0] ? new_master : "(none)", @@ -3542,11 +3542,11 @@ static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) } if (clctl_loaded && departed_node_id > 0) clctl.remove_node(cl->cluster_id, departed_node_id); - cc_apply_shtags(cl); + cl_ctr_apply_shtags(cl); } /** - * cc_handle_node_assign() - process a CC_PKT_NODE_ASSIGN from the master. + * cl_ctr_handle_node_assign() - process a CL_CTR_PKT_NODE_ASSIGN from the master. * * Payload: [node_id 2B BE][ip NUL][bin_count 1B][sock1 NUL]...[sockN NUL] * @@ -3555,29 +3555,29 @@ static void cc_handle_goodbye(int sock, const char *src_ip, cc_cluster_t *cl) * - Store node_id and BIN sockets. * - If ip == my_ip: record my_node_id. */ -static void cc_handle_node_assign(const char *payload, int payload_len, - const char *sender_ip, cc_cluster_t *cl) +static void cl_ctr_handle_node_assign(const char *payload, int payload_len, + const char *sender_ip, cl_ctr_cluster_t *cl) { const char *p = payload; const char *end = payload + payload_len; uint16_t node_id; - char ip[CC_MAX_IP_LEN + 1]; - char bin_socks[CC_MAX_BIN_SOCKETS][CC_MAX_BIN_SOCK_LEN]; + char ip[CL_CTR_MAX_IP_LEN + 1]; + char bin_socks[CL_CTR_MAX_BIN_SOCKETS][CL_CTR_MAX_BIN_SOCK_LEN]; uint8_t bin_cnt = 0; int ip_len, i; - if (payload_len < (int)(CC_NODE_ID_SZ + 2)) { + if (payload_len < (int)(CL_CTR_NODE_ID_SZ + 2)) { LM_WARN("clusterer_controller: NODE_ASSIGN payload too short\n"); return; } /* node_id (2B BE) */ - memcpy(&node_id, p, CC_NODE_ID_SZ); + memcpy(&node_id, p, CL_CTR_NODE_ID_SZ); node_id = ntohs(node_id); - p += CC_NODE_ID_SZ; + p += CL_CTR_NODE_ID_SZ; /* IP */ - ip_len = (int)strnlen(p, CC_MAX_IP_LEN); + ip_len = (int)strnlen(p, CL_CTR_MAX_IP_LEN); memcpy(ip, p, ip_len); ip[ip_len] = '\0'; p += ip_len + 1; @@ -3586,10 +3586,10 @@ static void cc_handle_node_assign(const char *payload, int payload_len, memset(bin_socks, 0, sizeof(bin_socks)); if (p < end) { bin_cnt = (uint8_t)*p++; - if (bin_cnt > CC_MAX_BIN_SOCKETS) - bin_cnt = CC_MAX_BIN_SOCKETS; + if (bin_cnt > CL_CTR_MAX_BIN_SOCKETS) + bin_cnt = CL_CTR_MAX_BIN_SOCKETS; for (i = 0; i < (int)bin_cnt && p < end; i++) { - int slen = (int)strnlen(p, CC_MAX_BIN_SOCK_LEN - 1); + int slen = (int)strnlen(p, CL_CTR_MAX_BIN_SOCK_LEN - 1); memcpy(bin_socks[i], p, slen); bin_socks[i][slen] = '\0'; p += slen + 1; @@ -3597,9 +3597,9 @@ static void cc_handle_node_assign(const char *payload, int payload_len, } lock_start_write(cl->peers->lock); - cc_upsert_peer_locked(ip, cl); - cc_update_peer_bin_locked(ip, node_id, bin_cnt, - (const char (*)[CC_MAX_BIN_SOCK_LEN])bin_socks, + cl_ctr_upsert_peer_locked(ip, cl); + cl_ctr_update_peer_bin_locked(ip, node_id, bin_cnt, + (const char (*)[CL_CTR_MAX_BIN_SOCK_LEN])bin_socks, cl); lock_stop_write(cl->peers->lock); @@ -3609,7 +3609,7 @@ static void cc_handle_node_assign(const char *payload, int payload_len, if (my_node_id == 0) { int is_joining; lock_start_read(cl->peers->lock); - is_joining = (cl->peers->node_state == CC_NODE_NEW); + is_joining = (cl->peers->node_state == CL_CTR_NODE_NEW); lock_stop_read(cl->peers->lock); if (is_joining) LM_INFO("clusterer_controller: [cluster %d] found existing master " @@ -3636,7 +3636,7 @@ static void cc_handle_node_assign(const char *payload, int payload_len, } /** - * cc_handle_master_alive() - process a master keepalive. + * cl_ctr_handle_master_alive() - process a master keepalive. * * Beyond rearming the master-dead watchdog, this keeps every node agreeing on * the master's identity (the 1s keepalive is more frequent than MEMBER_LIST, @@ -3646,28 +3646,28 @@ static void cc_handle_node_assign(const char *payload, int payload_len, * preemption modes, so two masters (e.g. after a network partition heals) can * never both stick. */ -static void cc_handle_master_alive(const char *sender_ip, cc_cluster_t *cl) +static void cl_ctr_handle_master_alive(const char *sender_ip, cl_ctr_cluster_t *cl) { int i_am_master, yielded = 0; int from_self = (strcmp(sender_ip, my_ip) == 0); lock_start_write(cl->peers->lock); - i_am_master = cc_i_am_master_locked(cl); + i_am_master = cl_ctr_i_am_master_locked(cl); if (i_am_master) { if (!from_self && ip_to_num(sender_ip) > ip_to_num(my_ip)) { /* Split-brain: a higher-IP node also claims mastership - yield. */ - cc_upsert_peer_locked(sender_ip, cl); - cc_apply_master_from_list_locked(sender_ip, cl); + cl_ctr_upsert_peer_locked(sender_ip, cl); + cl_ctr_apply_master_from_list_locked(sender_ip, cl); yielded = 1; } /* else: my own loopback, or a lower-IP claimant that will yield to us */ } else if (!from_self) { /* Track the announcing node as the current master so all nodes agree - * even if a MEMBER_LIST was missed, and so sticky cc_elect_master + * even if a MEMBER_LIST was missed, and so sticky cl_ctr_elect_master * preserves the correct current master. */ - cc_upsert_peer_locked(sender_ip, cl); - cc_apply_master_from_list_locked(sender_ip, cl); + cl_ctr_upsert_peer_locked(sender_ip, cl); + cl_ctr_apply_master_from_list_locked(sender_ip, cl); } lock_stop_write(cl->peers->lock); @@ -3675,60 +3675,60 @@ static void cc_handle_master_alive(const char *sender_ip, cc_cluster_t *cl) LM_INFO("clusterer_controller: [cluster %d] yielding mastership to " "higher-IP master %s (split-brain resolution)\n", cl->cluster_id, sender_ip); - cc_arm_master_timers(cl, 0); /* stop MASTER_ALIVE, arm dead watchdog */ + cl_ctr_arm_master_timers(cl, 0); /* stop MASTER_ALIVE, arm dead watchdog */ } /* Non-masters (including a node that just yielded) watch the keepalive. */ if (!i_am_master || yielded) - cc_arm_tfd(cl->master_dead_tfd, CC_MASTER_KA_TIMEOUT, 0); + cl_ctr_arm_tfd(cl->master_dead_tfd, CL_CTR_MASTER_KA_TIMEOUT, 0); } /** - * cc_rejoin_superior_master() - abandon our current allegiance and merge into + * cl_ctr_rejoin_superior_master() - abandon our current allegiance and merge into * the partition led by 'superior_ip', adopting its session key. * - * Used by the split-brain merge (cc_handle_master_beacon): our node - whether a + * Used by the split-brain merge (cl_ctr_handle_master_beacon): our node - whether a * lone/independent master or a member of a smaller partition - records * superior_ip as master, stops asserting mastership, and issues a JOIN_REQ so * the superior master answers with NODE_ASSIGN + KEY_GRANT. Once the KEY_GRANT * lands we can decrypt the superior partition's session traffic and are fully * merged. Call WITHOUT cl->peers->lock held. */ -static void cc_rejoin_superior_master(cc_cluster_t *cl, const char *superior_ip) +static void cl_ctr_rejoin_superior_master(cl_ctr_cluster_t *cl, const char *superior_ip) { int was_master; lock_start_write(cl->peers->lock); - was_master = cc_i_am_master_locked(cl); - cc_upsert_peer_locked(superior_ip, cl); - cc_apply_master_from_list_locked(superior_ip, cl); - cl->peers->node_state = CC_NODE_ACTIVE; + was_master = cl_ctr_i_am_master_locked(cl); + cl_ctr_upsert_peer_locked(superior_ip, cl); + cl_ctr_apply_master_from_list_locked(superior_ip, cl); + cl->peers->node_state = CL_CTR_NODE_ACTIVE; lock_stop_write(cl->peers->lock); if (was_master) { LM_INFO("clusterer_controller: [cluster %d] superior master %s found via " "beacon - demoting and merging (split-brain resolution)\n", cl->cluster_id, superior_ip); - cc_arm_master_timers(cl, 0); /* stop MASTER_ALIVE, arm dead watchdog */ + cl_ctr_arm_master_timers(cl, 0); /* stop MASTER_ALIVE, arm dead watchdog */ } else { LM_INFO("clusterer_controller: [cluster %d] moving to superior master %s " "via beacon (split-brain merge)\n", cl->cluster_id, superior_ip); } /* Drop any locally-held active shtag now that we are no longer master. */ - cc_apply_shtags(cl); + cl_ctr_apply_shtags(cl); /* Fetch the superior master's session key. join_pending guards the nonce * so a beacon storm cannot stomp an exchange already in flight. */ if (!cl->join_pending) { cl->join_pending = 1; - cc_send_join_req(cl->sock, cl); + cl_ctr_send_join_req(cl->sock, cl); } - cc_arm_tfd(cl->master_dead_tfd, CC_MASTER_KA_TIMEOUT, 0); + cl_ctr_arm_tfd(cl->master_dead_tfd, CL_CTR_MASTER_KA_TIMEOUT, 0); } /** - * cc_handle_master_beacon() - process a CC_PKT_MASTER_BEACON (bootstrap key). + * cl_ctr_handle_master_beacon() - process a CL_CTR_PKT_MASTER_BEACON (bootstrap key). * * A master announced itself on the bootstrap layer. If it outranks our own * partition we merge into it; otherwise we ignore it (that master will yield to @@ -3737,24 +3737,24 @@ static void cc_rejoin_superior_master(cc_cluster_t *cl, const char *superior_ip) * master survives. A healthy single-master cluster sees only its own master's * beacon (sender == our master) and never acts, so this adds no churn. */ -static void cc_handle_master_beacon(const char *sender_ip, uint16_t sender_count, - cc_cluster_t *cl) +static void cl_ctr_handle_master_beacon(const char *sender_ip, uint16_t sender_count, + cl_ctr_cluster_t *cl) { int i_am_master, is_new, our_count, superior; - char our_master[CC_MAX_IP_LEN + 1]; + char our_master[CL_CTR_MAX_IP_LEN + 1]; if (strcmp(sender_ip, my_ip) == 0) return; /* our own beacon looped back */ lock_start_read(cl->peers->lock); - is_new = (cl->peers->node_state == CC_NODE_NEW); - i_am_master = cc_i_am_master_locked(cl); + is_new = (cl->peers->node_state == CL_CTR_NODE_NEW); + i_am_master = cl_ctr_i_am_master_locked(cl); our_count = cl->peers->count; if (i_am_master) { - size_t l = strnlen(my_ip, CC_MAX_IP_LEN); + size_t l = strnlen(my_ip, CL_CTR_MAX_IP_LEN); memcpy(our_master, my_ip, l); our_master[l] = '\0'; } else { - size_t l = strnlen(cl->peers->last_master, CC_MAX_IP_LEN); + size_t l = strnlen(cl->peers->last_master, CL_CTR_MAX_IP_LEN); memcpy(our_master, cl->peers->last_master, l); our_master[l] = '\0'; } lock_stop_read(cl->peers->lock); @@ -3778,27 +3778,27 @@ static void cc_handle_master_beacon(const char *sender_ip, uint16_t sender_count if (!superior) return; /* we outrank the sender; it will yield to us on our beacon */ - cc_rejoin_superior_master(cl, sender_ip); + cl_ctr_rejoin_superior_master(cl, sender_ip); } /** - * cc_handle_key_grant() - process CC_PKT_KEY_GRANT from master. + * cl_ctr_handle_key_grant() - process CL_CTR_PKT_KEY_GRANT from master. * Payload: [target_ip NUL][master_pubkey 32B][join_nonce 16B][wrapped_salt 32B] * * Recover master_salt via ECDH unwrap, derive session_key, update shm. * Only processed if target_ip == my_ip (multicast - all nodes receive it). */ -static void cc_handle_key_grant(const char *payload, int payload_len, - const char *sender_ip, cc_cluster_t *cl) +static void cl_ctr_handle_key_grant(const char *payload, int payload_len, + const char *sender_ip, cl_ctr_cluster_t *cl) { const char *p = payload; const char *end = payload + payload_len; - char target_ip[CC_MAX_IP_LEN + 1]; - unsigned char new_salt[CC_MASTER_SALT_SZ]; + char target_ip[CL_CTR_MAX_IP_LEN + 1]; + unsigned char new_salt[CL_CTR_MASTER_SALT_SZ]; int ip_len; /* Parse target_ip */ - ip_len = (int)strnlen(p, CC_MAX_IP_LEN); + ip_len = (int)strnlen(p, CL_CTR_MAX_IP_LEN); if (p + ip_len >= end) return; memcpy(target_ip, p, ip_len); target_ip[ip_len] = '\0'; p += ip_len + 1; @@ -3808,13 +3808,13 @@ static void cc_handle_key_grant(const char *payload, int payload_len, /* Masters are the key source - they never accept KEY_GRANTs. * Without this guard a stale KEY_GRANT (from the previous master, - * responding to a JOIN_REQ we sent while CC_NODE_NEW) can arrive - * after we self-promoted via cc_on_join_tfd and overwrite the + * responding to a JOIN_REQ we sent while CL_CTR_NODE_NEW) can arrive + * after we self-promoted via cl_ctr_on_join_tfd and overwrite the * session key we just generated, causing a key-mismatch loop. */ { int _im; lock_start_read(cl->peers->lock); - _im = cc_i_am_master_locked(cl); + _im = cl_ctr_i_am_master_locked(cl); lock_stop_read(cl->peers->lock); if (_im) { LM_DBG("clusterer_controller: KEY_GRANT from %s ignored - " @@ -3823,7 +3823,7 @@ static void cc_handle_key_grant(const char *payload, int payload_len, } } - if (p + CC_NOISE_MSG2_SZ > end) { + if (p + CL_CTR_NOISE_MSG2_SZ > end) { LM_WARN("clusterer_controller: KEY_GRANT too short\n"); return; } @@ -3833,8 +3833,8 @@ static void cc_handle_key_grant(const char *payload, int payload_len, * KEY_GRANT (for a superseded JOIN_REQ) fails here and is dropped - this * subsumes the old join_nonce echo check. */ if (!cl->noise_hs_valid || - cc_noise_read2(&cl->noise_hs, cl->noise_e_priv, - (const unsigned char *)p, CC_NOISE_MSG2_SZ, new_salt) < 0) { + cl_ctr_noise_read2(&cl->noise_hs, cl->noise_e_priv, + (const unsigned char *)p, CL_CTR_NOISE_MSG2_SZ, new_salt) < 0) { LM_DBG("clusterer_controller: KEY_GRANT from %s did not complete the " "Noise handshake (stale or wrong key) - dropping\n", sender_ip); /* Clear join_pending so the next decryption failure or rejoin_tfd can @@ -3846,8 +3846,8 @@ static void cc_handle_key_grant(const char *payload, int payload_len, /* Apply: write to shm, derive session_key */ lock_start_write(cl->peers->lock); - memcpy(cl->peers->master_salt, new_salt, CC_MASTER_SALT_SZ); - cc_derive_session_key(cl); + memcpy(cl->peers->master_salt, new_salt, CL_CTR_MASTER_SALT_SZ); + cl_ctr_derive_session_key(cl); lock_stop_write(cl->peers->lock); cl->join_pending = 0; @@ -3860,20 +3860,20 @@ static void cc_handle_key_grant(const char *payload, int payload_len, } /** - * cc_handle_key_handoff() - process CC_PKT_KEY_HANDOFF. + * cl_ctr_handle_key_handoff() - process CL_CTR_PKT_KEY_HANDOFF. * Only acted on by the node that wins the next election (highest IP). * Payload: [next_master_ip NUL][crypto_box_seal(master_salt)] */ -static void cc_handle_key_handoff(const char *payload, int payload_len, - const char *sender_ip, cc_cluster_t *cl) +static void cl_ctr_handle_key_handoff(const char *payload, int payload_len, + const char *sender_ip, cl_ctr_cluster_t *cl) { const char *p = payload; const char *end = payload + payload_len; - char target_ip[CC_MAX_IP_LEN + 1]; - unsigned char new_salt[CC_MASTER_SALT_SZ]; + char target_ip[CL_CTR_MAX_IP_LEN + 1]; + unsigned char new_salt[CL_CTR_MASTER_SALT_SZ]; int ip_len; - ip_len = (int)strnlen(p, CC_MAX_IP_LEN); + ip_len = (int)strnlen(p, CL_CTR_MAX_IP_LEN); if (p + ip_len >= end) return; memcpy(target_ip, p, ip_len); target_ip[ip_len] = '\0'; p += ip_len + 1; @@ -3881,13 +3881,13 @@ static void cc_handle_key_handoff(const char *payload, int payload_len, /* Only the intended next master unwraps this */ if (strcmp(target_ip, my_ip) != 0) return; - if (p + (int)(crypto_box_SEALBYTES + CC_MASTER_SALT_SZ) > end) { + if (p + (int)(crypto_box_SEALBYTES + CL_CTR_MASTER_SALT_SZ) > end) { LM_WARN("clusterer_controller: KEY_HANDOFF too short\n"); return; } /* Open the anonymous sealed box with our own long-lived keypair. */ if (crypto_box_seal_open(new_salt, (const unsigned char *)p, - crypto_box_SEALBYTES + CC_MASTER_SALT_SZ, + crypto_box_SEALBYTES + CL_CTR_MASTER_SALT_SZ, cl->my_pubkey, cl->my_privkey) != 0) { LM_WARN("clusterer_controller: [cluster %d] KEY_HANDOFF from %s did not " "open - dropping\n", cl->cluster_id, sender_ip); @@ -3897,8 +3897,8 @@ static void cc_handle_key_handoff(const char *payload, int payload_len, /* Store salt and re-derive so session_key is guaranteed consistent with * the adopted salt (the incoming master's salt may differ from ours). */ lock_start_write(cl->peers->lock); - memcpy(cl->peers->master_salt, new_salt, CC_MASTER_SALT_SZ); - cc_derive_session_key(cl); /* sets have_session_key = 1 */ + memcpy(cl->peers->master_salt, new_salt, CL_CTR_MASTER_SALT_SZ); + cl_ctr_derive_session_key(cl); /* sets have_session_key = 1 */ lock_stop_write(cl->peers->lock); LM_INFO("clusterer_controller: [cluster %d] KEY_HANDOFF from %s - " @@ -3911,18 +3911,18 @@ static void cc_handle_key_handoff(const char *payload, int payload_len, * ========================================================================= */ /** - * cc_rate_check() - per-source-IP rate limiter, called before decryption. + * cl_ctr_rate_check() - per-source-IP rate limiter, called before decryption. * Finds or creates a 1-second sliding-window counter for src_ip. - * @return 0 if within CC_RATE_LIMIT packets/s, -1 to drop. + * @return 0 if within CL_CTR_RATE_LIMIT packets/s, -1 to drop. */ -static int cc_rate_check(cc_cluster_t *cl, uint32_t src_ip) +static int cl_ctr_rate_check(cl_ctr_cluster_t *cl, uint32_t src_ip) { time_t now = time(NULL); - cc_rate_entry_t *oldest = NULL; + cl_ctr_rate_entry_t *oldest = NULL; int i; - for (i = 0; i < CC_RATE_TBL_SZ; i++) { - cc_rate_entry_t *e = &cl->rate_tbl[i]; + for (i = 0; i < CL_CTR_RATE_TBL_SZ; i++) { + cl_ctr_rate_entry_t *e = &cl->rate_tbl[i]; if (e->ip == 0) { if (!oldest) oldest = e; /* prefer empty slot */ continue; @@ -3933,7 +3933,7 @@ static int cc_rate_check(cc_cluster_t *cl, uint32_t src_ip) e->count = 1; return 0; } - if (++e->count > CC_RATE_LIMIT) + if (++e->count > CL_CTR_RATE_LIMIT) return -1; return 0; } @@ -3951,31 +3951,31 @@ static int cc_rate_check(cc_cluster_t *cl, uint32_t src_ip) /* -- Join-reject helpers ---------------------------------------------------- * - * Security model: JOIN_REJECT is sent as a normal CC_BOOTSTRAP_MAGIC packet + * Security model: JOIN_REJECT is sent as a normal CL_CTR_BOOTSTRAP_MAGIC packet * (AES-256-GCM authenticated with the bootstrap key). An attacker without * the cluster password cannot forge a GCM-authenticated reject, so they - * cannot kick nodes out or block joins. cc_handle_join_reject() also guards - * on CC_NODE_NEW state so that even a legitimate cluster member with the + * cannot kick nodes out or block joins. cl_ctr_handle_join_reject() also guards + * on CL_CTR_NODE_NEW state so that even a legitimate cluster member with the * correct password cannot send a JOIN_REJECT to an already-active node. * - * cc_join_fail_check() - master: track per-IP BOOTSTRAP_MAGIC decrypt failures. - * Returns 1 the first time a source IP reaches CC_JOIN_FAIL_LIMIT failures. + * cl_ctr_join_fail_check() - master: track per-IP BOOTSTRAP_MAGIC decrypt failures. + * Returns 1 the first time a source IP reaches CL_CTR_JOIN_FAIL_LIMIT failures. * - * cc_send_join_reject() - master: send encrypted JOIN_REJECT via BOOTSTRAP_MAGIC. + * cl_ctr_send_join_reject() - master: send encrypted JOIN_REJECT via BOOTSTRAP_MAGIC. * Wire payload: [target_ip NUL] * - * cc_handle_join_reject() - joiner: stop OpenSIPS if the reject is for us and - * we are still in CC_NODE_NEW (i.e., have not successfully joined yet). + * cl_ctr_handle_join_reject() - joiner: stop OpenSIPS if the reject is for us and + * we are still in CL_CTR_NODE_NEW (i.e., have not successfully joined yet). * * WRONG-PASSWORD fallback: if the joiner has the wrong password it cannot * decrypt the encrypted JOIN_REJECT. Instead it detects the situation * through bootstrap_auth_fails (incremented on any BOOTSTRAP_MAGIC decrypt - * failure during CC_NODE_NEW) combined with join_attempt_count. After - * CC_JOIN_FAIL_LIMIT rejoin retries with at least one bootstrap failure + * failure during CL_CTR_NODE_NEW) combined with join_attempt_count. After + * CL_CTR_JOIN_FAIL_LIMIT rejoin retries with at least one bootstrap failure * observed, the joiner concludes the master rejected it and exits. */ -static int cc_join_fail_check(const char *src_ip, cc_cluster_t *cl) +static int cl_ctr_join_fail_check(const char *src_ip, cl_ctr_cluster_t *cl) { uint32_t ip_num = ip_to_num(src_ip); int evict = 0; /* index of lowest-count slot for eviction */ @@ -3984,13 +3984,13 @@ static int cc_join_fail_check(const char *src_ip, cc_cluster_t *cl) if (ip_num == 0) return 0; - for (i = 0; i < CC_JOIN_FAIL_TABLE_SZ; i++) { + for (i = 0; i < CL_CTR_JOIN_FAIL_TABLE_SZ; i++) { if (cl->join_fail_tbl[i].ip_num != ip_num) continue; if (cl->join_fail_tbl[i].rejected) return 0; /* reject already sent; don't repeat */ cl->join_fail_tbl[i].count++; - if (cl->join_fail_tbl[i].count >= CC_JOIN_FAIL_LIMIT) { + if (cl->join_fail_tbl[i].count >= CL_CTR_JOIN_FAIL_LIMIT) { cl->join_fail_tbl[i].rejected = 1; return 1; } @@ -3998,7 +3998,7 @@ static int cc_join_fail_check(const char *src_ip, cc_cluster_t *cl) } /* Not found - insert, evicting the slot with the smallest count */ - for (i = 1; i < CC_JOIN_FAIL_TABLE_SZ; i++) { + for (i = 1; i < CL_CTR_JOIN_FAIL_TABLE_SZ; i++) { if (cl->join_fail_tbl[i].count < cl->join_fail_tbl[evict].count) evict = i; } @@ -4010,35 +4010,35 @@ static int cc_join_fail_check(const char *src_ip, cc_cluster_t *cl) return 0; } -static void cc_send_join_reject(int sock, const char *target_ip, cc_cluster_t *cl, +static void cl_ctr_send_join_reject(int sock, const char *target_ip, cl_ctr_cluster_t *cl, int reason) { - char pkt[CC_SMALL_PKT_SZ + 1]; /* +1 for the reason byte */ + char pkt[CL_CTR_SMALL_PKT_SZ + 1]; /* +1 for the reason byte */ uint32_t seq = htonl(++cl->peers->my_seq); int ip_len, plain_len; - ip_len = (int)strnlen(target_ip, CC_MAX_IP_LEN); + ip_len = (int)strnlen(target_ip, CL_CTR_MAX_IP_LEN); - memcpy(pkt, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ); - pkt[CC_WIRE_HDR_SZ] = (char)CC_PKT_JOIN_REJECT; - memcpy(pkt + CC_WIRE_HDR_SZ + 1, &seq, CC_SEQ_SZ); - memcpy(pkt + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ, target_ip, ip_len); - pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + ip_len] = '\0'; + memcpy(pkt, CL_CTR_BOOTSTRAP_MAGIC, CL_CTR_MAGIC_SZ); + pkt[CL_CTR_WIRE_HDR_SZ] = (char)CL_CTR_PKT_JOIN_REJECT; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ, target_ip, ip_len); + pkt[CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + ip_len] = '\0'; /* reason byte follows the NUL-terminated target IP */ - pkt[CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + ip_len + 1] = (char)reason; + pkt[CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + ip_len + 1] = (char)reason; - plain_len = CC_PLAIN_HDR_SZ + ip_len + 1 + 1; - if (cc_seal_and_send(sock, cl, pkt, plain_len, cl->key, CC_PKT_JOIN_REJECT) == 0) + plain_len = CL_CTR_PLAIN_HDR_SZ + ip_len + 1 + 1; + if (cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->key, CL_CTR_PKT_JOIN_REJECT) == 0) LM_WARN("clusterer_controller: [cluster %d] sent JOIN_REJECT to %s (%s)\n", cl->cluster_id, target_ip, - reason == CC_REJECT_CONFIG ? "different cluster settings" + reason == CL_CTR_REJECT_CONFIG ? "different cluster settings" : "repeated auth failure - wrong password?"); } -static void cc_handle_join_reject(const char *payload, int payload_len, - const char *sender_ip, cc_cluster_t *cl) +static void cl_ctr_handle_join_reject(const char *payload, int payload_len, + const char *sender_ip, cl_ctr_cluster_t *cl) { - char target_ip[CC_MAX_IP_LEN + 1]; + char target_ip[CL_CTR_MAX_IP_LEN + 1]; int l, still_new; /* Only act during the initial join phase. An active member receiving a @@ -4046,11 +4046,11 @@ static void cc_handle_join_reject(const char *payload, int payload_len, * wanted to test the mechanism, or a stale in-flight packet) must ignore * it - this prevents any cluster member from silently evicting another. */ lock_start_read(cl->peers->lock); - still_new = (cl->peers->node_state == CC_NODE_NEW); + still_new = (cl->peers->node_state == CL_CTR_NODE_NEW); lock_stop_read(cl->peers->lock); if (!still_new) return; - l = (int)strnlen(payload, CC_MAX_IP_LEN); + l = (int)strnlen(payload, CL_CTR_MAX_IP_LEN); if (l >= payload_len) return; memcpy(target_ip, payload, l); target_ip[l] = '\0'; @@ -4060,10 +4060,10 @@ static void cc_handle_join_reject(const char *payload, int payload_len, /* reason byte follows the NUL-terminated target IP (older senders omit it) */ { - int reason = CC_REJECT_GENERIC; + int reason = CL_CTR_REJECT_GENERIC; if (payload_len > l + 1) reason = (unsigned char)payload[l + 1]; - if (reason == CC_REJECT_CONFIG) + if (reason == CL_CTR_REJECT_CONFIG) LM_CRIT("clusterer_controller: [cluster %d] JOIN_REJECT from %s - the " "running cluster has different settings than this node; fix the " "local config (manage_shtags/master_stickiness/query_time) to " @@ -4078,13 +4078,13 @@ static void cc_handle_join_reject(const char *payload, int payload_len, } /** - * cc_recv_one() - read one datagram, validate header, dispatch by type. + * cl_ctr_recv_one() - read one datagram, validate header, dispatch by type. */ -static void cc_recv_one(int sock, cc_cluster_t *cl) +static void cl_ctr_recv_one(int sock, cl_ctr_cluster_t *cl) { - /* Static buffer: avoids a 64 KB stack frame; safe because cc_recv_one - * is called only from the single-threaded cc_worker process. */ - static char buf[CC_RECV_BUF_SZ]; + /* Static buffer: avoids a 64 KB stack frame; safe because cl_ctr_recv_one + * is called only from the single-threaded cl_ctr_worker process. */ + static char buf[CL_CTR_RECV_BUF_SZ]; struct sockaddr_in src_addr; socklen_t src_len = sizeof(src_addr); ssize_t n; @@ -4101,14 +4101,14 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) } /* Minimum: magic(2)+cluster_id(2)+nonce(12)+type(1)+seq(4)+tag(16) = 37 */ - if (n < CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_TAG_SZ) { + if (n < CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_TAG_SZ) { LM_WARN("clusterer_controller: short packet (%zd bytes), dropping\n", n); return; } - if (memcmp(buf, CC_PACKET_MAGIC, CC_MAGIC_SZ) != 0 && - memcmp(buf, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ) != 0) { + if (memcmp(buf, CL_CTR_PACKET_MAGIC, CL_CTR_MAGIC_SZ) != 0 && + memcmp(buf, CL_CTR_BOOTSTRAP_MAGIC, CL_CTR_MAGIC_SZ) != 0) { LM_DBG("clusterer_controller: bad magic, dropping\n"); return; } @@ -4119,7 +4119,7 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) * packet is not a wrong-password attempt. */ { uint16_t pkt_cid_be; - memcpy(&pkt_cid_be, buf + CC_MAGIC_SZ, CC_CLUSTER_ID_SZ); + memcpy(&pkt_cid_be, buf + CL_CTR_MAGIC_SZ, CL_CTR_CLUSTER_ID_SZ); if (ntohs(pkt_cid_be) != (uint16_t)cl->cluster_id) { LM_DBG("clusterer_controller: [cluster %d] ignoring packet for " "cluster %u on shared group\n", cl->cluster_id, @@ -4129,7 +4129,7 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) } /* Rate-limit before any crypto work to shed floods cheaply. */ - if (cc_rate_check(cl, src_addr.sin_addr.s_addr) < 0) + if (cl_ctr_rate_check(cl, src_addr.sin_addr.s_addr) < 0) return; /* Resolve sender IP once - used for HMAC warning and MEMBER_LIST dispatch */ @@ -4140,18 +4140,18 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) sender_ip_buf, sizeof(sender_ip_buf)); /* Select decryption key by magic: - * CC_BOOTSTRAP_MAGIC -> bootstrap key (JOIN_REQ, KEY_GRANT) - * CC_PACKET_MAGIC -> session key (all normal traffic) + * CL_CTR_BOOTSTRAP_MAGIC -> bootstrap key (JOIN_REQ, KEY_GRANT) + * CL_CTR_PACKET_MAGIC -> session key (all normal traffic) * If session key decryption fails, schedule a re-JOIN to refresh it. */ - int is_bootstrap = (memcmp(buf, CC_BOOTSTRAP_MAGIC, CC_MAGIC_SZ) == 0); + int is_bootstrap = (memcmp(buf, CL_CTR_BOOTSTRAP_MAGIC, CL_CTR_MAGIC_SZ) == 0); dec_key = is_bootstrap ? cl->key : cl->session_key; - if (cc_decrypt_pkt(buf, n, sender_ip_buf, dec_key, is_bootstrap) < 0) { + if (cl_ctr_decrypt_pkt(buf, n, sender_ip_buf, dec_key, is_bootstrap) < 0) { int _im, _new; - char _lm[CC_MAX_IP_LEN + 1]; + char _lm[CL_CTR_MAX_IP_LEN + 1]; lock_start_read(cl->peers->lock); - _im = cc_i_am_master_locked(cl); - _new = (cl->peers->node_state == CC_NODE_NEW); + _im = cl_ctr_i_am_master_locked(cl); + _new = (cl->peers->node_state == CL_CTR_NODE_NEW); memcpy(_lm, cl->peers->last_master, sizeof(_lm)); lock_stop_read(cl->peers->lock); @@ -4163,7 +4163,7 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) * session-encrypted, so a wrong-password joiner would otherwise see too * little bootstrap traffic to notice the cluster and would self-promote * into a split-brain lone master. This is only *evidence*, never an - * immediate death sentence: cc_on_join_tfd defers and keeps re-joining, + * immediate death sentence: cl_ctr_on_join_tfd defers and keeps re-joining, * and a KEY_GRANT arriving in the grace window resets this counter, so a * healthy node whose key is merely slow is not affected. Our own * loopback decrypts fine, so guard on my_ip. */ @@ -4176,10 +4176,10 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) * JOIN_REJECT is encrypted (BOOTSTRAP_MAGIC/GCM) so only nodes with * the correct password can read it. Forgeries are impossible without * the bootstrap key. */ - if (_im && cc_join_fail_check(sender_ip_buf, cl)) - cc_send_join_reject(sock, sender_ip_buf, cl, CC_REJECT_GENERIC); + if (_im && cl_ctr_join_fail_check(sender_ip_buf, cl)) + cl_ctr_send_join_reject(sock, sender_ip_buf, cl, CL_CTR_REJECT_GENERIC); - /* Joiner fallback: count bootstrap decrypt failures during CC_NODE_NEW + /* Joiner fallback: count bootstrap decrypt failures during CL_CTR_NODE_NEW * only when the packet came from the known master. This filters out * rogue nodes on the multicast group whose JOIN_REQs (encrypted with * their own wrong key) would otherwise increment this counter and @@ -4201,7 +4201,7 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) "from master %s - sending JOIN_REQ to re-key\n", cl->cluster_id, sender_ip_buf); cl->join_pending = 1; - cc_send_join_req(cl->sock, cl); + cl_ctr_send_join_req(cl->sock, cl); } } return; @@ -4211,36 +4211,36 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) * GOODBYE. my_seq lives in shm so mod_destroy increments the same * counter the worker uses - GOODBYE gets a valid monotonic seq number * without any special-casing. - * Bootstrap packets (CC_BOOTSTRAP_MAGIC) use join_nonce instead. */ + * Bootstrap packets (CL_CTR_BOOTSTRAP_MAGIC) use join_nonce instead. */ if (!is_bootstrap) { uint32_t pkt_seq; - memcpy(&pkt_seq, buf + CC_WIRE_HDR_SZ + 1, CC_SEQ_SZ); + memcpy(&pkt_seq, buf + CL_CTR_WIRE_HDR_SZ + 1, CL_CTR_SEQ_SZ); pkt_seq = ntohl(pkt_seq); - if (cc_check_and_update_seq(sender_ip_buf, pkt_seq, cl) < 0) + if (cl_ctr_check_and_update_seq(sender_ip_buf, pkt_seq, cl) < 0) return; } - pkt_type = (unsigned char)buf[CC_WIRE_HDR_SZ]; - payload = buf + CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ; + pkt_type = (unsigned char)buf[CL_CTR_WIRE_HDR_SZ]; + payload = buf + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ; /* Non-negative: the minimum-length gate above guarantees - * n >= CC_WIRE_HDR_SZ + CC_PLAIN_HDR_SZ + CC_TAG_SZ. */ - payload_len = (int)(n - CC_WIRE_HDR_SZ - CC_PLAIN_HDR_SZ - CC_TAG_SZ); + * n >= CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_TAG_SZ. */ + payload_len = (int)(n - CL_CTR_WIRE_HDR_SZ - CL_CTR_PLAIN_HDR_SZ - CL_CTR_TAG_SZ); switch (pkt_type) { - case CC_PKT_ALIVE: { - char ip_buf[CC_MAX_IP_LEN + 1]; - int ip_len = (int)strnlen(payload, CC_MAX_IP_LEN); + case CL_CTR_PKT_ALIVE: { + char ip_buf[CL_CTR_MAX_IP_LEN + 1]; + int ip_len = (int)strnlen(payload, CL_CTR_MAX_IP_LEN); const unsigned char *pubkey = NULL; int cfg_present = 0, p_manage = 0, p_stick = 0, p_qt = 0; memcpy(ip_buf, payload, ip_len); ip_buf[ip_len] = '\0'; /* Pubkey appended after NUL-terminated IP */ - if (payload_len >= ip_len + 1 + (int)CC_PUBKEY_SZ) + if (payload_len >= ip_len + 1 + (int)CL_CTR_PUBKEY_SZ) pubkey = (const unsigned char *)payload + ip_len + 1; /* Config descriptor appended after the pubkey (optional) */ - if (payload_len >= ip_len + 1 + (int)CC_PUBKEY_SZ + CC_CONFIG_SZ) { - const char *c = payload + ip_len + 1 + CC_PUBKEY_SZ; + if (payload_len >= ip_len + 1 + (int)CL_CTR_PUBKEY_SZ + CL_CTR_CONFIG_SZ) { + const char *c = payload + ip_len + 1 + CL_CTR_PUBKEY_SZ; uint16_t qt_be; p_manage = (unsigned char)c[0]; p_stick = (unsigned char)c[1]; @@ -4248,54 +4248,54 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) p_qt = ntohs(qt_be); cfg_present = 1; } - cc_handle_alive(ip_buf, pubkey, cfg_present, p_manage, p_stick, p_qt, cl); + cl_ctr_handle_alive(ip_buf, pubkey, cfg_present, p_manage, p_stick, p_qt, cl); break; } - case CC_PKT_JOIN_REQ: - cc_handle_join_req(sock, payload, payload_len, cl); + case CL_CTR_PKT_JOIN_REQ: + cl_ctr_handle_join_req(sock, payload, payload_len, cl); break; - case CC_PKT_MEMBER_LIST: - cc_handle_member_list(payload, payload_len, sender_ip_buf, cl); + case CL_CTR_PKT_MEMBER_LIST: + cl_ctr_handle_member_list(payload, payload_len, sender_ip_buf, cl); break; - case CC_PKT_GOODBYE: { - char ip_buf[CC_MAX_IP_LEN + 1]; - int ip_len = payload_len > CC_MAX_IP_LEN ? CC_MAX_IP_LEN : payload_len; + case CL_CTR_PKT_GOODBYE: { + char ip_buf[CL_CTR_MAX_IP_LEN + 1]; + int ip_len = payload_len > CL_CTR_MAX_IP_LEN ? CL_CTR_MAX_IP_LEN : payload_len; memcpy(ip_buf, payload, ip_len); ip_buf[ip_len] = '\0'; - cc_handle_goodbye(sock, ip_buf, cl); + cl_ctr_handle_goodbye(sock, ip_buf, cl); break; } - case CC_PKT_NODE_ASSIGN: - cc_handle_node_assign(payload, payload_len, sender_ip_buf, cl); + case CL_CTR_PKT_NODE_ASSIGN: + cl_ctr_handle_node_assign(payload, payload_len, sender_ip_buf, cl); break; - case CC_PKT_MASTER_ALIVE: - cc_handle_master_alive(sender_ip_buf, cl); + case CL_CTR_PKT_MASTER_ALIVE: + cl_ctr_handle_master_alive(sender_ip_buf, cl); break; - case CC_PKT_KEY_GRANT: - cc_handle_key_grant(payload, payload_len, sender_ip_buf, cl); + case CL_CTR_PKT_KEY_GRANT: + cl_ctr_handle_key_grant(payload, payload_len, sender_ip_buf, cl); break; - case CC_PKT_KEY_HANDOFF: - cc_handle_key_handoff(payload, payload_len, sender_ip_buf, cl); + case CL_CTR_PKT_KEY_HANDOFF: + cl_ctr_handle_key_handoff(payload, payload_len, sender_ip_buf, cl); break; - case CC_PKT_JOIN_REJECT: - cc_handle_join_reject(payload, payload_len, sender_ip_buf, cl); + case CL_CTR_PKT_JOIN_REJECT: + cl_ctr_handle_join_reject(payload, payload_len, sender_ip_buf, cl); break; - case CC_PKT_MASTER_BEACON: { + case CL_CTR_PKT_MASTER_BEACON: { uint16_t cnt_be, sender_count = 0; if (payload_len >= 2) { memcpy(&cnt_be, payload, 2); sender_count = ntohs(cnt_be); } - cc_handle_master_beacon(sender_ip_buf, sender_count, cl); + cl_ctr_handle_master_beacon(sender_ip_buf, sender_count, cl); break; } @@ -4315,35 +4315,35 @@ static void cc_recv_one(int sock, cc_cluster_t *cl) * Reactor callbacks - one per event source * ========================================================================= */ -static int cc_on_sock(int fd, void *param, int was_timeout) +static int cl_ctr_on_sock(int fd, void *param, int was_timeout) { - cc_recv_one(fd, (cc_cluster_t *)param); + cl_ctr_recv_one(fd, (cl_ctr_cluster_t *)param); return 0; } -static int cc_on_alive_tfd(int fd, void *param, int was_timeout) +static int cl_ctr_on_alive_tfd(int fd, void *param, int was_timeout) { - cc_cluster_t *cl = (cc_cluster_t *)param; + cl_ctr_cluster_t *cl = (cl_ctr_cluster_t *)param; int prev_master, now_master; - cc_drain_tfd(fd); - cc_send_alive(cl->sock, cl); + cl_ctr_drain_tfd(fd); + cl_ctr_send_alive(cl->sock, cl); lock_start_write(cl->peers->lock); - prev_master = cc_i_am_master_locked(cl); - cc_prune_stale(cl); - cc_elect_master(cl); - now_master = cc_i_am_master_locked(cl); + prev_master = cl_ctr_i_am_master_locked(cl); + cl_ctr_prune_stale(cl); + cl_ctr_elect_master(cl); + now_master = cl_ctr_i_am_master_locked(cl); lock_stop_write(cl->peers->lock); /* Do not broadcast MASTER_ALIVE before we hold the cluster key - * request a re-key from the current key-holder instead. */ if (now_master && !cl->have_session_key) { - cc_request_rekey(cl); + cl_ctr_request_rekey(cl); return 0; } if (prev_master != now_master) - cc_arm_master_timers(cl, now_master); + cl_ctr_arm_master_timers(cl, now_master); /* Belt-and-suspenders identity registration (covers paths where - * cc_handle_node_assign has not yet run). */ + * cl_ctr_handle_node_assign has not yet run). */ if (!cl->identity_registered && clctl_loaded && my_node_id > 0) { str url = {cl->bin_socket, (int)strlen(cl->bin_socket)}; clctl.update_identity(cl->cluster_id, my_node_id, &url); @@ -4356,24 +4356,24 @@ static int cc_on_alive_tfd(int fd, void *param, int was_timeout) && clctl.activate_backup_shtags) { int _im; lock_start_read(cl->peers->lock); - _im = cc_i_am_master_locked(cl); + _im = cl_ctr_i_am_master_locked(cl); lock_stop_read(cl->peers->lock); if (_im) { - cc_apply_shtags(cl); /* override-aware */ + cl_ctr_apply_shtags(cl); /* override-aware */ cl->shtag_bootstrapped = 1; } } return 0; } -static int cc_on_join_tfd(int fd, void *param, int was_timeout) +static int cl_ctr_on_join_tfd(int fd, void *param, int was_timeout) { - cc_cluster_t *cl = (cc_cluster_t *)param; - cc_drain_tfd(fd); + cl_ctr_cluster_t *cl = (cl_ctr_cluster_t *)param; + cl_ctr_drain_tfd(fd); int was_new, auth_fails; lock_start_write(cl->peers->lock); - was_new = (cl->peers->node_state == CC_NODE_NEW); + was_new = (cl->peers->node_state == CL_CTR_NODE_NEW); auth_fails = cl->auth_fail_pkts; /* Wrong-password / unauthorized guard: if we are still joining at the @@ -4382,18 +4382,18 @@ static int cc_on_join_tfd(int fd, void *param, int was_timeout) * Self-promoting would create a split-brain lone master (and, with managed * shtags, a duplicate active tag). A wrong-password node also cannot read * the master's JOIN_REJECT, so this is where it must shut down. */ - if (was_new && auth_fails >= CC_JOIN_FAIL_LIMIT) { + if (was_new && auth_fails >= CL_CTR_JOIN_FAIL_LIMIT) { /* We could not decrypt traffic from peers on our cluster_id: a cluster * using a different password (or we do) shares this group. Do NOT * self-terminate on the first deadline - a KEY_GRANT that is merely slow, a * brief burst of start-up noise, or a flood of crafted garbage would * otherwise kill a healthy node. Defer and keep re-joining: a real * KEY_GRANT arriving in the grace window resets auth_fail_pkts - * (cc_handle_key_grant) and we join normally. Only give up after - * CC_JOIN_DEFER_MAX rounds still short of authentication - by then the + * (cl_ctr_handle_key_grant) and we join normally. Only give up after + * CL_CTR_JOIN_DEFER_MAX rounds still short of authentication - by then the * wrong-password / foreign-cluster condition is sustained, not transient, * and self-promoting would create a split-brain lone master. */ - if (cl->auth_defer_count < CC_JOIN_DEFER_MAX) { + if (cl->auth_defer_count < CL_CTR_JOIN_DEFER_MAX) { cl->auth_defer_count++; lock_stop_write(cl->peers->lock); LM_WARN("clusterer_controller: [cluster %d] %d undecryptable packet(s) " @@ -4401,10 +4401,10 @@ static int cc_on_join_tfd(int fd, void *param, int was_timeout) "self-promotion (%d/%d); a slow KEY_GRANT would clear this\n", cl->cluster_id, auth_fails, cl->cluster_id, cl->multicast_address, cl->multicast_port, - cl->auth_defer_count, CC_JOIN_DEFER_MAX); + cl->auth_defer_count, CL_CTR_JOIN_DEFER_MAX); cl->join_pending = 0; - cc_send_join_req(cl->sock, cl); - cc_arm_tfd(cl->join_tfd, CC_JOIN_DEFER_SECS, 0); + cl_ctr_send_join_req(cl->sock, cl); + cl_ctr_arm_tfd(cl->join_tfd, CL_CTR_JOIN_DEFER_SECS, 0); return 0; } lock_stop_write(cl->peers->lock); @@ -4420,10 +4420,10 @@ static int cc_on_join_tfd(int fd, void *param, int was_timeout) /* Split-brain PREVENTION: if a higher-IP node is also still joining (we * learned it from its JOIN_REQ), defer self-promotion so it becomes the * single master and we join it, rather than both self-promoting with - * independent session keys. Bounded by CC_JOIN_DEFER_MAX so a higher-IP + * independent session keys. Bounded by CL_CTR_JOIN_DEFER_MAX so a higher-IP * node that was heard but never finished starting cannot stall us. */ - if (was_new && cl->join_defer_count < CC_JOIN_DEFER_MAX - && cl->join_defer_total < CC_JOIN_DEFER_HARDMAX) { + if (was_new && cl->join_defer_count < CL_CTR_JOIN_DEFER_MAX + && cl->join_defer_total < CL_CTR_JOIN_DEFER_HARDMAX) { unsigned int my_ipn = ip_to_num(my_ip); int higher_seen = 0, _i; for (_i = 0; _i < cl->peers->count; _i++) { @@ -4441,46 +4441,46 @@ static int cc_on_join_tfd(int fd, void *param, int was_timeout) LM_INFO("clusterer_controller: [cluster %d] join deadline: a higher-IP " "node is still joining - deferring self-promotion (%d/%d) to " "avoid split brain\n", - cl->cluster_id, cl->join_defer_count, CC_JOIN_DEFER_MAX); + cl->cluster_id, cl->join_defer_count, CL_CTR_JOIN_DEFER_MAX); /* Re-send a JOIN_REQ now (clear join_pending so it is not suppressed) * so the higher-IP node answers as soon as it becomes master, then * extend the join window for one more short round. */ cl->join_pending = 0; - cc_send_join_req(cl->sock, cl); - cc_arm_tfd(cl->join_tfd, CC_JOIN_DEFER_SECS, 0); + cl_ctr_send_join_req(cl->sock, cl); + cl_ctr_arm_tfd(cl->join_tfd, CL_CTR_JOIN_DEFER_SECS, 0); return 0; } } if (was_new) { LM_INFO("clusterer_controller: [cluster %d] join deadline expired, " - "no master found - transitioning to CC_NODE_ACTIVE\n", + "no master found - transitioning to CL_CTR_NODE_ACTIVE\n", cl->cluster_id); cl->join_defer_count = 0; /* leaving NEW state */ cl->join_defer_total = 0; cl->shtag_bootstrapped = -1; /* fresh cluster: eligible to claim active */ - cc_upsert_peer_locked(my_ip, cl); - my_node_id = cc_alloc_node_id_locked(cl); + cl_ctr_upsert_peer_locked(my_ip, cl); + my_node_id = cl_ctr_alloc_node_id_locked(cl); { - char self_sock[1][CC_MAX_BIN_SOCK_LEN]; - memcpy(self_sock[0], cl->bin_socket, CC_MAX_BIN_SOCK_LEN); - cc_update_peer_bin_locked(my_ip, my_node_id, 1, - (const char (*)[CC_MAX_BIN_SOCK_LEN]) + char self_sock[1][CL_CTR_MAX_BIN_SOCK_LEN]; + memcpy(self_sock[0], cl->bin_socket, CL_CTR_MAX_BIN_SOCK_LEN); + cl_ctr_update_peer_bin_locked(my_ip, my_node_id, 1, + (const char (*)[CL_CTR_MAX_BIN_SOCK_LEN]) self_sock, cl); } - cl->peers->node_state = CC_NODE_ACTIVE; + cl->peers->node_state = CL_CTR_NODE_ACTIVE; /* Run election first so is_master and last_master are set before - * cc_on_became_master. Without this, cc_handle_alive would see - * is_master=0 on the loopback ALIVE and call cc_on_became_master + * cl_ctr_on_became_master. Without this, cl_ctr_handle_alive would see + * is_master=0 on the loopback ALIVE and call cl_ctr_on_became_master * a second time, regenerating the session key unnecessarily. */ - cc_elect_master(cl); - cc_on_became_master(cl); + cl_ctr_elect_master(cl); + cl_ctr_on_became_master(cl); } lock_stop_write(cl->peers->lock); if (!was_new) return 0; /* already active via MEMBER_LIST - nothing to do */ - cc_transition_to_active(cl); - cc_arm_master_timers(cl, 1); + cl_ctr_transition_to_active(cl); + cl_ctr_arm_master_timers(cl, 1); if (!cl->identity_registered && clctl_loaded && my_node_id > 0) { str url = {cl->bin_socket, (int)strlen(cl->bin_socket)}; @@ -4490,24 +4490,24 @@ static int cc_on_join_tfd(int fd, void *param, int was_timeout) return 0; } -static int cc_on_rejoin_tfd(int fd, void *param, int was_timeout) +static int cl_ctr_on_rejoin_tfd(int fd, void *param, int was_timeout) { - cc_cluster_t *cl = (cc_cluster_t *)param; + cl_ctr_cluster_t *cl = (cl_ctr_cluster_t *)param; int still_new; - cc_drain_tfd(fd); + cl_ctr_drain_tfd(fd); lock_start_read(cl->peers->lock); - still_new = (cl->peers->node_state == CC_NODE_NEW); + still_new = (cl->peers->node_state == CL_CTR_NODE_NEW); lock_stop_read(cl->peers->lock); if (still_new && !cl->join_pending) { cl->join_attempt_count++; /* Wrong-password fallback: if the master keeps sending bootstrap * packets we cannot decrypt (bootstrap_auth_fails > 0) and we have - * already retried CC_JOIN_FAIL_LIMIT times, give up. This fires when + * already retried CL_CTR_JOIN_FAIL_LIMIT times, give up. This fires when * the encrypted JOIN_REJECT from the master was lost in transit (if it - * arrived intact, cc_handle_join_reject would have already exited). */ - if (cl->join_attempt_count >= CC_JOIN_FAIL_LIMIT + * arrived intact, cl_ctr_handle_join_reject would have already exited). */ + if (cl->join_attempt_count >= CL_CTR_JOIN_FAIL_LIMIT && cl->bootstrap_auth_fails > 0) { LM_CRIT("clusterer_controller: [cluster %d] join failed after %d " "attempts with %d bootstrap auth error(s) - wrong password? " @@ -4517,7 +4517,7 @@ static int cc_on_rejoin_tfd(int fd, void *param, int was_timeout) exit(-1); } - cc_send_join_req(cl->sock, cl); + cl_ctr_send_join_req(cl->sock, cl); cl->join_pending = 1; LM_DBG("clusterer_controller: [cluster %d] resending JOIN_REQ " "(attempt %d)\n", cl->cluster_id, cl->join_attempt_count); @@ -4525,35 +4525,35 @@ static int cc_on_rejoin_tfd(int fd, void *param, int was_timeout) return 0; } -static int cc_on_master_alive_tfd(int fd, void *param, int was_timeout) +static int cl_ctr_on_master_alive_tfd(int fd, void *param, int was_timeout) { - cc_cluster_t *cl = (cc_cluster_t *)param; - cc_drain_tfd(fd); - cc_send_master_alive(cl->sock, cl); - /* Emit a bootstrap-key beacon every CC_MASTER_BEACON_EVERY ticks so any + cl_ctr_cluster_t *cl = (cl_ctr_cluster_t *)param; + cl_ctr_drain_tfd(fd); + cl_ctr_send_master_alive(cl->sock, cl); + /* Emit a bootstrap-key beacon every CL_CTR_MASTER_BEACON_EVERY ticks so any * peer master holding a different session key can find us and merge. */ - if (++cl->beacon_tick >= CC_MASTER_BEACON_EVERY) { + if (++cl->beacon_tick >= CL_CTR_MASTER_BEACON_EVERY) { cl->beacon_tick = 0; - cc_send_master_beacon(cl->sock, cl); + cl_ctr_send_master_beacon(cl->sock, cl); } return 0; } -static int cc_on_master_dead_tfd(int fd, void *param, int was_timeout) +static int cl_ctr_on_master_dead_tfd(int fd, void *param, int was_timeout) { - cc_cluster_t *cl = (cc_cluster_t *)param; + cl_ctr_cluster_t *cl = (cl_ctr_cluster_t *)param; int now_master; - char dead_master[CC_MAX_IP_LEN + 1]; + char dead_master[CL_CTR_MAX_IP_LEN + 1]; - cc_drain_tfd(fd); + cl_ctr_drain_tfd(fd); dead_master[0] = '\0'; lock_start_write(cl->peers->lock); - /* The master has been silent for CC_MASTER_KA_TIMEOUT (3s). Age the silent + /* The master has been silent for CL_CTR_MASTER_KA_TIMEOUT (3s). Age the silent * master OUT of the election window before re-electing, otherwise - * cc_elect_master would just re-select it: the election window is - * query_time * CC_ELECT_FACTOR (~15s), far longer than the keepalive + * cl_ctr_elect_master would just re-select it: the election window is + * query_time * CL_CTR_ELECT_FACTOR (~15s), far longer than the keepalive * timeout, so a just-declared-dead master stays "eligible" and keeps * winning - delaying real failover by ~12s. Zeroing its last_seen makes * the immediate re-election pick the next-highest LIVE peer. If the @@ -4564,7 +4564,7 @@ static int cc_on_master_dead_tfd(int fd, void *param, int was_timeout) for (_i = 0; _i < cl->peers->count; _i++) { if (cl->peers->entries[_i].is_master && strcmp(cl->peers->entries[_i].ip, my_ip) != 0) { - size_t _l = strnlen(cl->peers->entries[_i].ip, CC_MAX_IP_LEN); + size_t _l = strnlen(cl->peers->entries[_i].ip, CL_CTR_MAX_IP_LEN); memcpy(dead_master, cl->peers->entries[_i].ip, _l); dead_master[_l] = '\0'; cl->peers->entries[_i].last_seen = 0; @@ -4579,11 +4579,11 @@ static int cc_on_master_dead_tfd(int fd, void *param, int was_timeout) "(no keepalive for %ds) - re-electing\n", cl->cluster_id, dead_master[0] ? dead_master : "(unknown)", - CC_MASTER_KA_TIMEOUT); + CL_CTR_MASTER_KA_TIMEOUT); - /* cc_elect_master logs the resulting MASTER/BACKUP roles and why. */ - cc_elect_master(cl); - now_master = cc_i_am_master_locked(cl); + /* cl_ctr_elect_master logs the resulting MASTER/BACKUP roles and why. */ + cl_ctr_elect_master(cl); + now_master = cl_ctr_i_am_master_locked(cl); lock_stop_write(cl->peers->lock); /* Preserve-key recovery: every surviving member already holds the session @@ -4596,46 +4596,46 @@ static int cc_on_master_dead_tfd(int fd, void *param, int was_timeout) return 0; } - cc_arm_master_timers(cl, now_master); + cl_ctr_arm_master_timers(cl, now_master); /* The new master announces itself; all members already hold the session * key so no re-keying is needed. */ if (now_master) - cc_send_member_list(cl->sock, cl); + cl_ctr_send_member_list(cl->sock, cl); return 0; } /** - * cc_worker() - the single dedicated background process. + * cl_ctr_worker() - the single dedicated background process. * * JOIN PROTOCOL: * 1. Open socket, join multicast group. - * 2. Send CC_PKT_JOIN_REQ and set state = CC_NODE_NEW with a deadline + * 2. Send CL_CTR_PKT_JOIN_REQ and set state = CL_CTR_NODE_NEW with a deadline * of (now + query_time). - * 3. Listen for incoming packets. If CC_PKT_MEMBER_LIST arrives: - * -> cc_handle_member_list() sets state = CC_NODE_ACTIVE. + * 3. Listen for incoming packets. If CL_CTR_PKT_MEMBER_LIST arrives: + * -> cl_ctr_handle_member_list() sets state = CL_CTR_NODE_ACTIVE. * If deadline expires with no MEMBER_LIST: - * -> no master exists yet; transition to CC_NODE_ACTIVE and + * -> no master exists yet; transition to CL_CTR_NODE_ACTIVE and * join the normal election cycle. * * ACTIVE LOOP: * Fully event-driven via OpenSIPS reactor (epoll by default). * Each event source is a registered fd with a dedicated callback: - * cc_on_sock - incoming UDP packet - * cc_on_alive_tfd - periodic ALIVE heartbeat (query_time seconds) - * cc_on_join_tfd - one-shot join deadline - * cc_on_rejoin_tfd - 1-second JOIN_REQ retry while in CC_NODE_NEW + * cl_ctr_on_sock - incoming UDP packet + * cl_ctr_on_alive_tfd - periodic ALIVE heartbeat (query_time seconds) + * cl_ctr_on_join_tfd - one-shot join deadline + * cl_ctr_on_rejoin_tfd - 1-second JOIN_REQ retry while in CL_CTR_NODE_NEW * reactor_proc_init() also wires in IPC (shutdown, load stats, reload). */ -static void cc_worker(int rank) +static void cl_ctr_worker(int rank) { - cc_cluster_t *cl; + cl_ctr_cluster_t *cl; - if (rank >= cc_cluster_count) { + if (rank >= cl_ctr_cluster_count) { /* Extra process slot - no cluster assigned, exit cleanly */ return; } - cl = &cc_clusters[rank]; + cl = &cl_ctr_clusters[rank]; LM_INFO("clusterer_controller: [cluster %d] worker started (pid=%d)\n", cl->cluster_id, getpid()); @@ -4647,7 +4647,7 @@ static void cc_worker(int rank) cl->shtag_last_active = -1; /* unknown - first decision logs its reason */ cl->shtag_last_forced = 0; - cl->sock = cc_setup_socket(cl); + cl->sock = cl_ctr_setup_socket(cl); if (cl->sock < 0) { LM_CRIT("clusterer_controller: [cluster %d] cannot open multicast socket, " "worker exits\n", cl->cluster_id); @@ -4655,7 +4655,7 @@ static void cc_worker(int rank) } /* Generate ephemeral X25519 keypair - private key never leaves this process */ - if (cc_gen_ecdh_keypair(cl->my_privkey, cl->my_pubkey) < 0) { + if (cl_ctr_gen_ecdh_keypair(cl->my_privkey, cl->my_pubkey) < 0) { LM_CRIT("clusterer_controller: [cluster %d] ECDH keypair generation failed\n", cl->cluster_id); exit(-1); @@ -4667,7 +4667,7 @@ static void cc_worker(int rank) int _i; for (_i = 0; _i < cl->peers->count; _i++) { if (strcmp(cl->peers->entries[_i].ip, my_ip) == 0) { - memcpy(cl->peers->entries[_i].pubkey, cl->my_pubkey, CC_PUBKEY_SZ); + memcpy(cl->peers->entries[_i].pubkey, cl->my_pubkey, CL_CTR_PUBKEY_SZ); break; } } @@ -4686,24 +4686,24 @@ static void cc_worker(int rank) exit(-1); } - cl->rate_tbl = pkg_malloc(CC_RATE_TBL_SZ * sizeof(cc_rate_entry_t)); + cl->rate_tbl = pkg_malloc(CL_CTR_RATE_TBL_SZ * sizeof(cl_ctr_rate_entry_t)); if (!cl->rate_tbl) { LM_CRIT("clusterer_controller: [cluster %d] no pkg memory for rate table\n", cl->cluster_id); exit(-1); } - memset(cl->rate_tbl, 0, CC_RATE_TBL_SZ * sizeof(cc_rate_entry_t)); + memset(cl->rate_tbl, 0, CL_CTR_RATE_TBL_SZ * sizeof(cl_ctr_rate_entry_t)); /* ---- Phase 1: join protocol ---- */ lock_start_write(cl->peers->lock); - cl->peers->node_state = CC_NODE_NEW; + cl->peers->node_state = CL_CTR_NODE_NEW; cl->peers->join_deadline = time(NULL) + (time_t)query_time; lock_stop_write(cl->peers->lock); - cc_send_join_req(cl->sock, cl); - cc_arm_tfd(cl->join_tfd, (time_t)query_time, 0); /* one-shot deadline */ - cc_arm_tfd(cl->rejoin_tfd, 1, 1); /* retry every 1 s */ - /* alive_tfd left disarmed - armed by cc_transition_to_active() */ + cl_ctr_send_join_req(cl->sock, cl); + cl_ctr_arm_tfd(cl->join_tfd, (time_t)query_time, 0); /* one-shot deadline */ + cl_ctr_arm_tfd(cl->rejoin_tfd, 1, 1); /* retry every 1 s */ + /* alive_tfd left disarmed - armed by cl_ctr_transition_to_active() */ LM_INFO("clusterer_controller: [cluster %d] sent JOIN_REQ, waiting up to %ds " "for master response\n", cl->cluster_id, query_time); @@ -4715,12 +4715,12 @@ static void cc_worker(int rank) exit(-1); } - if (reactor_proc_add_fd(cl->sock, cc_on_sock, cl) < 0 || - reactor_proc_add_fd(cl->alive_tfd, cc_on_alive_tfd, cl) < 0 || - reactor_proc_add_fd(cl->join_tfd, cc_on_join_tfd, cl) < 0 || - reactor_proc_add_fd(cl->rejoin_tfd, cc_on_rejoin_tfd, cl) < 0 || - reactor_proc_add_fd(cl->master_alive_tfd, cc_on_master_alive_tfd, cl) < 0 || - reactor_proc_add_fd(cl->master_dead_tfd, cc_on_master_dead_tfd, cl) < 0) { + if (reactor_proc_add_fd(cl->sock, cl_ctr_on_sock, cl) < 0 || + reactor_proc_add_fd(cl->alive_tfd, cl_ctr_on_alive_tfd, cl) < 0 || + reactor_proc_add_fd(cl->join_tfd, cl_ctr_on_join_tfd, cl) < 0 || + reactor_proc_add_fd(cl->rejoin_tfd, cl_ctr_on_rejoin_tfd, cl) < 0 || + reactor_proc_add_fd(cl->master_alive_tfd, cl_ctr_on_master_alive_tfd, cl) < 0 || + reactor_proc_add_fd(cl->master_dead_tfd, cl_ctr_on_master_dead_tfd, cl) < 0) { LM_CRIT("clusterer_controller: [cluster %d] reactor_proc_add_fd failed\n", cl->cluster_id); exit(-1); @@ -4732,7 +4732,7 @@ static void cc_worker(int rank) { int i_am_master; lock_start_read(cl->peers->lock); - i_am_master = cc_i_am_master_locked(cl); + i_am_master = cl_ctr_i_am_master_locked(cl); lock_stop_read(cl->peers->lock); if (i_am_master && cl->peers->count > 1) { @@ -4742,17 +4742,17 @@ static void cc_worker(int rank) int _i; lock_start_read(cl->peers->lock); for (_i = 0; _i < cl->peers->count; _i++) { - cc_peer_t *e = &cl->peers->entries[_i]; + cl_ctr_peer_t *e = &cl->peers->entries[_i]; if (strcmp(e->ip, my_ip) == 0) continue; if (e->ip_num > best_ip) { best_ip = e->ip_num; best_idx = _i; } } if (best_idx >= 0) { - char next_ip[CC_MAX_IP_LEN + 1]; - unsigned char next_pub[CC_PUBKEY_SZ]; - memcpy(next_ip, cl->peers->entries[best_idx].ip, CC_MAX_IP_LEN + 1); - memcpy(next_pub, cl->peers->entries[best_idx].pubkey, CC_PUBKEY_SZ); + char next_ip[CL_CTR_MAX_IP_LEN + 1]; + unsigned char next_pub[CL_CTR_PUBKEY_SZ]; + memcpy(next_ip, cl->peers->entries[best_idx].ip, CL_CTR_MAX_IP_LEN + 1); + memcpy(next_pub, cl->peers->entries[best_idx].pubkey, CL_CTR_PUBKEY_SZ); lock_stop_read(cl->peers->lock); - cc_send_key_handoff(cl->sock, next_ip, next_pub, cl); + cl_ctr_send_key_handoff(cl->sock, next_ip, next_pub, cl); } else { lock_stop_read(cl->peers->lock); } @@ -4783,7 +4783,7 @@ static void cc_worker(int rank) * ] * * Only peers within the current quantized election window are shown, - * consistent with what cc_elect_master(cl) considers. + * consistent with what cl_ctr_elect_master(cl) considers. */ static mi_response_t *mi_cl_ctr_members(const mi_params_t *params, struct mi_handler *hdl) @@ -4791,14 +4791,14 @@ static mi_response_t *mi_cl_ctr_members(const mi_params_t *params, mi_response_t *resp; mi_item_t *arr, *cl_obj, *members_arr, *peer_obj, *bin_arr; int i, j, ci; - cc_cluster_t *cl; + cl_ctr_cluster_t *cl; resp = init_mi_result_array(&arr); if (!resp) return NULL; - for (ci = 0; ci < cc_cluster_count; ci++) { - cl = &cc_clusters[ci]; + for (ci = 0; ci < cl_ctr_cluster_count; ci++) { + cl = &cl_ctr_clusters[ci]; if (!cl->peers) continue; @@ -4814,7 +4814,7 @@ static mi_response_t *mi_cl_ctr_members(const mi_params_t *params, lock_start_read(cl->peers->lock); for (i = 0; i < cl->peers->count; i++) { - cc_peer_t *e = &cl->peers->entries[i]; + cl_ctr_peer_t *e = &cl->peers->entries[i]; peer_obj = add_mi_object(members_arr, NULL, 0); if (!peer_obj) { @@ -4869,14 +4869,14 @@ static mi_response_t *mi_cl_ctr_node_info(const mi_params_t *params, mi_item_t *root, *bin_arr; int target_id; int i, j, ci; - cc_cluster_t *cl; - cc_peer_t *e; + cl_ctr_cluster_t *cl; + cl_ctr_peer_t *e; if (get_mi_int_param(params, "node_id", &target_id) < 0) return init_mi_param_error(); - for (ci = 0; ci < cc_cluster_count; ci++) { - cl = &cc_clusters[ci]; + for (ci = 0; ci < cl_ctr_cluster_count; ci++) { + cl = &cl_ctr_clusters[ci]; if (!cl->peers) continue; @@ -4944,14 +4944,14 @@ static mi_response_t *mi_cl_ctr_config(const mi_params_t *params, int ci, members; char mcast[INET_ADDRSTRLEN + 8]; /* "IP:PORT" */ char shtag_mode[24]; /* "auto" / "override:" */ - cc_cluster_t *cl; + cl_ctr_cluster_t *cl; resp = init_mi_result_array(&arr); if (!resp) return NULL; - for (ci = 0; ci < cc_cluster_count; ci++) { - cl = &cc_clusters[ci]; + for (ci = 0; ci < cl_ctr_cluster_count; ci++) { + cl = &cl_ctr_clusters[ci]; cl_obj = add_mi_object(arr, NULL, 0); if (!cl_obj) goto error; @@ -5010,7 +5010,7 @@ static mi_response_t *mi_cl_ctr_config(const mi_params_t *params, } /** - * cc_rpc_apply_shtags() - IPC job run inside the cc_worker process. + * cl_ctr_rpc_apply_shtags() - IPC job run inside the cl_ctr_worker process. * * An MI handler (running in a different process) has already updated * cl->peers->shtag_forced_node_id in shm; this job makes the change take @@ -5018,35 +5018,35 @@ static mi_response_t *mi_cl_ctr_config(const mi_params_t *params, * master, re-broadcasts the MEMBER_LIST so every member learns the new * override without waiting for the next periodic announcement. */ -static void cc_rpc_apply_shtags(int sender, void *param) +static void cl_ctr_rpc_apply_shtags(int sender, void *param) { - cc_cluster_t *cl = (cc_cluster_t *)param; + cl_ctr_cluster_t *cl = (cl_ctr_cluster_t *)param; int i_am_master; (void)sender; if (!cl || !cl->peers) return; - cc_apply_shtags(cl); + cl_ctr_apply_shtags(cl); lock_start_read(cl->peers->lock); - i_am_master = cc_i_am_master_locked(cl); + i_am_master = cl_ctr_i_am_master_locked(cl); lock_stop_read(cl->peers->lock); if (i_am_master && cl->sock >= 0) - cc_send_member_list(cl->sock, cl); + cl_ctr_send_member_list(cl->sock, cl); } /** - * cc_mi_find_cluster() - locate a configured cluster by its cluster_id. + * cl_ctr_mi_find_cluster() - locate a configured cluster by its cluster_id. * Returns NULL if no cluster matches. */ -static cc_cluster_t *cc_mi_find_cluster(int cluster_id) +static cl_ctr_cluster_t *cl_ctr_mi_find_cluster(int cluster_id) { int ci; - for (ci = 0; ci < cc_cluster_count; ci++) - if (cc_clusters[ci].cluster_id == cluster_id) - return &cc_clusters[ci]; + for (ci = 0; ci < cl_ctr_cluster_count; ci++) + if (cl_ctr_clusters[ci].cluster_id == cluster_id) + return &cl_ctr_clusters[ci]; return NULL; } @@ -5065,7 +5065,7 @@ static mi_response_t *mi_cl_ctr_shtag_force(const mi_params_t *params, struct mi_handler *hdl) { int cluster_id, node_id; - cc_cluster_t *cl; + cl_ctr_cluster_t *cl; int i_am_master, found = 0, proc_no; if (get_mi_int_param(params, "cluster_id", &cluster_id) < 0 || @@ -5075,7 +5075,7 @@ static mi_response_t *mi_cl_ctr_shtag_force(const mi_params_t *params, if (node_id <= 0 || node_id > 0xFFFF) return init_mi_error(400, MI_SSTR("node_id out of range")); - cl = cc_mi_find_cluster(cluster_id); + cl = cl_ctr_mi_find_cluster(cluster_id); if (!cl || !cl->peers) return init_mi_error(404, MI_SSTR("cluster_id not found")); @@ -5084,7 +5084,7 @@ static mi_response_t *mi_cl_ctr_shtag_force(const mi_params_t *params, MI_SSTR("shtag management is disabled for this cluster")); lock_start_write(cl->peers->lock); - i_am_master = cc_i_am_master_locked(cl); + i_am_master = cl_ctr_i_am_master_locked(cl); if (i_am_master) { int i; for (i = 0; i < cl->peers->count; i++) { @@ -5109,7 +5109,7 @@ static mi_response_t *mi_cl_ctr_shtag_force(const mi_params_t *params, /* Apply locally and broadcast the new override from the worker process. */ if (proc_no >= 0) - ipc_send_rpc(proc_no, cc_rpc_apply_shtags, cl); + ipc_send_rpc(proc_no, cl_ctr_rpc_apply_shtags, cl); LM_INFO("clusterer_controller: [cluster %d] operator forced shtag onto " "node %d\n", cluster_id, node_id); @@ -5128,18 +5128,18 @@ static mi_response_t *mi_cl_ctr_shtag_auto(const mi_params_t *params, struct mi_handler *hdl) { int cluster_id; - cc_cluster_t *cl; + cl_ctr_cluster_t *cl; int i_am_master, proc_no; if (get_mi_int_param(params, "cluster_id", &cluster_id) < 0) return init_mi_param_error(); - cl = cc_mi_find_cluster(cluster_id); + cl = cl_ctr_mi_find_cluster(cluster_id); if (!cl || !cl->peers) return init_mi_error(404, MI_SSTR("cluster_id not found")); lock_start_write(cl->peers->lock); - i_am_master = cc_i_am_master_locked(cl); + i_am_master = cl_ctr_i_am_master_locked(cl); if (i_am_master) cl->peers->shtag_forced_node_id = 0; proc_no = cl->peers->worker_proc_no; @@ -5150,7 +5150,7 @@ static mi_response_t *mi_cl_ctr_shtag_auto(const mi_params_t *params, MI_SSTR("not the master - issue cl_ctr_shtag_auto on the master node")); if (proc_no >= 0) - ipc_send_rpc(proc_no, cc_rpc_apply_shtags, cl); + ipc_send_rpc(proc_no, cl_ctr_rpc_apply_shtags, cl); LM_INFO("clusterer_controller: [cluster %d] operator resumed automatic " "shtag allocation\n", cluster_id); @@ -5161,7 +5161,7 @@ static mi_response_t *mi_cl_ctr_shtag_auto(const mi_params_t *params, * ========================================================================= */ /** - * cc_resolve_local_identity() - determine my_ip and my_interface_buf. + * cl_ctr_resolve_local_identity() - determine my_ip and my_interface_buf. * * Three modes depending on which modparams were provided: * @@ -5184,7 +5184,7 @@ static mi_response_t *mi_cl_ctr_shtag_auto(const mi_params_t *params, * reverse lookup failed in mode 3 - non-fatal). */ /** - * cc_parse_cluster_str() - parse one "cluster" modparam string into cl. + * cl_ctr_parse_cluster_str() - parse one "cluster" modparam string into cl. * * Format: "id=N,multicast=A.B.C.D:PORT[,password=STRING][,bin_socket=bin:IP:PORT]" * - id= required, positive integer @@ -5193,7 +5193,7 @@ static mi_response_t *mi_cl_ctr_shtag_auto(const mi_params_t *params, * - bin_socket= optional, BIN socket for this cluster; falls back to * first discovered socket (or only socket if one exists) */ -static int cc_parse_cluster_str(const char *str, cc_cluster_t *cl) +static int cl_ctr_parse_cluster_str(const char *str, cl_ctr_cluster_t *cl) { char buf[2048]; char *p, *tok, *key, *val, *colon; @@ -5255,12 +5255,12 @@ static int cc_parse_cluster_str(const char *str, cc_cluster_t *cl) cl->password[sizeof(cl->password) - 1] = '\0'; } else if (strcmp(key, "bin_socket") == 0) { - if (strlen(val) >= CC_MAX_BIN_SOCK_LEN) { + if (strlen(val) >= CL_CTR_MAX_BIN_SOCK_LEN) { LM_ERR("clusterer_controller: bin_socket value too long\n"); return -1; } - strncpy(cl->bin_socket, val, CC_MAX_BIN_SOCK_LEN - 1); - cl->bin_socket[CC_MAX_BIN_SOCK_LEN - 1] = '\0'; + strncpy(cl->bin_socket, val, CL_CTR_MAX_BIN_SOCK_LEN - 1); + cl->bin_socket[CL_CTR_MAX_BIN_SOCK_LEN - 1] = '\0'; } else if (strcmp(key, "manage_shtags") == 0) { cl->manage_shtags = atoi(val) ? 1 : 0; @@ -5281,7 +5281,7 @@ static int cc_parse_cluster_str(const char *str, cc_cluster_t *cl) return 0; } -static int cc_resolve_local_identity(void) +static int cl_ctr_resolve_local_identity(void) { struct ifaddrs *ifap = NULL, *ifa; int found = 0; @@ -5365,8 +5365,8 @@ static int cc_resolve_local_identity(void) memset(&dest, 0, sizeof(dest)); dest.sin_family = AF_INET; - dest.sin_port = htons((uint16_t)cc_clusters[0].multicast_port); - dest.sin_addr.s_addr = inet_addr(cc_clusters[0].multicast_address); + dest.sin_port = htons((uint16_t)cl_ctr_clusters[0].multicast_port); + dest.sin_addr.s_addr = inet_addr(cl_ctr_clusters[0].multicast_address); probe = socket(AF_INET, SOCK_DGRAM, 0); if (probe < 0) { @@ -5420,7 +5420,7 @@ static int cc_resolve_local_identity(void) } /** - * cc_discover_bin_sockets() - enumerate BIN listeners via proto_bin. + * cl_ctr_discover_bin_sockets() - enumerate BIN listeners via proto_bin. * * Walks the protos[PROTO_BIN].listeners list and collects every * entry with proto == PROTO_BIN. proto_bin must be loaded before this @@ -5429,11 +5429,11 @@ static int cc_resolve_local_identity(void) * Populates my_bin_sockets[] and my_bin_count. * Returns 0 on success, -1 if no BIN sockets are found. */ -static int cc_discover_bin_sockets(void) +static int cl_ctr_discover_bin_sockets(void) { struct socket_info_full *sif; struct socket_info *si; - char buf[CC_MAX_BIN_SOCK_LEN]; + char buf[CL_CTR_MAX_BIN_SOCK_LEN]; int len; /* On devel, protos[].listeners is a list of socket_info_full (next/prev), @@ -5455,15 +5455,15 @@ static int cc_discover_bin_sockets(void) si->port_no, my_ip ? my_ip : "YOUR_IP", si->port_no); return -1; } - if (my_bin_count >= CC_MAX_BIN_SOCKETS) { + if (my_bin_count >= CL_CTR_MAX_BIN_SOCKETS) { LM_WARN("clusterer_controller: more than %d BIN sockets, " - "ignoring the rest\n", CC_MAX_BIN_SOCKETS); + "ignoring the rest\n", CL_CTR_MAX_BIN_SOCKETS); break; } len = snprintf(buf, sizeof(buf), "bin:%.*s:%u", si->address_str.len, si->address_str.s, si->port_no); - if (len <= 0 || len >= CC_MAX_BIN_SOCK_LEN) { + if (len <= 0 || len >= CL_CTR_MAX_BIN_SOCK_LEN) { LM_WARN("clusterer_controller: BIN socket name too long, " "skipping\n"); continue; @@ -5497,11 +5497,11 @@ static int mod_init(void) /* Resolve the on_config_mismatch policy string. */ if (on_config_mismatch_s) { if (strcasecmp(on_config_mismatch_s, "warn") == 0) - on_config_mismatch = CC_CFGMISMATCH_WARN; + on_config_mismatch = CL_CTR_CFGMISMATCH_WARN; else if (strcasecmp(on_config_mismatch_s, "reject") == 0) - on_config_mismatch = CC_CFGMISMATCH_REJECT; + on_config_mismatch = CL_CTR_CFGMISMATCH_REJECT; else if (strcasecmp(on_config_mismatch_s, "adopt") == 0) - on_config_mismatch = CC_CFGMISMATCH_ADOPT; + on_config_mismatch = CL_CTR_CFGMISMATCH_ADOPT; else { LM_ERR("clusterer_controller: invalid on_config_mismatch '%s' " "(expected warn|reject|adopt)\n", on_config_mismatch_s); @@ -5511,7 +5511,7 @@ static int mod_init(void) /* ---- Require at least one cluster ---------------------------------- */ - if (cc_cluster_str_count == 0) { + if (cl_ctr_cluster_str_count == 0) { LM_ERR("clusterer_controller: no 'cluster' modparam defined\n"); return -1; } @@ -5530,9 +5530,9 @@ static int mod_init(void) /* ---- Parse and validate all cluster strings ------------------------ */ - for (i = 0; i < cc_cluster_str_count; i++) { - cc_cluster_t *cl = &cc_clusters[i]; - if (cc_parse_cluster_str(cc_cluster_strs[i], cl) < 0) + for (i = 0; i < cl_ctr_cluster_str_count; i++) { + cl_ctr_cluster_t *cl = &cl_ctr_clusters[i]; + if (cl_ctr_parse_cluster_str(cl_ctr_cluster_strs[i], cl) < 0) return -1; /* Resolve the multicast destination once, in the main process, so both * the forked workers (via fork) and mod_destroy's GOODBYE path (main @@ -5541,43 +5541,43 @@ static int mod_init(void) cl->mcast_dest.sin_family = AF_INET; cl->mcast_dest.sin_port = htons((uint16_t)cl->multicast_port); cl->mcast_dest.sin_addr.s_addr = inet_addr(cl->multicast_address); - cc_cluster_count++; - pkg_free(cc_cluster_strs[i]); - cc_cluster_strs[i] = NULL; + cl_ctr_cluster_count++; + pkg_free(cl_ctr_cluster_strs[i]); + cl_ctr_cluster_strs[i] = NULL; } /* Validate cluster_id uniqueness and (multicast, port) uniqueness */ - for (i = 0; i < cc_cluster_count; i++) { - for (j = i + 1; j < cc_cluster_count; j++) { - if (cc_clusters[i].cluster_id == cc_clusters[j].cluster_id) { + for (i = 0; i < cl_ctr_cluster_count; i++) { + for (j = i + 1; j < cl_ctr_cluster_count; j++) { + if (cl_ctr_clusters[i].cluster_id == cl_ctr_clusters[j].cluster_id) { LM_ERR("clusterer_controller: duplicate cluster_id %d\n", - cc_clusters[i].cluster_id); + cl_ctr_clusters[i].cluster_id); return -1; } - if (strcmp(cc_clusters[i].multicast_address, - cc_clusters[j].multicast_address) == 0 && - cc_clusters[i].multicast_port == cc_clusters[j].multicast_port) { + if (strcmp(cl_ctr_clusters[i].multicast_address, + cl_ctr_clusters[j].multicast_address) == 0 && + cl_ctr_clusters[i].multicast_port == cl_ctr_clusters[j].multicast_port) { LM_ERR("clusterer_controller: duplicate multicast %s:%d\n", - cc_clusters[i].multicast_address, - cc_clusters[i].multicast_port); + cl_ctr_clusters[i].multicast_address, + cl_ctr_clusters[i].multicast_port); return -1; } } } /* ---- Discover BIN sockets from opensips config file --------------- */ - /* Called after cc_resolve_local_identity() so my_ip is available for */ + /* Called after cl_ctr_resolve_local_identity() so my_ip is available for */ /* wildcard substitution (bin:*:PORT -> bin:my_ip:PORT). */ /* ---- Resolve local identity using first cluster for Mode 3 probe --- */ - if (cc_resolve_local_identity() < 0) + if (cl_ctr_resolve_local_identity() < 0) return -1; - if (cc_discover_bin_sockets() < 0) + if (cl_ctr_discover_bin_sockets() < 0) return -1; - if (strlen(my_ip) > CC_MAX_IP_LEN) { + if (strlen(my_ip) > CL_CTR_MAX_IP_LEN) { LM_ERR("clusterer_controller: resolved my_ip too long\n"); return -1; } @@ -5588,13 +5588,13 @@ static int mod_init(void) /* ---- Multi-cluster: each cluster must name its BIN socket ---------- */ - if (cc_cluster_count > 1) { - for (i = 0; i < cc_cluster_count; i++) { - if (cc_clusters[i].bin_socket[0] == '\0') { + if (cl_ctr_cluster_count > 1) { + for (i = 0; i < cl_ctr_cluster_count; i++) { + if (cl_ctr_clusters[i].bin_socket[0] == '\0') { LM_ERR("clusterer_controller: cluster %d has no bin_socket= " "defined - required when multiple clusters are configured " "(e.g. id=%d,multicast=...,bin_socket=bin:IP:PORT)\n", - cc_clusters[i].cluster_id, cc_clusters[i].cluster_id); + cl_ctr_clusters[i].cluster_id, cl_ctr_clusters[i].cluster_id); return -1; } } @@ -5602,8 +5602,8 @@ static int mod_init(void) /* ---- Per-cluster: resolve BIN socket, derive key, allocate peers --- */ - for (i = 0; i < cc_cluster_count; i++) { - cc_cluster_t *cl = &cc_clusters[i]; + for (i = 0; i < cl_ctr_cluster_count; i++) { + cl_ctr_cluster_t *cl = &cl_ctr_clusters[i]; /* Resolve sentinels to the global default when the cluster string did * not set them explicitly. Done here (unconditionally, before workers @@ -5629,7 +5629,7 @@ static int mod_init(void) } } if (!found_bs) { - char _disc[CC_MAX_BIN_SOCKETS * (CC_MAX_BIN_SOCK_LEN + 2)]; + char _disc[CL_CTR_MAX_BIN_SOCKETS * (CL_CTR_MAX_BIN_SOCK_LEN + 2)]; int _o = 0, _b; _disc[0] = '\0'; for (_b = 0; _b < my_bin_count; _b++) @@ -5645,14 +5645,14 @@ static int mod_init(void) } else if (my_bin_count == 1) { /* Only one socket - unambiguous */ { - size_t _l = strnlen(my_bin_sockets[0], CC_MAX_BIN_SOCK_LEN - 1); + size_t _l = strnlen(my_bin_sockets[0], CL_CTR_MAX_BIN_SOCK_LEN - 1); memcpy(cl->bin_socket, my_bin_sockets[0], _l); cl->bin_socket[_l] = '\0'; } } else { /* Multiple sockets, no explicit override - use first, warn */ { - size_t _l = strnlen(my_bin_sockets[0], CC_MAX_BIN_SOCK_LEN - 1); + size_t _l = strnlen(my_bin_sockets[0], CL_CTR_MAX_BIN_SOCK_LEN - 1); memcpy(cl->bin_socket, my_bin_sockets[0], _l); cl->bin_socket[_l] = '\0'; } @@ -5664,18 +5664,18 @@ static int mod_init(void) LM_INFO("clusterer_controller: cluster %d: bin_socket=%s\n", cl->cluster_id, cl->bin_socket); - if (cc_derive_key(cl) < 0) + if (cl_ctr_derive_key(cl) < 0) return -1; - cl->peers = shm_malloc(sizeof(cc_peers_t)); + cl->peers = shm_malloc(sizeof(cl_ctr_peers_t)); if (!cl->peers) { LM_ERR("clusterer_controller: no shm memory for cluster %d peer table\n", cl->cluster_id); return -1; } - memset(cl->peers, 0, sizeof(cc_peers_t)); - cl->peers->node_state = CC_NODE_NEW; - cl->peers->worker_proc_no = -1; /* published by cc_worker after fork */ + memset(cl->peers, 0, sizeof(cl_ctr_peers_t)); + cl->peers->node_state = CL_CTR_NODE_NEW; + cl->peers->worker_proc_no = -1; /* published by cl_ctr_worker after fork */ cl->peers->eff_manage_shtags = cl->manage_shtags; cl->peers->eff_master_stickiness = cl->master_stickiness; cl->peers->eff_query_time = query_time; @@ -5697,10 +5697,10 @@ static int mod_init(void) LM_INFO("clusterer_controller: my_ip=%s interface=%s query_time=%ds " "clusters=%d bin_sockets=%d crypto=%s\n", my_ip, my_interface_buf[0] ? my_interface_buf : "(unknown)", - query_time, cc_cluster_count, my_bin_count, CC_CRYPTO_SUITE); + query_time, cl_ctr_cluster_count, my_bin_count, CL_CTR_CRYPTO_SUITE); /* Set worker process count dynamically - one per cluster */ - procs[0].no = cc_cluster_count; + procs[0].no = cl_ctr_cluster_count; /* Load clusterer controller API if clusterer.so is present and * use_controller=1 is set. Soft dependency - controller works @@ -5729,8 +5729,8 @@ static int mod_init(void) int m, k, found; for (m = 0; m < clctl.managed_count; m++) { found = 0; - for (k = 0; k < cc_cluster_count; k++) - if (cc_clusters[k].cluster_id == clctl.managed_ids[m]) { + for (k = 0; k < cl_ctr_cluster_count; k++) + if (cl_ctr_clusters[k].cluster_id == clctl.managed_ids[m]) { found = 1; break; } @@ -5750,10 +5750,10 @@ static int mod_init(void) /* (b) we have a config for a cluster the clusterer did not mark managed */ { int k, m, managed; - for (k = 0; k < cc_cluster_count; k++) { + for (k = 0; k < cl_ctr_cluster_count; k++) { managed = 0; for (m = 0; m < clctl.managed_count; m++) - if (clctl.managed_ids[m] == cc_clusters[k].cluster_id) { + if (clctl.managed_ids[m] == cl_ctr_clusters[k].cluster_id) { managed = 1; break; } @@ -5763,7 +5763,7 @@ static int mod_init(void) "- add modparam(\"clusterer\", \"cluster_options\", " "\"cluster_id=%d, use_controller=1\"), or remove this " "module's 'cluster' config for it.\n", - cc_clusters[k].cluster_id, cc_clusters[k].cluster_id); + cl_ctr_clusters[k].cluster_id, cl_ctr_clusters[k].cluster_id); return -1; } } @@ -5775,13 +5775,13 @@ static int mod_init(void) * resolved to concrete 0/1 in the unconditional loop above.) */ { int _ci; - for (_ci = 0; _ci < cc_cluster_count; _ci++) { - if (!cc_clusters[_ci].manage_shtags) + for (_ci = 0; _ci < cl_ctr_cluster_count; _ci++) { + if (!cl_ctr_clusters[_ci].manage_shtags) continue; if (clctl.force_backup_shtags) - clctl.force_backup_shtags(cc_clusters[_ci].cluster_id); + clctl.force_backup_shtags(cl_ctr_clusters[_ci].cluster_id); if (clctl.set_shtag_managed) - clctl.set_shtag_managed(cc_clusters[_ci].cluster_id); + clctl.set_shtag_managed(cl_ctr_clusters[_ci].cluster_id); } } @@ -5794,7 +5794,7 @@ static int mod_init(void) return 0; } -static int cc_child_init(int rank) +static int cl_ctr_child_init(int rank) { /* Re-seed the CSPRNG after fork - each worker must have independent state. */ randombytes_stir(); @@ -5811,14 +5811,14 @@ static void mod_destroy(void) { int i, sock; unsigned char ttl = 32; - cc_cluster_t *cl; + cl_ctr_cluster_t *cl; if (!my_ip) goto cleanup; /* Send GOODBYE on each cluster's multicast group so peers re-elect */ - for (i = 0; i < cc_cluster_count; i++) { - cl = &cc_clusters[i]; + for (i = 0; i < cl_ctr_cluster_count; i++) { + cl = &cl_ctr_clusters[i]; if (!cl->peers) continue; if (cl->peers->count <= 1) { @@ -5837,19 +5837,19 @@ static void mod_destroy(void) * GOODBYE - the worker's cl->session_key is in a different process. */ { size_t plen = strlen(cl->password); - cc_hkdf_sha256((unsigned char *)cl->password, plen, - cl->peers->master_salt, CC_MASTER_SALT_SZ, - "cc_session", cl->session_key); + cl_ctr_hkdf_sha256((unsigned char *)cl->password, plen, + cl->peers->master_salt, CL_CTR_MASTER_SALT_SZ, + "cl_ctr_session", cl->session_key); } - cc_send_pkt_with_ip(sock, CC_PKT_GOODBYE, cl); + cl_ctr_send_pkt_with_ip(sock, CL_CTR_PKT_GOODBYE, cl); close(sock); LM_INFO("clusterer_controller: [cluster %d] GOODBYE sent\n", cl->cluster_id); } cleanup: - for (i = 0; i < cc_cluster_count; i++) { - cl = &cc_clusters[i]; + for (i = 0; i < cl_ctr_cluster_count; i++) { + cl = &cl_ctr_clusters[i]; if (!cl->peers) continue; if (cl->peers->lock) { diff --git a/modules/clusterer_controller/doc/clusterer_controller_admin.xml b/modules/clusterer_controller/doc/clusterer_controller_admin.xml index 16c889e89cb..51e0311a5dd 100644 --- a/modules/clusterer_controller/doc/clusterer_controller_admin.xml +++ b/modules/clusterer_controller/doc/clusterer_controller_admin.xml @@ -171,7 +171,7 @@ JOIN_REJECT — sent by the master to a joining node whose JOIN_REQ repeatedly fails authentication (wrong password). After - CC_JOIN_FAIL_LIMIT (3) consecutive + CL_CTR_JOIN_FAIL_LIMIT (3) consecutive bootstrap-key decryption failures from the same source IP the master sends a JOIN_REJECT to that IP. Encrypted with the bootstrap key so it cannot be forged by a node that does not @@ -353,8 +353,8 @@ Join authentication and rejection: the master tracks consecutive bootstrap-key decryption failures per source IP in a small worker-local table - (CC_JOIN_FAIL_TABLE_SZ = 8 slots). When any - source IP accumulates CC_JOIN_FAIL_LIMIT (3) + (CL_CTR_JOIN_FAIL_TABLE_SZ = 8 slots). When any + source IP accumulates CL_CTR_JOIN_FAIL_LIMIT (3) consecutive failures — indicating a node attempting to join with the wrong password — the master sends an encrypted JOIN_REJECT packet and stops responding to further JOIN_REQs from that IP. @@ -362,7 +362,7 @@ On the joining side, a received JOIN_REJECT is only acted on while the node is still in the initial join phase - (CC_NODE_NEW state) and is addressed to this + (CL_CTR_NODE_NEW state) and is addressed to this node; an already-active cluster member ignores any JOIN_REJECT unconditionally, so a node with the correct password can never be evicted by a peer. @@ -373,7 +373,7 @@ key), so it relies on a self-contained signal instead: while joining it counts packets received from other peers that it cannot decrypt. If, at the join deadline, the node is still unjoined and has seen - CC_JOIN_FAIL_LIMIT or more such undecryptable + CL_CTR_JOIN_FAIL_LIMIT or more such undecryptable packets, it concludes that a cluster it cannot authenticate to exists on the group and shuts down OpenSIPS with a critical log message — rather than promoting itself into a lone, split-brain master (which, @@ -393,7 +393,7 @@ Peer table exhaustion defence: the - peer table is bounded at CC_MAX_PEERS (256) + peer table is bounded at CL_CTR_MAX_PEERS (256) entries. When the table is full, the master rejects JOIN_REQ packets from unknown IPs with a JOIN_REJECT response. Known peers that are reconnecting after a restart continue to be admitted @@ -1787,7 +1787,7 @@ modparam("dispatcher", "cluster_probing_mode", "distributed") networks this brief re-flood/prune cycle could cause a gap in MASTER_ALIVE delivery and trigger a spurious master re-election. If this occurs, increase the effective timeout - by raising CC_MASTER_KA_MISSED in the + by raising CL_CTR_MASTER_KA_MISSED in the source from 3 to 5 or higher. PIM Sparse Mode (PIM-SM) or networks with IGMP snooping do not have this issue. @@ -1800,7 +1800,7 @@ modparam("dispatcher", "cluster_probing_mode", "distributed") case the controller will not function, as it has no unicast fallback. Encapsulation overhead also adds latency and jitter; on high-latency overlays consider raising - CC_MASTER_KA_MISSED to avoid spurious + CL_CTR_MASTER_KA_MISSED to avoid spurious re-elections. diff --git a/modules/clusterer_controller/test/cc_join_reject_test.py b/modules/clusterer_controller/test/cl_ctr_join_reject_test.py similarity index 97% rename from modules/clusterer_controller/test/cc_join_reject_test.py rename to modules/clusterer_controller/test/cl_ctr_join_reject_test.py index 6096a1fdc2e..f5f44a80593 100755 --- a/modules/clusterer_controller/test/cc_join_reject_test.py +++ b/modules/clusterer_controller/test/cl_ctr_join_reject_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -cc_join_reject_test.py - rogue-joiner security test for clusterer_controller. +cl_ctr_join_reject_test.py - rogue-joiner security test for clusterer_controller. Simulates an unauthorized node on the cluster's multicast group WITHOUT touching any node config. It exercises two defences: @@ -20,7 +20,7 @@ Run this on a host on the multicast segment that is NOT a live cluster member (e.g. one node with `systemctl stop opensips`). Requires only python3. - ./cc_join_reject_test.py --group 239.0.90.1 --port 3333 + ./cl_ctr_join_reject_test.py --group 239.0.90.1 --port 3333 """ import argparse, collections, os, socket, struct, threading, time From 8561f25c2007bd1b9a02d4c8d41740b5c9387479 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Sun, 19 Jul 2026 20:06:22 +1000 Subject: [PATCH 14/21] clusterer_controller: unicast the KEY_GRANT to the joining node KEY_GRANT is strictly 1:1 - the master answers one joiner's JOIN_REQ with the Noise msg 2 carrying that joiner's master_salt. It was multicast to the whole group, so every other member spent an AEAD decrypt only to find the packet was not addressed to it and discard it. Send it to the joiner's own address instead: the datagram source of its JOIN_REQ, threaded from recvfrom through to the sender as a generic struct sockaddr. The controller socket is bound to INADDR_ANY on the group port, so it already receives unicast to the node's own address on that port - no socket change is needed - and echoing the received sockaddr rather than rebuilding it from a string keeps this correct for IPv6 (it carries the scope id link-local traffic needs). Add cl_ctr_seal_and_send_to(), which takes an explicit destination; cl_ctr_seal_and_send() becomes a thin multicast wrapper over it for the group packets. The KEY_GRANT wire format is unchanged (target_ip is still carried), so this interoperates with peers that still multicast it. Turns the per-join KEY_GRANT cost from one AEAD decrypt on every member into one on the joiner alone. --- .../clusterer_controller.c | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index cc5e9cb5816..956f5bcf0ed 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -779,7 +779,8 @@ static mi_response_t *mi_cl_ctr_members(const mi_params_t *params, static void cl_ctr_handle_member_list(const char *payload, int payload_len, const char *sender_ip, cl_ctr_cluster_t *cl); static void cl_ctr_handle_join_req(int sock, const char *payload, int payload_len, - cl_ctr_cluster_t *cl); + cl_ctr_cluster_t *cl, + const struct sockaddr *src, socklen_t src_len); static void cl_ctr_handle_node_assign(const char *payload, int payload_len, const char *sender_ip, cl_ctr_cluster_t *cl); static void cl_ctr_handle_goodbye(int sock, const char *src_ip, cl_ctr_cluster_t *cl); @@ -2298,26 +2299,32 @@ static int cl_ctr_setup_socket(cl_ctr_cluster_t *cl) * ========================================================================= */ /** - * cl_ctr_seal_and_send() - encrypt a prepared packet in place and multicast it. + * cl_ctr_seal_and_send_to() - encrypt a prepared packet in place and send it to + * an explicit destination. * * On entry @pkt holds the cleartext framing (magic already stamped) plus the * @plain_len-byte plaintext starting at CL_CTR_WIRE_HDR_SZ; cl_ctr_encrypt_pkt() fills - * the cluster_id + nonce and appends the AEAD tag. Every controller packet - * targets the cluster's multicast group (cached in cl->mcast_dest), so all - * senders share this tail. @type is only used for logging. + * the cluster_id + nonce and appends the AEAD tag. @dest/@destlen is the target + * sockaddr: the cluster's multicast group for group packets (see the + * cl_ctr_seal_and_send() wrapper), or a single peer for the strictly 1:1 packets + * of the join handshake (e.g. KEY_GRANT), which every other member would + * otherwise decrypt only to discard. @dest is passed straight to sendto(), so + * its address family is whatever the caller supplies - v4 today, v6-ready. + * @type is only used for logging. * * @return 0 on success, -1 on encrypt or send failure. */ -static int cl_ctr_seal_and_send(int sock, cl_ctr_cluster_t *cl, char *pkt, int plain_len, - const unsigned char *key, unsigned char type) +static int cl_ctr_seal_and_send_to(int sock, cl_ctr_cluster_t *cl, char *pkt, + int plain_len, const unsigned char *key, + unsigned char type, + const struct sockaddr *dest, socklen_t destlen) { int total_len = cl_ctr_encrypt_pkt(pkt, CL_CTR_WIRE_HDR_SZ, plain_len, key, cl->cluster_id); if (total_len < 0) return -1; - if (sendto(sock, pkt, total_len, 0, - (struct sockaddr *)&cl->mcast_dest, sizeof(cl->mcast_dest)) < 0) { + if (sendto(sock, pkt, total_len, 0, dest, destlen) < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK) LM_DBG("clusterer_controller: [cluster %d] sendto (type=0x%02x) " "would block\n", cl->cluster_id, type); @@ -2330,6 +2337,20 @@ static int cl_ctr_seal_and_send(int sock, cl_ctr_cluster_t *cl, char *pkt, int p return 0; } +/** + * cl_ctr_seal_and_send() - multicast wrapper over cl_ctr_seal_and_send_to(). + * + * Sends to the cluster's multicast group (cached in cl->mcast_dest); used by + * every group packet (ALIVE, MASTER_ALIVE, MEMBER_LIST, NODE_ASSIGN, ...). + */ +static int cl_ctr_seal_and_send(int sock, cl_ctr_cluster_t *cl, char *pkt, int plain_len, + const unsigned char *key, unsigned char type) +{ + return cl_ctr_seal_and_send_to(sock, cl, pkt, plain_len, key, type, + (const struct sockaddr *)&cl->mcast_dest, + sizeof(cl->mcast_dest)); +} + /** * cl_ctr_send_pkt_with_ip() - build and multicast a small (ALIVE/GOODBYE) packet. * JOIN_REQ is handled by cl_ctr_send_join_req_pkt() which carries BIN socket info. @@ -2626,7 +2647,11 @@ static void cl_ctr_send_master_beacon(int sock, cl_ctr_cluster_t *cl) /** * cl_ctr_send_key_grant() - send ECDH-wrapped master_salt to a joining node. * Encrypted with bootstrap key (CL_CTR_BOOTSTRAP_MAGIC) so the joining node can - * read it before having the session key. + * read it before having the session key. Unicast to the joiner (@dest, its + * JOIN_REQ datagram source): it is strictly 1:1, so multicasting it only made + * every other member spend an AEAD decrypt to discard a KEY_GRANT not addressed + * to it. The wire format is unchanged (target_ip still carried), so this + * interoperates with peers that still multicast it. * * Runs the Noise responder over the joiner's msg 1 and answers with msg 2, * whose AEAD payload is the master_salt. The Noise handshake authenticates the @@ -2636,7 +2661,8 @@ static void cl_ctr_send_master_beacon(int sock, cl_ctr_cluster_t *cl) * Payload: [target_ip NUL][noise_msg2 64B] */ static void cl_ctr_send_key_grant(int sock, const char *target_ip, cl_ctr_cluster_t *cl, - const unsigned char *msg1, int msg1_len) + const unsigned char *msg1, int msg1_len, + const struct sockaddr *dest, socklen_t destlen) { char pkt[CL_CTR_KEY_GRANT_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); @@ -2663,8 +2689,9 @@ static void cl_ctr_send_key_grant(int sock, const char *target_ip, cl_ctr_cluste memcpy(p, msg2, m2); p += m2; plain_len = (int)(p - (pkt + CL_CTR_WIRE_HDR_SZ)); - if (cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->key, CL_CTR_PKT_KEY_GRANT) == 0) - LM_INFO("clusterer_controller: [cluster %d] sent KEY_GRANT to %s\n", + if (cl_ctr_seal_and_send_to(sock, cl, pkt, plain_len, cl->key, + CL_CTR_PKT_KEY_GRANT, dest, destlen) == 0) + LM_INFO("clusterer_controller: [cluster %d] sent KEY_GRANT to %s (unicast)\n", cl->cluster_id, target_ip); } @@ -2983,7 +3010,8 @@ static void cl_ctr_handle_alive(const char *src_ip, * 5. If joining IP <= own IP: send MEMBER_LIST (self still master). */ static void cl_ctr_handle_join_req(int sock, const char *payload, int payload_len, - cl_ctr_cluster_t *cl) + cl_ctr_cluster_t *cl, + const struct sockaddr *src, socklen_t src_len) { const char *p = payload; const char *end = payload + payload_len; @@ -3165,7 +3193,8 @@ static void cl_ctr_handle_join_req(int sock, const char *payload, int payload_le /* Send KEY_GRANT first (bootstrap key) so joiner can derive session_key, * then NODE_ASSIGN + MEMBER_LIST (session key). */ if (noise_msg1) /* Noise msg 1 present -> run responder, answer msg 2 */ - cl_ctr_send_key_grant(sock, src_ip, cl, noise_msg1, noise_msg1_len); + cl_ctr_send_key_grant(sock, src_ip, cl, noise_msg1, noise_msg1_len, + src, src_len); lock_start_write(cl->peers->lock); @@ -4253,7 +4282,8 @@ static void cl_ctr_recv_one(int sock, cl_ctr_cluster_t *cl) } case CL_CTR_PKT_JOIN_REQ: - cl_ctr_handle_join_req(sock, payload, payload_len, cl); + cl_ctr_handle_join_req(sock, payload, payload_len, cl, + (const struct sockaddr *)&src_addr, src_len); break; case CL_CTR_PKT_MEMBER_LIST: From aea959b7c8f1b1003339e78d40ab45b5f85992fe Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Sun, 19 Jul 2026 20:07:04 +1000 Subject: [PATCH 15/21] clusterer_controller: ACK + bounded retransmit for the join handshake The 1:1 handshake packets ride unreliable UDP: a lost KEY_GRANT left the joiner waiting out its whole join window before re-trying. Add positive acknowledgement with a bounded retransmit so a single loss is recovered in milliseconds rather than seconds. The receiver of a 1:1 packet replies with CL_CTR_PKT_ACK echoing the packet's seq, sealed in the same key class as the packet it acks (a KEY_GRANT is bootstrap-keyed, so its ACK is too - the joiner need not already hold the session key). The sender keeps the sealed bytes in a small worker-local queue and, if no ACK arrives, resends them verbatim; after CL_CTR_RETX_MAX_RETRIES it gives up and lets the joiner's JOIN_REQ retry restart the handshake. A joiner that already holds the key re-ACKs a duplicate KEY_GRANT, so a lost ACK is recovered on the next retransmit instead of burning the whole budget. The budget is pinned to the join timers - CL_CTR_RETX_INTERVAL_US is half of CL_CTR_JOIN_REQ_MIN_US and a compile-time check keeps the total retransmit window shorter than one JOIN_REQ cycle - so the fast per-packet ARQ nests inside the slower JOIN_REQ backstop instead of racing it. Two invariants keep this strictly below the election layer, so it can never cause split brain: - ACKs are retransmit accounting only. They never gate membership or election; a never-acked packet is simply dropped, and a node joins through the normal JOIN_REQ -> ACTIVE -> ALIVE path regardless. So a lost ACK or a departed joiner can stall nothing. - The retransmit queue is flushed on loss of mastership and on session-key rotation, so a demoted or re-keyed node never keeps delivering a stale master-keyed KEY_GRANT that would push a joiner onto a dead key. Only the strictly 1:1 KEY_GRANT is wired to this here; the multicast group packets are deliberately left unacked (acking them from every member would be ack implosion - their loss is better healed by a membership digest). This adds a new packet type, so all nodes of a cluster must run this build; a node that does not understand CL_CTR_PKT_ACK simply draws extra retransmits, which is harmless. --- .../clusterer_controller.c | 268 +++++++++++++++++- 1 file changed, 260 insertions(+), 8 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index 956f5bcf0ed..248fb382e27 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -241,6 +241,7 @@ static const unsigned char CL_CTR_BOOTSTRAP_MAGIC[CL_CTR_MAGIC_SZ] = { 0xCC, 0x0 #define CL_CTR_PKT_KEY_GRANT 0x07 /* master -> joiner: ECDH-wrapped master_salt */ #define CL_CTR_PKT_KEY_HANDOFF 0x08 /* outgoing master -> next master: salt handoff */ #define CL_CTR_PKT_JOIN_REJECT 0x09 /* master -> joiner: authentication rejected */ +#define CL_CTR_PKT_ACK 0x0B /* receiver -> sender: ack of a 1:1 handshake pkt */ #define CL_CTR_PKT_MASTER_BEACON 0x0A /* master-only announce (BOOTSTRAP key) so * masters with divergent session keys can * still discover each other and merge a @@ -316,6 +317,29 @@ static const unsigned char CL_CTR_BOOTSTRAP_MAGIC[CL_CTR_MAGIC_SZ] = { 0xCC, 0x0 /* a peer stuck joining can't stall forever */ #define CL_CTR_JOIN_REQ_MIN_US 500000 /* min microseconds between JOIN_REQ sends */ +/* + * Reliable delivery (ACK + bounded retransmit) for the strictly 1:1 packets of + * the join handshake - KEY_GRANT today, the joiner's NODE_ASSIGN burst next. + * The receiver ACKs each one (CL_CTR_PKT_ACK, echoing the packet's seq); the + * sender retransmits the cached bytes until ACKed or the budget runs out. + * + * Multicast/group packets are deliberately NOT ACKed - ACKing from every member + * is ACK implosion; their loss is healed by the periodic membership digest. + * + * The budget is pinned to the join timers so master-side ARQ nests inside one + * JOIN_REQ retry cycle: CL_CTR_RETX_MAX_RETRIES x CL_CTR_RETX_INTERVAL_US of + * retransmit (750 ms), then give up and let the joiner's own JOIN_REQ retry + * (~1 s) restart the handshake with fresh packets. ACKs are ONLY retransmit + * accounting - a never-ACKed packet is dropped, never gating a join or any + * cluster state, so a lost ACK or a dead joiner can never stall anything. + */ +#define CL_CTR_RETX_INTERVAL_US (CL_CTR_JOIN_REQ_MIN_US / 2) /* 250 ms */ +#define CL_CTR_RETX_MAX_RETRIES 3 /* then give up */ +#define CL_CTR_RETX_QUEUE_SZ 128 /* max outstanding unacked 1:1 packets */ +#if (CL_CTR_RETX_MAX_RETRIES * CL_CTR_RETX_INTERVAL_US) >= (CL_CTR_JOIN_DEFER_SECS * 1000000) +#error "retransmit budget must stay shorter than the JOIN_REQ retry interval" +#endif + /* Per-source-IP rate limiter: checked before decryption to shed floods cheaply. * Tracks up to CL_CTR_RATE_TBL_SZ source IPs with a 1-second sliding window. */ #define CL_CTR_RATE_TBL_SZ 256 /* one slot per peer; matches max cluster size */ @@ -457,6 +481,25 @@ static inline const char *cl_ctr_role_name(int r) } } +/* + * One outstanding 1:1 handshake packet awaiting an ACK. The full sealed bytes + * are cached so a retransmit is a single sendto() with no re-encryption: the + * same seq/nonce is resent, which a peer that already got it (and ACKed) will + * not see again, while one that lost it still has an older last_seq and accepts + * it. Worker-local (only the controller worker touches the queue), so no lock. + */ +typedef struct { + int used; + uint32_t seq; /* packet seq, echoed by the ACK */ + unsigned char type; /* packet type, for logging */ + int retries_left; + utime_t next_due_us; /* get_uticks() deadline for next send */ + struct sockaddr_storage dest; /* unicast destination */ + socklen_t destlen; + int pkt_len; /* sealed length */ + unsigned char pkt[CL_CTR_NODE_ASSIGN_MAX_SZ]; /* cached sealed bytes */ +} cl_ctr_retx_entry_t; + /** * cl_ctr_cluster_t - per-cluster runtime state. * One instance per "cluster" modparam; one worker process per instance. @@ -504,6 +547,11 @@ typedef struct cl_ctr_cluster_ { * (broadcast MASTER_ALIVE) while this is 0, or it would encrypt with an * underived key that no member can decrypt. */ int have_session_key; + /* Reliable-delivery queue for 1:1 handshake packets (worker-local, no lock). + * retx_tfd sweeps it every CL_CTR_RETX_INTERVAL_US while retx_count > 0. */ + int retx_tfd; + int retx_count; + cl_ctr_retx_entry_t retx_q[CL_CTR_RETX_QUEUE_SZ]; /* Master-side per-IP table tracking bootstrap-decrypt failures. * Worker-local (no shm, no lock needed). After CL_CTR_JOIN_FAIL_LIMIT * failures from the same source IP the master sends JOIN_REJECT. */ @@ -768,6 +816,8 @@ static int cl_ctr_child_init(int rank); static void mod_destroy(void); static void cl_ctr_worker(int rank); static int cl_ctr_on_sock(int fd, void *param, int was_timeout); +static void cl_ctr_retx_flush(cl_ctr_cluster_t *cl); +static int cl_ctr_on_retx_tfd(int fd, void *param, int was_timeout); static int cl_ctr_on_alive_tfd(int fd, void *param, int was_timeout); static void cl_ctr_arm_master_timers(cl_ctr_cluster_t *cl, int i_am_master); static int cl_ctr_on_join_tfd(int fd, void *param, int was_timeout); @@ -786,7 +836,9 @@ static void cl_ctr_handle_node_assign(const char *payload, int payload_len, static void cl_ctr_handle_goodbye(int sock, const char *src_ip, cl_ctr_cluster_t *cl); static void cl_ctr_handle_master_alive(const char *sender_ip, cl_ctr_cluster_t *cl); static void cl_ctr_handle_key_grant(const char *payload, int payload_len, - const char *sender_ip, cl_ctr_cluster_t *cl); + const char *sender_ip, cl_ctr_cluster_t *cl, + uint32_t in_seq, const struct sockaddr *src, + socklen_t src_len); static void cl_ctr_handle_key_handoff(const char *payload, int payload_len, const char *sender_ip, cl_ctr_cluster_t *cl); static void cl_ctr_handle_join_reject(const char *payload, int payload_len, @@ -1974,6 +2026,8 @@ static int cl_ctr_derive_session_key(cl_ctr_cluster_t *cl) for (i = 0; i < cl->peers->count; i++) cl->peers->entries[i].last_seq = 0; cl->have_session_key = 1; /* a valid group key now exists */ + /* The salt (and my_seq) just changed, so any queued retransmit is now stale. */ + cl_ctr_retx_flush(cl); return 0; } @@ -2351,6 +2405,175 @@ static int cl_ctr_seal_and_send(int sock, cl_ctr_cluster_t *cl, char *pkt, int p sizeof(cl->mcast_dest)); } +/* ========================================================================= + * Reliable delivery: ACK + bounded retransmit for 1:1 handshake packets. + * All of this runs only in the controller worker, so the queue needs no lock. + * ========================================================================= */ + +/* Arm a timerfd with microsecond resolution (one-shot; 0/0 disarms). */ +static void cl_ctr_arm_tfd_us(int tfd, uint64_t usec_value, uint64_t usec_interval) +{ + struct itimerspec its; + memset(&its, 0, sizeof(its)); + its.it_value.tv_sec = (time_t)(usec_value / 1000000); + its.it_value.tv_nsec = (long)((usec_value % 1000000) * 1000); + its.it_interval.tv_sec = (time_t)(usec_interval / 1000000); + its.it_interval.tv_nsec = (long)((usec_interval % 1000000) * 1000); + if (timerfd_settime(tfd, 0, &its, NULL) < 0) + LM_WARN("clusterer_controller: timerfd_settime(us): %s\n", strerror(errno)); +} + +/* + * Drop every outstanding retransmit. Called on loss of mastership and on + * session-key rotation, so a demoted or re-keyed node never keeps delivering a + * stale master-keyed KEY_GRANT/NODE_ASSIGN that would push a joiner onto a dead + * key - the split-brain guard for the ARQ layer. ACKs never influence election + * or membership, so dropping these entries can only stop redundant sends. + */ +static void cl_ctr_retx_flush(cl_ctr_cluster_t *cl) +{ + if (cl->retx_count == 0) + return; + LM_DBG("clusterer_controller: [cluster %d] flushing %d pending retransmit(s)\n", + cl->cluster_id, cl->retx_count); + memset(cl->retx_q, 0, sizeof(cl->retx_q)); + cl->retx_count = 0; + cl_ctr_arm_tfd_us(cl->retx_tfd, 0, 0); /* disarm */ +} + +/* + * Queue a just-sent 1:1 packet (already sealed in @pkt, @pkt_len bytes) for + * retransmit until ACKed. Best-effort: if the queue is full the packet still + * went out once and the joiner's JOIN_REQ retry remains the backstop. + */ +static void cl_ctr_retx_enqueue(cl_ctr_cluster_t *cl, uint32_t seq, unsigned char type, + const unsigned char *pkt, int pkt_len, + const struct sockaddr *dest, socklen_t destlen) +{ + cl_ctr_retx_entry_t *e; + int i, slot = -1, was_empty; + + if (pkt_len <= 0 || pkt_len > (int)sizeof(cl->retx_q[0].pkt) || + destlen == 0 || destlen > (socklen_t)sizeof(cl->retx_q[0].dest)) + return; + + for (i = 0; i < CL_CTR_RETX_QUEUE_SZ; i++) + if (!cl->retx_q[i].used) { slot = i; break; } + if (slot < 0) { + LM_DBG("clusterer_controller: [cluster %d] retransmit queue full, " + "0x%02x sent best-effort\n", cl->cluster_id, type); + return; + } + + was_empty = (cl->retx_count == 0); + e = &cl->retx_q[slot]; + memset(e, 0, sizeof(*e)); + e->used = 1; + e->seq = seq; + e->type = type; + e->retries_left = CL_CTR_RETX_MAX_RETRIES; + e->next_due_us = get_uticks() + CL_CTR_RETX_INTERVAL_US; + memcpy(&e->dest, dest, destlen); + e->destlen = destlen; + memcpy(e->pkt, pkt, pkt_len); + e->pkt_len = pkt_len; + cl->retx_count++; + if (was_empty) + cl_ctr_arm_tfd_us(cl->retx_tfd, CL_CTR_RETX_INTERVAL_US, 0); +} + +/* An ACK arrived: drop the queued packet whose seq it echoes. */ +static void cl_ctr_handle_ack(const char *payload, int payload_len, cl_ctr_cluster_t *cl) +{ + uint32_t acked_be, acked; + int i; + + if (payload_len < (int)sizeof(uint32_t)) + return; + memcpy(&acked_be, payload, sizeof(acked_be)); + acked = ntohl(acked_be); + + for (i = 0; i < CL_CTR_RETX_QUEUE_SZ; i++) { + cl_ctr_retx_entry_t *e = &cl->retx_q[i]; + if (!e->used || e->seq != acked) + continue; + /* seq is unique within a key epoch (a rekey flushes the queue), so this + * is the acknowledged packet; drop it. */ + LM_DBG("clusterer_controller: [cluster %d] ACK for 0x%02x seq %u\n", + cl->cluster_id, e->type, acked); + e->used = 0; + cl->retx_count--; + break; + } + if (cl->retx_count == 0) + cl_ctr_arm_tfd_us(cl->retx_tfd, 0, 0); /* nothing pending - disarm */ +} + +/* + * Retransmit sweep: resend every entry whose deadline has passed, one step of + * its budget at a time; drop an entry once the budget is spent (the joiner's + * JOIN_REQ retry is the outer backstop). Re-arms only while work remains. + */ +static int cl_ctr_on_retx_tfd(int fd, void *param, int was_timeout) +{ + cl_ctr_cluster_t *cl = (cl_ctr_cluster_t *)param; + utime_t now = get_uticks(); + int i; + (void)was_timeout; + + cl_ctr_drain_tfd(fd); + + for (i = 0; i < CL_CTR_RETX_QUEUE_SZ; i++) { + cl_ctr_retx_entry_t *e = &cl->retx_q[i]; + if (!e->used || now < e->next_due_us) + continue; + + if (sendto(cl->sock, e->pkt, e->pkt_len, 0, + (struct sockaddr *)&e->dest, e->destlen) < 0 && + errno != EAGAIN && errno != EWOULDBLOCK) + LM_DBG("clusterer_controller: [cluster %d] retransmit 0x%02x: %s\n", + cl->cluster_id, e->type, strerror(errno)); + + if (--e->retries_left <= 0) { + LM_DBG("clusterer_controller: [cluster %d] 0x%02x (seq %u) unacked " + "after %d retransmits, giving up - joiner will re-JOIN_REQ\n", + cl->cluster_id, e->type, e->seq, CL_CTR_RETX_MAX_RETRIES); + e->used = 0; + cl->retx_count--; + } else { + e->next_due_us = now + CL_CTR_RETX_INTERVAL_US; + } + } + + if (cl->retx_count > 0) + cl_ctr_arm_tfd_us(cl->retx_tfd, CL_CTR_RETX_INTERVAL_US, 0); + return 0; +} + +/* + * Send an ACK for @acked_seq back to @dest. Sealed in the same key class as the + * packet being acknowledged (@use_bootstrap: KEY_GRANT is bootstrap-keyed), so + * the sender can read it without assuming the receiver already holds a key. + */ +static void cl_ctr_send_ack(int sock, cl_ctr_cluster_t *cl, uint32_t acked_seq, + int use_bootstrap, const struct sockaddr *dest, socklen_t destlen) +{ + char pkt[CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + sizeof(uint32_t) + + CL_CTR_TAG_SZ]; + uint32_t seq = htonl(++cl->peers->my_seq); + uint32_t acked_be = htonl(acked_seq); + const unsigned char *key = use_bootstrap ? cl->key : cl->session_key; + + memcpy(pkt, use_bootstrap ? CL_CTR_BOOTSTRAP_MAGIC : CL_CTR_PACKET_MAGIC, + CL_CTR_MAGIC_SZ); + pkt[CL_CTR_WIRE_HDR_SZ] = (char)CL_CTR_PKT_ACK; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ, &acked_be, sizeof(acked_be)); + + cl_ctr_seal_and_send_to(sock, cl, pkt, CL_CTR_PLAIN_HDR_SZ + (int)sizeof(acked_be), + key, CL_CTR_PKT_ACK, dest, destlen); +} + /** * cl_ctr_send_pkt_with_ip() - build and multicast a small (ALIVE/GOODBYE) packet. * JOIN_REQ is handled by cl_ctr_send_join_req_pkt() which carries BIN socket info. @@ -2665,7 +2888,8 @@ static void cl_ctr_send_key_grant(int sock, const char *target_ip, cl_ctr_cluste const struct sockaddr *dest, socklen_t destlen) { char pkt[CL_CTR_KEY_GRANT_SZ]; - uint32_t seq = htonl(++cl->peers->my_seq); + uint32_t seq_h = ++cl->peers->my_seq; + uint32_t seq = htonl(seq_h); unsigned char msg2[32 + CL_CTR_MASTER_SALT_SZ + CL_CTR_TAG_SZ]; char *p; int ip_len, plain_len, m2; @@ -2690,9 +2914,13 @@ static void cl_ctr_send_key_grant(int sock, const char *target_ip, cl_ctr_cluste plain_len = (int)(p - (pkt + CL_CTR_WIRE_HDR_SZ)); if (cl_ctr_seal_and_send_to(sock, cl, pkt, plain_len, cl->key, - CL_CTR_PKT_KEY_GRANT, dest, destlen) == 0) + CL_CTR_PKT_KEY_GRANT, dest, destlen) == 0) { LM_INFO("clusterer_controller: [cluster %d] sent KEY_GRANT to %s (unicast)\n", cl->cluster_id, target_ip); + /* Critical 1:1 packet - retransmit until the joiner ACKs it. */ + cl_ctr_retx_enqueue(cl, seq_h, CL_CTR_PKT_KEY_GRANT, (const unsigned char *)pkt, + CL_CTR_WIRE_HDR_SZ + plain_len + CL_CTR_TAG_SZ, dest, destlen); + } } /** @@ -2774,6 +3002,9 @@ static void cl_ctr_arm_master_timers(cl_ctr_cluster_t *cl, int i_am_master) cl_ctr_arm_tfd(cl->master_alive_tfd, CL_CTR_MASTER_KA_INTERVAL, CL_CTR_MASTER_KA_INTERVAL); cl_ctr_arm_tfd(cl->master_dead_tfd, 0, 0); /* disarm watchdog */ } else { + /* No longer master: drop any queued master->joiner retransmits so a + * demoted node can't keep pushing a joiner onto a now-stale key. */ + cl_ctr_retx_flush(cl); cl_ctr_arm_tfd(cl->master_alive_tfd, 0, 0); /* disarm sender */ cl_ctr_arm_tfd(cl->master_dead_tfd, CL_CTR_MASTER_KA_TIMEOUT, 0); } @@ -3818,7 +4049,9 @@ static void cl_ctr_handle_master_beacon(const char *sender_ip, uint16_t sender_c * Only processed if target_ip == my_ip (multicast - all nodes receive it). */ static void cl_ctr_handle_key_grant(const char *payload, int payload_len, - const char *sender_ip, cl_ctr_cluster_t *cl) + const char *sender_ip, cl_ctr_cluster_t *cl, + uint32_t in_seq, const struct sockaddr *src, + socklen_t src_len) { const char *p = payload; const char *end = payload + payload_len; @@ -3869,6 +4102,10 @@ static void cl_ctr_handle_key_grant(const char *payload, int payload_len, /* Clear join_pending so the next decryption failure or rejoin_tfd can * issue a fresh JOIN_REQ. */ cl->join_pending = 0; + /* If we already hold a session key, this is a duplicate of a KEY_GRANT + * we handled whose ACK was lost - re-ACK so the master stops retrying. */ + if (cl->have_session_key) + cl_ctr_send_ack(cl->sock, cl, in_seq, 1/*bootstrap*/, src, src_len); return; } cl->noise_hs_valid = 0; /* handshake consumed */ @@ -3886,6 +4123,10 @@ static void cl_ctr_handle_key_grant(const char *payload, int payload_len, cl->auth_defer_count = 0; LM_INFO("clusterer_controller: [cluster %d] KEY_GRANT from %s - " "session key updated\n", cl->cluster_id, sender_ip); + /* Acknowledge to the master (bootstrap key class, matching KEY_GRANT) so it + * stops retransmitting. Best-effort: a lost ACK just draws a retransmit, + * which we re-ACK via the duplicate path above; never gates our join. */ + cl_ctr_send_ack(cl->sock, cl, in_seq, 1/*bootstrap*/, src, src_len); } /** @@ -4307,8 +4548,17 @@ static void cl_ctr_recv_one(int sock, cl_ctr_cluster_t *cl) cl_ctr_handle_master_alive(sender_ip_buf, cl); break; - case CL_CTR_PKT_KEY_GRANT: - cl_ctr_handle_key_grant(payload, payload_len, sender_ip_buf, cl); + case CL_CTR_PKT_KEY_GRANT: { + uint32_t in_seq; + memcpy(&in_seq, buf + CL_CTR_WIRE_HDR_SZ + 1, CL_CTR_SEQ_SZ); + in_seq = ntohl(in_seq); + cl_ctr_handle_key_grant(payload, payload_len, sender_ip_buf, cl, + in_seq, (const struct sockaddr *)&src_addr, src_len); + break; + } + + case CL_CTR_PKT_ACK: + cl_ctr_handle_ack(payload, payload_len, cl); break; case CL_CTR_PKT_KEY_HANDOFF: @@ -4709,8 +4959,9 @@ static void cl_ctr_worker(int rank) cl->rejoin_tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); cl->master_alive_tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); cl->master_dead_tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); + cl->retx_tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); if (cl->alive_tfd < 0 || cl->join_tfd < 0 || cl->rejoin_tfd < 0 || - cl->master_alive_tfd < 0 || cl->master_dead_tfd < 0) { + cl->master_alive_tfd < 0 || cl->master_dead_tfd < 0 || cl->retx_tfd < 0) { LM_CRIT("clusterer_controller: [cluster %d] timerfd_create: %s\n", cl->cluster_id, strerror(errno)); exit(-1); @@ -4750,7 +5001,8 @@ static void cl_ctr_worker(int rank) reactor_proc_add_fd(cl->join_tfd, cl_ctr_on_join_tfd, cl) < 0 || reactor_proc_add_fd(cl->rejoin_tfd, cl_ctr_on_rejoin_tfd, cl) < 0 || reactor_proc_add_fd(cl->master_alive_tfd, cl_ctr_on_master_alive_tfd, cl) < 0 || - reactor_proc_add_fd(cl->master_dead_tfd, cl_ctr_on_master_dead_tfd, cl) < 0) { + reactor_proc_add_fd(cl->master_dead_tfd, cl_ctr_on_master_dead_tfd, cl) < 0 || + reactor_proc_add_fd(cl->retx_tfd, cl_ctr_on_retx_tfd, cl) < 0) { LM_CRIT("clusterer_controller: [cluster %d] reactor_proc_add_fd failed\n", cl->cluster_id); exit(-1); From 781a6441ac2a89125412eaa489401c0d2b81e78a Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Sun, 19 Jul 2026 20:24:40 +1000 Subject: [PATCH 16/21] clusterer_controller: membership digest in MASTER_ALIVE + RESYNC repair A peer's node_id and BIN mapping travel only in NODE_ASSIGN, which is multicast and so best-effort: a member that drops one knows the peer exists (from its ALIVE) but not its node_id, leaving its clusterer BIN mesh to that peer incomplete with no repair. Until now the only thing that fixed it was the master re-multicasting every peer's NODE_ASSIGN on the next join - an accident, not a mechanism, and useless in a stable cluster. Add an explicit anti-entropy repair. Every MASTER_ALIVE now carries a membership digest - the active-peer count plus an order-independent XOR of a per-peer FNV-1a over (node_id, ip). A member that computes a different digest from the master's has missed something - a whole peer (count differs) or just a peer's node_id (count matches, hash differs) - and sends a rate-limited RESYNC (new packet type). The master answers by re-broadcasting every NODE_ASSIGN plus the MEMBER_LIST, coalescing a burst of RESYNCs into one re-broadcast per interval. Roles are left out of the digest so transient master disagreement does not cause spurious resyncs. This is the multicast counterpart to the 1:1 ACK path: 1:1 handshake packets get positive-ack retransmit, group packets get digest-driven pull. It also unblocks unicasting the join-time NODE_ASSIGN burst to the joiner - a member that would have relied on the incidental re-multicast now self-heals through the digest instead. Adds a packet type, so all nodes of a cluster must run this build; a node that does not understand it simply advertises no digest and is never asked to resync. --- .../clusterer_controller.c | 201 +++++++++++++++++- 1 file changed, 191 insertions(+), 10 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index 248fb382e27..40743bf9560 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -242,6 +242,7 @@ static const unsigned char CL_CTR_BOOTSTRAP_MAGIC[CL_CTR_MAGIC_SZ] = { 0xCC, 0x0 #define CL_CTR_PKT_KEY_HANDOFF 0x08 /* outgoing master -> next master: salt handoff */ #define CL_CTR_PKT_JOIN_REJECT 0x09 /* master -> joiner: authentication rejected */ #define CL_CTR_PKT_ACK 0x0B /* receiver -> sender: ack of a 1:1 handshake pkt */ +#define CL_CTR_PKT_RESYNC 0x0C /* member -> master: my view differs, resend state */ #define CL_CTR_PKT_MASTER_BEACON 0x0A /* master-only announce (BOOTSTRAP key) so * masters with divergent session keys can * still discover each other and merge a @@ -340,6 +341,21 @@ static const unsigned char CL_CTR_BOOTSTRAP_MAGIC[CL_CTR_MAGIC_SZ] = { 0xCC, 0x0 #error "retransmit budget must stay shorter than the JOIN_REQ retry interval" #endif +/* + * Membership digest, carried in every MASTER_ALIVE so a member can notice it is + * out of sync and pull a resend. A peer's node_id/BIN mapping travels only in + * NODE_ASSIGN, which is multicast (best-effort) - a member that dropped one + * knows the peer exists (from its ALIVE) but not its node_id, so its clusterer + * BIN mesh to that peer is incomplete with no other repair. The digest closes + * that: [member_count 2B BE][set_hash 8B BE] over the active peers; a mismatch + * triggers a rate-limited RESYNC, to which the master re-broadcasts the full + * NODE_ASSIGN set + MEMBER_LIST. Multicast/group packets stay unacked - this + * is their anti-entropy repair, the counterpart to the 1:1 ACK path. + */ +#define CL_CTR_DIGEST_SZ (2 + 8) +#define CL_CTR_RESYNC_MIN_US 1000000 /* member: min between RESYNCs; master: */ + /* min between full-state re-broadcasts (1 s) */ + /* Per-source-IP rate limiter: checked before decryption to shed floods cheaply. * Tracks up to CL_CTR_RATE_TBL_SZ source IPs with a 1-second sliding window. */ #define CL_CTR_RATE_TBL_SZ 256 /* one slot per peer; matches max cluster size */ @@ -552,6 +568,11 @@ typedef struct cl_ctr_cluster_ { int retx_tfd; int retx_count; cl_ctr_retx_entry_t retx_q[CL_CTR_RETX_QUEUE_SZ]; + /* Membership-digest resync throttles (worker-local): a member sends at most + * one RESYNC, and a master re-broadcasts full state at most once, per + * CL_CTR_RESYNC_MIN_US. */ + utime_t last_resync_us; + utime_t last_full_state_us; /* Master-side per-IP table tracking bootstrap-decrypt failures. * Worker-local (no shm, no lock needed). After CL_CTR_JOIN_FAIL_LIMIT * failures from the same source IP the master sends JOIN_REJECT. */ @@ -818,6 +839,10 @@ static void cl_ctr_worker(int rank); static int cl_ctr_on_sock(int fd, void *param, int was_timeout); static void cl_ctr_retx_flush(cl_ctr_cluster_t *cl); static int cl_ctr_on_retx_tfd(int fd, void *param, int was_timeout); +static void cl_ctr_membership_digest(cl_ctr_cluster_t *cl, uint16_t *count, uint64_t *hash); +static void cl_ctr_send_resync(int sock, cl_ctr_cluster_t *cl, + const struct sockaddr *dest, socklen_t destlen); +static void cl_ctr_broadcast_full_state(cl_ctr_cluster_t *cl); static int cl_ctr_on_alive_tfd(int fd, void *param, int was_timeout); static void cl_ctr_arm_master_timers(cl_ctr_cluster_t *cl, int i_am_master); static int cl_ctr_on_join_tfd(int fd, void *param, int was_timeout); @@ -834,7 +859,9 @@ static void cl_ctr_handle_join_req(int sock, const char *payload, int payload_le static void cl_ctr_handle_node_assign(const char *payload, int payload_len, const char *sender_ip, cl_ctr_cluster_t *cl); static void cl_ctr_handle_goodbye(int sock, const char *src_ip, cl_ctr_cluster_t *cl); -static void cl_ctr_handle_master_alive(const char *sender_ip, cl_ctr_cluster_t *cl); +static void cl_ctr_handle_master_alive(const char *sender_ip, cl_ctr_cluster_t *cl, + const char *payload, int payload_len, + const struct sockaddr *src, socklen_t src_len); static void cl_ctr_handle_key_grant(const char *payload, int payload_len, const char *sender_ip, cl_ctr_cluster_t *cl, uint32_t in_seq, const struct sockaddr *src, @@ -2820,22 +2847,144 @@ static void cl_ctr_send_node_assign(int sock, const char *ip, uint16_t node_id, } +/* + * Order-independent digest of the active peer set: the XOR of a per-peer FNV-1a + * over (node_id, ip_num). A member that missed a peer's NODE_ASSIGN carries + * node_id 0 for it and so computes a different hash from the master's - which is + * how the gap is detected. Roles are deliberately excluded (they reconcile via + * MASTER_ALIVE itself) to avoid spurious resyncs on transient role churn. + * Takes the read lock itself - the caller must NOT hold cl->peers->lock. + */ +static void cl_ctr_membership_digest(cl_ctr_cluster_t *cl, uint16_t *count, uint64_t *hash) +{ + uint64_t h = 0; + uint16_t c = 0; + time_t cutoff = time(NULL) - (time_t)(query_time * CL_CTR_ELECT_FACTOR); + int i, k; + + lock_start_read(cl->peers->lock); + for (i = 0; i < cl->peers->count; i++) { + cl_ctr_peer_t *e = &cl->peers->entries[i]; + uint64_t ph = 1469598103934665603ULL; /* FNV-1a offset basis */ + unsigned char b[6]; + if (e->last_seen < cutoff) + continue; + b[0] = (unsigned char)(e->node_id >> 8); b[1] = (unsigned char)e->node_id; + b[2] = (unsigned char)(e->ip_num >> 24); b[3] = (unsigned char)(e->ip_num >> 16); + b[4] = (unsigned char)(e->ip_num >> 8); b[5] = (unsigned char)e->ip_num; + for (k = 0; k < 6; k++) { ph ^= b[k]; ph *= 1099511628211ULL; } + h ^= ph; + c++; + } + lock_stop_read(cl->peers->lock); + *count = c; + *hash = h; +} + /** - * cl_ctr_send_master_alive() - master-only keepalive, no payload beyond the header. - * Encrypted with session_key (CL_CTR_PACKET_MAGIC). + * cl_ctr_send_master_alive() - master-only keepalive. Encrypted with session_key + * (CL_CTR_PACKET_MAGIC). Carries the membership digest ([count 2B][hash 8B]) so + * members can detect a missed NODE_ASSIGN and pull a RESYNC. */ static void cl_ctr_send_master_alive(int sock, cl_ctr_cluster_t *cl) { - char pkt[CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_TAG_SZ]; - uint32_t seq = htonl(++cl->peers->my_seq); + char pkt[CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + + CL_CTR_DIGEST_SZ + CL_CTR_TAG_SZ]; + uint32_t seq = htonl(++cl->peers->my_seq); + uint16_t cnt; + uint64_t hash; + char *p; + int k; + + cl_ctr_membership_digest(cl, &cnt, &hash); memcpy(pkt, CL_CTR_PACKET_MAGIC, CL_CTR_MAGIC_SZ); pkt[CL_CTR_WIRE_HDR_SZ] = (char)CL_CTR_PKT_MASTER_ALIVE; memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); - /* no payload beyond type+seq */ + p = pkt + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ; + *p++ = (char)(cnt >> 8); *p++ = (char)cnt; + for (k = 0; k < 8; k++) *p++ = (char)(hash >> (56 - 8 * k)); + + cl_ctr_seal_and_send(sock, cl, pkt, CL_CTR_PLAIN_HDR_SZ + CL_CTR_DIGEST_SZ, + cl->session_key, CL_CTR_PKT_MASTER_ALIVE); +} + +/* + * cl_ctr_send_resync() - member -> master (unicast, session-keyed): "my membership + * view differs from your digest, please resend". Carries our own digest for + * the master's logs. Best-effort and rate-limited by the caller. + */ +static void cl_ctr_send_resync(int sock, cl_ctr_cluster_t *cl, + const struct sockaddr *dest, socklen_t destlen) +{ + char pkt[CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ + CL_CTR_DIGEST_SZ + + CL_CTR_TAG_SZ]; + uint32_t seq = htonl(++cl->peers->my_seq); + uint16_t cnt; + uint64_t hash; + char *p; + int k; + + cl_ctr_membership_digest(cl, &cnt, &hash); + memcpy(pkt, CL_CTR_PACKET_MAGIC, CL_CTR_MAGIC_SZ); + pkt[CL_CTR_WIRE_HDR_SZ] = (char)CL_CTR_PKT_RESYNC; + memcpy(pkt + CL_CTR_WIRE_HDR_SZ + 1, &seq, CL_CTR_SEQ_SZ); + p = pkt + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ; + *p++ = (char)(cnt >> 8); *p++ = (char)cnt; + for (k = 0; k < 8; k++) *p++ = (char)(hash >> (56 - 8 * k)); + + cl_ctr_seal_and_send_to(sock, cl, pkt, CL_CTR_PLAIN_HDR_SZ + CL_CTR_DIGEST_SZ, + cl->session_key, CL_CTR_PKT_RESYNC, dest, destlen); +} - cl_ctr_seal_and_send(sock, cl, pkt, CL_CTR_PLAIN_HDR_SZ, cl->session_key, - CL_CTR_PKT_MASTER_ALIVE); +/* + * cl_ctr_broadcast_full_state() - master re-multicasts every peer's NODE_ASSIGN + * (node_id + BIN mapping) followed by the MEMBER_LIST. This is the same state a + * join emits; here it repairs members that missed a NODE_ASSIGN. Master path. + */ +static void cl_ctr_broadcast_full_state(cl_ctr_cluster_t *cl) +{ + int i; + + lock_start_read(cl->peers->lock); + for (i = 0; i < cl->peers->count; i++) { + cl_ctr_peer_t *e = &cl->peers->entries[i]; + if (e->node_id == 0) + continue; + cl_ctr_send_node_assign(cl->sock, e->ip, e->node_id, e->bin_count, + (const char (*)[CL_CTR_MAX_BIN_SOCK_LEN])e->bin_sockets, + cl); + } + lock_stop_read(cl->peers->lock); + cl_ctr_send_member_list(cl->sock, cl); +} + +/* + * cl_ctr_handle_resync() - a member reported a membership mismatch. Only the + * master answers, and coalesces a burst of RESYNCs into one full-state + * re-broadcast per CL_CTR_RESYNC_MIN_US. + */ +static void cl_ctr_handle_resync(const char *sender_ip, cl_ctr_cluster_t *cl) +{ + int i_am_master; + utime_t now; + + lock_start_read(cl->peers->lock); + i_am_master = cl_ctr_i_am_master_locked(cl); + lock_stop_read(cl->peers->lock); + if (!i_am_master) + return; + + now = get_uticks(); + if ((utime_t)(now - cl->last_full_state_us) < CL_CTR_RESYNC_MIN_US) { + LM_DBG("clusterer_controller: [cluster %d] RESYNC from %s coalesced\n", + cl->cluster_id, sender_ip); + return; + } + cl->last_full_state_us = now; + LM_INFO("clusterer_controller: [cluster %d] RESYNC from %s - re-broadcasting " + "full state\n", cl->cluster_id, sender_ip); + cl_ctr_broadcast_full_state(cl); } /** @@ -3906,7 +4055,9 @@ static void cl_ctr_handle_node_assign(const char *payload, int payload_len, * preemption modes, so two masters (e.g. after a network partition heals) can * never both stick. */ -static void cl_ctr_handle_master_alive(const char *sender_ip, cl_ctr_cluster_t *cl) +static void cl_ctr_handle_master_alive(const char *sender_ip, cl_ctr_cluster_t *cl, + const char *payload, int payload_len, + const struct sockaddr *src, socklen_t src_len) { int i_am_master, yielded = 0; int from_self = (strcmp(sender_ip, my_ip) == 0); @@ -3941,6 +4092,31 @@ static void cl_ctr_handle_master_alive(const char *sender_ip, cl_ctr_cluster_t * /* Non-masters (including a node that just yielded) watch the keepalive. */ if (!i_am_master || yielded) cl_ctr_arm_tfd(cl->master_dead_tfd, CL_CTR_MASTER_KA_TIMEOUT, 0); + + /* Membership digest: if the master advertises a set we do not match, we may + * have missed a NODE_ASSIGN (a peer known from its ALIVE but with node_id/BIN + * unknown). Pull a rate-limited RESYNC. Only as a member holding the key, + * and never off our own loopback. */ + if (!i_am_master && !from_self && cl->have_session_key && + payload_len >= CL_CTR_DIGEST_SZ) { + const unsigned char *d = (const unsigned char *)payload; + uint16_t adv_cnt = (uint16_t)((d[0] << 8) | d[1]); + uint16_t my_cnt; + uint64_t adv_hash = 0, my_hash; + int k; + for (k = 0; k < 8; k++) adv_hash = (adv_hash << 8) | d[2 + k]; + cl_ctr_membership_digest(cl, &my_cnt, &my_hash); + if (adv_cnt != my_cnt || adv_hash != my_hash) { + utime_t now = get_uticks(); + if ((utime_t)(now - cl->last_resync_us) >= CL_CTR_RESYNC_MIN_US) { + cl->last_resync_us = now; + LM_DBG("clusterer_controller: [cluster %d] membership digest " + "mismatch with master %s (theirs %u, ours %u) - RESYNC\n", + cl->cluster_id, sender_ip, adv_cnt, my_cnt); + cl_ctr_send_resync(cl->sock, cl, src, src_len); + } + } + } } /** @@ -4545,7 +4721,12 @@ static void cl_ctr_recv_one(int sock, cl_ctr_cluster_t *cl) break; case CL_CTR_PKT_MASTER_ALIVE: - cl_ctr_handle_master_alive(sender_ip_buf, cl); + cl_ctr_handle_master_alive(sender_ip_buf, cl, payload, payload_len, + (const struct sockaddr *)&src_addr, src_len); + break; + + case CL_CTR_PKT_RESYNC: + cl_ctr_handle_resync(sender_ip_buf, cl); break; case CL_CTR_PKT_KEY_GRANT: { From bfb808a88ad5a5b8f7b4e39cf0a46e3c084a2491 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Sun, 19 Jul 2026 20:30:10 +1000 Subject: [PATCH 17/21] clusterer_controller: unicast the join-time state to the joiner On every join the master re-multicast a NODE_ASSIGN for each existing peer plus the MEMBER_LIST - one packet per member, decrypted by every member, of which only the joiner needed all but the newcomer's own assignment. So a join into an N-node cluster cost O(N) group packets and O(N^2) cluster-wide AEAD decrypts, nearly all wasted. Send the joiner-only state to the joiner. cl_ctr_send_node_assign() and cl_ctr_send_list_pkt() take an optional unicast destination (NULL keeps the multicast behaviour); the existing-peer NODE_ASSIGN burst and the MEMBER_LIST snapshot now go to the joiner's own address, while the newcomer's NODE_ASSIGN stays multicast so every member still learns it. Per join the group now sees one NODE_ASSIGN instead of N. Dropping the re-multicast removes the incidental repair it used to give a member that had missed a peer's original assignment; that is now covered by the membership digest added earlier - such a member notices the mismatch and pulls a RESYNC. The unicast snapshot is not acked: if the joiner loses it, it stays CL_CTR_NODE_NEW and its JOIN_REQ retry brings another, exactly as a lost multicast MEMBER_LIST behaved before. --- .../clusterer_controller.c | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index 40743bf9560..4002e4ad61e 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -2655,7 +2655,8 @@ static void cl_ctr_send_pkt_with_ip(int sock, unsigned char type, cl_ctr_cluster * * Wire: [magic 2B][cluster_id 2B][nonce 12B][AES-256-GCM([type 1B][seq 4B][count 2B][entries...])][tag 16B] */ -static void cl_ctr_send_list_pkt(int sock, unsigned char type, cl_ctr_cluster_t *cl) +static void cl_ctr_send_list_pkt(int sock, unsigned char type, cl_ctr_cluster_t *cl, + const struct sockaddr *dest, socklen_t destlen) { char pkt[CL_CTR_LIST_PKT_MAX_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); @@ -2702,12 +2703,14 @@ static void cl_ctr_send_list_pkt(int sock, unsigned char type, cl_ctr_cluster_t plain_len = 1 + CL_CTR_SEQ_SZ + CL_CTR_LIST_COUNT_SZ + CL_CTR_NODE_ID_SZ + count * CL_CTR_IP_ENTRY_SZ; - if (cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, type) == 0) - LM_INFO("clusterer_controller: [cluster %d] sent MEMBER_LIST (%d members)\n", - cl->cluster_id, count); + if ((dest ? cl_ctr_seal_and_send_to(sock, cl, pkt, plain_len, cl->session_key, + type, dest, destlen) + : cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, type)) == 0) + LM_INFO("clusterer_controller: [cluster %d] sent MEMBER_LIST (%d members)%s\n", + cl->cluster_id, count, dest ? " (unicast)" : ""); } -#define cl_ctr_send_member_list(sock, cl) cl_ctr_send_list_pkt((sock), CL_CTR_PKT_MEMBER_LIST, (cl)) +#define cl_ctr_send_member_list(sock, cl) cl_ctr_send_list_pkt((sock), CL_CTR_PKT_MEMBER_LIST, (cl), NULL, 0) /** * cl_ctr_send_join_req_pkt() - send CL_CTR_PKT_JOIN_REQ with BIN socket info. @@ -2801,7 +2804,8 @@ static void cl_ctr_send_join_req_pkt(int sock, cl_ctr_cluster_t *cl) static void cl_ctr_send_node_assign(int sock, const char *ip, uint16_t node_id, uint8_t bin_count, const char (*bin_sockets)[CL_CTR_MAX_BIN_SOCK_LEN], - cl_ctr_cluster_t *cl) + cl_ctr_cluster_t *cl, + const struct sockaddr *dest, socklen_t destlen) { char pkt[CL_CTR_NODE_ASSIGN_MAX_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); @@ -2841,9 +2845,12 @@ static void cl_ctr_send_node_assign(int sock, const char *ip, uint16_t node_id, * GCM auth on receipt ("session key mismatch"), driving a JOIN_REQ storm. * KEY_GRANT is sent before NODE_ASSIGN so the joiner already holds the * session key by the time this arrives. */ - if (cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, CL_CTR_PKT_NODE_ASSIGN) == 0) - LM_INFO("clusterer_controller: [cluster %d] NODE_ASSIGN node_id=%u ip=%s\n", - cl->cluster_id, node_id, ip); + if ((dest ? cl_ctr_seal_and_send_to(sock, cl, pkt, plain_len, cl->session_key, + CL_CTR_PKT_NODE_ASSIGN, dest, destlen) + : cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, + CL_CTR_PKT_NODE_ASSIGN)) == 0) + LM_INFO("clusterer_controller: [cluster %d] NODE_ASSIGN node_id=%u ip=%s%s\n", + cl->cluster_id, node_id, ip, dest ? " (unicast)" : ""); } @@ -2953,7 +2960,7 @@ static void cl_ctr_broadcast_full_state(cl_ctr_cluster_t *cl) continue; cl_ctr_send_node_assign(cl->sock, e->ip, e->node_id, e->bin_count, (const char (*)[CL_CTR_MAX_BIN_SOCK_LEN])e->bin_sockets, - cl); + cl, NULL, 0); /* re-broadcast to the group */ } lock_stop_read(cl->peers->lock); cl_ctr_send_member_list(cl->sock, cl); @@ -3579,18 +3586,23 @@ static void cl_ctr_handle_join_req(int sock, const char *payload, int payload_le lock_start_write(cl->peers->lock); /* Send NODE_ASSIGN for joining node */ + /* the newcomer's assignment must reach every member - multicast (a member + * that misses it self-heals via the membership digest). */ cl_ctr_send_node_assign(sock, src_ip, new_id, bin_cnt, - (const char (*)[CL_CTR_MAX_BIN_SOCK_LEN])bin_socks, cl); + (const char (*)[CL_CTR_MAX_BIN_SOCK_LEN])bin_socks, cl, NULL, 0); - /* Send NODE_ASSIGN for each existing peer so joining node learns - * all current node_ids and BIN sockets */ + /* Send NODE_ASSIGN for each existing peer so the joining node learns all + * current node_ids and BIN sockets. Only the joiner needs these, so unicast + * them to it rather than re-multicasting to the whole cluster; a member that + * would once have re-learned a peer from this burst now self-heals via the + * membership digest instead. */ for (i = 0; i < cl->peers->count; i++) { cl_ctr_peer_t *e = &cl->peers->entries[i]; if (strcmp(e->ip, src_ip) == 0 || e->node_id == 0) continue; cl_ctr_send_node_assign(sock, e->ip, e->node_id, e->bin_count, (const char (*)[CL_CTR_MAX_BIN_SOCK_LEN])e->bin_sockets, - cl); + cl, src, src_len); } /* A current master NEVER hands mastership to a joining node during the @@ -3608,7 +3620,9 @@ static void cl_ctr_handle_join_req(int sock, const char *payload, int payload_le LM_INFO("clusterer_controller: [cluster %d] I am master, " "new node %s assigned node_id=%u\n", cl->cluster_id, src_ip, new_id); - cl_ctr_send_member_list(sock, cl); + /* the joiner's authoritative snapshot - unicast to it (a member that misses + * it self-heals via the membership digest; the joiner, via its JOIN_REQ retry). */ + cl_ctr_send_list_pkt(sock, CL_CTR_PKT_MEMBER_LIST, cl, src, src_len); } /** From 6773aedbf619085058a1ea0e3327245b68fca783 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Sun, 19 Jul 2026 20:46:27 +1000 Subject: [PATCH 18/21] clusterer_controller: drop the redundant MEMBER_LIST re-broadcast on failover When a node departed (GOODBYE) or a master went silent, the newly elected master re-broadcast the full MEMBER_LIST to assert itself. That was redundant: every surviving member runs the same deterministic election locally off the very same GOODBYE or keepalive timeout, so they already agree on the winner; MASTER_ALIVE re-asserts it within a keepalive; and the membership digest reconciles anyone who missed the event. A node still joining reaches the new master through its own JOIN_REQ retry. Drop both re-broadcasts. This removes an O(N) packet - which fragments above ~80 nodes and is dropped wholesale by any middlebox that blocks fragmented UDP - from the failover path, where it bought nothing. Verified on a 3-node cluster: killing the master leaves the two survivors converging to the same new master (highest IP) with the other as backup, no split brain and no stuck state, purely via local re-election plus MASTER_ALIVE. The forced-shtag MEMBER_LIST is untouched - it carries the forced node_id, which nothing else propagates. --- .../clusterer_controller.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index 4002e4ad61e..bc769fb89bc 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -3950,10 +3950,14 @@ static void cl_ctr_handle_goodbye(int sock, const char *src_ip, cl_ctr_cluster_t } else { LM_INFO("clusterer_controller: re-election complete - " "I reclaimed mastership after %s departed " - "(%d node(s) remaining) - sending MEMBER_LIST\n", + "(%d node(s) remaining)\n", src_ip, remaining); cl_ctr_arm_master_timers(cl, 1); - cl_ctr_send_member_list(sock, cl); + /* No full MEMBER_LIST is broadcast: every member re-elected the same + * deterministic master locally on this same GOODBYE, MASTER_ALIVE + * re-asserts it within a keepalive, and the membership digest + * reconciles anyone who missed the departure - so the O(N) re-broadcast + * (which fragments at large N) is redundant. */ } } else { /* This node's own role change (if any) is logged separately by @@ -5073,10 +5077,12 @@ static int cl_ctr_on_master_dead_tfd(int fd, void *param, int was_timeout) cl_ctr_arm_master_timers(cl, now_master); - /* The new master announces itself; all members already hold the session - * key so no re-keying is needed. */ - if (now_master) - cl_ctr_send_member_list(cl->sock, cl); + /* No full MEMBER_LIST is broadcast: every surviving member ran the same + * deterministic re-election on its own keepalive timeout, MASTER_ALIVE + * (which the new master now emits) re-asserts the winner, and the + * membership digest reconciles any divergence - so the O(N) re-broadcast + * (which fragments at large N) is redundant. A node still joining reaches + * the new master through its own JOIN_REQ retry. */ return 0; } From 9eb92a5fee5d31d8e2821effef75af1dda220842 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Mon, 20 Jul 2026 13:42:40 +1000 Subject: [PATCH 19/21] clusterer_controller: master-mediated ALIVE liveness (O(N^2) -> O(N)) Every active node multicast an ALIVE every query_time carrying its pubkey and config, and every node ran an election off every one it received - so steady-state liveness cost O(N^2) packets and cluster-wide AEAD decrypts. Route it through the master instead. A settled non-master unicasts its ALIVE to the master (still multicast during formation, with no stable master/key yet, for discovery); the master learns every peer's last_seen, pubkey and config in O(N). The master then relays liveness to the whole group by appending a per-node-id bitmap to its MASTER_ALIVE - built from the ELECTION-window cutoff, not the purge window, so a departed node leaves the bitmap on the same schedule it leaves an election. A non-master refreshes last_seen for the peers the master reports alive, keeping its election window populated without hearing the peers' ALIVEs. Election itself is unchanged and stays local. The master keeps multicasting its own ALIVE so its liveness/config still reach everyone, and config-drift detection is preserved (the master sees every peer's config, non-masters see the master's). A settled non-master no longer hears a multicast loopback of its own ALIVE, so it must refresh its own last_seen locally in on_alive_tfd - otherwise it ages itself out of its own election window once the master is gone and briefly finds "no eligible peers" during failover. The KEY_HANDOFF pubkey a promoted backup needs is re-collected from the now-incoming unicast ALIVEs within a query_time; a handoff attempted in that brief window degrades to a rejoin, which the successor already handles. Validated on 3-node netns rigs: steady-state unicast+bitmap (no O(N^2)), single failover, chained failover, rapid double failure (master + backup killed together -> the survivor self-corrects through one extra watchdog round, converging to sole master), higher-IP join -> sticky backup, and simultaneous cold start -> deterministic single master, no split brain. --- .../clusterer_controller.c | 118 ++++++++++++++++-- 1 file changed, 111 insertions(+), 7 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index bc769fb89bc..f2c9411dcd7 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -355,6 +355,17 @@ static const unsigned char CL_CTR_BOOTSTRAP_MAGIC[CL_CTR_MAGIC_SZ] = { 0xCC, 0x0 #define CL_CTR_DIGEST_SZ (2 + 8) #define CL_CTR_RESYNC_MIN_US 1000000 /* member: min between RESYNCs; master: */ /* min between full-state re-broadcasts (1 s) */ +/* + * Liveness bitmap relayed by the master in every MASTER_ALIVE: one bit per + * node_id. A settled non-master unicasts its ALIVE to the master and no longer + * hears peers' ALIVEs directly, so the master reports which node_ids it has seen + * within the election window and non-masters refresh those peers' last_seen from + * it - turning the old all-to-all O(N^2) liveness gossip into O(N). Built from + * the ELECTION-window cutoff (not the purge window) so a departed node drops out + * of the bitmap on the same schedule it drops out of an election. Sized to + * cover node_id 1..CL_CTR_MAX_PEERS. + */ +#define CL_CTR_ALIVE_BITMAP_SZ ((CL_CTR_MAX_PEERS / 8) + 1) /* Per-source-IP rate limiter: checked before decryption to shed floods cheaply. * Tracks up to CL_CTR_RATE_TBL_SZ source IPs with a 1-second sliding window. */ @@ -2608,7 +2619,8 @@ static void cl_ctr_send_ack(int sock, cl_ctr_cluster_t *cl, uint32_t acked_seq, * ALIVE: [type 1B][seq 4B][ip NUL][pubkey 32B] - peers learn our pubkey here * GOODBYE: [type 1B][seq 4B][ip NUL] - no pubkey needed */ -static void cl_ctr_send_pkt_with_ip(int sock, unsigned char type, cl_ctr_cluster_t *cl) +static void cl_ctr_send_pkt_with_ip(int sock, unsigned char type, cl_ctr_cluster_t *cl, + const struct sockaddr *dest, socklen_t destlen) { /* Sized for ALIVE which carries an extra pubkey + config descriptor */ char pkt[CL_CTR_SMALL_PKT_SZ + CL_CTR_PUBKEY_SZ + CL_CTR_CONFIG_SZ]; @@ -2644,10 +2656,16 @@ static void cl_ctr_send_pkt_with_ip(int sock, unsigned char type, cl_ctr_cluster } } - cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, type); + if (dest) + cl_ctr_seal_and_send_to(sock, cl, pkt, plain_len, cl->session_key, type, + dest, destlen); + else + cl_ctr_seal_and_send(sock, cl, pkt, plain_len, cl->session_key, type); } -#define cl_ctr_send_alive(sock, cl) cl_ctr_send_pkt_with_ip((sock), CL_CTR_PKT_ALIVE, (cl)) +#define cl_ctr_send_alive(sock, cl) cl_ctr_send_pkt_with_ip((sock), CL_CTR_PKT_ALIVE, (cl), NULL, 0) +#define cl_ctr_send_alive_to(sock, cl, dest, dlen) \ + cl_ctr_send_pkt_with_ip((sock), CL_CTR_PKT_ALIVE, (cl), (dest), (dlen)) #define cl_ctr_send_join_req(sock, cl) cl_ctr_send_join_req_pkt((sock), (cl)) /** @@ -2888,6 +2906,54 @@ static void cl_ctr_membership_digest(cl_ctr_cluster_t *cl, uint16_t *count, uint *hash = h; } +/* Build a sockaddr_in for an IPv4 dotted string + port (for unicast sends). */ +static void cl_ctr_sockaddr_in(const char *ip, int port, struct sockaddr_in *out) +{ + memset(out, 0, sizeof(*out)); + out->sin_family = AF_INET; + out->sin_port = htons((uint16_t)port); + out->sin_addr.s_addr = inet_addr(ip); +} + +/* Fill @bm (CL_CTR_ALIVE_BITMAP_SZ bytes) with a bit set for every peer the + * master has seen inside the election window - the liveness it relays. */ +static void cl_ctr_build_alive_bitmap(cl_ctr_cluster_t *cl, unsigned char *bm) +{ + time_t cutoff = time(NULL) - (time_t)(query_time * CL_CTR_ELECT_FACTOR); + int i; + + memset(bm, 0, CL_CTR_ALIVE_BITMAP_SZ); + lock_start_read(cl->peers->lock); + for (i = 0; i < cl->peers->count; i++) { + cl_ctr_peer_t *e = &cl->peers->entries[i]; + uint16_t nid = e->node_id; + if (e->last_seen < cutoff || nid == 0 || nid > CL_CTR_MAX_PEERS) + continue; + bm[nid >> 3] |= (unsigned char)(1u << (nid & 7)); + } + lock_stop_read(cl->peers->lock); +} + +/* Refresh last_seen for every peer whose node_id the master reports alive in @bm, + * so a settled non-master keeps its election window populated without hearing the + * peers' ALIVEs directly. Only a non-master applies this. */ +static void cl_ctr_apply_alive_bitmap(cl_ctr_cluster_t *cl, const unsigned char *bm) +{ + time_t now = time(NULL); + int i; + + lock_start_write(cl->peers->lock); + for (i = 0; i < cl->peers->count; i++) { + cl_ctr_peer_t *e = &cl->peers->entries[i]; + uint16_t nid = e->node_id; + if (nid == 0 || nid > CL_CTR_MAX_PEERS) + continue; + if (bm[nid >> 3] & (unsigned char)(1u << (nid & 7))) + e->last_seen = now; + } + lock_stop_write(cl->peers->lock); +} + /** * cl_ctr_send_master_alive() - master-only keepalive. Encrypted with session_key * (CL_CTR_PACKET_MAGIC). Carries the membership digest ([count 2B][hash 8B]) so @@ -2896,7 +2962,7 @@ static void cl_ctr_membership_digest(cl_ctr_cluster_t *cl, uint16_t *count, uint static void cl_ctr_send_master_alive(int sock, cl_ctr_cluster_t *cl) { char pkt[CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ - + CL_CTR_DIGEST_SZ + CL_CTR_TAG_SZ]; + + CL_CTR_DIGEST_SZ + CL_CTR_ALIVE_BITMAP_SZ + CL_CTR_TAG_SZ]; uint32_t seq = htonl(++cl->peers->my_seq); uint16_t cnt; uint64_t hash; @@ -2911,8 +2977,13 @@ static void cl_ctr_send_master_alive(int sock, cl_ctr_cluster_t *cl) p = pkt + CL_CTR_WIRE_HDR_SZ + CL_CTR_PLAIN_HDR_SZ; *p++ = (char)(cnt >> 8); *p++ = (char)cnt; for (k = 0; k < 8; k++) *p++ = (char)(hash >> (56 - 8 * k)); + /* relay the per-node liveness bitmap so settled non-masters can refresh + * their election window without hearing peers' ALIVEs directly */ + cl_ctr_build_alive_bitmap(cl, (unsigned char *)p); + p += CL_CTR_ALIVE_BITMAP_SZ; - cl_ctr_seal_and_send(sock, cl, pkt, CL_CTR_PLAIN_HDR_SZ + CL_CTR_DIGEST_SZ, + cl_ctr_seal_and_send(sock, cl, pkt, + CL_CTR_PLAIN_HDR_SZ + CL_CTR_DIGEST_SZ + CL_CTR_ALIVE_BITMAP_SZ, cl->session_key, CL_CTR_PKT_MASTER_ALIVE); } @@ -4111,6 +4182,14 @@ static void cl_ctr_handle_master_alive(const char *sender_ip, cl_ctr_cluster_t * if (!i_am_master || yielded) cl_ctr_arm_tfd(cl->master_dead_tfd, CL_CTR_MASTER_KA_TIMEOUT, 0); + /* Liveness relay: the master reports which node_ids it has seen in-window; + * refresh those peers' last_seen so our election window stays populated even + * though a settled non-master no longer hears the peers' ALIVEs directly. */ + if (!i_am_master && !from_self && + payload_len >= CL_CTR_DIGEST_SZ + CL_CTR_ALIVE_BITMAP_SZ) + cl_ctr_apply_alive_bitmap(cl, + (const unsigned char *)payload + CL_CTR_DIGEST_SZ); + /* Membership digest: if the master advertises a set we do not match, we may * have missed a NODE_ASSIGN (a peer known from its ALIVE but with node_id/BIN * unknown). Pull a rate-limited RESYNC. Only as a member holding the key, @@ -4805,8 +4884,33 @@ static int cl_ctr_on_alive_tfd(int fd, void *param, int was_timeout) cl_ctr_cluster_t *cl = (cl_ctr_cluster_t *)param; int prev_master, now_master; cl_ctr_drain_tfd(fd); - cl_ctr_send_alive(cl->sock, cl); + /* ALIVE transport: a settled non-master unicasts its heartbeat to the master, + * which relays liveness to the whole group via the MASTER_ALIVE bitmap - so + * the old all-to-all O(N^2) becomes O(N). During formation (no settled master + * or key yet) fall back to multicast for discovery; the master itself keeps + * multicasting its own ALIVE so its liveness and config still reach everyone. */ + { + int _im, _active; + char _lm[CL_CTR_MAX_IP_LEN + 1]; + lock_start_read(cl->peers->lock); + _im = cl_ctr_i_am_master_locked(cl); + _active = (cl->peers->node_state == CL_CTR_NODE_ACTIVE); + memcpy(_lm, cl->peers->last_master, sizeof(_lm)); + lock_stop_read(cl->peers->lock); + if (!_im && _active && cl->have_session_key && _lm[0] != '\0') { + struct sockaddr_in d; + cl_ctr_sockaddr_in(_lm, cl->multicast_port, &d); + cl_ctr_send_alive_to(cl->sock, cl, (struct sockaddr *)&d, sizeof(d)); + } else { + cl_ctr_send_alive(cl->sock, cl); + } + } lock_start_write(cl->peers->lock); + /* Refresh our own liveness first: a settled non-master unicasts its ALIVE to + * the master and no longer hears a multicast loopback, so without this it + * would age itself out of its own election window once the master is gone + * and briefly find "no eligible peers" during a failover. */ + cl_ctr_upsert_peer_locked(my_ip, cl); prev_master = cl_ctr_i_am_master_locked(cl); cl_ctr_prune_stale(cl); cl_ctr_elect_master(cl); @@ -6324,7 +6428,7 @@ static void mod_destroy(void) cl->peers->master_salt, CL_CTR_MASTER_SALT_SZ, "cl_ctr_session", cl->session_key); } - cl_ctr_send_pkt_with_ip(sock, CL_CTR_PKT_GOODBYE, cl); + cl_ctr_send_pkt_with_ip(sock, CL_CTR_PKT_GOODBYE, cl, NULL, 0); close(sock); LM_INFO("clusterer_controller: [cluster %d] GOODBYE sent\n", cl->cluster_id); From fabcdd10da06fa4d2dadb7ce417754e057c6babd Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Mon, 20 Jul 2026 13:49:36 +1000 Subject: [PATCH 20/21] clusterer_controller: fall back to multicast when clusters share a port The unicast work assumed my_ip:multicast_port uniquely identifies a cluster's socket. It does not when several controller clusters run on one node: every socket binds INADDR_ANY:multicast_port, and the config allows two clusters to share a port with distinct multicast groups (the duplicate check only rejected an identical address+port). Multicast is unaffected there - IP group membership demultiplexes - but a unicast to my_ip:port is routed by port alone, so it can land on the wrong cluster's socket and be dropped by the cluster_id filter, losing the KEY_GRANT / ACK / NODE_ASSIGN / MEMBER_LIST / RESYNC / ALIVE. Detect the condition at mod_init (a per-cluster unicast_ok flag: cleared for any cluster that shares its multicast port with another local cluster) and redirect every unicast send to the group in cl_ctr_seal_and_send_to() when the flag is clear. Correct in all cases: single-cluster and distinct-port deployments keep unicast; shared-port clusters transparently use multicast for their 1:1 packets too, only losing the optimisation. A one-line warning at startup points operators at distinct ports. Hybrid (native + controller) nodes are unaffected: a native clusterer cluster uses the BIN protocol on its own TCP socket, not the controller's UDP port, so it never collides - only fellow controller clusters count. Verified: two controller clusters sharing port 3333 log the warning, both sockets come up, and their 1:1 traffic goes multicast. --- .../clusterer_controller.c | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index f2c9411dcd7..d3965f0531e 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -536,6 +536,9 @@ typedef struct cl_ctr_cluster_ { char multicast_address[INET_ADDRSTRLEN]; int multicast_port; struct sockaddr_in mcast_dest; /* resolved once in cl_ctr_setup_socket */ + int unicast_ok; /* 0 if this cluster shares its multicast */ + /* port with another local cluster (then the */ + /* 1:1 packets fall back to multicast) */ char password[1025]; unsigned char key[32]; /* bootstrap key = SHA256(password); JOIN only */ unsigned char session_key[32]; /* group key = HKDF(password, master_salt) */ @@ -2411,7 +2414,19 @@ static int cl_ctr_seal_and_send_to(int sock, cl_ctr_cluster_t *cl, char *pkt, unsigned char type, const struct sockaddr *dest, socklen_t destlen) { - int total_len = cl_ctr_encrypt_pkt(pkt, CL_CTR_WIRE_HDR_SZ, plain_len, key, + int total_len; + + /* If this cluster cannot use unicast (it shares its multicast port with + * another local cluster, so a unicast to my_ip:port could not be demuxed to + * the right socket), redirect any unicast dest to the group - IP membership + * is the only reliable demultiplexer there. Multicast sends already pass + * mcast_dest, so this is a no-op for them. */ + if (!cl->unicast_ok) { + dest = (const struct sockaddr *)&cl->mcast_dest; + destlen = sizeof(cl->mcast_dest); + } + + total_len = cl_ctr_encrypt_pkt(pkt, CL_CTR_WIRE_HDR_SZ, plain_len, key, cl->cluster_id); if (total_len < 0) return -1; @@ -6133,7 +6148,16 @@ static int mod_init(void) cl_ctr_cluster_strs[i] = NULL; } - /* Validate cluster_id uniqueness and (multicast, port) uniqueness */ + /* Validate cluster_id / (multicast,port) uniqueness, and decide per cluster + * whether unicast is safe. Every local controller socket binds + * INADDR_ANY:multicast_port, so a unicast to my_ip:port is demultiplexed by + * port alone - if two local clusters share a port (distinct groups), a + * unicast could land on the wrong cluster's socket and be dropped by the + * cluster_id filter. Such clusters fall back to multicast for their 1:1 + * packets too (correct, only unoptimised); the common single-cluster and + * distinct-port cases keep unicast. */ + for (i = 0; i < cl_ctr_cluster_count; i++) + cl_ctr_clusters[i].unicast_ok = 1; for (i = 0; i < cl_ctr_cluster_count; i++) { for (j = i + 1; j < cl_ctr_cluster_count; j++) { if (cl_ctr_clusters[i].cluster_id == cl_ctr_clusters[j].cluster_id) { @@ -6141,14 +6165,24 @@ static int mod_init(void) cl_ctr_clusters[i].cluster_id); return -1; } + if (cl_ctr_clusters[i].multicast_port != cl_ctr_clusters[j].multicast_port) + continue; if (strcmp(cl_ctr_clusters[i].multicast_address, - cl_ctr_clusters[j].multicast_address) == 0 && - cl_ctr_clusters[i].multicast_port == cl_ctr_clusters[j].multicast_port) { + cl_ctr_clusters[j].multicast_address) == 0) { LM_ERR("clusterer_controller: duplicate multicast %s:%d\n", cl_ctr_clusters[i].multicast_address, cl_ctr_clusters[i].multicast_port); return -1; } + /* same port, different group: unicast cannot be demuxed to the + * right cluster - both fall back to multicast for their 1:1 packets */ + cl_ctr_clusters[i].unicast_ok = 0; + cl_ctr_clusters[j].unicast_ok = 0; + LM_WARN("clusterer_controller: clusters %d and %d share multicast " + "port %d - using multicast for their 1:1 packets; give them " + "distinct ports to enable unicast\n", + cl_ctr_clusters[i].cluster_id, cl_ctr_clusters[j].cluster_id, + cl_ctr_clusters[i].multicast_port); } } From db39f15eee7afb68d4b9f451b1cc14929e8716a9 Mon Sep 17 00:00:00 2001 From: Yury Kirsanov Date: Mon, 20 Jul 2026 14:15:29 +1000 Subject: [PATCH 21/21] clusterer_controller: route shared-port unicast by cluster_id When several controller clusters on one node share a multicast_port (distinct groups), every cluster socket binds INADDR_ANY:port, so the kernel demultiplexes multicast by group membership but unicast by port alone. A 1:1 reply meant for a co-located cluster can therefore be delivered to the wrong sibling's socket, where the cluster_id filter would drop it. fabcdd10da worked around this by making such clusters fall back to multicast for their 1:1 packets - correct, but it gives up the unicast scalability win exactly where several clusters coexist. Instead, recover the packet at the receiver: on a cluster_id mismatch the receiving worker looks up the local cluster the packet's cleartext cluster_id belongs to and forwards the still-encrypted datagram to that cluster's worker via ipc_send_rpc(); the target decrypts it with its own key and processes it exactly as if it had arrived on its own socket. Senders now always unicast; the multicast fallback and the per-cluster unicast_ok flag are gone. cl_ctr_recv_one() grows a forwarded-buffer path (fwd_buf) so the RPC handler reuses the full validate/dispatch logic without duplicating it. Only the original receiver forwards (a forwarded packet is never re-forwarded), so no loop is possible; the forward is size-bounded and rate-limited against the source to bound flood work. Single-cluster and distinct-port deployments never hit a cluster_id mismatch, so the forwarding path stays entirely inert for them. Tested on a netns rig: single-cluster 2-node failover unchanged; two clusters sharing port 3333 across two nodes both converge to full membership, with 61 misdelivered unicast packets correctly re-routed. --- .../clusterer_controller.c | 181 ++++++++++++++---- 1 file changed, 140 insertions(+), 41 deletions(-) diff --git a/modules/clusterer_controller/clusterer_controller.c b/modules/clusterer_controller/clusterer_controller.c index d3965f0531e..a7b8528df75 100644 --- a/modules/clusterer_controller/clusterer_controller.c +++ b/modules/clusterer_controller/clusterer_controller.c @@ -536,9 +536,6 @@ typedef struct cl_ctr_cluster_ { char multicast_address[INET_ADDRSTRLEN]; int multicast_port; struct sockaddr_in mcast_dest; /* resolved once in cl_ctr_setup_socket */ - int unicast_ok; /* 0 if this cluster shares its multicast */ - /* port with another local cluster (then the */ - /* 1:1 packets fall back to multicast) */ char password[1025]; unsigned char key[32]; /* bootstrap key = SHA256(password); JOIN only */ unsigned char session_key[32]; /* group key = HKDF(password, master_salt) */ @@ -2416,16 +2413,11 @@ static int cl_ctr_seal_and_send_to(int sock, cl_ctr_cluster_t *cl, char *pkt, { int total_len; - /* If this cluster cannot use unicast (it shares its multicast port with - * another local cluster, so a unicast to my_ip:port could not be demuxed to - * the right socket), redirect any unicast dest to the group - IP membership - * is the only reliable demultiplexer there. Multicast sends already pass - * mcast_dest, so this is a no-op for them. */ - if (!cl->unicast_ok) { - dest = (const struct sockaddr *)&cl->mcast_dest; - destlen = sizeof(cl->mcast_dest); - } - + /* Always unicast to @dest as the caller asked. When several local clusters + * share a multicast port a unicast reply may be delivered by the kernel to + * the wrong sibling's socket (unicast is demuxed by port alone); the + * receiver recovers it by cluster_id in cl_ctr_maybe_forward(), so we no + * longer fall back to the multicast group here. */ total_len = cl_ctr_encrypt_pkt(pkt, CL_CTR_WIRE_HDR_SZ, plain_len, key, cl->cluster_id); if (total_len < 0) @@ -4638,7 +4630,22 @@ static void cl_ctr_handle_join_reject(const char *payload, int payload_len, /** * cl_ctr_recv_one() - read one datagram, validate header, dispatch by type. */ -static void cl_ctr_recv_one(int sock, cl_ctr_cluster_t *cl) +/* A datagram received on a shared ip:port for a sibling local cluster, handed + * to that cluster's worker (still encrypted) by its cleartext cluster_id. */ +struct cl_ctr_fwd_pkt { + cl_ctr_cluster_t *cl; /* target cluster (owns the reply socket) */ + struct sockaddr_in src; /* original datagram source */ + int n; /* length of buf */ + unsigned char buf[1]; /* the encrypted packet (flexible array) */ +}; +static void cl_ctr_rpc_forward(int sender, void *param); +static void cl_ctr_maybe_forward(const char *buf, int n, + const struct sockaddr_in *src, uint16_t pkt_cid, + cl_ctr_cluster_t *from); + +static void cl_ctr_recv_one(int sock, cl_ctr_cluster_t *cl, + const char *fwd_buf, int fwd_n, + const struct sockaddr_in *fwd_src) { /* Static buffer: avoids a 64 KB stack frame; safe because cl_ctr_recv_one * is called only from the single-threaded cl_ctr_worker process. */ @@ -4650,12 +4657,22 @@ static void cl_ctr_recv_one(int sock, cl_ctr_cluster_t *cl) const char *payload; int payload_len; - n = recvfrom(sock, buf, sizeof(buf) - 1, 0, - (struct sockaddr *)&src_addr, &src_len); - if (n < 0) { - if (errno != EAGAIN && errno != EWOULDBLOCK) - LM_ERR("clusterer_controller: recvfrom(): %s\n", strerror(errno)); - return; + if (fwd_buf) { + /* Forwarded from a sibling cluster's worker that received this on our + * shared ip:port (see cl_ctr_maybe_forward). Process it exactly as if + * it had arrived on our own socket. */ + n = (fwd_n > (int)sizeof(buf) - 1) ? (int)sizeof(buf) - 1 : fwd_n; + memcpy(buf, fwd_buf, (size_t)n); + src_addr = *fwd_src; + (void)src_len; + } else { + n = recvfrom(sock, buf, sizeof(buf) - 1, 0, + (struct sockaddr *)&src_addr, &src_len); + if (n < 0) { + if (errno != EAGAIN && errno != EWOULDBLOCK) + LM_ERR("clusterer_controller: recvfrom(): %s\n", strerror(errno)); + return; + } } /* Minimum: magic(2)+cluster_id(2)+nonce(12)+type(1)+seq(4)+tag(16) = 37 */ @@ -4679,9 +4696,17 @@ static void cl_ctr_recv_one(int sock, cl_ctr_cluster_t *cl) uint16_t pkt_cid_be; memcpy(&pkt_cid_be, buf + CL_CTR_MAGIC_SZ, CL_CTR_CLUSTER_ID_SZ); if (ntohs(pkt_cid_be) != (uint16_t)cl->cluster_id) { - LM_DBG("clusterer_controller: [cluster %d] ignoring packet for " - "cluster %u on shared group\n", cl->cluster_id, - ntohs(pkt_cid_be)); + /* Not ours. On a shared ip:port the kernel routes unicast by port + * alone, so it may have delivered a sibling local cluster's packet + * to us - hand it to that cluster's worker by its cleartext + * cluster_id (still encrypted; the sibling decrypts with its own + * key). A packet we were already forwarded is never re-forwarded. */ + if (!fwd_buf) + cl_ctr_maybe_forward(buf, (int)n, &src_addr, ntohs(pkt_cid_be), cl); + else + LM_DBG("clusterer_controller: [cluster %d] forwarded packet for " + "cluster %u is not ours, dropping\n", cl->cluster_id, + ntohs(pkt_cid_be)); return; } } @@ -4880,6 +4905,86 @@ static void cl_ctr_recv_one(int sock, cl_ctr_cluster_t *cl) } } +/* + * cl_ctr_maybe_forward() - hand a datagram received on a shared ip:port to the + * sibling local cluster it actually belongs to. + * + * On a shared multicast_port every controller cluster on this node binds the + * same ip:port; the kernel demultiplexes multicast by group membership but + * unicast by port alone, so a 1:1 reply meant for a co-located cluster can be + * delivered to the wrong worker. We recover it here: find the local cluster + * whose cluster_id matches the packet's cleartext header and forward the still- + * encrypted bytes to its worker via IPC. The target decrypts with its own key. + * + * Called only from the original receiver (never re-entered for a forwarded + * packet), so no forwarding loop is possible. + */ +static void cl_ctr_maybe_forward(const char *buf, int n, + const struct sockaddr_in *src, uint16_t pkt_cid, + cl_ctr_cluster_t *from) +{ + cl_ctr_cluster_t *target = NULL; + struct cl_ctr_fwd_pkt *fp; + int i, proc_no; + + /* Bound the work: only a legitimately-sized unicast is worth forwarding. */ + if (n <= 0 || n > CL_CTR_LIST_PKT_MAX_SZ) + return; + + for (i = 0; i < cl_ctr_cluster_count; i++) { + if ((uint16_t)cl_ctr_clusters[i].cluster_id == pkt_cid && + &cl_ctr_clusters[i] != from) { + target = &cl_ctr_clusters[i]; + break; + } + } + if (!target) { + LM_DBG("clusterer_controller: [cluster %d] no local cluster %u for " + "packet on shared port, dropping\n", from->cluster_id, pkt_cid); + return; + } + + /* Shed floods before allocating: charge the source against our own limiter + * (the target re-checks after it receives the forward). */ + if (cl_ctr_rate_check(from, src->sin_addr.s_addr) < 0) + return; + + /* worker_proc_no is written once at worker fork and stable thereafter. */ + proc_no = target->peers ? target->peers->worker_proc_no : -1; + if (proc_no < 0) + return; /* target worker not up yet */ + + fp = shm_malloc(sizeof(*fp) + (size_t)n - 1); + if (!fp) { + LM_ERR("clusterer_controller: out of shm forwarding to cluster %u\n", + pkt_cid); + return; + } + fp->cl = target; + fp->src = *src; + fp->n = n; + memcpy(fp->buf, buf, (size_t)n); + + if (ipc_send_rpc(proc_no, cl_ctr_rpc_forward, fp) < 0) { + LM_ERR("clusterer_controller: ipc forward to cluster %u failed\n", + pkt_cid); + shm_free(fp); + } +} + +/* + * cl_ctr_rpc_forward() - runs in the target cluster's worker; process a datagram + * a sibling worker forwarded to us (see cl_ctr_maybe_forward). + */ +static void cl_ctr_rpc_forward(int sender, void *param) +{ + struct cl_ctr_fwd_pkt *fp = (struct cl_ctr_fwd_pkt *)param; + (void)sender; + + cl_ctr_recv_one(fp->cl->sock, fp->cl, (char *)fp->buf, fp->n, &fp->src); + shm_free(fp); +} + /* ========================================================================= * Background worker process * ========================================================================= */ @@ -4890,7 +4995,7 @@ static void cl_ctr_recv_one(int sock, cl_ctr_cluster_t *cl) static int cl_ctr_on_sock(int fd, void *param, int was_timeout) { - cl_ctr_recv_one(fd, (cl_ctr_cluster_t *)param); + cl_ctr_recv_one(fd, (cl_ctr_cluster_t *)param, NULL, 0, NULL); return 0; } @@ -6148,16 +6253,13 @@ static int mod_init(void) cl_ctr_cluster_strs[i] = NULL; } - /* Validate cluster_id / (multicast,port) uniqueness, and decide per cluster - * whether unicast is safe. Every local controller socket binds - * INADDR_ANY:multicast_port, so a unicast to my_ip:port is demultiplexed by - * port alone - if two local clusters share a port (distinct groups), a - * unicast could land on the wrong cluster's socket and be dropped by the - * cluster_id filter. Such clusters fall back to multicast for their 1:1 - * packets too (correct, only unoptimised); the common single-cluster and - * distinct-port cases keep unicast. */ - for (i = 0; i < cl_ctr_cluster_count; i++) - cl_ctr_clusters[i].unicast_ok = 1; + /* Validate cluster_id / (multicast,port) uniqueness. Every local controller + * socket binds INADDR_ANY:multicast_port, so a unicast to my_ip:port is + * demultiplexed by port alone - if two local clusters share a port (distinct + * groups), a unicast can land on the wrong cluster's socket. That is now + * recovered at the receiver by the packet's cleartext cluster_id (see + * cl_ctr_maybe_forward()), so shared ports stay fully unicast; we only note + * it. */ for (i = 0; i < cl_ctr_cluster_count; i++) { for (j = i + 1; j < cl_ctr_cluster_count; j++) { if (cl_ctr_clusters[i].cluster_id == cl_ctr_clusters[j].cluster_id) { @@ -6174,13 +6276,10 @@ static int mod_init(void) cl_ctr_clusters[i].multicast_port); return -1; } - /* same port, different group: unicast cannot be demuxed to the - * right cluster - both fall back to multicast for their 1:1 packets */ - cl_ctr_clusters[i].unicast_ok = 0; - cl_ctr_clusters[j].unicast_ok = 0; - LM_WARN("clusterer_controller: clusters %d and %d share multicast " - "port %d - using multicast for their 1:1 packets; give them " - "distinct ports to enable unicast\n", + /* same port, different group: unicast is demuxed by port alone and + * routed to the right cluster by cluster_id at the receiver */ + LM_INFO("clusterer_controller: clusters %d and %d share multicast " + "port %d - unicast is routed by cluster_id\n", cl_ctr_clusters[i].cluster_id, cl_ctr_clusters[j].cluster_id, cl_ctr_clusters[i].multicast_port); }