From 396d20b57688f937db24fd1cb24ea4143e9d85c2 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 6 May 2026 15:49:19 +0800 Subject: [PATCH 1/8] 1. ingress support multi hosts 2. egress failover optimise --- .../main/java/io/questdb/client/Sender.java | 60 ++++- .../cutlass/http/client/WebSocketClient.java | 32 +++ .../qwp/client/QwpHostHealthTracker.java | 164 +++++++++++++ .../QwpIngressRoleRejectedException.java | 72 ++++++ .../cutlass/qwp/client/QwpQueryClient.java | 226 ++++++++++++++---- .../qwp/client/QwpWebSocketSender.java | 177 ++++++++++---- .../WebSocketClient503RoleHeaderTest.java | 99 ++++++++ .../LineSenderBuilderWebSocketTest.java | 18 +- .../qwp/client/QwpHostHealthTrackerTest.java | 156 ++++++++++++ .../QwpIngressRoleRejectedExceptionTest.java | 63 +++++ .../client/QwpQueryClientAuthTimeoutTest.java | 88 +++++++ .../client/QwpQueryClientFromConfigTest.java | 59 +++++ .../QwpQueryClientPostConnectGuardTest.java | 6 + 13 files changed, 1123 insertions(+), 97 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient503RoleHeaderTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpIngressRoleRejectedExceptionTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientAuthTimeoutTest.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index f008f807..a26f2e6b 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -729,7 +729,9 @@ public int getTimeout() { // build() time. 0 or negative is a documented "disable" value, so // a Long.MIN_VALUE sentinel keeps it distinguishable from "unset". private static final long DURABLE_ACK_KEEPALIVE_NOT_SET = Long.MIN_VALUE; + private long authTimeoutMillis = QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS; private long durableAckKeepaliveIntervalMillis = DURABLE_ACK_KEEPALIVE_NOT_SET; + private boolean gorillaEnabled = true; // Drives the initial-connect strategy. OFF is fail-fast (default). // SYNC retries on the user thread up to the reconnect cap. ASYNC // returns immediately and lets the I/O thread retry in the @@ -1029,8 +1031,8 @@ public Sender build() { } if (protocol == PROTOCOL_WEBSOCKET) { - if (hosts.size() != 1 || ports.size() != 1) { - throw new LineSenderException("only a single address (host:port) is supported for WebSocket transport"); + if (hosts.size() < 1 || ports.size() != hosts.size()) { + throw new LineSenderException("WebSocket transport requires at least one host:port pair"); } int actualAutoFlushRows = autoFlushRows == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_WS_AUTO_FLUSH_ROWS : autoFlushRows; @@ -1134,11 +1136,15 @@ public Sender build() { int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY ? errorInboxCapacity : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; + java.util.List wsEndpoints = + new java.util.ArrayList<>(hosts.size()); + for (int i = 0, n = hosts.size(); i < n; i++) { + wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); + } QwpWebSocketSender connected; try { connected = QwpWebSocketSender.connect( - hosts.getQuick(0), - ports.getQuick(0), + wsEndpoints, wsTlsConfig, actualAutoFlushRows, actualAutoFlushBytes, @@ -1155,7 +1161,8 @@ public Sender build() { initialConnectMode, errorHandler, actualErrorInboxCapacity, - actualDurableAckKeepaliveIntervalMillis + actualDurableAckKeepaliveIntervalMillis, + authTimeoutMillis ); } catch (Throwable t) { // connect() failed before ownership of cursorEngine @@ -1167,6 +1174,7 @@ public Sender build() { } throw t; } + connected.setGorillaEnabled(gorillaEnabled); // connect() succeeded — `connected` now owns cursorEngine // via setCursorEngine(engine, true). From here on, ANY // failure must close `connected` (which closes the engine @@ -1971,6 +1979,30 @@ public LineSenderBuilder durableAckKeepaliveIntervalMillis(long millis) { return this; } + /** + * Per-endpoint timeout on the WebSocket upgrade response read. Default + * {@value QwpWebSocketSender#DEFAULT_AUTH_TIMEOUT_MS} ms. + */ + public LineSenderBuilder authTimeoutMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException( + "auth_timeout is only supported for WebSocket transport"); + } + if (millis <= 0L) { + throw new LineSenderException("auth_timeout must be > 0: ").put(millis); + } + this.authTimeoutMillis = millis; + return this; + } + + public LineSenderBuilder gorilla(boolean enabled) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("gorilla is only supported for WebSocket transport"); + } + this.gorillaEnabled = enabled; + return this; + } + /** * Per-outage cap on the cursor I/O loop's reconnect retry budget. * Once a wire failure occurs, the loop retries with exponential @@ -2666,6 +2698,24 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } pos = getValue(configurationString, pos, sink, "close_flush_timeout_millis"); closeFlushTimeoutMillis(parseLongValue(sink, "close_flush_timeout_millis")); + } else if (Chars.equals("auth_timeout", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("auth_timeout is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "auth_timeout"); + authTimeoutMillis(parseLongValue(sink, "auth_timeout")); + } else if (Chars.equals("gorilla", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("gorilla is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "gorilla"); + if (Chars.equals("on", sink) || Chars.equals("true", sink)) { + gorilla(true); + } else if (Chars.equals("off", sink) || Chars.equals("false", sink)) { + gorilla(false); + } else { + throw new LineSenderException("invalid gorilla [value=").put(sink).put(", allowed=[on, off]]"); + } } else if (Chars.equals("durable_ack_keepalive_interval_millis", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException( diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 488449d9..579334b5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -75,6 +75,7 @@ public abstract class WebSocketClient implements QuietCloseable { private static final int PARSE_INCOMPLETE = 0; private static final int PARSE_NEED_MORE = -1; private static final int PARSE_OK = 1; + private static final String QUESTDB_ROLE_HEADER_NAME = "X-QuestDB-Role:"; private static final String QWP_DURABLE_ACK_ENABLED_VALUE = "enabled"; private static final String QWP_DURABLE_ACK_HEADER_NAME = "X-QWP-Durable-Ack:"; private static final String QWP_VERSION_HEADER_NAME = "X-QWP-Version:"; @@ -133,6 +134,7 @@ public abstract class WebSocketClient implements QuietCloseable { // setQwpRequestDurableAck) is the early-fail signal. private boolean serverDurableAckEnabled; private int serverQwpVersion = 1; + private String upgradeRejectRole; private boolean upgraded; public WebSocketClient(HttpClientConfiguration configuration, SocketFactory socketFactory) { @@ -296,6 +298,16 @@ public int getServerQwpVersion() { return serverQwpVersion; } + /** + * If the most recent {@link #upgrade} was rejected with a 503 carrying an + * {@code X-QuestDB-Role} header, returns that role (e.g. {@code REPLICA}, + * {@code PRIMARY_CATCHUP}). Returns null otherwise. Read after a failed + * upgrade to classify the rejection by replication role. + */ + public String getUpgradeRejectRole() { + return upgradeRejectRole; + } + /** * Returns whether the WebSocket is connected and upgraded. */ @@ -624,6 +636,23 @@ private static boolean extractDurableAckEnabled(String response) { return false; } + private static String extractRoleHeader(String response) { + int headerLen = QUESTDB_ROLE_HEADER_NAME.length(); + int responseLen = response.length(); + for (int i = 0; i <= responseLen - headerLen; i++) { + if (response.regionMatches(true, i, QUESTDB_ROLE_HEADER_NAME, 0, headerLen)) { + int valueStart = i + headerLen; + int lineEnd = response.indexOf('\r', valueStart); + if (lineEnd < 0) { + lineEnd = responseLen; + } + String value = response.substring(valueStart, lineEnd).trim(); + return value.isEmpty() ? null : value; + } + } + return null; + } + private static int extractQwpVersion(String response) { int headerLen = QWP_VERSION_HEADER_NAME.length(); int responseLen = response.length(); @@ -1031,6 +1060,9 @@ private void validateUpgradeResponse(int headerEnd) { // Check status line if (!response.startsWith("HTTP/1.1 101")) { String statusLine = response.split("\r\n")[0]; + if (statusLine.startsWith("HTTP/1.1 503")) { + upgradeRejectRole = extractRoleHeader(response); + } throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java new file mode 100644 index 00000000..8c433c4e --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java @@ -0,0 +1,164 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Per-client bookkeeping that ranks the configured endpoint list when picking + * the next host to try. Mirrors the .NET client's QwpHostHealthTracker. + *

+ * Within a round, {@link #pickNext()} returns the highest-priority host that + * has not yet been attempted; the caller advances the round via + * {@link #beginRound(boolean)}. + */ +public final class QwpHostHealthTracker { + public enum HostState { + UNKNOWN, + HEALTHY, + TRANSIENT_REJECT, + TRANSPORT_ERROR, + TOPOLOGY_REJECT, + } + + private static final HostState[] PRIORITY_ORDER = { + HostState.HEALTHY, + HostState.UNKNOWN, + HostState.TRANSIENT_REJECT, + HostState.TRANSPORT_ERROR, + HostState.TOPOLOGY_REJECT, + }; + + private final boolean[] attemptedThisRound; + private final int hostCount; + private final Object lock = new Object(); + private final HostState[] states; + + public QwpHostHealthTracker(int hostCount) { + if (hostCount <= 0) { + throw new IllegalArgumentException("hostCount must be > 0"); + } + this.hostCount = hostCount; + this.states = new HostState[hostCount]; + this.attemptedThisRound = new boolean[hostCount]; + for (int i = 0; i < hostCount; i++) { + states[i] = HostState.UNKNOWN; + } + } + + /** + * Resets attempted flags. With {@code forgetClassifications}, every host + * except the last-known {@link HostState#HEALTHY} entry is reset to + * {@link HostState#UNKNOWN}; the sticky-Healthy keeps the last successful + * host first in line on the next round. + */ + public void beginRound(boolean forgetClassifications) { + synchronized (lock) { + int stickyIndex = -1; + if (forgetClassifications) { + for (int i = 0; i < hostCount; i++) { + if (states[i] == HostState.HEALTHY) { + stickyIndex = i; + } + } + } + for (int i = 0; i < hostCount; i++) { + attemptedThisRound[i] = false; + if (forgetClassifications && i != stickyIndex) { + states[i] = HostState.UNKNOWN; + } + } + } + } + + public int count() { + return hostCount; + } + + public HostState getState(int idx) { + synchronized (lock) { + return states[idx]; + } + } + + public boolean isRoundExhausted() { + synchronized (lock) { + for (int i = 0; i < hostCount; i++) { + if (!attemptedThisRound[i]) { + return false; + } + } + return true; + } + } + + /** + * Returns the highest-priority host not yet attempted this round, or -1 + * when the round is exhausted. + */ + public int pickNext() { + synchronized (lock) { + for (HostState p : PRIORITY_ORDER) { + for (int i = 0; i < hostCount; i++) { + if (!attemptedThisRound[i] && states[i] == p) { + return i; + } + } + } + return -1; + } + } + + /** + * Demotes a previously-healthy host on send/receive failure so a subsequent + * sticky-Healthy reset doesn't preserve it as the priority entry. + */ + public void recordMidStreamFailure(int idx) { + synchronized (lock) { + if (states[idx] == HostState.HEALTHY) { + states[idx] = HostState.TRANSPORT_ERROR; + } + } + } + + public void recordRoleReject(int idx, boolean isTransient) { + synchronized (lock) { + states[idx] = isTransient ? HostState.TRANSIENT_REJECT : HostState.TOPOLOGY_REJECT; + attemptedThisRound[idx] = true; + } + } + + public void recordSuccess(int idx) { + synchronized (lock) { + states[idx] = HostState.HEALTHY; + attemptedThisRound[idx] = true; + } + } + + public void recordTransportError(int idx) { + synchronized (lock) { + states[idx] = HostState.TRANSPORT_ERROR; + attemptedThisRound[idx] = true; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java new file mode 100644 index 00000000..f66fea66 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java @@ -0,0 +1,72 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; + +/** + * Raised when the server rejects a {@code /write/v4} WebSocket upgrade with a + * {@code 503 Service Unavailable} carrying an {@code X-QuestDB-Role} header. + * Carries the role name so the host-health tracker can classify the endpoint + * as transiently unavailable ({@code PRIMARY_CATCHUP}) versus structurally + * unwritable ({@code REPLICA}). + */ +public final class QwpIngressRoleRejectedException extends HttpClientException { + public static final String ROLE_PRIMARY = "PRIMARY"; + public static final String ROLE_PRIMARY_CATCHUP = "PRIMARY_CATCHUP"; + public static final String ROLE_REPLICA = "REPLICA"; + public static final String ROLE_STANDALONE = "STANDALONE"; + + private final String host; + private final int port; + private final String role; + + public QwpIngressRoleRejectedException(String role, String host, int port) { + super("WebSocket ingress upgrade rejected by role=" + role + " at " + host + ":" + port); + this.role = role; + this.host = host; + this.port = port; + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } + + public String getRole() { + return role; + } + + public boolean isTopological() { + return ROLE_REPLICA.equals(role); + } + + public boolean isTransient() { + return ROLE_PRIMARY_CATCHUP.equals(role); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index f6202799..3569c98c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -134,8 +134,12 @@ public class QwpQueryClient implements QuietCloseable { * in bounded wall time: with 8 attempts and a 1 s cap the client gives * up after roughly 5 s of cumulative sleep. */ + private static final long DEFAULT_AUTH_TIMEOUT_MS = 15_000L; private static final long DEFAULT_FAILOVER_MAX_BACKOFF_MS = 1_000L; + private static final long DEFAULT_FAILOVER_MAX_DURATION_MS = 30_000L; private static final int DEFAULT_IO_BUFFER_POOL_SIZE = 4; + private static final String LB_FIRST = "first"; + private static final String LB_RANDOM = "random"; /** * How long {@link #connect()} waits to read the v2 {@code SERVER_INFO} frame * from each endpoint before giving up and moving to the next. 5 seconds is @@ -157,6 +161,7 @@ public class QwpQueryClient implements QuietCloseable { // scratch. private final AtomicBoolean closedFlag = new AtomicBoolean(); private final List endpoints = new ArrayList<>(); + private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; private String authorizationHeader; private int bufferPoolSize = DEFAULT_IO_BUFFER_POOL_SIZE; private String clientId; @@ -168,7 +173,7 @@ public class QwpQueryClient implements QuietCloseable { // {@code compression=zstd} (demands zstd) or {@code compression=auto} // (advertises zstd,raw and lets the server pick). private String compressionPreference = "raw"; - // Published by connect() / reconnectSkippingIndex() and read by cancel(), + // Published by connect() / reconnectViaTracker() and read by cancel(), // close(), and the pre-connect-guard on the configuration setters. Volatile // both for the standard happens-before relationship against pre-connect // configuration writes and so a second thread calling cancel() or close() @@ -202,6 +207,9 @@ public class QwpQueryClient implements QuietCloseable { private long failoverInitialBackoffMs = DEFAULT_FAILOVER_INITIAL_BACKOFF_MS; private int failoverMaxAttempts = DEFAULT_FAILOVER_MAX_ATTEMPTS; private long failoverMaxBackoffMs = DEFAULT_FAILOVER_MAX_BACKOFF_MS; + private long failoverMaxDurationMs = DEFAULT_FAILOVER_MAX_DURATION_MS; + private QwpHostHealthTracker hostTracker; + private String lbStrategy = LB_RANDOM; // Credit-flow send-ahead budget. 0 = unbounded (Phase-1 default, no CREDIT // bookkeeping on either side). A positive value puts the stream under byte- // based flow control: the server emits at most this many bytes of result @@ -344,6 +352,9 @@ public static QwpQueryClient fromConfig(CharSequence configurationString) { Integer failoverMaxAttempts = null; Long failoverBackoffInitialMs = null; Long failoverBackoffMaxMs = null; + Long failoverMaxDurationMs = null; + Long authTimeoutMs = null; + String lbStrategy = null; String auth = null; String username = null; String password = null; @@ -421,6 +432,35 @@ public static QwpQueryClient fromConfig(CharSequence configurationString) { throw new IllegalArgumentException("failover_backoff_max_ms must be >= 0"); } break; + case "failover_max_duration_ms": + try { + long parsed = Long.parseLong(value); + if (parsed < 0L) { + throw new IllegalArgumentException("failover_max_duration_ms must be >= 0"); + } + failoverMaxDurationMs = parsed; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid failover_max_duration_ms: " + value); + } + break; + case "lb_strategy": + if (!LB_RANDOM.equals(value) && !LB_FIRST.equals(value)) { + throw new IllegalArgumentException( + "invalid lb_strategy: " + value + " (expected random or first)"); + } + lbStrategy = value; + break; + case "auth_timeout_ms": + try { + long parsed = Long.parseLong(value); + if (parsed <= 0L) { + throw new IllegalArgumentException("auth_timeout_ms must be > 0"); + } + authTimeoutMs = parsed; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid auth_timeout_ms: " + value); + } + break; case "path": path = value; break; @@ -544,6 +584,15 @@ public static QwpQueryClient fromConfig(CharSequence configurationString) { : Math.max(initial, DEFAULT_FAILOVER_MAX_BACKOFF_MS); client.withFailoverBackoff(initial, max); } + if (failoverMaxDurationMs != null) { + client.withFailoverMaxDuration(failoverMaxDurationMs); + } + if (lbStrategy != null) { + client.withLbStrategy(lbStrategy); + } + if (authTimeoutMs != null) { + client.withAuthTimeout(authTimeoutMs); + } client.withEndpointPath(path); client.withBufferPoolSize(poolSize); client.withCompression(compression, compressionLevel); @@ -698,27 +747,41 @@ public void connect() { if (connected) { return; } + if (hostTracker == null) { + if (LB_RANDOM.equals(lbStrategy) && endpoints.size() > 1) { + java.util.Collections.shuffle(endpoints, java.util.concurrent.ThreadLocalRandom.current()); + } + hostTracker = new QwpHostHealthTracker(endpoints.size()); + } QwpServerInfo lastObservedMismatch = null; boolean sawV1Mismatch = false; Throwable lastTransportError = null; - for (int i = 0; i < endpoints.size(); i++) { + while (true) { + int i = hostTracker.pickNext(); + if (i < 0) { + break; + } Endpoint ep = endpoints.get(i); try { connectToEndpoint(ep); + } catch (QwpIngressRoleRejectedException re) { + lastTransportError = re; + hostTracker.recordRoleReject(i, re.isTransient()); + LOG.info("QwpQueryClient {}:{} rejected upgrade with role={}; trying next", + ep.host, ep.port, re.getRole()); + cleanupFailedConnect(); + continue; } catch (RuntimeException e) { lastTransportError = e; + hostTracker.recordTransportError(i); LOG.warn("QwpQueryClient connect failed for {}:{} -- {}", ep.host, ep.port, e.getMessage()); cleanupFailedConnect(); continue; } QwpServerInfo info = serverInfo; if (!TARGET_ANY.equals(target) && info == null) { - // v1 server (no SERVER_INFO frame) cannot satisfy a specific-role - // filter. target=primary/replica asks for a role guarantee; a - // silent no-SERVER_INFO bind would give the caller false - // confidence that the connected endpoint is the role they asked - // for. sawV1Mismatch = true; + hostTracker.recordRoleReject(i, false); LOG.info("QwpQueryClient {}:{} negotiated v1 (no SERVER_INFO) and target={} requires v2; trying next", ep.host, ep.port, target); cleanupFailedConnect(); @@ -726,11 +789,14 @@ public void connect() { } if (info != null && !matchesTarget(info.getRole(), target)) { lastObservedMismatch = info; + boolean isTransient = info.getRole() == QwpEgressMsgKind.ROLE_PRIMARY_CATCHUP; + hostTracker.recordRoleReject(i, isTransient); LOG.info("QwpQueryClient {}:{} role={} does not match target={}, trying next", ep.host, ep.port, QwpServerInfo.roleName(info.getRole()), target); cleanupFailedConnect(); continue; } + hostTracker.recordSuccess(i); currentEndpointIndex = i; connected = true; return; @@ -790,6 +856,10 @@ public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handl if (!connected) { throw new IllegalStateException("QwpQueryClient not connected; call connect() first"); } + hostTracker.beginRound(true); + long failoverDeadlineMs = failoverMaxDurationMs > 0L + ? System.currentTimeMillis() + failoverMaxDurationMs + : Long.MAX_VALUE; int attempt = 0; while (true) { attempt++; @@ -799,12 +869,10 @@ public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handl return; } if (!failoverEnabled) { - // failover disabled: surface the transport failure to the user - // and leave the client in a broken state (per documented contract). handler.onError(probe.interceptedStatus, probe.interceptedMessage); return; } - if (attempt >= failoverMaxAttempts) { + if (attempt >= failoverMaxAttempts || System.currentTimeMillis() >= failoverDeadlineMs) { int failovers = Math.max(0, attempt - 1); handler.onError(probe.interceptedStatus, "transport failure after " + attempt + " execute attempt" @@ -814,30 +882,19 @@ public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handl + probe.interceptedMessage); return; } - // Snapshot the endpoint we were just bound to before cleanup - // clobbers currentEndpointIndex. reconnectSkippingIndex uses it - // to start the walk at the NEXT entry -- without this, a transport - // failure against a primary whose server is still accepting new - // sockets (our debug hook does exactly that, but so would a brief - // WebSocket hiccup in production) would pick the same primary - // back up on reconnect and never exercise the replica. int failedIndex = currentEndpointIndex; - // Tear the broken connection down before reconnecting. We don't want - // to leak the current I/O thread or WebSocket. cleanupFailedConnect - // also orphans the outgoing generation's terminal-failure listener - // so a late callback from the dying I/O thread cannot pollute the - // new connection's state. + if (hostTracker != null && failedIndex >= 0) { + hostTracker.recordMidStreamFailure(failedIndex); + } cleanupFailedConnect(); connected = false; - // Exponential backoff between failover reconnects, doubling per - // attempt and capped at failoverMaxBackoffMs. attempt=1 is the - // original execute and never sleeps; attempt=2 sleeps the initial - // amount, attempt=3 double, etc. Sleep is interruptible: a thread - // interrupt aborts failover and surfaces as an onError so a - // blocking execute() can still be cancelled by the user. if (failoverInitialBackoffMs > 0L) { long base = failoverInitialBackoffMs << Math.min(attempt - 1, 30); - long delay = Math.min(base, failoverMaxBackoffMs); + long capped = Math.min(base, failoverMaxBackoffMs); + long jitter = capped > 0L + ? java.util.concurrent.ThreadLocalRandom.current().nextLong(capped) + : 0L; + long delay = Math.min(capped + jitter, failoverMaxBackoffMs); if (delay > 0L) { try { Thread.sleep(delay); @@ -851,7 +908,7 @@ public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handl } } try { - reconnectSkippingIndex(failedIndex); + reconnectViaTracker(); } catch (RuntimeException reconnectErr) { handler.onError(probe.interceptedStatus, "failover reconnect failed after " + attempt + " attempt" @@ -861,7 +918,6 @@ public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handl return; } handler.onFailoverReset(serverInfo); - // loop back: next iteration re-executes the query on the fresh connection } } @@ -1029,6 +1085,52 @@ public QwpQueryClient withFailoverMaxAttempts(int attempts) { return this; } + /** + * Total wall-clock cap on the failover loop; {@code 0} disables. + * Whichever of this or {@link #withFailoverMaxAttempts} fires first ends + * the loop. Default {@value #DEFAULT_FAILOVER_MAX_DURATION_MS} ms. + */ + public QwpQueryClient withFailoverMaxDuration(long maxDurationMs) { + checkPreConnect("withFailoverMaxDuration"); + if (maxDurationMs < 0L) { + throw new IllegalArgumentException("failoverMaxDurationMs must be >= 0"); + } + this.failoverMaxDurationMs = maxDurationMs; + return this; + } + + /** + * Initial address-pick strategy: {@code "random"} (default) shuffles the + * endpoint list so N clients spread across N hosts; {@code "first"} keeps + * the connection-string order, useful for tests and primary-preferred + * topologies. Failover walks deterministically by tracker priority in + * either case. + */ + public QwpQueryClient withLbStrategy(String strategy) { + checkPreConnect("withLbStrategy"); + if (!LB_RANDOM.equals(strategy) && !LB_FIRST.equals(strategy)) { + throw new IllegalArgumentException( + "lbStrategy must be \"random\" or \"first\", got " + strategy); + } + this.lbStrategy = strategy; + return this; + } + + /** + * Per-endpoint timeout on the HTTP upgrade response read. Bounds the common + * "host accepts TCP but never replies" blackhole. Does NOT bound the TCP + * connect itself (no native knob); a routing blackhole with no SYN-ACK + * still falls back to OS defaults. Default {@value #DEFAULT_AUTH_TIMEOUT_MS} ms. + */ + public QwpQueryClient withAuthTimeout(long authTimeoutMs) { + checkPreConnect("withAuthTimeout"); + if (authTimeoutMs <= 0L) { + throw new IllegalArgumentException("authTimeoutMs must be > 0"); + } + this.authTimeoutMs = authTimeoutMs; + return this; + } + /** * Opts the next {@link #execute} into credit-based flow control with * {@code bytes} of initial send-ahead budget. The server streams at most @@ -1333,6 +1435,38 @@ private void cleanupFailedConnect() { currentEndpointIndex = -1; } + private void runUpgradeWithTimeout(Endpoint ep) { + if (authTimeoutMs <= 0L) { + webSocketClient.connect(ep.host, ep.port); + webSocketClient.upgrade(endpointPath, authorizationHeader); + return; + } + // The TCP connect itself isn't bounded -- cutlass native socket has no connect-timeout knob, + // so a routing blackhole (no SYN-ACK) still falls back to the OS default. The HTTP upgrade + // response read IS bounded via WebSocketClient's own per-call timeout; that covers the common + // "accepts TCP but never replies" blackhole scenario auth_timeout targets. + webSocketClient.connect(ep.host, ep.port); + int timeoutMs = (int) Math.min(authTimeoutMs, Integer.MAX_VALUE); + try { + webSocketClient.upgrade(endpointPath, timeoutMs, authorizationHeader); + } catch (HttpClientException ex) { + if (ex.isTimeout()) { + HttpClientException timeout = new HttpClientException("WebSocket upgrade to ") + .put(ep.host).put(':').put(ep.port) + .put(" exceeded auth_timeout=").put(authTimeoutMs).put("ms"); + timeout.initCause(ex); + throw timeout; + } + String role = webSocketClient.getUpgradeRejectRole(); + if (role != null) { + QwpIngressRoleRejectedException re = new QwpIngressRoleRejectedException(role, ep.host, ep.port); + re.initCause(ex); + throw re; + } + throw ex; + } + } + private void connectToEndpoint(Endpoint ep) { if (tlsEnabled) { webSocketClient = WebSocketClientFactory.newTlsInstance( @@ -1344,8 +1478,7 @@ private void connectToEndpoint(Endpoint ep) { webSocketClient.setQwpClientId(clientId != null ? clientId : defaultClientId()); webSocketClient.setQwpAcceptEncoding(buildAcceptEncodingHeader()); webSocketClient.setQwpMaxBatchRows(maxBatchRows); - webSocketClient.connect(ep.host, ep.port); - webSocketClient.upgrade(endpointPath, authorizationHeader); + runUpgradeWithTimeout(ep); negotiatedQwpVersion = webSocketClient.getServerQwpVersion(); // v2 servers send SERVER_INFO as the first WebSocket frame after the @@ -1568,39 +1701,46 @@ private QwpServerInfo receiveServerInfoSync() { * {@code serverInfo} populated. On exhaustion, raises the same exceptions * as {@link #connect()}. */ - private void reconnectSkippingIndex(int failedIndex) { + private void reconnectViaTracker() { int total = endpoints.size(); - // When failedIndex is known, walk the other (total - 1) entries only. - // When it is not (defensive path: currentEndpointIndex was never set - // before the failure), fall back to the full (total) walk. - int startFrom = failedIndex < 0 ? 0 : failedIndex + 1; - int stepCount = failedIndex < 0 ? total : total - 1; + hostTracker.beginRound(false); QwpServerInfo lastMismatch = null; boolean sawV1Mismatch = false; Throwable lastError = null; - for (int step = 0; step < stepCount; step++) { - int i = (startFrom + step) % total; + while (true) { + int i = hostTracker.pickNext(); + if (i < 0) { + break; + } Endpoint ep = endpoints.get(i); try { connectToEndpoint(ep); + } catch (QwpIngressRoleRejectedException re) { + lastError = re; + hostTracker.recordRoleReject(i, re.isTransient()); + cleanupFailedConnect(); + continue; } catch (RuntimeException e) { lastError = e; + hostTracker.recordTransportError(i); cleanupFailedConnect(); continue; } QwpServerInfo info = serverInfo; if (!TARGET_ANY.equals(target) && info == null) { - // v1 cannot satisfy a specific-role filter (see the matching - // branch in connect()). sawV1Mismatch = true; + hostTracker.recordRoleReject(i, false); cleanupFailedConnect(); continue; } if (info != null && !matchesTarget(info.getRole(), target)) { lastMismatch = info; + boolean isTransient = info.getRole() == QwpEgressMsgKind.ROLE_PRIMARY_CATCHUP; + hostTracker.recordRoleReject(i, isTransient); cleanupFailedConnect(); continue; } + hostTracker.recordSuccess(i); currentEndpointIndex = i; connected = true; return; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index a5d379e9..2c81a881 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -30,6 +30,7 @@ import io.questdb.client.SenderErrorHandler; import io.questdb.client.SenderProgressHandler; import io.questdb.client.cairo.TableUtils; +import io.questdb.client.cutlass.http.client.HttpClientException; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.http.client.WebSocketClientFactory; import io.questdb.client.cutlass.line.LineSenderException; @@ -59,6 +60,9 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -108,6 +112,7 @@ */ public class QwpWebSocketSender implements Sender { + public static final long DEFAULT_AUTH_TIMEOUT_MS = 15_000L; public static final int DEFAULT_AUTO_FLUSH_BYTES = 0; public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms public static final int DEFAULT_AUTO_FLUSH_ROWS = 1_000; @@ -130,10 +135,13 @@ public class QwpWebSocketSender implements Sender { private final QwpWebSocketEncoder encoder; // Global symbol dictionary for delta encoding private final GlobalSymbolDictionary globalSymbolDictionary; + private final List endpoints; + private final QwpHostHealthTracker hostTracker; private final String host; private final int inFlightWindowSize; private final int maxSchemasPerConnection; private final int port; + private volatile int currentEndpointIdx = -1; private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) private final ClientTlsConfiguration tlsConfig; @@ -203,6 +211,7 @@ public class QwpWebSocketSender implements Sender { // values; Sender.build can override via the new connect overload. private long reconnectMaxDurationMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_DURATION_MILLIS; + private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; private boolean requestDurableAck; // Keepalive PING cadence used by the I/O loop while // request_durable_ack=on AND there are pending durable-ack @@ -212,8 +221,7 @@ public class QwpWebSocketSender implements Sender { CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS; private QwpWebSocketSender( - String host, - int port, + List endpoints, ClientTlsConfiguration tlsConfig, int autoFlushRows, int autoFlushBytes, @@ -222,9 +230,14 @@ private QwpWebSocketSender( String authorizationHeader, int maxSchemasPerConnection ) { + if (endpoints == null || endpoints.isEmpty()) { + throw new IllegalArgumentException("endpoints must be non-empty"); + } + this.endpoints = Collections.unmodifiableList(new ArrayList<>(endpoints)); + this.hostTracker = new QwpHostHealthTracker(this.endpoints.size()); this.authorizationHeader = authorizationHeader; - this.host = host; - this.port = port; + this.host = this.endpoints.get(0).host; + this.port = this.endpoints.get(0).port; this.tlsConfig = tlsConfig; this.encoder = new QwpWebSocketEncoder(DEFAULT_BUFFER_SIZE); this.tableBuffers = new CharSequenceObjHashMap<>(); @@ -476,14 +489,51 @@ public static QwpWebSocketSender connect( SenderErrorHandler errorHandler, int errorInboxCapacity, long durableAckKeepaliveIntervalMillis + ) { + return connect( + singleEndpoint(host, port), tlsConfig, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize, authorizationHeader, maxSchemasPerConnection, + requestDurableAck, cursorEngine, + closeFlushTimeoutMillis, reconnectMaxDurationMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, + initialConnectMode, errorHandler, errorInboxCapacity, + durableAckKeepaliveIntervalMillis, DEFAULT_AUTH_TIMEOUT_MS); + } + + /** + * Multi-endpoint variant. {@code endpoints} must be non-empty; the order is the failover + * preference (single-primary cluster: walk the list until one accepts the upgrade). + */ + public static QwpWebSocketSender connect( + List endpoints, + ClientTlsConfiguration tlsConfig, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize, + String authorizationHeader, + int maxSchemasPerConnection, + boolean requestDurableAck, + CursorSendEngine cursorEngine, + long closeFlushTimeoutMillis, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + Sender.InitialConnectMode initialConnectMode, + SenderErrorHandler errorHandler, + int errorInboxCapacity, + long durableAckKeepaliveIntervalMillis, + long authTimeoutMs ) { QwpWebSocketSender sender = new QwpWebSocketSender( - host, port, tlsConfig, + endpoints, tlsConfig, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, authorizationHeader, maxSchemasPerConnection ); try { sender.requestDurableAck = requestDurableAck; + sender.authTimeoutMs = authTimeoutMs; sender.closeFlushTimeoutMillis = closeFlushTimeoutMillis; sender.reconnectMaxDurationMillis = reconnectMaxDurationMillis; sender.reconnectInitialBackoffMillis = reconnectInitialBackoffMillis; @@ -524,7 +574,7 @@ public static QwpWebSocketSender createForTesting(String host, int port, int inF public static QwpWebSocketSender createForTesting(String host, int port, int inFlightWindowSize, String authorizationHeader) { return new QwpWebSocketSender( - host, port, null, + singleEndpoint(host, port), null, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, inFlightWindowSize, authorizationHeader, DEFAULT_MAX_SCHEMAS_PER_CONNECTION ); @@ -570,7 +620,7 @@ public static QwpWebSocketSender createForTesting( int maxSchemasPerConnection ) { return new QwpWebSocketSender( - host, port, null, + singleEndpoint(host, port), null, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, null, maxSchemasPerConnection ); @@ -1845,43 +1895,71 @@ private void atNanos(long timestampNanos) { * within the per-outage time cap). */ private WebSocketClient buildAndConnect() { - WebSocketClient newClient; - if (tlsConfig != null) { - newClient = WebSocketClientFactory.newTlsInstance(tlsConfig); - } else { - newClient = WebSocketClientFactory.newPlainTextInstance(); - } - try { - newClient.setQwpMaxVersion(QwpConstants.MAX_SUPPORTED_INGEST_VERSION); - newClient.setQwpClientId(QwpConstants.CLIENT_ID); - newClient.setQwpRequestDurableAck(requestDurableAck); - newClient.connect(host, port); - newClient.upgrade(WRITE_PATH, authorizationHeader); - } catch (Exception e) { - newClient.close(); - throw new LineSenderException("Failed to connect to " + host + ":" + port, e); - } - // Fail at connect when the user opted into durable acks but landed on - // a server that did not echo the X-QWP-Durable-Ack: enabled confirmation. - // Without this check, store-and-forward would never receive trim signals - // and the on-disk store would grow unbounded -- silent storage exhaustion - // is a worse outcome than a loud connect-time failure. - if (requestDurableAck && !newClient.isServerDurableAckEnabled()) { - newClient.close(); - // The "WebSocket upgrade failed:" prefix is load-bearing: the cursor I/O - // loop's isTerminalUpgradeError() sniffs for that exact substring to - // classify a connect-time throw as terminal (won't retry). Without the - // prefix the loop would treat this misconfig as transient and burn the - // full reconnect budget before surfacing it. - throw new LineSenderException( - "WebSocket upgrade failed: server does not support durable ack [host=" - + host + ", port=" + port - + "]. The client opted in via request_durable_ack=on but the server " - + "did not echo X-QWP-Durable-Ack: enabled in the upgrade response. " - + "Either disable request_durable_ack or connect to a server with " - + "primary replication configured."); + int previousIdx = currentEndpointIdx; + if (previousIdx >= 0) { + hostTracker.recordMidStreamFailure(previousIdx); + currentEndpointIdx = -1; + } + if (hostTracker.isRoundExhausted()) { + hostTracker.beginRound(true); + } + Throwable lastError = null; + Endpoint lastEndpoint = null; + while (true) { + int idx = hostTracker.pickNext(); + if (idx < 0) break; + Endpoint ep = endpoints.get(idx); + lastEndpoint = ep; + WebSocketClient newClient = tlsConfig != null + ? WebSocketClientFactory.newTlsInstance(tlsConfig) + : WebSocketClientFactory.newPlainTextInstance(); + try { + newClient.setQwpMaxVersion(QwpConstants.MAX_SUPPORTED_INGEST_VERSION); + newClient.setQwpClientId(QwpConstants.CLIENT_ID); + newClient.setQwpRequestDurableAck(requestDurableAck); + newClient.connect(ep.host, ep.port); + int upgradeTimeoutMs = (int) Math.min(authTimeoutMs, Integer.MAX_VALUE); + newClient.upgrade(WRITE_PATH, upgradeTimeoutMs, authorizationHeader); + } catch (HttpClientException e) { + String role = newClient.getUpgradeRejectRole(); + newClient.close(); + if (role != null) { + boolean isTransient = QwpIngressRoleRejectedException.ROLE_PRIMARY_CATCHUP.equals(role); + hostTracker.recordRoleReject(idx, isTransient); + QwpIngressRoleRejectedException re = new QwpIngressRoleRejectedException(role, ep.host, ep.port); + re.initCause(e); + lastError = re; + continue; + } + hostTracker.recordTransportError(idx); + lastError = e; + continue; + } catch (Exception e) { + newClient.close(); + hostTracker.recordTransportError(idx); + lastError = e; + continue; + } + if (requestDurableAck && !newClient.isServerDurableAckEnabled()) { + newClient.close(); + hostTracker.recordTransportError(idx); + throw new LineSenderException( + "WebSocket upgrade failed: server does not support durable ack [host=" + + ep.host + ", port=" + ep.port + + "]. The client opted in via request_durable_ack=on but the server " + + "did not echo X-QWP-Durable-Ack: enabled in the upgrade response. " + + "Either disable request_durable_ack or connect to a server with " + + "primary replication configured."); + } + hostTracker.recordSuccess(idx); + currentEndpointIdx = idx; + return newClient; } - return newClient; + String summary = lastEndpoint == null + ? "no endpoints available" + : "all " + endpoints.size() + " endpoint(s) unreachable; last=" + + lastEndpoint.host + ":" + lastEndpoint.port; + throw new LineSenderException("Failed to connect: " + summary, lastError); } private void checkConnectionError() { @@ -2377,4 +2455,17 @@ private void validateTableName(CharSequence name) { } } + private static List singleEndpoint(String host, int port) { + return Collections.singletonList(new Endpoint(host, port)); + } + + public static final class Endpoint { + public final String host; + public final int port; + + public Endpoint(String host, int port) { + this.host = host; + this.port = port; + } + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient503RoleHeaderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient503RoleHeaderTest.java new file mode 100644 index 00000000..281c5884 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient503RoleHeaderTest.java @@ -0,0 +1,99 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.http.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; + +public class WebSocketClient503RoleHeaderTest { + + @Test(timeout = 10_000) + public void testReplicaReject() throws Exception { + assertCapturedRole("REPLICA", "X-QuestDB-Role: REPLICA"); + } + + @Test(timeout = 10_000) + public void testPrimaryCatchupReject() throws Exception { + assertCapturedRole("PRIMARY_CATCHUP", "X-QuestDB-Role: PRIMARY_CATCHUP"); + } + + @Test(timeout = 10_000) + public void testRoleHeaderAbsentOn503_NullCaptured() throws Exception { + assertCapturedRole(null, null); + } + + @Test(timeout = 10_000) + public void testCaseInsensitiveHeaderName() throws Exception { + assertCapturedRole("REPLICA", "x-questdb-role: REPLICA"); + } + + private static void assertCapturedRole(String expected, String roleHeaderLine) throws Exception { + ServerSocket listener = new ServerSocket(0, 1, InetAddress.getLoopbackAddress()); + int port = listener.getLocalPort(); + Thread serverThread = new Thread(() -> { + try (Socket s = listener.accept()) { + byte[] discardBuf = new byte[8192]; + int n = s.getInputStream().read(discardBuf); + if (n < 0) return; + StringBuilder resp = new StringBuilder(); + resp.append("HTTP/1.1 503 Service Unavailable\r\n"); + if (roleHeaderLine != null) resp.append(roleHeaderLine).append("\r\n"); + resp.append("Content-Length: 0\r\n\r\n"); + OutputStream os = s.getOutputStream(); + os.write(resp.toString().getBytes(StandardCharsets.US_ASCII)); + os.flush(); + } catch (Exception ignored) { + } + }, "fake-503"); + serverThread.setDaemon(true); + serverThread.start(); + + try (WebSocketClient client = WebSocketClientFactory.newPlainTextInstance()) { + client.setQwpMaxVersion(1); + client.connect("127.0.0.1", port); + try { + client.upgrade("/write/v4", null); + Assert.fail("expected upgrade to fail with 503"); + } catch (HttpClientException ex) { + Assert.assertTrue( + "expected 'WebSocket upgrade failed:' message, got: " + ex.getMessage(), + ex.getMessage().contains("WebSocket upgrade failed:")); + Assert.assertEquals(expected, client.getUpgradeRejectRole()); + } + } finally { + listener.close(); + serverThread.join(500); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 5290b09f..f0658855 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -530,12 +530,11 @@ public void testMinRequestThroughput_notSupportedForWebSocket() { } @Test - public void testMultipleAddresses_fails() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST + ":9000") - .address(LOCALHOST + ":9001"), - "single address"); + public void testMultipleAddresses_buildable() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .address(LOCALHOST + ":9001"); + Assert.assertNotNull(builder); } @Test @@ -717,6 +716,13 @@ public void testWsConfigString_uppercaseNotSupported() { assertBadConfig("WS::addr=localhost:9000;", "invalid schema"); } + @Test + public void testWsConfigString_multipleAddresses() { + Sender.LineSenderBuilder builder = + Sender.builder("ws::addr=a:9000,b:9001,c:9002;"); + Assert.assertNotNull(builder); + } + @Test public void testWsConfigString_withAutoFlushBytes() { Sender.LineSenderBuilder builder = Sender.builder("ws::addr=localhost:9000;auto_flush_bytes=1024;"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java new file mode 100644 index 00000000..3ef66f6c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java @@ -0,0 +1,156 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpHostHealthTracker; +import org.junit.Assert; +import org.junit.Test; + +public class QwpHostHealthTrackerTest { + + @Test + public void testAllUnknown_PicksByIdxOrder() { + QwpHostHealthTracker t = new QwpHostHealthTracker(3); + Assert.assertEquals(0, t.pickNext()); + Assert.assertEquals(QwpHostHealthTracker.HostState.UNKNOWN, t.getState(0)); + } + + @Test + public void testBeginRoundFalseClearsAttemptedKeepsClassifications() { + QwpHostHealthTracker t = new QwpHostHealthTracker(2); + t.recordTransportError(0); + t.recordSuccess(1); + Assert.assertTrue(t.isRoundExhausted()); + + t.beginRound(false); + Assert.assertEquals(QwpHostHealthTracker.HostState.TRANSPORT_ERROR, t.getState(0)); + Assert.assertEquals(QwpHostHealthTracker.HostState.HEALTHY, t.getState(1)); + Assert.assertFalse(t.isRoundExhausted()); + Assert.assertEquals(1, t.pickNext()); + } + + @Test + public void testBeginRoundTrue_PreservesHealthyResetsRest() { + QwpHostHealthTracker t = new QwpHostHealthTracker(3); + t.recordSuccess(0); + t.recordTransportError(1); + t.recordRoleReject(2, false); + + t.beginRound(true); + + Assert.assertEquals(QwpHostHealthTracker.HostState.HEALTHY, t.getState(0)); + Assert.assertEquals(QwpHostHealthTracker.HostState.UNKNOWN, t.getState(1)); + Assert.assertEquals(QwpHostHealthTracker.HostState.UNKNOWN, t.getState(2)); + Assert.assertEquals(0, t.pickNext()); + } + + @Test + public void testCount() { + Assert.assertEquals(4, new QwpHostHealthTracker(4).count()); + } + + @Test + public void testCtorRejectsZeroHosts() { + try { + new QwpHostHealthTracker(0); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException ignored) { + } + } + + @Test + public void testIsRoundExhausted() { + QwpHostHealthTracker t = new QwpHostHealthTracker(2); + Assert.assertFalse(t.isRoundExhausted()); + t.recordSuccess(0); + Assert.assertFalse(t.isRoundExhausted()); + t.recordTransportError(1); + Assert.assertTrue(t.isRoundExhausted()); + } + + @Test + public void testMidStreamFailure_HealthyDemoted() { + QwpHostHealthTracker t = new QwpHostHealthTracker(2); + t.recordSuccess(0); + t.recordMidStreamFailure(0); + Assert.assertEquals(QwpHostHealthTracker.HostState.TRANSPORT_ERROR, t.getState(0)); + } + + @Test + public void testMidStreamFailure_NonHealthyUnchanged() { + QwpHostHealthTracker t = new QwpHostHealthTracker(2); + t.recordRoleReject(0, false); + t.recordMidStreamFailure(0); + Assert.assertEquals(QwpHostHealthTracker.HostState.TOPOLOGY_REJECT, t.getState(0)); + } + + @Test + public void testPickNextReturnsMinusOneOnExhaustion() { + QwpHostHealthTracker t = new QwpHostHealthTracker(1); + Assert.assertEquals(0, t.pickNext()); + t.recordSuccess(0); + Assert.assertEquals(-1, t.pickNext()); + } + + @Test + public void testPriorityOrder_HealthyBeforeUnknown() { + QwpHostHealthTracker t = new QwpHostHealthTracker(3); + t.recordSuccess(2); + t.beginRound(false); + Assert.assertEquals(2, t.pickNext()); + } + + @Test + public void testPriorityOrder_TopologyRejectLast() { + QwpHostHealthTracker t = new QwpHostHealthTracker(3); + t.recordRoleReject(0, false); // TOPOLOGY_REJECT + t.recordTransportError(1); // TRANSPORT_ERROR + // 2 stays UNKNOWN + t.beginRound(false); + Assert.assertEquals(2, t.pickNext()); // UNKNOWN first + t.recordSuccess(2); + Assert.assertEquals(1, t.pickNext()); // TRANSPORT_ERROR before TOPOLOGY_REJECT + t.recordTransportError(1); + Assert.assertEquals(0, t.pickNext()); // TOPOLOGY_REJECT last + } + + @Test + public void testRecordRoleReject_Transient_TransientCategory() { + QwpHostHealthTracker t = new QwpHostHealthTracker(2); + t.recordRoleReject(0, true); + t.recordRoleReject(1, false); + Assert.assertEquals(QwpHostHealthTracker.HostState.TRANSIENT_REJECT, t.getState(0)); + Assert.assertEquals(QwpHostHealthTracker.HostState.TOPOLOGY_REJECT, t.getState(1)); + } + + @Test + public void testStickyHealthyAcrossRounds() { + QwpHostHealthTracker t = new QwpHostHealthTracker(2); + t.recordTransportError(0); + t.recordSuccess(1); + t.beginRound(true); + Assert.assertEquals(1, t.pickNext()); // sticky-Healthy + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpIngressRoleRejectedExceptionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpIngressRoleRejectedExceptionTest.java new file mode 100644 index 00000000..838d54bb --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpIngressRoleRejectedExceptionTest.java @@ -0,0 +1,63 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; +import org.junit.Assert; +import org.junit.Test; + +public class QwpIngressRoleRejectedExceptionTest { + + @Test + public void testReplicaIsTopological() { + QwpIngressRoleRejectedException ex = new QwpIngressRoleRejectedException("REPLICA", "h", 9); + Assert.assertTrue(ex.isTopological()); + Assert.assertFalse(ex.isTransient()); + } + + @Test + public void testPrimaryCatchupIsTransient() { + QwpIngressRoleRejectedException ex = new QwpIngressRoleRejectedException("PRIMARY_CATCHUP", "h", 9); + Assert.assertTrue(ex.isTransient()); + Assert.assertFalse(ex.isTopological()); + } + + @Test + public void testStandaloneIsNeither() { + QwpIngressRoleRejectedException ex = new QwpIngressRoleRejectedException("STANDALONE", "h", 9); + Assert.assertFalse(ex.isTransient()); + Assert.assertFalse(ex.isTopological()); + } + + @Test + public void testFieldsRetained() { + QwpIngressRoleRejectedException ex = new QwpIngressRoleRejectedException("REPLICA", "db", 9000); + Assert.assertEquals("REPLICA", ex.getRole()); + Assert.assertEquals("db", ex.getHost()); + Assert.assertEquals(9000, ex.getPort()); + Assert.assertTrue(ex.getMessage().contains("REPLICA")); + Assert.assertTrue(ex.getMessage().contains("db:9000")); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientAuthTimeoutTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientAuthTimeoutTest.java new file mode 100644 index 00000000..cd62a3aa --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientAuthTimeoutTest.java @@ -0,0 +1,88 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import org.junit.Assert; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.ArrayList; +import java.util.List; + +public class QwpQueryClientAuthTimeoutTest { + + @Test(timeout = 10_000) + public void testAuthTimeoutBoundsConnectAttemptToBlackholeHost() throws Exception { + ServerSocket listener = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); + int port = listener.getLocalPort(); + List held = new ArrayList<>(); + Thread acceptor = new Thread(() -> { + try { + while (!listener.isClosed()) { + Socket s = listener.accept(); + synchronized (held) { + held.add(s); + } + } + } catch (Exception ignored) { + } + }, "blackhole-acceptor"); + acceptor.setDaemon(true); + acceptor.start(); + + try (QwpQueryClient client = QwpQueryClient.fromConfig( + "ws::addr=127.0.0.1:" + port + + ";auth_timeout_ms=300;failover=off;target=any;")) { + long start = System.currentTimeMillis(); + try { + client.connect(); + Assert.fail("expected connect to fail with auth_timeout"); + } catch (HttpClientException ex) { + long elapsed = System.currentTimeMillis() - start; + Assert.assertTrue( + "expected message to mention auth_timeout, got: " + ex.getMessage(), + ex.getMessage().contains("auth_timeout")); + Assert.assertTrue( + "auth_timeout=300ms should bail in <5s, took " + elapsed + "ms", + elapsed < 5_000); + } + } finally { + listener.close(); + synchronized (held) { + for (Socket s : held) { + try { + s.close(); + } catch (Exception ignored) { + } + } + } + acceptor.join(500); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java index 5ce009f7..3f7102ac 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java @@ -648,6 +648,65 @@ public void testWhitespaceOnlyAddrEntryRejected() { assertReject("ws::addr=a:9000, ,b:9000;", "empty addr entry"); } + @Test + public void testFailoverMaxDuration_AcceptsZero() { + assertParses("ws::addr=a:9000;failover_max_duration_ms=0;"); + } + + @Test + public void testFailoverMaxDuration_AcceptsPositive() { + assertParses("ws::addr=a:9000;failover_max_duration_ms=5000;"); + } + + @Test + public void testFailoverMaxDuration_NegativeRejected() { + assertReject("ws::addr=a:9000;failover_max_duration_ms=-1;", + "failover_max_duration_ms must be >= 0"); + } + + @Test + public void testFailoverMaxDuration_NonNumericRejected() { + assertReject("ws::addr=a:9000;failover_max_duration_ms=forever;", + "invalid failover_max_duration_ms: forever"); + } + + @Test + public void testLbStrategy_AcceptsRandom() { + assertParses("ws::addr=a:9000,b:9000;lb_strategy=random;"); + } + + @Test + public void testLbStrategy_AcceptsFirst() { + assertParses("ws::addr=a:9000,b:9000;lb_strategy=first;"); + } + + @Test + public void testLbStrategy_OtherRejected() { + assertReject("ws::addr=a:9000;lb_strategy=roundrobin;", + "invalid lb_strategy: roundrobin (expected random or first)"); + } + + @Test + public void testAuthTimeout_AcceptsPositive() { + assertParses("ws::addr=a:9000;auth_timeout_ms=2500;"); + } + + @Test + public void testAuthTimeout_ZeroRejected() { + assertReject("ws::addr=a:9000;auth_timeout_ms=0;", "auth_timeout_ms must be > 0"); + } + + @Test + public void testAuthTimeout_NegativeRejected() { + assertReject("ws::addr=a:9000;auth_timeout_ms=-50;", "auth_timeout_ms must be > 0"); + } + + @Test + public void testAuthTimeout_NonNumericRejected() { + assertReject("ws::addr=a:9000;auth_timeout_ms=forever;", + "invalid auth_timeout_ms: forever"); + } + /** * Asserts that {@code conf} parses into a non-null {@link QwpQueryClient} * and closes the result on the way out. Centralising both checks here diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientPostConnectGuardTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientPostConnectGuardTest.java index 69847682..612faba2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientPostConnectGuardTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientPostConnectGuardTest.java @@ -67,6 +67,12 @@ public void testAllSettersRejectAfterConnect() throws Exception { assertRejects(c -> c.withFailoverBackoff(100L, 500L), "withFailoverBackoff"); // withFailoverMaxAttempts assertRejects(c -> c.withFailoverMaxAttempts(3), "withFailoverMaxAttempts"); + // withFailoverMaxDuration + assertRejects(c -> c.withFailoverMaxDuration(15_000L), "withFailoverMaxDuration"); + // withLbStrategy + assertRejects(c -> c.withLbStrategy("first"), "withLbStrategy"); + // withAuthTimeout + assertRejects(c -> c.withAuthTimeout(5_000L), "withAuthTimeout"); // withInitialCredit assertRejects(c -> c.withInitialCredit(1024L), "withInitialCredit"); // withInsecureTls From 9463d6afe76289baf41263a72e47c9c1271a4d96 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 6 May 2026 19:43:39 +0800 Subject: [PATCH 2/8] 1. ingress support multi hosts 2. egress failover optimise --- .../main/java/io/questdb/client/Sender.java | 132 ++++++++++++++++-- .../cutlass/qwp/client/QwpQueryClient.java | 6 +- .../qwp/client/QwpWebSocketSender.java | 29 ++-- .../LineSenderBuilderWebSocketTest.java | 119 ++++++++++++++++ 4 files changed, 257 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index a26f2e6b..1470e7f6 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -835,6 +835,93 @@ public LineSenderBuilder address(CharSequence address) { return this; } + private void addAddressEntry(CharSequence src, int start, int end) { + int hostStart; + int hostEnd; + int portStart; + if (src.charAt(start) == '[') { + int closeBracket = Chars.indexOf(src, start + 1, end, ']'); + if (closeBracket < 0) { + throw new LineSenderException("missing closing ']' in IPv6 addr entry [address=") + .put(src.subSequence(start, end)).put("]"); + } + hostStart = start + 1; + hostEnd = closeBracket; + if (closeBracket == end - 1) { + portStart = -1; + } else if (src.charAt(closeBracket + 1) != ':') { + throw new LineSenderException("expected ':' after ']' in IPv6 addr entry [address=") + .put(src.subSequence(start, end)).put("]"); + } else { + portStart = closeBracket + 2; + } + } else { + int firstColon = Chars.indexOf(src, start, end, ':'); + int lastColon = Chars.indexOf(src, start, end, ':', -1); + if (firstColon != lastColon) { + hostStart = start; + hostEnd = end; + portStart = -1; + } else if (firstColon < 0) { + hostStart = start; + hostEnd = end; + portStart = -1; + } else { + hostStart = start; + hostEnd = firstColon; + portStart = firstColon + 1; + } + } + if (hostStart == hostEnd) { + throw new LineSenderException("empty host in addr entry [address=") + .put(src.subSequence(start, end)).put("]"); + } + int parsedPort = -1; + if (portStart >= 0) { + if (portStart >= end) { + throw new LineSenderException("invalid address, use IPv4 address or a domain name [address=") + .put(src.subSequence(start, end)).put("]"); + } + try { + parsedPort = Numbers.parseInt(src, portStart, end); + if (parsedPort < 1 || parsedPort > 65535) { + throw new LineSenderException("invalid port [port=").put(parsedPort).put("]"); + } + } catch (NumericException e) { + throw new LineSenderException("cannot parse a port from the address, use IPv4 address or a domain name") + .put(" [address=").put(src.subSequence(start, end)).put("]"); + } + } + if (parsedPort != -1) { + for (int i = 0, n = hosts.size(); i < n; i++) { + String storedHost = hosts.get(i); + if (charsEqualsRange(storedHost, src, hostStart, hostEnd)) { + if (ports.size() > i && ports.getQuick(i) == parsedPort) { + throw new LineSenderException("duplicated addresses are not allowed [address=") + .put(src.subSequence(start, end)).put("]"); + } + } + } + } + hosts.add(src.subSequence(hostStart, hostEnd).toString()); + if (parsedPort != -1) { + ports.add(parsedPort); + } + } + + private static boolean charsEqualsRange(CharSequence a, CharSequence b, int bStart, int bEnd) { + int len = bEnd - bStart; + if (a.length() != len) { + return false; + } + for (int i = 0; i < len; i++) { + if (a.charAt(i) != b.charAt(bStart + i)) { + return false; + } + } + return true; + } + /** * Advanced TLS configuration. Most users should not need to use this. * @@ -1162,7 +1249,8 @@ public Sender build() { errorHandler, actualErrorInboxCapacity, actualDurableAckKeepaliveIntervalMillis, - authTimeoutMillis + authTimeoutMillis, + gorillaEnabled ); } catch (Throwable t) { // connect() failed before ownership of cursorEngine @@ -1174,7 +1262,6 @@ public Sender build() { } throw t; } - connected.setGorillaEnabled(gorillaEnabled); // connect() succeeded — `connected` now owns cursorEngine // via setCursorEngine(engine, true). From here on, ANY // failure must close `connected` (which closes the engine @@ -1986,10 +2073,10 @@ public LineSenderBuilder durableAckKeepaliveIntervalMillis(long millis) { public LineSenderBuilder authTimeoutMillis(long millis) { if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException( - "auth_timeout is only supported for WebSocket transport"); + "auth_timeout_ms is only supported for WebSocket transport"); } if (millis <= 0L) { - throw new LineSenderException("auth_timeout must be > 0: ").put(millis); + throw new LineSenderException("auth_timeout_ms must be > 0: ").put(millis); } this.authTimeoutMillis = millis; return this; @@ -2466,13 +2553,28 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } if (Chars.equals("addr", sink)) { pos = getValue(configurationString, pos, sink, "address"); - address(sink); - if (ports.size() == hosts.size() - 1) { - // not set - port(protocol == PROTOCOL_HTTP ? DEFAULT_HTTP_PORT - : protocol == PROTOCOL_UDP ? DEFAULT_UDP_PORT - : protocol == PROTOCOL_WEBSOCKET ? DEFAULT_WEBSOCKET_PORT - : DEFAULT_TCP_PORT); + int defaultPort = protocol == PROTOCOL_HTTP ? DEFAULT_HTTP_PORT + : protocol == PROTOCOL_UDP ? DEFAULT_UDP_PORT + : protocol == PROTOCOL_WEBSOCKET ? DEFAULT_WEBSOCKET_PORT + : DEFAULT_TCP_PORT; + int valLen = sink.length(); + int entryStart = 0; + for (int i = 0; i <= valLen; i++) { + if (i == valLen || sink.charAt(i) == ',') { + int s = entryStart; + int e = i; + while (s < e && sink.charAt(s) == ' ') s++; + while (e > s && sink.charAt(e - 1) == ' ') e--; + if (s == e) { + throw new LineSenderException("empty addr entry"); + } + int portsBefore = ports.size(); + addAddressEntry(sink, s, e); + if (ports.size() == portsBefore) { + port(defaultPort); + } + entryStart = i + 1; + } } } else if (Chars.equals("user", sink)) { // deprecated key: user, new key: username @@ -2698,12 +2800,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } pos = getValue(configurationString, pos, sink, "close_flush_timeout_millis"); closeFlushTimeoutMillis(parseLongValue(sink, "close_flush_timeout_millis")); - } else if (Chars.equals("auth_timeout", sink)) { + } else if (Chars.equals("auth_timeout_ms", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { - throw new LineSenderException("auth_timeout is only supported for WebSocket transport"); + throw new LineSenderException("auth_timeout_ms is only supported for WebSocket transport"); } - pos = getValue(configurationString, pos, sink, "auth_timeout"); - authTimeoutMillis(parseLongValue(sink, "auth_timeout")); + pos = getValue(configurationString, pos, sink, "auth_timeout_ms"); + authTimeoutMillis(parseLongValue(sink, "auth_timeout_ms")); } else if (Chars.equals("gorilla", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("gorilla is only supported for WebSocket transport"); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index 3569c98c..f616bab8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -42,6 +42,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.List; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -891,10 +892,9 @@ public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handl if (failoverInitialBackoffMs > 0L) { long base = failoverInitialBackoffMs << Math.min(attempt - 1, 30); long capped = Math.min(base, failoverMaxBackoffMs); - long jitter = capped > 0L - ? java.util.concurrent.ThreadLocalRandom.current().nextLong(capped) + long delay = capped > 0L + ? ThreadLocalRandom.current().nextLong(capped + 1L) : 0L; - long delay = Math.min(capped + jitter, failoverMaxBackoffMs); if (delay > 0L) { try { Thread.sleep(delay); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 2c81a881..1af50cbb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -137,10 +137,8 @@ public class QwpWebSocketSender implements Sender { private final GlobalSymbolDictionary globalSymbolDictionary; private final List endpoints; private final QwpHostHealthTracker hostTracker; - private final String host; private final int inFlightWindowSize; private final int maxSchemasPerConnection; - private final int port; private volatile int currentEndpointIdx = -1; private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) @@ -236,8 +234,6 @@ private QwpWebSocketSender( this.endpoints = Collections.unmodifiableList(new ArrayList<>(endpoints)); this.hostTracker = new QwpHostHealthTracker(this.endpoints.size()); this.authorizationHeader = authorizationHeader; - this.host = this.endpoints.get(0).host; - this.port = this.endpoints.get(0).port; this.tlsConfig = tlsConfig; this.encoder = new QwpWebSocketEncoder(DEFAULT_BUFFER_SIZE); this.tableBuffers = new CharSequenceObjHashMap<>(); @@ -498,7 +494,7 @@ public static QwpWebSocketSender connect( closeFlushTimeoutMillis, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, initialConnectMode, errorHandler, errorInboxCapacity, - durableAckKeepaliveIntervalMillis, DEFAULT_AUTH_TIMEOUT_MS); + durableAckKeepaliveIntervalMillis, DEFAULT_AUTH_TIMEOUT_MS, true); } /** @@ -524,7 +520,8 @@ public static QwpWebSocketSender connect( SenderErrorHandler errorHandler, int errorInboxCapacity, long durableAckKeepaliveIntervalMillis, - long authTimeoutMs + long authTimeoutMs, + boolean gorillaEnabled ) { QwpWebSocketSender sender = new QwpWebSocketSender( endpoints, tlsConfig, @@ -534,6 +531,8 @@ public static QwpWebSocketSender connect( try { sender.requestDurableAck = requestDurableAck; sender.authTimeoutMs = authTimeoutMs; + sender.gorillaEnabled = gorillaEnabled; + sender.encoder.setGorillaEnabled(gorillaEnabled); sender.closeFlushTimeoutMillis = closeFlushTimeoutMillis; sender.reconnectMaxDurationMillis = reconnectMaxDurationMillis; sender.reconnectInitialBackoffMillis = reconnectInitialBackoffMillis; @@ -1894,7 +1893,7 @@ private void atNanos(long timestampNanos) { * attempt (and, in the follow-up commit, schedules a backoff retry * within the per-outage time cap). */ - private WebSocketClient buildAndConnect() { + private synchronized WebSocketClient buildAndConnect() { int previousIdx = currentEndpointIdx; if (previousIdx >= 0) { hostTracker.recordMidStreamFailure(previousIdx); @@ -2146,22 +2145,25 @@ private void ensureConnected() { client.close(); client = null; } + Endpoint ep = currentEndpoint(); throw new LineSenderException( - "Failed to start cursor I/O thread for " + host + ":" + port, t); + "Failed to start cursor I/O thread for " + ep.host + ":" + ep.port, t); } if (client != null) { + Endpoint ep = currentEndpoint(); encoder.setVersion((byte) client.getServerQwpVersion()); LOG.info("Connected to WebSocket [host={}, port={}, windowSize={}, qwpVersion={}]", - host, port, inFlightWindowSize, client.getServerQwpVersion()); + ep.host, ep.port, inFlightWindowSize, client.getServerQwpVersion()); } else { // Async mode: I/O thread will drive the connect. Encoder uses // its default version (V1). Schema state still gets reset for // consistency with the sync path; the post-connect replay path // does not need a producer-side reset signal because every // cursor frame is self-sufficient. - LOG.info("Async initial connect deferred to I/O thread [host={}, port={}, windowSize={}]", - host, port, inFlightWindowSize); + Endpoint ep = endpoints.get(0); + LOG.info("Async initial connect deferred to I/O thread [host={}, port={}, endpointCount={}, windowSize={}]", + ep.host, ep.port, endpoints.size(), inFlightWindowSize); } // Server starts fresh on each connection — discard any schema IDs // retained from prior state. Cursor frames are self-sufficient (every @@ -2459,6 +2461,11 @@ private static List singleEndpoint(String host, int port) { return Collections.singletonList(new Endpoint(host, port)); } + private Endpoint currentEndpoint() { + int idx = currentEndpointIdx; + return endpoints.get(Math.max(idx, 0)); + } + public static final class Endpoint { public final String host; public final int port; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index f0658855..a769dede 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -339,6 +339,125 @@ public void testDurableAckKeepaliveIntervalNotSupportedForTcp() { .durableAckKeepaliveIntervalMillis(100)); } + @Test + public void testAuthTimeoutConfig_acceptsPositive() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=localhost:9000;auth_timeout_ms=2500;"); + Assert.assertNotNull(builder); + } + + @Test + public void testAuthTimeoutConfig_zeroRejected() { + assertBadConfig("ws::addr=localhost:9000;auth_timeout_ms=0;", "auth_timeout_ms must be > 0"); + } + + @Test + public void testAuthTimeoutConfig_negativeRejected() { + assertBadConfig("ws::addr=localhost:9000;auth_timeout_ms=-50;", "auth_timeout_ms must be > 0"); + } + + @Test + public void testAuthTimeoutConfig_notSupportedForHttp() { + assertBadConfig("http::addr=localhost:9000;auth_timeout_ms=1000;", + "auth_timeout_ms is only supported for WebSocket transport"); + } + + @Test + public void testAuthTimeoutBuilder_notSupportedForTcp() { + assertThrows("auth_timeout_ms is only supported for WebSocket transport", + () -> Sender.builder(Sender.Transport.TCP) + .address(LOCALHOST) + .authTimeoutMillis(1000)); + } + + @Test + public void testGorillaConfig_acceptsOn() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=localhost:9000;gorilla=on;"); + Assert.assertNotNull(builder); + } + + @Test + public void testGorillaConfig_acceptsOff() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=localhost:9000;gorilla=off;"); + Assert.assertNotNull(builder); + } + + @Test + public void testGorillaConfig_acceptsTrue() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=localhost:9000;gorilla=true;"); + Assert.assertNotNull(builder); + } + + @Test + public void testGorillaConfig_acceptsFalse() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=localhost:9000;gorilla=false;"); + Assert.assertNotNull(builder); + } + + @Test + public void testGorillaConfig_unknownValueRejected() { + assertBadConfig("ws::addr=localhost:9000;gorilla=maybe;", + "invalid gorilla [value=maybe"); + } + + @Test + public void testGorillaConfig_notSupportedForHttp() { + assertBadConfig("http::addr=localhost:9000;gorilla=on;", + "gorilla is only supported for WebSocket transport"); + } + + @Test + public void testGorillaBuilder_notSupportedForTcp() { + assertThrows("gorilla is only supported for WebSocket transport", + () -> Sender.builder(Sender.Transport.TCP) + .address(LOCALHOST) + .gorilla(false)); + } + + @Test + public void testWsConfigString_ipv6Bracketed_withPort() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=[::1]:9000;"); + Assert.assertNotNull(builder); + } + + @Test + public void testWsConfigString_ipv6Bracketed_defaultPort() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=[fe80::1];"); + Assert.assertNotNull(builder); + } + + @Test + public void testWsConfigString_ipv6BareUnbracketed_defaultPort() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=fe80::1;"); + Assert.assertNotNull(builder); + } + + @Test + public void testWsConfigString_ipv6Mixed_multiHost() { + Sender.LineSenderBuilder builder = + Sender.builder("ws::addr=[::1]:9000,a:9001,[fe80::2]:9002;"); + Assert.assertNotNull(builder); + } + + @Test + public void testWsConfigString_ipv6Bracketed_missingClose_fails() { + assertBadConfig("ws::addr=[::1:9000;", "missing closing ']'"); + } + + @Test + public void testWsConfigString_ipv6Bracketed_junkAfterBracket_fails() { + assertBadConfig("ws::addr=[::1]x:9000;", "expected ':' after ']'"); + } + + @Test + public void testWsConfigString_emptyHost_fails() { + assertBadConfig("ws::addr=:9000;", "empty host in addr entry"); + } + + @Test + public void testWsConfigString_emptyBracketedHost_fails() { + assertBadConfig("ws::addr=[]:9000;", "empty host in addr entry"); + } + @Test public void testFullAsyncConfiguration() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) From 6671888a1adc1e808c2d6a333bedb2395790ebe0 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 7 May 2026 00:23:36 +0800 Subject: [PATCH 3/8] code review --- .../main/java/io/questdb/client/Sender.java | 81 ++++++------------- .../cutlass/http/client/WebSocketClient.java | 35 ++++++-- .../client/cutlass/qwp/client/QueryEvent.java | 15 ++++ .../qwp/client/QwpAuthFailedException.java | 59 ++++++++++++++ .../cutlass/qwp/client/QwpEgressIoThread.java | 17 ++-- .../QwpIngressRoleRejectedException.java | 5 +- .../client/QwpProtocolVersionException.java | 38 +++++++++ .../cutlass/qwp/client/QwpQueryClient.java | 69 ++++++++++++---- .../qwp/client/QwpResultBatchDecoder.java | 2 +- .../qwp/client/QwpWebSocketSender.java | 29 ++++++- .../sf/cursor/CursorWebSocketSendLoop.java | 37 +++++++-- .../LineSenderBuilderWebSocketTest.java | 40 --------- 12 files changed, 288 insertions(+), 139 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 1470e7f6..a4a11840 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -62,7 +62,9 @@ import java.security.PrivateKey; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Base64; +import java.util.List; import java.util.concurrent.TimeUnit; /** @@ -835,55 +837,17 @@ public LineSenderBuilder address(CharSequence address) { return this; } - private void addAddressEntry(CharSequence src, int start, int end) { - int hostStart; - int hostEnd; - int portStart; - if (src.charAt(start) == '[') { - int closeBracket = Chars.indexOf(src, start + 1, end, ']'); - if (closeBracket < 0) { - throw new LineSenderException("missing closing ']' in IPv6 addr entry [address=") - .put(src.subSequence(start, end)).put("]"); - } - hostStart = start + 1; - hostEnd = closeBracket; - if (closeBracket == end - 1) { - portStart = -1; - } else if (src.charAt(closeBracket + 1) != ':') { - throw new LineSenderException("expected ':' after ']' in IPv6 addr entry [address=") - .put(src.subSequence(start, end)).put("]"); - } else { - portStart = closeBracket + 2; - } - } else { - int firstColon = Chars.indexOf(src, start, end, ':'); - int lastColon = Chars.indexOf(src, start, end, ':', -1); - if (firstColon != lastColon) { - hostStart = start; - hostEnd = end; - portStart = -1; - } else if (firstColon < 0) { - hostStart = start; - hostEnd = end; - portStart = -1; - } else { - hostStart = start; - hostEnd = firstColon; - portStart = firstColon + 1; - } - } - if (hostStart == hostEnd) { - throw new LineSenderException("empty host in addr entry [address=") + private void addAddressEntry(CharSequence src, int start, int end, int defaultPort) { + int colon = Chars.indexOf(src, start, end, ':'); + if (colon == end - 1) { + throw new LineSenderException("invalid address, use IPv4 address or a domain name [address=") .put(src.subSequence(start, end)).put("]"); } + int hostEnd = colon < 0 ? end : colon; int parsedPort = -1; - if (portStart >= 0) { - if (portStart >= end) { - throw new LineSenderException("invalid address, use IPv4 address or a domain name [address=") - .put(src.subSequence(start, end)).put("]"); - } + if (colon >= 0) { try { - parsedPort = Numbers.parseInt(src, portStart, end); + parsedPort = Numbers.parseInt(src, colon + 1, end); if (parsedPort < 1 || parsedPort > 65535) { throw new LineSenderException("invalid port [port=").put(parsedPort).put("]"); } @@ -892,18 +856,21 @@ private void addAddressEntry(CharSequence src, int start, int end) { .put(" [address=").put(src.subSequence(start, end)).put("]"); } } - if (parsedPort != -1) { - for (int i = 0, n = hosts.size(); i < n; i++) { - String storedHost = hosts.get(i); - if (charsEqualsRange(storedHost, src, hostStart, hostEnd)) { - if (ports.size() > i && ports.getQuick(i) == parsedPort) { - throw new LineSenderException("duplicated addresses are not allowed [address=") - .put(src.subSequence(start, end)).put("]"); - } + if (hostEnd == start) { + throw new LineSenderException("empty host in addr entry [address=") + .put(src.subSequence(start, end)).put("]"); + } + int effectivePort = parsedPort != -1 ? parsedPort : defaultPort; + for (int i = 0, n = hosts.size(); i < n; i++) { + String storedHost = hosts.get(i); + if (charsEqualsRange(storedHost, src, start, hostEnd)) { + if (ports.size() > i && ports.getQuick(i) == effectivePort) { + throw new LineSenderException("duplicated addresses are not allowed [address=") + .put(src.subSequence(start, end)).put("]"); } } } - hosts.add(src.subSequence(hostStart, hostEnd).toString()); + hosts.add(src.subSequence(start, hostEnd).toString()); if (parsedPort != -1) { ports.add(parsedPort); } @@ -1223,8 +1190,8 @@ public Sender build() { int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY ? errorInboxCapacity : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; - java.util.List wsEndpoints = - new java.util.ArrayList<>(hosts.size()); + List wsEndpoints = + new ArrayList<>(hosts.size()); for (int i = 0, n = hosts.size(); i < n; i++) { wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); } @@ -2569,7 +2536,7 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { throw new LineSenderException("empty addr entry"); } int portsBefore = ports.size(); - addAddressEntry(sink, s, e); + addAddressEntry(sink, s, e, defaultPort); if (ports.size() == portsBefore) { port(defaultPort); } diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 579334b5..c8cfa4e5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -135,6 +135,7 @@ public abstract class WebSocketClient implements QuietCloseable { private boolean serverDurableAckEnabled; private int serverQwpVersion = 1; private String upgradeRejectRole; + private int upgradeStatusCode; private boolean upgraded; public WebSocketClient(HttpClientConfiguration configuration, SocketFactory socketFactory) { @@ -299,15 +300,21 @@ public int getServerQwpVersion() { } /** - * If the most recent {@link #upgrade} was rejected with a 503 carrying an - * {@code X-QuestDB-Role} header, returns that role (e.g. {@code REPLICA}, - * {@code PRIMARY_CATCHUP}). Returns null otherwise. Read after a failed - * upgrade to classify the rejection by replication role. + * Role from {@code X-QuestDB-Role} on the most recent rejected upgrade, + * or null when no such header was present. */ public String getUpgradeRejectRole() { return upgradeRejectRole; } + /** + * HTTP status code from the most recent rejected upgrade, or 0 if no + * upgrade rejection has been observed yet. + */ + public int getUpgradeStatusCode() { + return upgradeStatusCode; + } + /** * Returns whether the WebSocket is connected and upgraded. */ @@ -516,6 +523,8 @@ public void upgrade(CharSequence path, int timeout, CharSequence authorizationHe if (upgraded) { return; // Already upgraded } + upgradeRejectRole = null; + upgradeStatusCode = 0; // Generate random key byte[] keyBytes = new byte[16]; @@ -636,6 +645,18 @@ private static boolean extractDurableAckEnabled(String response) { return false; } + private static int parseStatusCode(String statusLine) { + int sp1 = statusLine.indexOf(' '); + if (sp1 < 0 || sp1 + 4 > statusLine.length()) return 0; + int code = 0; + for (int i = sp1 + 1; i < sp1 + 4; i++) { + char c = statusLine.charAt(i); + if (c < '0' || c > '9') return 0; + code = code * 10 + (c - '0'); + } + return code; + } + private static String extractRoleHeader(String response) { int headerLen = QUESTDB_ROLE_HEADER_NAME.length(); int responseLen = response.length(); @@ -647,7 +668,7 @@ private static String extractRoleHeader(String response) { lineEnd = responseLen; } String value = response.substring(valueStart, lineEnd).trim(); - return value.isEmpty() ? null : value; + return value.isEmpty() ? null : value.toUpperCase(java.util.Locale.ROOT); } } return null; @@ -1057,10 +1078,10 @@ private void validateUpgradeResponse(int headerEnd) { } String response = new String(responseBytes, StandardCharsets.US_ASCII); - // Check status line if (!response.startsWith("HTTP/1.1 101")) { String statusLine = response.split("\r\n")[0]; - if (statusLine.startsWith("HTTP/1.1 503")) { + upgradeStatusCode = parseStatusCode(statusLine); + if (upgradeStatusCode == 421) { upgradeRejectRole = extractRoleHeader(response); } throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java index d3083e4c..354ebf7f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java @@ -44,6 +44,13 @@ public class QueryEvent { * having to reconstruct the classification from a side-channel latch. */ public static final int KIND_TRANSPORT_ERROR = 4; + /** + * Permanent protocol-level disagreement (unsupported QWP version, framing + * corruption that won't recover). {@code execute()} surfaces this directly + * instead of triggering failover -- version mismatch is cluster-wide and + * retrying against another endpoint masks the disagreement. + */ + public static final int KIND_PROTOCOL_ERROR = 5; public QwpBatchBuffer buffer; // valid for KIND_BATCH (must be released to pool by consumer) public String errorMessage; // valid for KIND_ERROR @@ -90,6 +97,14 @@ public QueryEvent asTransportError(byte status, String message) { return this; } + public QueryEvent asProtocolError(byte status, String message) { + this.kind = KIND_PROTOCOL_ERROR; + this.buffer = null; + this.errorStatus = status; + this.errorMessage = message; + return this; + } + /** * Clears object references and resets primitive fields so a pooled event is * safe to reuse across queries. The I/O thread calls the {@code asX(...)} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java new file mode 100644 index 00000000..b9e3f444 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java @@ -0,0 +1,59 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; + +/** + * WebSocket upgrade rejected with {@code 401}/{@code 403}/{@code 404}. Terminal + * across all configured endpoints: a rejected credential is uniformly rejected + * across the cluster, so failing fast keeps server logs clean and surfaces the + * configuration error immediately. + */ +public final class QwpAuthFailedException extends HttpClientException { + private final String host; + private final int port; + private final int statusCode; + + public QwpAuthFailedException(int statusCode, String host, int port) { + super("WebSocket upgrade rejected with HTTP "); + put(statusCode).put(" for ").put(host).put(':').put(port); + this.statusCode = statusCode; + this.host = host; + this.port = port; + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } + + public int getStatusCode() { + return statusCode; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java index 0569c631..8560e11d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java @@ -535,6 +535,11 @@ private void emitTerminalTransportError(String message) { events.offer(new QueryEvent().asTransportError(WebSocketResponse.STATUS_INTERNAL_ERROR, message)); } + private void emitTerminalProtocolError(String message) { + notifyTerminalFailure(message); + events.offer(new QueryEvent().asProtocolError(WebSocketResponse.STATUS_INTERNAL_ERROR, message)); + } + /** * Like {@link #emitError} but emits a {@code KIND_TRANSPORT_ERROR} event * rather than {@code KIND_ERROR}, and retries until the event is enqueued. @@ -594,15 +599,17 @@ private void handleResultBatch(long payloadPtr, int payloadLen) { // buffer) directly, skipping the previous per-batch memcpy into buf.scratchAddr. try { decoder.decode(buf, payloadPtr, payloadLen); + } catch (QwpProtocolVersionException e) { + if (!freeBuffers.offer(buf)) { + buf.close(); + } + emitTerminalProtocolError(e.getMessage()); + currentQueryDone = true; + return; } catch (QwpDecodeException e) { - // Same invariant as releaseBuffer: a slot is always free for a buf - // we took out of the pool moments ago. Close-on-failure is a - // defensive guard against future refactors breaking that invariant. if (!freeBuffers.offer(buf)) { buf.close(); } - // A decode failure leaves the client-side decoder out of step with - // the server's byte stream: the next frame cannot be trusted. emitTerminalTransportError("decode failure: " + e.getMessage()); currentQueryDone = true; return; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java index f66fea66..1f1c4c6a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java @@ -28,7 +28,7 @@ /** * Raised when the server rejects a {@code /write/v4} WebSocket upgrade with a - * {@code 503 Service Unavailable} carrying an {@code X-QuestDB-Role} header. + * {@code 421 Misdirected Request} carrying an {@code X-QuestDB-Role} header. * Carries the role name so the host-health tracker can classify the endpoint * as transiently unavailable ({@code PRIMARY_CATCHUP}) versus structurally * unwritable ({@code REPLICA}). @@ -44,7 +44,8 @@ public final class QwpIngressRoleRejectedException extends HttpClientException { private final String role; public QwpIngressRoleRejectedException(String role, String host, int port) { - super("WebSocket ingress upgrade rejected by role=" + role + " at " + host + ":" + port); + super("WebSocket ingress upgrade rejected by role="); + put(role).put(" at ").put(host).put(':').put(port); this.role = role; this.host = host; this.port = port; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java new file mode 100644 index 00000000..c824318f --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java @@ -0,0 +1,38 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Server negotiated a QWP version outside {@code [VERSION_1, MAX_SUPPORTED_VERSION]}. + * Terminal: version negotiation is cluster-wide, so failover masks the disagreement. + */ +public class QwpProtocolVersionException extends QwpDecodeException { + public static final QwpProtocolVersionException UNSUPPORTED = new QwpProtocolVersionException( + "unsupported QWP version"); + + public QwpProtocolVersionException(String message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index f616bab8..2392f3a9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -41,6 +41,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Base64; +import java.util.Collections; import java.util.List; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; @@ -161,6 +162,7 @@ public class QwpQueryClient implements QuietCloseable { // full shutdown path again and double-free {@link #bindValues} native // scratch. private final AtomicBoolean closedFlag = new AtomicBoolean(); + private final AtomicBoolean executing = new AtomicBoolean(); private final List endpoints = new ArrayList<>(); private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; private String authorizationHeader; @@ -750,9 +752,11 @@ public void connect() { } if (hostTracker == null) { if (LB_RANDOM.equals(lbStrategy) && endpoints.size() > 1) { - java.util.Collections.shuffle(endpoints, java.util.concurrent.ThreadLocalRandom.current()); + Collections.shuffle(endpoints, ThreadLocalRandom.current()); } hostTracker = new QwpHostHealthTracker(endpoints.size()); + } else { + hostTracker.beginRound(false); } QwpServerInfo lastObservedMismatch = null; boolean sawV1Mismatch = false; @@ -765,6 +769,9 @@ public void connect() { Endpoint ep = endpoints.get(i); try { connectToEndpoint(ep); + } catch (QwpAuthFailedException ae) { + cleanupFailedConnect(); + throw ae; } catch (QwpIngressRoleRejectedException re) { lastTransportError = re; hostTracker.recordRoleReject(i, re.isTransient()); @@ -854,10 +861,22 @@ public void execute(String sql, QwpColumnBatchHandler handler) { * defeats this reuse. */ public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handler) { + if (!executing.compareAndSet(false, true)) { + throw new IllegalStateException( + "QwpQueryClient.execute called while another execute is in flight; one query at a time per client"); + } + try { + executeImpl(sql, binds, handler); + } finally { + executing.set(false); + } + } + + private void executeImpl(String sql, QwpBindSetter binds, QwpColumnBatchHandler handler) { if (!connected) { throw new IllegalStateException("QwpQueryClient not connected; call connect() first"); } - hostTracker.beginRound(true); + hostTracker.beginRound(false); long failoverDeadlineMs = failoverMaxDurationMs > 0L ? System.currentTimeMillis() + failoverMaxDurationMs : Long.MAX_VALUE; @@ -893,8 +912,22 @@ public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handl long base = failoverInitialBackoffMs << Math.min(attempt - 1, 30); long capped = Math.min(base, failoverMaxBackoffMs); long delay = capped > 0L - ? ThreadLocalRandom.current().nextLong(capped + 1L) + ? ThreadLocalRandom.current().nextLong(capped) : 0L; + long remaining = failoverDeadlineMs - System.currentTimeMillis(); + if (remaining <= 0L) { + int failovers = Math.max(0, attempt - 1); + handler.onError(probe.interceptedStatus, + "transport failure after " + attempt + " execute attempt" + + (attempt == 1 ? "" : "s") + " (" + + failovers + " failover reconnect" + + (failovers == 1 ? "" : "s") + "); last error: " + + probe.interceptedMessage); + return; + } + if (delay > remaining) { + delay = remaining; + } if (delay > 0L) { try { Thread.sleep(delay); @@ -909,6 +942,8 @@ public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handl } try { reconnectViaTracker(); + } catch (QwpAuthFailedException ae) { + throw ae; } catch (RuntimeException reconnectErr) { handler.onError(probe.interceptedStatus, "failover reconnect failed after " + attempt + " attempt" @@ -1436,15 +1471,6 @@ private void cleanupFailedConnect() { } private void runUpgradeWithTimeout(Endpoint ep) { - if (authTimeoutMs <= 0L) { - webSocketClient.connect(ep.host, ep.port); - webSocketClient.upgrade(endpointPath, authorizationHeader); - return; - } - // The TCP connect itself isn't bounded -- cutlass native socket has no connect-timeout knob, - // so a routing blackhole (no SYN-ACK) still falls back to the OS default. The HTTP upgrade - // response read IS bounded via WebSocketClient's own per-call timeout; that covers the common - // "accepts TCP but never replies" blackhole scenario auth_timeout targets. webSocketClient.connect(ep.host, ep.port); int timeoutMs = (int) Math.min(authTimeoutMs, Integer.MAX_VALUE); try { @@ -1455,6 +1481,7 @@ private void runUpgradeWithTimeout(Endpoint ep) { .put(ep.host).put(':').put(ep.port) .put(" exceeded auth_timeout=").put(authTimeoutMs).put("ms"); timeout.initCause(ex); + timeout.flagAsTimeout(); throw timeout; } String role = webSocketClient.getUpgradeRejectRole(); @@ -1463,6 +1490,12 @@ private void runUpgradeWithTimeout(Endpoint ep) { re.initCause(ex); throw re; } + int status = webSocketClient.getUpgradeStatusCode(); + if (status == 401 || status == 403 || status == 404) { + QwpAuthFailedException ae = new QwpAuthFailedException(status, ep.host, ep.port); + ae.initCause(ex); + throw ae; + } throw ex; } } @@ -1596,12 +1629,13 @@ private void executeOnce(String sql, QwpBindSetter binds, FailoverProbeHandler p probe.onError(ev.errorStatus, ev.errorMessage); return; case QueryEvent.KIND_TRANSPORT_ERROR: - // Transport / protocol-level failure synthesised by the - // I/O thread. Tag as a transport failure so the outer - // execute loop can decide whether to replay (failover=on) - // or surface as a final onError (failover=off). + // Transport-level failure -- replay candidate. probe.markTransportFailure(ev.errorStatus, ev.errorMessage); return; + case QueryEvent.KIND_PROTOCOL_ERROR: + // Permanent protocol disagreement -- surface to user, no failover. + probe.onError(ev.errorStatus, ev.errorMessage); + return; default: probe.onError(WebSocketResponse.STATUS_INTERNAL_ERROR, "unknown event kind " + ev.kind); return; @@ -1715,6 +1749,9 @@ private void reconnectViaTracker() { Endpoint ep = endpoints.get(i); try { connectToEndpoint(ep); + } catch (QwpAuthFailedException ae) { + cleanupFailedConnect(); + throw ae; } catch (QwpIngressRoleRejectedException re) { lastError = re; hostTracker.recordRoleReject(i, re.isTransient()); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java index 3fadb6e0..55b16f54 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java @@ -298,7 +298,7 @@ private void decodePayload(QwpBatchBuffer buffer, long payload, int payloadLen) } byte version = Unsafe.getUnsafe().getByte(payload + 4); if (version < QwpConstants.VERSION_1 || version > QwpConstants.MAX_SUPPORTED_VERSION) { - throw new QwpDecodeException("unsupported version " + (version & 0xFF)); + throw QwpProtocolVersionException.UNSUPPORTED; } byte flags = Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS); deltaMode = (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 1af50cbb..1e47dd21 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -139,7 +139,6 @@ public class QwpWebSocketSender implements Sender { private final QwpHostHealthTracker hostTracker; private final int inFlightWindowSize; private final int maxSchemasPerConnection; - private volatile int currentEndpointIdx = -1; private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) private final ClientTlsConfiguration tlsConfig; @@ -155,10 +154,11 @@ public class QwpWebSocketSender implements Sender { // 0 or -1 means "fast close" (skip the drain); otherwise close blocks // up to this many millis for ackedFsn to catch up to publishedFsn. private long closeFlushTimeoutMillis = 5_000L; - private boolean closed; + private volatile boolean closed; private boolean connected; // Track max global symbol ID used in current batch (for delta calculation) private int currentBatchMaxSymbolId = -1; + private volatile int currentEndpointIdx = -1; private QwpTableBuffer currentTableBuffer; private String currentTableName; // Cursor SF engine: the producer (user thread) writes encoded QWP frames @@ -1903,8 +1903,12 @@ private synchronized WebSocketClient buildAndConnect() { hostTracker.beginRound(true); } Throwable lastError = null; + Throwable terminalUpgradeError = null; Endpoint lastEndpoint = null; while (true) { + if (closed) { + throw new LineSenderException("sender closed during connect"); + } int idx = hostTracker.pickNext(); if (idx < 0) break; Endpoint ep = endpoints.get(idx); @@ -1921,6 +1925,7 @@ private synchronized WebSocketClient buildAndConnect() { newClient.upgrade(WRITE_PATH, upgradeTimeoutMs, authorizationHeader); } catch (HttpClientException e) { String role = newClient.getUpgradeRejectRole(); + int status = newClient.getUpgradeStatusCode(); newClient.close(); if (role != null) { boolean isTransient = QwpIngressRoleRejectedException.ROLE_PRIMARY_CATCHUP.equals(role); @@ -1930,8 +1935,16 @@ private synchronized WebSocketClient buildAndConnect() { lastError = re; continue; } + if (status == 401 || status == 403 || status == 404) { + QwpAuthFailedException ae = new QwpAuthFailedException(status, ep.host, ep.port); + ae.initCause(e); + throw ae; + } hostTracker.recordTransportError(idx); lastError = e; + if (terminalUpgradeError == null && isUpgradeFailedSentinel(e)) { + terminalUpgradeError = e; + } continue; } catch (Exception e) { newClient.close(); @@ -1954,6 +1967,11 @@ private synchronized WebSocketClient buildAndConnect() { currentEndpointIdx = idx; return newClient; } + if (terminalUpgradeError != null) { + throw new LineSenderException( + "Failed to connect: WebSocket upgrade failed across " + endpoints.size() + " endpoint(s)", + terminalUpgradeError); + } String summary = lastEndpoint == null ? "no endpoints available" : "all " + endpoints.size() + " endpoint(s) unreachable; last=" @@ -1961,6 +1979,11 @@ private synchronized WebSocketClient buildAndConnect() { throw new LineSenderException("Failed to connect: " + summary, lastError); } + private static boolean isUpgradeFailedSentinel(Throwable e) { + String msg = e.getMessage(); + return msg != null && msg.contains("WebSocket upgrade failed:"); + } + private void checkConnectionError() { LineSenderException error = connectionError.get(); if (error != null) { @@ -2162,7 +2185,7 @@ private void ensureConnected() { // does not need a producer-side reset signal because every // cursor frame is self-sufficient. Endpoint ep = endpoints.get(0); - LOG.info("Async initial connect deferred to I/O thread [host={}, port={}, endpointCount={}, windowSize={}]", + LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}, windowSize={}]", ep.host, ep.port, endpoints.size(), inFlightWindowSize); } // Server starts fresh on each connection — discard any schema IDs diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 5fe71075..024694fe 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -605,15 +605,20 @@ private void connectLoop(Throwable initial, String phase) { System.nanoTime() ); totalServerErrors.incrementAndGet(); - // recordFatal MUST run before dispatchError: the spec - // requires signal.terminalError to be latched BEFORE the - // handler is invoked, so a handler that synchronously - // probes getLastTerminalError() (or calls flush()) sees - // the typed error rather than null. recordFatal(new LineSenderServerException(err), err); dispatchError(err); return; } + if (isRoleReject(e)) { + backoffMillis = reconnectInitialBackoffMillis; + long roleSleepStart = System.nanoTime(); + if (running) { + LockSupport.parkNanos(reconnectInitialBackoffMillis * 1_000_000L); + } + deadlineNanos += System.nanoTime() - roleSleepStart; + lastReconnectError = e; + continue; + } lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -621,9 +626,6 @@ private void connectLoop(Throwable initial, String phase) { lastLogNanos = now; } } - // Backoff with jitter: sleep [backoff, 2*backoff). Cap the - // sleep at the remaining budget so we don't oversleep past - // the deadline. if (running) { long jitter = ThreadLocalRandom.current().nextLong(backoffMillis); long sleepMillis = backoffMillis + jitter; @@ -732,9 +734,28 @@ private void recordFatal(Throwable t, SenderError serverError) { * handshake) are treated as transient. */ private static boolean isTerminalUpgradeError(Throwable t) { + if (isRoleReject(t)) { + return false; + } + for (Throwable cur = t; cur != null; cur = cur.getCause()) { + if (cur instanceof io.questdb.client.cutlass.qwp.client.QwpAuthFailedException) { + return true; + } + if (cur.getCause() == cur) break; + } return findUpgradeFailureMessage(t) != null; } + private static boolean isRoleReject(Throwable t) { + for (Throwable cur = t; cur != null; cur = cur.getCause()) { + if (cur instanceof io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException) { + return true; + } + if (cur.getCause() == cur) break; + } + return false; + } + /** * Walks the cause chain looking for the WebSocketClient's * "WebSocket upgrade failed:" sentinel and returns its message, or diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index a769dede..fc3c1391 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -413,51 +413,11 @@ public void testGorillaBuilder_notSupportedForTcp() { .gorilla(false)); } - @Test - public void testWsConfigString_ipv6Bracketed_withPort() { - Sender.LineSenderBuilder builder = Sender.builder("ws::addr=[::1]:9000;"); - Assert.assertNotNull(builder); - } - - @Test - public void testWsConfigString_ipv6Bracketed_defaultPort() { - Sender.LineSenderBuilder builder = Sender.builder("ws::addr=[fe80::1];"); - Assert.assertNotNull(builder); - } - - @Test - public void testWsConfigString_ipv6BareUnbracketed_defaultPort() { - Sender.LineSenderBuilder builder = Sender.builder("ws::addr=fe80::1;"); - Assert.assertNotNull(builder); - } - - @Test - public void testWsConfigString_ipv6Mixed_multiHost() { - Sender.LineSenderBuilder builder = - Sender.builder("ws::addr=[::1]:9000,a:9001,[fe80::2]:9002;"); - Assert.assertNotNull(builder); - } - - @Test - public void testWsConfigString_ipv6Bracketed_missingClose_fails() { - assertBadConfig("ws::addr=[::1:9000;", "missing closing ']'"); - } - - @Test - public void testWsConfigString_ipv6Bracketed_junkAfterBracket_fails() { - assertBadConfig("ws::addr=[::1]x:9000;", "expected ':' after ']'"); - } - @Test public void testWsConfigString_emptyHost_fails() { assertBadConfig("ws::addr=:9000;", "empty host in addr entry"); } - @Test - public void testWsConfigString_emptyBracketedHost_fails() { - assertBadConfig("ws::addr=[]:9000;", "empty host in addr entry"); - } - @Test public void testFullAsyncConfiguration() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) From fad90cc4af44ec8f2d65752ffc50734ab5e6860a Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 7 May 2026 00:24:55 +0800 Subject: [PATCH 4/8] use 421 instead of 503 --- ...Test.java => WebSocketClient421RoleHeaderTest.java} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename core/src/test/java/io/questdb/client/test/cutlass/http/client/{WebSocketClient503RoleHeaderTest.java => WebSocketClient421RoleHeaderTest.java} (93%) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient503RoleHeaderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient421RoleHeaderTest.java similarity index 93% rename from core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient503RoleHeaderTest.java rename to core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient421RoleHeaderTest.java index 281c5884..533e8bfe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient503RoleHeaderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient421RoleHeaderTest.java @@ -36,7 +36,7 @@ import java.net.Socket; import java.nio.charset.StandardCharsets; -public class WebSocketClient503RoleHeaderTest { +public class WebSocketClient421RoleHeaderTest { @Test(timeout = 10_000) public void testReplicaReject() throws Exception { @@ -49,7 +49,7 @@ public void testPrimaryCatchupReject() throws Exception { } @Test(timeout = 10_000) - public void testRoleHeaderAbsentOn503_NullCaptured() throws Exception { + public void testRoleHeaderAbsentOn421_NullCaptured() throws Exception { assertCapturedRole(null, null); } @@ -67,7 +67,7 @@ private static void assertCapturedRole(String expected, String roleHeaderLine) t int n = s.getInputStream().read(discardBuf); if (n < 0) return; StringBuilder resp = new StringBuilder(); - resp.append("HTTP/1.1 503 Service Unavailable\r\n"); + resp.append("HTTP/1.1 421 Misdirected Request\r\n"); if (roleHeaderLine != null) resp.append(roleHeaderLine).append("\r\n"); resp.append("Content-Length: 0\r\n\r\n"); OutputStream os = s.getOutputStream(); @@ -75,7 +75,7 @@ private static void assertCapturedRole(String expected, String roleHeaderLine) t os.flush(); } catch (Exception ignored) { } - }, "fake-503"); + }, "fake-421"); serverThread.setDaemon(true); serverThread.start(); @@ -84,7 +84,7 @@ private static void assertCapturedRole(String expected, String roleHeaderLine) t client.connect("127.0.0.1", port); try { client.upgrade("/write/v4", null); - Assert.fail("expected upgrade to fail with 503"); + Assert.fail("expected upgrade to fail with 421"); } catch (HttpClientException ex) { Assert.assertTrue( "expected 'WebSocket upgrade failed:' message, got: " + ex.getMessage(), From cab44291ae00d601d15ed355672a477f62b2d934 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 7 May 2026 00:32:33 +0800 Subject: [PATCH 5/8] fix tests --- .../client/cutlass/qwp/client/QueryEvent.java | 2 +- .../qwp/client/QwpProtocolVersionException.java | 2 +- .../client/LineSenderBuilderWebSocketTest.java | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java index 354ebf7f..1eb299dc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java @@ -45,7 +45,7 @@ public class QueryEvent { */ public static final int KIND_TRANSPORT_ERROR = 4; /** - * Permanent protocol-level disagreement (unsupported QWP version, framing + * Permanent protocol-level disagreement (unsupported version, framing * corruption that won't recover). {@code execute()} surfaces this directly * instead of triggering failover -- version mismatch is cluster-wide and * retrying against another endpoint masks the disagreement. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java index c824318f..9905cfc5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java @@ -30,7 +30,7 @@ */ public class QwpProtocolVersionException extends QwpDecodeException { public static final QwpProtocolVersionException UNSUPPORTED = new QwpProtocolVersionException( - "unsupported QWP version"); + "unsupported version"); public QwpProtocolVersionException(String message) { super(message); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index fc3c1391..16f55dce 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -418,6 +418,21 @@ public void testWsConfigString_emptyHost_fails() { assertBadConfig("ws::addr=:9000;", "empty host in addr entry"); } + @Test + public void testWsConfigString_dupAddr_explicitThenDefaultPort_fails() { + assertBadConfig("ws::addr=a:9000,a;", "duplicated addresses are not allowed"); + } + + @Test + public void testWsConfigString_dupAddr_defaultThenExplicitPort_fails() { + assertBadConfig("ws::addr=a,a:9000;", "duplicated addresses are not allowed"); + } + + @Test + public void testWsConfigString_dupAddr_bothDefaultPort_fails() { + assertBadConfig("ws::addr=a,a;", "duplicated addresses are not allowed"); + } + @Test public void testFullAsyncConfiguration() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) From 33b4420c782bf0d2a4858fe329959541be4499c6 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 7 May 2026 10:15:38 +0800 Subject: [PATCH 6/8] code review --- .../main/java/io/questdb/client/Sender.java | 29 ++-- .../cutlass/http/client/WebSocketClient.java | 15 ++- .../qwp/client/QwpAuthFailedException.java | 9 +- .../qwp/client/QwpHostHealthTracker.java | 24 +++- .../QwpIngressRoleRejectedException.java | 4 +- .../client/QwpProtocolVersionException.java | 11 +- .../cutlass/qwp/client/QwpQueryClient.java | 47 +++++-- .../qwp/client/QwpResultBatchDecoder.java | 2 +- .../qwp/client/QwpWebSocketSender.java | 61 ++++++--- .../sf/cursor/CursorWebSocketSendLoop.java | 14 +- .../WebSocketClient421RoleHeaderTest.java | 55 ++++++++ .../cutlass/line/LineSenderBuilderTest.java | 2 +- .../LineSenderBuilderWebSocketTest.java | 36 +++++ .../qwp/client/QwpHostHealthTrackerTest.java | 14 ++ .../QwpQueryClientExecutingFlagTest.java | 95 +++++++++++++ .../QwpQueryClientUpgradeStatusTest.java | 126 ++++++++++++++++++ 16 files changed, 480 insertions(+), 64 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientExecutingFlagTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUpgradeStatusTest.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index a4a11840..4e574524 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1085,7 +1085,7 @@ public Sender build() { } if (protocol == PROTOCOL_WEBSOCKET) { - if (hosts.size() < 1 || ports.size() != hosts.size()) { + if (hosts.size() < 1) { throw new LineSenderException("WebSocket transport requires at least one host:port pair"); } @@ -2405,16 +2405,19 @@ private void configureDefaults() { if (maximumBufferCapacity == PARAMETER_NOT_SET_EXPLICITLY) { maximumBufferCapacity = protocol == PROTOCOL_HTTP ? DEFAULT_MAXIMUM_BUFFER_CAPACITY : bufferCapacity; } - if (ports.size() == 0) { - if (protocol == PROTOCOL_HTTP) { - ports.add(DEFAULT_HTTP_PORT); - } else if (protocol == PROTOCOL_UDP) { - ports.add(DEFAULT_UDP_PORT); - } else if (protocol == PROTOCOL_WEBSOCKET) { - ports.add(DEFAULT_WEBSOCKET_PORT); - } else { - ports.add(DEFAULT_TCP_PORT); - } + int defaultPort; + if (protocol == PROTOCOL_HTTP) { + defaultPort = DEFAULT_HTTP_PORT; + } else if (protocol == PROTOCOL_UDP) { + defaultPort = DEFAULT_UDP_PORT; + } else if (protocol == PROTOCOL_WEBSOCKET) { + defaultPort = DEFAULT_WEBSOCKET_PORT; + } else { + defaultPort = DEFAULT_TCP_PORT; + } + int hostsCount = Math.max(hosts.size(), 1); + while (ports.size() < hostsCount) { + ports.add(defaultPort); } if (tlsValidationMode == null) { tlsValidationMode = TlsValidationMode.DEFAULT; @@ -2530,8 +2533,8 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { if (i == valLen || sink.charAt(i) == ',') { int s = entryStart; int e = i; - while (s < e && sink.charAt(s) == ' ') s++; - while (e > s && sink.charAt(e - 1) == ' ') e--; + while (s < e && Character.isWhitespace(sink.charAt(s))) s++; + while (e > s && Character.isWhitespace(sink.charAt(e - 1))) e--; if (s == e) { throw new LineSenderException("empty addr entry"); } diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index c8cfa4e5..aaf78053 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -648,6 +648,10 @@ private static boolean extractDurableAckEnabled(String response) { private static int parseStatusCode(String statusLine) { int sp1 = statusLine.indexOf(' '); if (sp1 < 0 || sp1 + 4 > statusLine.length()) return 0; + char afterCode = statusLine.charAt(sp1 + 4); + if (afterCode != ' ' && afterCode != '\r' && afterCode != '\n') { + return 0; + } int code = 0; for (int i = sp1 + 1; i < sp1 + 4; i++) { char c = statusLine.charAt(i); @@ -660,16 +664,19 @@ private static int parseStatusCode(String statusLine) { private static String extractRoleHeader(String response) { int headerLen = QUESTDB_ROLE_HEADER_NAME.length(); int responseLen = response.length(); - for (int i = 0; i <= responseLen - headerLen; i++) { - if (response.regionMatches(true, i, QUESTDB_ROLE_HEADER_NAME, 0, headerLen)) { - int valueStart = i + headerLen; + int lineStart = response.indexOf("\r\n"); + while (lineStart >= 0 && lineStart + 2 + headerLen <= responseLen) { + int hStart = lineStart + 2; + if (response.regionMatches(true, hStart, QUESTDB_ROLE_HEADER_NAME, 0, headerLen)) { + int valueStart = hStart + headerLen; int lineEnd = response.indexOf('\r', valueStart); if (lineEnd < 0) { lineEnd = responseLen; } String value = response.substring(valueStart, lineEnd).trim(); - return value.isEmpty() ? null : value.toUpperCase(java.util.Locale.ROOT); + return value.isEmpty() ? null : value; } + lineStart = response.indexOf("\r\n", hStart); } return null; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java index b9e3f444..555670e6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java @@ -27,10 +27,11 @@ import io.questdb.client.cutlass.http.client.HttpClientException; /** - * WebSocket upgrade rejected with {@code 401}/{@code 403}/{@code 404}. Terminal - * across all configured endpoints: a rejected credential is uniformly rejected - * across the cluster, so failing fast keeps server logs clean and surfaces the - * configuration error immediately. + * WebSocket upgrade rejected with {@code 401} or {@code 403}. Terminal across all + * configured endpoints: a rejected credential is uniformly rejected across the + * cluster, so failing fast surfaces the configuration error immediately. Path + * mismatches ({@code 404}) are NOT routed through this exception because a single + * misconfigured node mid-deploy can return 404 while peers are healthy. */ public final class QwpAuthFailedException extends HttpClientException { private final String host; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java index 8c433c4e..9efd517f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java @@ -31,6 +31,11 @@ * Within a round, {@link #pickNext()} returns the highest-priority host that * has not yet been attempted; the caller advances the round via * {@link #beginRound(boolean)}. + *

+ * Each method is internally synchronized, but pickNext + recordX is not atomic + * across the pair. Callers must externally serialize a pick → record sequence + * (the QWP clients do this via the sender's {@code synchronized buildAndConnect} + * and the query client's documented one-execute-at-a-time contract). */ public final class QwpHostHealthTracker { public enum HostState { @@ -51,8 +56,10 @@ public enum HostState { private final boolean[] attemptedThisRound; private final int hostCount; + private final long[] lastSuccessEpoch; private final Object lock = new Object(); private final HostState[] states; + private long successEpoch; public QwpHostHealthTracker(int hostCount) { if (hostCount <= 0) { @@ -61,6 +68,7 @@ public QwpHostHealthTracker(int hostCount) { this.hostCount = hostCount; this.states = new HostState[hostCount]; this.attemptedThisRound = new boolean[hostCount]; + this.lastSuccessEpoch = new long[hostCount]; for (int i = 0; i < hostCount; i++) { states[i] = HostState.UNKNOWN; } @@ -68,16 +76,19 @@ public QwpHostHealthTracker(int hostCount) { /** * Resets attempted flags. With {@code forgetClassifications}, every host - * except the last-known {@link HostState#HEALTHY} entry is reset to - * {@link HostState#UNKNOWN}; the sticky-Healthy keeps the last successful - * host first in line on the next round. + * except the most-recently-successful {@link HostState#HEALTHY} entry is + * reset to {@link HostState#UNKNOWN}; the sticky-Healthy keeps the last + * successful host first in line on the next round. Recency uses the + * {@code recordSuccess} epoch counter, not array order. */ public void beginRound(boolean forgetClassifications) { synchronized (lock) { int stickyIndex = -1; if (forgetClassifications) { + long bestEpoch = -1L; for (int i = 0; i < hostCount; i++) { - if (states[i] == HostState.HEALTHY) { + if (states[i] == HostState.HEALTHY && lastSuccessEpoch[i] > bestEpoch) { + bestEpoch = lastSuccessEpoch[i]; stickyIndex = i; } } @@ -114,7 +125,9 @@ public boolean isRoundExhausted() { /** * Returns the highest-priority host not yet attempted this round, or -1 - * when the round is exhausted. + * when the round is exhausted. The caller is expected to be externally + * serialized (see class doc): the returned index is intended to be paired + * with a follow-up {@code recordX(idx)} on the same logical thread. */ public int pickNext() { synchronized (lock) { @@ -152,6 +165,7 @@ public void recordSuccess(int idx) { synchronized (lock) { states[idx] = HostState.HEALTHY; attemptedThisRound[idx] = true; + lastSuccessEpoch[idx] = ++successEpoch; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java index 1f1c4c6a..6d520eb7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpIngressRoleRejectedException.java @@ -64,10 +64,10 @@ public String getRole() { } public boolean isTopological() { - return ROLE_REPLICA.equals(role); + return ROLE_REPLICA.equalsIgnoreCase(role); } public boolean isTransient() { - return ROLE_PRIMARY_CATCHUP.equals(role); + return ROLE_PRIMARY_CATCHUP.equalsIgnoreCase(role); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java index 9905cfc5..03f2bcaa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java @@ -24,15 +24,22 @@ package io.questdb.client.cutlass.qwp.client; +import io.questdb.client.std.Misc; +import io.questdb.client.std.str.StringSink; + /** * Server negotiated a QWP version outside {@code [VERSION_1, MAX_SUPPORTED_VERSION]}. * Terminal: version negotiation is cluster-wide, so failover masks the disagreement. */ public class QwpProtocolVersionException extends QwpDecodeException { - public static final QwpProtocolVersionException UNSUPPORTED = new QwpProtocolVersionException( - "unsupported version"); public QwpProtocolVersionException(String message) { super(message); } + + public static QwpProtocolVersionException unsupported(int version) { + StringSink sink = Misc.getThreadLocalSink(); + sink.put("unsupported version ").put(version); + return new QwpProtocolVersionException(sink.toString()); + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index 2392f3a9..7b959ed4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -746,13 +746,19 @@ public void close() { * endpoints unreachable" (the latter surfaces as a plain * {@link HttpClientException}). */ - public void connect() { + public synchronized void connect() { + if (closedFlag.get()) { + throw new IllegalStateException("QwpQueryClient is closed"); + } if (connected) { return; } if (hostTracker == null) { if (LB_RANDOM.equals(lbStrategy) && endpoints.size() > 1) { - Collections.shuffle(endpoints, ThreadLocalRandom.current()); + List shuffled = new ArrayList<>(endpoints); + Collections.shuffle(shuffled, ThreadLocalRandom.current()); + endpoints.clear(); + endpoints.addAll(shuffled); } hostTracker = new QwpHostHealthTracker(endpoints.size()); } else { @@ -761,9 +767,15 @@ public void connect() { QwpServerInfo lastObservedMismatch = null; boolean sawV1Mismatch = false; Throwable lastTransportError = null; + boolean retriedAfterReset = false; while (true) { int i = hostTracker.pickNext(); if (i < 0) { + if (!retriedAfterReset) { + hostTracker.beginRound(true); + retriedAfterReset = true; + continue; + } break; } Endpoint ep = endpoints.get(i); @@ -873,13 +885,25 @@ public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handl } private void executeImpl(String sql, QwpBindSetter binds, QwpColumnBatchHandler handler) { + if (closedFlag.get()) { + throw new IllegalStateException("QwpQueryClient is closed"); + } if (!connected) { throw new IllegalStateException("QwpQueryClient not connected; call connect() first"); } hostTracker.beginRound(false); - long failoverDeadlineMs = failoverMaxDurationMs > 0L - ? System.currentTimeMillis() + failoverMaxDurationMs - : Long.MAX_VALUE; + long failoverDeadlineNanos; + if (failoverMaxDurationMs > 0L) { + long durationNanos = failoverMaxDurationMs > Long.MAX_VALUE / 1_000_000L + ? Long.MAX_VALUE + : failoverMaxDurationMs * 1_000_000L; + long start = System.nanoTime(); + failoverDeadlineNanos = start + durationNanos < start + ? Long.MAX_VALUE + : start + durationNanos; + } else { + failoverDeadlineNanos = Long.MAX_VALUE; + } int attempt = 0; while (true) { attempt++; @@ -892,7 +916,7 @@ private void executeImpl(String sql, QwpBindSetter binds, QwpColumnBatchHandler handler.onError(probe.interceptedStatus, probe.interceptedMessage); return; } - if (attempt >= failoverMaxAttempts || System.currentTimeMillis() >= failoverDeadlineMs) { + if (attempt >= failoverMaxAttempts || System.nanoTime() >= failoverDeadlineNanos) { int failovers = Math.max(0, attempt - 1); handler.onError(probe.interceptedStatus, "transport failure after " + attempt + " execute attempt" @@ -910,11 +934,12 @@ private void executeImpl(String sql, QwpBindSetter binds, QwpColumnBatchHandler connected = false; if (failoverInitialBackoffMs > 0L) { long base = failoverInitialBackoffMs << Math.min(attempt - 1, 30); + if (base < 0L) base = failoverMaxBackoffMs; long capped = Math.min(base, failoverMaxBackoffMs); long delay = capped > 0L ? ThreadLocalRandom.current().nextLong(capped) : 0L; - long remaining = failoverDeadlineMs - System.currentTimeMillis(); + long remaining = (failoverDeadlineNanos - System.nanoTime()) / 1_000_000L; if (remaining <= 0L) { int failovers = Math.max(0, attempt - 1); handler.onError(probe.interceptedStatus, @@ -1491,7 +1516,7 @@ private void runUpgradeWithTimeout(Endpoint ep) { throw re; } int status = webSocketClient.getUpgradeStatusCode(); - if (status == 401 || status == 403 || status == 404) { + if (status == 401 || status == 403) { QwpAuthFailedException ae = new QwpAuthFailedException(status, ep.host, ep.port); ae.initCause(ex); throw ae; @@ -1741,9 +1766,15 @@ private void reconnectViaTracker() { QwpServerInfo lastMismatch = null; boolean sawV1Mismatch = false; Throwable lastError = null; + boolean retriedAfterReset = false; while (true) { int i = hostTracker.pickNext(); if (i < 0) { + if (!retriedAfterReset) { + hostTracker.beginRound(true); + retriedAfterReset = true; + continue; + } break; } Endpoint ep = endpoints.get(i); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java index 55b16f54..46dfe558 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java @@ -298,7 +298,7 @@ private void decodePayload(QwpBatchBuffer buffer, long payload, int payloadLen) } byte version = Unsafe.getUnsafe().getByte(payload + 4); if (version < QwpConstants.VERSION_1 || version > QwpConstants.MAX_SUPPORTED_VERSION) { - throw QwpProtocolVersionException.UNSUPPORTED; + throw QwpProtocolVersionException.unsupported(version & 0xFF); } byte flags = Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS); deltaMode = (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 1e47dd21..e01bd468 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1671,7 +1671,7 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { * Should be called once, immediately after {@code connect()} returns. * Subsequent calls add more drainers to the same pool. */ - public synchronized void startOrphanDrainers( + public void startOrphanDrainers( io.questdb.client.std.ObjList orphanSlotPaths, int maxBackgroundDrainers, long segmentSizeBytes, @@ -1928,14 +1928,14 @@ private synchronized WebSocketClient buildAndConnect() { int status = newClient.getUpgradeStatusCode(); newClient.close(); if (role != null) { - boolean isTransient = QwpIngressRoleRejectedException.ROLE_PRIMARY_CATCHUP.equals(role); + boolean isTransient = QwpIngressRoleRejectedException.ROLE_PRIMARY_CATCHUP.equalsIgnoreCase(role); hostTracker.recordRoleReject(idx, isTransient); QwpIngressRoleRejectedException re = new QwpIngressRoleRejectedException(role, ep.host, ep.port); re.initCause(e); lastError = re; continue; } - if (status == 401 || status == 403 || status == 404) { + if (status == 401 || status == 403) { QwpAuthFailedException ae = new QwpAuthFailedException(status, ep.host, ep.port); ae.initCause(e); throw ae; @@ -1955,28 +1955,43 @@ private synchronized WebSocketClient buildAndConnect() { if (requestDurableAck && !newClient.isServerDurableAckEnabled()) { newClient.close(); hostTracker.recordTransportError(idx); - throw new LineSenderException( - "WebSocket upgrade failed: server does not support durable ack [host=" - + ep.host + ", port=" + ep.port - + "]. The client opted in via request_durable_ack=on but the server " + LineSenderException ackErr = new LineSenderException( + "WebSocket upgrade failed: server does not support durable ack [host=") + .put(ep.host).put(", port=").put(ep.port) + .put("]. The client opted in via request_durable_ack=on but the server " + "did not echo X-QWP-Durable-Ack: enabled in the upgrade response. " + "Either disable request_durable_ack or connect to a server with " + "primary replication configured."); + if (terminalUpgradeError == null) { + terminalUpgradeError = ackErr; + } + lastError = ackErr; + continue; } hostTracker.recordSuccess(idx); currentEndpointIdx = idx; return newClient; } if (terminalUpgradeError != null) { - throw new LineSenderException( - "Failed to connect: WebSocket upgrade failed across " + endpoints.size() + " endpoint(s)", - terminalUpgradeError); + LineSenderException ex = new LineSenderException(terminalUpgradeError); + ex.put("Failed to connect: WebSocket upgrade failed across ") + .put(endpoints.size()).put(" endpoint(s); cause: ") + .put(terminalUpgradeError.getMessage()); + throw ex; + } + LineSenderException ex = new LineSenderException(lastError); + ex.put("Failed to connect: "); + if (lastEndpoint == null) { + ex.put("no endpoints available"); + } else if (lastError instanceof QwpIngressRoleRejectedException) { + ex.put("all ").put(endpoints.size()) + .put(" endpoint(s) rejected the upgrade by role; last=") + .put(lastEndpoint.host).put(':').put(lastEndpoint.port); + } else { + ex.put("all ").put(endpoints.size()).put(" endpoint(s) unreachable; last=") + .put(lastEndpoint.host).put(':').put(lastEndpoint.port); } - String summary = lastEndpoint == null - ? "no endpoints available" - : "all " + endpoints.size() + " endpoint(s) unreachable; last=" - + lastEndpoint.host + ":" + lastEndpoint.port; - throw new LineSenderException("Failed to connect: " + summary, lastError); + throw ex; } private static boolean isUpgradeFailedSentinel(Throwable e) { @@ -2169,15 +2184,23 @@ private void ensureConnected() { client = null; } Endpoint ep = currentEndpoint(); - throw new LineSenderException( - "Failed to start cursor I/O thread for " + ep.host + ":" + ep.port, t); + LineSenderException ex = new LineSenderException(t); + ex.put("Failed to start cursor I/O thread for "); + if (ep == null) { + ex.put(""); + } else { + ex.put(ep.host).put(':').put(ep.port); + } + throw ex; } if (client != null) { Endpoint ep = currentEndpoint(); + String host = ep == null ? "" : ep.host; + int port = ep == null ? -1 : ep.port; encoder.setVersion((byte) client.getServerQwpVersion()); LOG.info("Connected to WebSocket [host={}, port={}, windowSize={}, qwpVersion={}]", - ep.host, ep.port, inFlightWindowSize, client.getServerQwpVersion()); + host, port, inFlightWindowSize, client.getServerQwpVersion()); } else { // Async mode: I/O thread will drive the connect. Encoder uses // its default version (V1). Schema state still gets reset for @@ -2486,7 +2509,7 @@ private static List singleEndpoint(String host, int port) { private Endpoint currentEndpoint() { int idx = currentEndpointIdx; - return endpoints.get(Math.max(idx, 0)); + return idx < 0 ? null : endpoints.get(idx); } public static final class Endpoint { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 024694fe..3ddd70d2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -202,7 +202,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // config typo or firewall block) from "lost connection after we were // up" (looks transient). private volatile boolean hasEverConnected; - private Thread ioThread; + private volatile Thread ioThread; /** * Full constructor with explicit reconnect-policy knobs. When @@ -346,6 +346,7 @@ public synchronized void close() { running = false; Thread t = ioThread; if (t != null) { + LockSupport.unpark(t); // Only await the shutdown latch if the I/O thread actually ran. // If start() failed after assigning ioThread but before t.start() // succeeded (e.g. native stack OOM), ioLoop never ran and its @@ -611,12 +612,15 @@ private void connectLoop(Throwable initial, String phase) { } if (isRoleReject(e)) { backoffMillis = reconnectInitialBackoffMillis; - long roleSleepStart = System.nanoTime(); + lastReconnectError = e; if (running) { - LockSupport.parkNanos(reconnectInitialBackoffMillis * 1_000_000L); + long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0L) { + break; + } + long parkNanos = Math.min(reconnectInitialBackoffMillis * 1_000_000L, remainingNanos); + LockSupport.parkNanos(parkNanos); } - deadlineNanos += System.nanoTime() - roleSleepStart; - lastReconnectError = e; continue; } lastReconnectError = e; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient421RoleHeaderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient421RoleHeaderTest.java index 533e8bfe..a70b4aa2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient421RoleHeaderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClient421RoleHeaderTest.java @@ -58,6 +58,61 @@ public void testCaseInsensitiveHeaderName() throws Exception { assertCapturedRole("REPLICA", "x-questdb-role: REPLICA"); } + @Test(timeout = 10_000) + public void testRoleValuePreservesCase() throws Exception { + // Parser must NOT uppercase the value -- predicates compare case-insensitive. + assertCapturedRole("replica", "X-QuestDB-Role: replica"); + } + + @Test(timeout = 10_000) + public void testRoleValueTrailingWhitespaceTrimmed() throws Exception { + assertCapturedRole("REPLICA", "X-QuestDB-Role: REPLICA "); + } + + @Test(timeout = 10_000) + public void testRoleValueEmpty_NullCaptured() throws Exception { + assertCapturedRole(null, "X-QuestDB-Role: "); + } + + @Test(timeout = 10_000) + public void testHeaderMatchAnchoredAtLineStart() throws Exception { + // A response body or another header value containing the literal + // "X-QuestDB-Role:" must NOT be picked up by the parser. The custom + // header line below appears as a non-header value -- only headers at + // CRLF boundaries should match. + ServerSocket listener = new ServerSocket(0, 1, InetAddress.getLoopbackAddress()); + int port = listener.getLocalPort(); + Thread serverThread = new Thread(() -> { + try (Socket s = listener.accept()) { + byte[] discardBuf = new byte[8192]; + int n = s.getInputStream().read(discardBuf); + if (n < 0) return; + String resp = "HTTP/1.1 421 Misdirected Request\r\n" + + "X-Echoed-Header: X-QuestDB-Role: REPLICA\r\n" + + "Content-Length: 0\r\n\r\n"; + OutputStream os = s.getOutputStream(); + os.write(resp.getBytes(StandardCharsets.US_ASCII)); + os.flush(); + } catch (Exception ignored) { + } + }, "fake-421-smuggle"); + serverThread.setDaemon(true); + serverThread.start(); + try (WebSocketClient client = WebSocketClientFactory.newPlainTextInstance()) { + client.setQwpMaxVersion(1); + client.connect("127.0.0.1", port); + try { + client.upgrade("/write/v4", null); + Assert.fail("expected upgrade to fail"); + } catch (HttpClientException ex) { + Assert.assertNull(client.getUpgradeRejectRole()); + } + } finally { + listener.close(); + serverThread.join(500); + } + } + private static void assertCapturedRole(String expected, String roleHeaderLine) throws Exception { ServerSocket listener = new ServerSocket(0, 1, InetAddress.getLoopbackAddress()); int port = listener.getLocalPort(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java index 2a312995..2d2a4974 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java @@ -44,7 +44,7 @@ public class LineSenderBuilderTest { @Test public void testAddressDoubleSet_firstAddressThenAddress() throws Exception { - assertMemoryLeak(() -> assertThrows("mismatch", + assertMemoryLeak(() -> assertThrows("only a single address", Sender.builder(Sender.Transport.TCP).address(LOCALHOST).address("127.0.0.1"))); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 16f55dce..bab5834a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -631,6 +631,42 @@ public void testMultipleAddresses_buildable() { Assert.assertNotNull(builder); } + @Test + public void testMultipleAddresses_noPorts_buildPastValidation() { + // configureDefaults() must pad ports so hosts.size == ports.size for + // multi-host builders that omit explicit ports. The build() then fails + // on connection (no server), NOT on the "host:port pair" validation. + try { + Sender.builder(Sender.Transport.WEBSOCKET) + .address("127.0.0.1") + .address("127.0.0.2") + .build() + .close(); + Assert.fail("expected build to fail with connection error"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertFalse( + "build failed on host/port count validation: " + msg, + msg.contains("host:port pair") || msg.contains("host/port count mismatch")); + } + } + + @Test + public void testMixedPortAddresses_unevenCounts_rejected() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .address(LOCALHOST + ":9001") + .port(9099), + "mismatch between number of hosts and number of ports"); + } + + @Test + public void testWsConfigString_sameHostDifferentPorts_passes() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=a:9000,a:9001;"); + Assert.assertNotNull(builder); + } + @Test public void testNoAddress_fails() { assertThrowsAny( diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java index 3ef66f6c..a98fe068 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java @@ -153,4 +153,18 @@ public void testStickyHealthyAcrossRounds() { t.beginRound(true); Assert.assertEquals(1, t.pickNext()); // sticky-Healthy } + + @Test + public void testStickyHealthyPicksMostRecentSuccess_NotHighestIndex() { + // Two hosts simultaneously HEALTHY (consecutive recordSuccess without + // intervening demotion). beginRound(true) must keep the most recently + // successful entry, not the highest index. + QwpHostHealthTracker t = new QwpHostHealthTracker(3); + t.recordSuccess(2); + t.recordSuccess(0); + t.beginRound(true); + Assert.assertEquals(QwpHostHealthTracker.HostState.HEALTHY, t.getState(0)); + Assert.assertEquals(QwpHostHealthTracker.HostState.UNKNOWN, t.getState(2)); + Assert.assertEquals(0, t.pickNext()); + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientExecutingFlagTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientExecutingFlagTest.java new file mode 100644 index 00000000..86f3c98a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientExecutingFlagTest.java @@ -0,0 +1,95 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import org.junit.Assert; +import org.junit.Test; + +public class QwpQueryClientExecutingFlagTest { + + private static final QwpColumnBatchHandler NOOP_HANDLER = new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + } + }; + + @Test(timeout = 10_000) + public void testExecutingFlagClearedAfterIllegalState() { + try (QwpQueryClient client = QwpQueryClient.fromConfig( + "ws::addr=127.0.0.1:1;failover=off;target=any;")) { + for (int i = 0; i < 3; i++) { + try { + client.execute("SELECT 1", NOOP_HANDLER); + Assert.fail("expected IllegalStateException"); + } catch (IllegalStateException e) { + Assert.assertTrue( + "iteration " + i + ": expected 'not connected', got: " + e.getMessage(), + e.getMessage().contains("not connected")); + } + } + } + } + + @Test(timeout = 10_000) + public void testExecuteAfterCloseRejected() { + QwpQueryClient client = QwpQueryClient.fromConfig( + "ws::addr=127.0.0.1:1;failover=off;target=any;"); + client.close(); + try { + client.execute("SELECT 1", NOOP_HANDLER); + Assert.fail("expected execute on closed client to throw"); + } catch (IllegalStateException e) { + Assert.assertTrue( + "expected 'closed', got: " + e.getMessage(), + e.getMessage().contains("closed")); + } + } + + @Test(timeout = 10_000) + public void testConnectAfterCloseRejected() { + QwpQueryClient client = QwpQueryClient.fromConfig( + "ws::addr=127.0.0.1:1;failover=off;target=any;"); + client.close(); + try { + client.connect(); + Assert.fail("expected connect on closed client to throw"); + } catch (IllegalStateException e) { + Assert.assertTrue( + "expected 'closed', got: " + e.getMessage(), + e.getMessage().contains("closed")); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUpgradeStatusTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUpgradeStatusTest.java new file mode 100644 index 00000000..63344166 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUpgradeStatusTest.java @@ -0,0 +1,126 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import org.junit.Assert; +import org.junit.Test; + +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicBoolean; + +public class QwpQueryClientUpgradeStatusTest { + + @Test(timeout = 10_000) + public void test401_classifiedAsAuthFailed_terminal() throws Exception { + runOneShotStatus(401, true); + } + + @Test(timeout = 10_000) + public void test403_classifiedAsAuthFailed_terminal() throws Exception { + runOneShotStatus(403, true); + } + + @Test(timeout = 10_000) + public void test404_NOT_classifiedAsAuthFailed() throws Exception { + // 404 indicates path mismatch (a single mid-deploy node may 404 while + // peers are healthy). It must NOT short-circuit failover. + runOneShotStatus(404, false); + } + + private static void runOneShotStatus(int statusCode, boolean expectAuthFailed) throws Exception { + ServerSocket listener = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); + int port = listener.getLocalPort(); + AtomicBoolean accepted = new AtomicBoolean(); + String response = "HTTP/1.1 " + statusCode + ' ' + reason(statusCode) + "\r\n" + + "Content-Length: 0\r\n\r\n"; + byte[] respBytes = response.getBytes(StandardCharsets.US_ASCII); + Thread serverThread = new Thread(() -> { + while (!listener.isClosed()) { + try { + Socket s = listener.accept(); + accepted.set(true); + Thread t = new Thread(() -> { + try (Socket sock = s) { + byte[] buf = new byte[8192]; + int n = sock.getInputStream().read(buf); + if (n < 0) return; + OutputStream os = sock.getOutputStream(); + os.write(respBytes); + os.flush(); + } catch (Exception ignored) { + } + }, "fake-status-handler-" + statusCode); + t.setDaemon(true); + t.start(); + } catch (Exception ignored) { + return; + } + } + }, "fake-status-" + statusCode); + serverThread.setDaemon(true); + serverThread.start(); + + try (QwpQueryClient client = QwpQueryClient.fromConfig( + "ws::addr=127.0.0.1:" + port + ";failover=off;target=any;")) { + try { + client.connect(); + Assert.fail("expected connect to fail with HTTP " + statusCode); + } catch (QwpAuthFailedException ae) { + if (!expectAuthFailed) { + Assert.fail("status " + statusCode + " should NOT be auth-failed; got: " + ae.getMessage()); + } + Assert.assertEquals(statusCode, ae.getStatusCode()); + } catch (HttpClientException ex) { + if (expectAuthFailed) { + Assert.fail("status " + statusCode + " should be auth-failed; got generic: " + ex.getMessage()); + } + } + Assert.assertTrue("server never accepted a connection", accepted.get()); + } finally { + listener.close(); + serverThread.join(500); + } + } + + private static String reason(int code) { + switch (code) { + case 401: + return "Unauthorized"; + case 403: + return "Forbidden"; + case 404: + return "Not Found"; + default: + return "Status"; + } + } +} From 055d0b2bbb1bc02a999ab622383a1c4d6e3fe453 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 7 May 2026 12:02:55 +0800 Subject: [PATCH 7/8] fix serval bugs --- .../main/java/io/questdb/client/Sender.java | 56 +++-- .../cutlass/http/client/WebSocketClient.java | 8 +- .../client/cutlass/qwp/client/QueryEvent.java | 8 +- .../qwp/client/QwpAuthFailedException.java | 2 +- .../cutlass/qwp/client/QwpEgressIoThread.java | 18 +- .../qwp/client/QwpHostHealthTracker.java | 2 +- .../client/QwpProtocolVersionException.java | 2 +- .../cutlass/qwp/client/QwpQueryClient.java | 90 ++++---- .../qwp/client/QwpUpgradeFailures.java | 56 +++++ .../qwp/client/QwpWebSocketSender.java | 76 ++++--- .../QwpEgressIoThreadCloseRaceTest.java | 2 +- .../client/QwpInPlaceDecodeAliasingTest.java | 2 +- .../client/QwpProtocolErrorRoutingTest.java | 75 +++++++ .../QwpQueryClientExecutingFlagTest.java | 2 +- .../QwpQueryClientMultiHostFailoverTest.java | 162 ++++++++++++++ .../QwpQueryClientUpgradeStatusTest.java | 2 +- .../QwpResultBatchDecoderHardeningTest.java | 6 +- .../client/QwpRoleRejectCloseRaceTest.java | 135 ++++++++++++ .../QwpWebSocketSenderMultiEndpointTest.java | 206 ++++++++++++++++++ 19 files changed, 778 insertions(+), 132 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUpgradeFailures.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpProtocolErrorRoutingTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientMultiHostFailoverTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpRoleRejectCloseRaceTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderMultiEndpointTest.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 4e574524..927e8298 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -627,6 +627,7 @@ final class LineSenderBuilder { // We want to fail-fast even when an explicitly configured options happens to be same value as the default value, // because this still indicates a user error and silently ignoring it could lead to hard-to-debug issues. private static final int PARAMETER_NOT_SET_EXPLICITLY = -1; + private static final int PORT_NOT_SET = -1; private static final int PROTOCOL_HTTP = 1; private static final int PROTOCOL_TCP = 0; private static final int PROTOCOL_UDP = 3; @@ -809,30 +810,23 @@ public LineSenderBuilder address(CharSequence address) { hostSansPort = address.toString(); } - // best effort dup detection, we might have incomplete information at this point, - // for example port or protocol might not be configured yet. so we are conservative - // and only detect dups when we have full information about the address if (parsedPort != -1) { - // we have a port, so we can do a full dup check for (int i = 0, n = hosts.size(); i < n; i++) { String storedHost = hosts.get(i); if (Chars.equals(storedHost, hostSansPort)) { - // given host is already configured, let's see if the port is the same - if (ports.size() > i) { - // ok, the previous address had a port explicitly configured, let's see if it's the same - if (ports.getQuick(i) == parsedPort) { - throw new LineSenderException("duplicated addresses are not allowed ") - .put("[address=").put(address).put("]"); - } + if (ports.size() > i && ports.getQuick(i) == parsedPort) { + throw new LineSenderException("duplicated addresses are not allowed ") + .put("[address=").put(address).put("]"); } } } - - } - this.hosts.add(hostSansPort); - if (parsedPort != -1) { - // port was specified in the address, so we use it + while (ports.size() < hosts.size()) { + ports.add(PORT_NOT_SET); + } + this.hosts.add(hostSansPort); this.ports.add(parsedPort); + } else { + this.hosts.add(hostSansPort); } return this; } @@ -864,16 +858,19 @@ private void addAddressEntry(CharSequence src, int start, int end, int defaultPo for (int i = 0, n = hosts.size(); i < n; i++) { String storedHost = hosts.get(i); if (charsEqualsRange(storedHost, src, start, hostEnd)) { - if (ports.size() > i && ports.getQuick(i) == effectivePort) { + int storedEffectivePort = ports.size() > i && ports.getQuick(i) != PORT_NOT_SET + ? ports.getQuick(i) : defaultPort; + if (storedEffectivePort == effectivePort) { throw new LineSenderException("duplicated addresses are not allowed [address=") .put(src.subSequence(start, end)).put("]"); } } } - hosts.add(src.subSequence(start, hostEnd).toString()); - if (parsedPort != -1) { - ports.add(parsedPort); + while (ports.size() < hosts.size()) { + ports.add(PORT_NOT_SET); } + hosts.add(src.subSequence(start, hostEnd).toString()); + ports.add(parsedPort != -1 ? parsedPort : PORT_NOT_SET); } private static boolean charsEqualsRange(CharSequence a, CharSequence b, int bStart, int bEnd) { @@ -2416,6 +2413,11 @@ private void configureDefaults() { defaultPort = DEFAULT_TCP_PORT; } int hostsCount = Math.max(hosts.size(), 1); + for (int i = 0, n = ports.size(); i < n; i++) { + if (ports.getQuick(i) == PORT_NOT_SET) { + ports.set(i, defaultPort); + } + } while (ports.size() < hostsCount) { ports.add(defaultPort); } @@ -2538,11 +2540,7 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { if (s == e) { throw new LineSenderException("empty addr entry"); } - int portsBefore = ports.size(); addAddressEntry(sink, s, e, defaultPort); - if (ports.size() == portsBefore) { - port(defaultPort); - } entryStart = i + 1; } } @@ -2935,6 +2933,16 @@ private void validateParameters() { if (hosts.size() != ports.size()) { throw new LineSenderException("mismatch between number of hosts and number of ports"); } + for (int i = 0, n = hosts.size(); i < n; i++) { + String host = hosts.get(i); + int port = ports.getQuick(i); + for (int j = i + 1; j < n; j++) { + if (ports.getQuick(j) == port && Chars.equals(host, hosts.get(j))) { + throw new LineSenderException("duplicated addresses are not allowed [address=") + .put(host).put(':').put(port).put("]"); + } + } + } if (!tlsEnabled && trustStorePath != null) { throw new LineSenderException("custom trust store configured, but TLS was not enabled ") .put("[path=").put(LineSenderBuilder.this.trustStorePath).put("]"); diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index aaf78053..9d415de6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -648,9 +648,11 @@ private static boolean extractDurableAckEnabled(String response) { private static int parseStatusCode(String statusLine) { int sp1 = statusLine.indexOf(' '); if (sp1 < 0 || sp1 + 4 > statusLine.length()) return 0; - char afterCode = statusLine.charAt(sp1 + 4); - if (afterCode != ' ' && afterCode != '\r' && afterCode != '\n') { - return 0; + if (sp1 + 4 < statusLine.length()) { + char afterCode = statusLine.charAt(sp1 + 4); + if (afterCode != ' ' && afterCode != '\r' && afterCode != '\n') { + return 0; + } } int code = 0; for (int i = sp1 + 1; i < sp1 + 4; i++) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java index 1eb299dc..3cd8ea81 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java @@ -89,16 +89,16 @@ public QueryEvent asExecDone(short opType, long rowsAffected) { return this; } - public QueryEvent asTransportError(byte status, String message) { - this.kind = KIND_TRANSPORT_ERROR; + public QueryEvent asProtocolError(byte status, String message) { + this.kind = KIND_PROTOCOL_ERROR; this.buffer = null; this.errorStatus = status; this.errorMessage = message; return this; } - public QueryEvent asProtocolError(byte status, String message) { - this.kind = KIND_PROTOCOL_ERROR; + public QueryEvent asTransportError(byte status, String message) { + this.kind = KIND_TRANSPORT_ERROR; this.buffer = null; this.errorStatus = status; this.errorMessage = message; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java index 555670e6..d2a5714a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpAuthFailedException.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java index 8560e11d..dd4c5ef1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java @@ -530,14 +530,14 @@ private void emitError(byte status, String message) { * latch -- the latch stays strictly for short-circuiting subsequent * {@code execute()} calls on a broken client. */ - private void emitTerminalTransportError(String message) { - notifyTerminalFailure(message); - events.offer(new QueryEvent().asTransportError(WebSocketResponse.STATUS_INTERNAL_ERROR, message)); + private void emitTerminalProtocolError(String message) { + notifyTerminalFailure(message, true); + events.offer(new QueryEvent().asProtocolError(WebSocketResponse.STATUS_INTERNAL_ERROR, message)); } - private void emitTerminalProtocolError(String message) { + private void emitTerminalTransportError(String message) { notifyTerminalFailure(message); - events.offer(new QueryEvent().asProtocolError(WebSocketResponse.STATUS_INTERNAL_ERROR, message)); + events.offer(new QueryEvent().asTransportError(WebSocketResponse.STATUS_INTERNAL_ERROR, message)); } /** @@ -650,9 +650,13 @@ private void handleResultBatch(long payloadPtr, int payloadLen) { } private void notifyTerminalFailure(String message) { + notifyTerminalFailure(message, false); + } + + private void notifyTerminalFailure(String message, boolean isProtocol) { if (terminalFailureListener != null) { try { - terminalFailureListener.onTerminalFailure(WebSocketResponse.STATUS_INTERNAL_ERROR, message); + terminalFailureListener.onTerminalFailure(WebSocketResponse.STATUS_INTERNAL_ERROR, message, isProtocol); } catch (Throwable ignored) { // Listener must not bring down the I/O thread. A first-failure-wins // CAS in the listener cannot throw in practice; defensive anyway. @@ -748,7 +752,7 @@ void closePool() { @FunctionalInterface public interface TerminalFailureListener { - void onTerminalFailure(byte status, String message); + void onTerminalFailure(byte status, String message, boolean isProtocol); } private static final class QueryRequest { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java index 9efd517f..2fd899b9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java index 03f2bcaa..21431170 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index 7b959ed4..40a9f064 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -162,8 +162,8 @@ public class QwpQueryClient implements QuietCloseable { // full shutdown path again and double-free {@link #bindValues} native // scratch. private final AtomicBoolean closedFlag = new AtomicBoolean(); - private final AtomicBoolean executing = new AtomicBoolean(); private final List endpoints = new ArrayList<>(); + private final AtomicBoolean executing = new AtomicBoolean(); private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; private String authorizationHeader; private int bufferPoolSize = DEFAULT_IO_BUFFER_POOL_SIZE; @@ -212,13 +212,13 @@ public class QwpQueryClient implements QuietCloseable { private long failoverMaxBackoffMs = DEFAULT_FAILOVER_MAX_BACKOFF_MS; private long failoverMaxDurationMs = DEFAULT_FAILOVER_MAX_DURATION_MS; private QwpHostHealthTracker hostTracker; - private String lbStrategy = LB_RANDOM; // Credit-flow send-ahead budget. 0 = unbounded (Phase-1 default, no CREDIT // bookkeeping on either side). A positive value puts the stream under byte- // based flow control: the server emits at most this many bytes of result // payload before it parks, and the client auto-replenishes by the size of // each batch as the user releases it. private long initialCreditBytes; + private String lbStrategy = LB_RANDOM; // Volatile so a cancel() call from a thread other than the one that ran // connect() sees the published reference (and a concurrent null-out from // close() is observed without a stale-reference race). The thread-safety @@ -269,7 +269,7 @@ public class QwpQueryClient implements QuietCloseable { private int tlsValidationMode = ClientTlsConfiguration.TLS_VALIDATION_MODE_FULL; private char[] trustStorePassword; private String trustStorePath; - private WebSocketClient webSocketClient; + private volatile WebSocketClient webSocketClient; private QwpQueryClient(String host, int port) { this.endpoints.add(new Endpoint(host, port)); @@ -435,17 +435,19 @@ public static QwpQueryClient fromConfig(CharSequence configurationString) { throw new IllegalArgumentException("failover_backoff_max_ms must be >= 0"); } break; - case "failover_max_duration_ms": + case "failover_max_duration_ms": { + long parsed; try { - long parsed = Long.parseLong(value); - if (parsed < 0L) { - throw new IllegalArgumentException("failover_max_duration_ms must be >= 0"); - } - failoverMaxDurationMs = parsed; + parsed = Long.parseLong(value); } catch (NumberFormatException e) { throw new IllegalArgumentException("invalid failover_max_duration_ms: " + value); } + if (parsed < 0L) { + throw new IllegalArgumentException("failover_max_duration_ms must be >= 0"); + } + failoverMaxDurationMs = parsed; break; + } case "lb_strategy": if (!LB_RANDOM.equals(value) && !LB_FIRST.equals(value)) { throw new IllegalArgumentException( @@ -453,17 +455,19 @@ public static QwpQueryClient fromConfig(CharSequence configurationString) { } lbStrategy = value; break; - case "auth_timeout_ms": + case "auth_timeout_ms": { + long parsed; try { - long parsed = Long.parseLong(value); - if (parsed <= 0L) { - throw new IllegalArgumentException("auth_timeout_ms must be > 0"); - } - authTimeoutMs = parsed; + parsed = Long.parseLong(value); } catch (NumberFormatException e) { throw new IllegalArgumentException("invalid auth_timeout_ms: " + value); } + if (parsed <= 0L) { + throw new IllegalArgumentException("auth_timeout_ms must be > 0"); + } + authTimeoutMs = parsed; break; + } case "path": path = value; break; @@ -916,7 +920,7 @@ private void executeImpl(String sql, QwpBindSetter binds, QwpColumnBatchHandler handler.onError(probe.interceptedStatus, probe.interceptedMessage); return; } - if (attempt >= failoverMaxAttempts || System.nanoTime() >= failoverDeadlineNanos) { + if (attempt >= failoverMaxAttempts || System.nanoTime() - failoverDeadlineNanos >= 0) { int failovers = Math.max(0, attempt - 1); handler.onError(probe.interceptedStatus, "transport failure after " + attempt + " execute attempt" @@ -939,8 +943,9 @@ private void executeImpl(String sql, QwpBindSetter binds, QwpColumnBatchHandler long delay = capped > 0L ? ThreadLocalRandom.current().nextLong(capped) : 0L; - long remaining = (failoverDeadlineNanos - System.nanoTime()) / 1_000_000L; - if (remaining <= 0L) { + long remainingNanos = failoverDeadlineNanos - System.nanoTime(); + long remaining = remainingNanos <= 0L ? 0L : remainingNanos / 1_000_000L; + if (remainingNanos <= 0L) { int failovers = Math.max(0, attempt - 1); handler.onError(probe.interceptedStatus, "transport failure after " + attempt + " execute attempt" @@ -967,8 +972,6 @@ private void executeImpl(String sql, QwpBindSetter binds, QwpColumnBatchHandler } try { reconnectViaTracker(); - } catch (QwpAuthFailedException ae) { - throw ae; } catch (RuntimeException reconnectErr) { handler.onError(probe.interceptedStatus, "failover reconnect failed after " + attempt + " attempt" @@ -1496,9 +1499,9 @@ private void cleanupFailedConnect() { } private void runUpgradeWithTimeout(Endpoint ep) { - webSocketClient.connect(ep.host, ep.port); int timeoutMs = (int) Math.min(authTimeoutMs, Integer.MAX_VALUE); try { + webSocketClient.connect(ep.host, ep.port); webSocketClient.upgrade(endpointPath, timeoutMs, authorizationHeader); } catch (HttpClientException ex) { if (ex.isTimeout()) { @@ -1509,19 +1512,7 @@ private void runUpgradeWithTimeout(Endpoint ep) { timeout.flagAsTimeout(); throw timeout; } - String role = webSocketClient.getUpgradeRejectRole(); - if (role != null) { - QwpIngressRoleRejectedException re = new QwpIngressRoleRejectedException(role, ep.host, ep.port); - re.initCause(ex); - throw re; - } - int status = webSocketClient.getUpgradeStatusCode(); - if (status == 401 || status == 403) { - QwpAuthFailedException ae = new QwpAuthFailedException(status, ep.host, ep.port); - ae.initCause(ex); - throw ae; - } - throw ex; + throw QwpUpgradeFailures.classify(webSocketClient, ep.host, ep.port, ex); } } @@ -1601,13 +1592,11 @@ private void executeOnce(String sql, QwpBindSetter binds, FailoverProbeHandler p GenerationListener listener = currentGenerationListener; TerminalFailure tf = listener != null ? listener.get() : null; if (tf != null) { - // I/O thread already reported a transport- or protocol-level failure - // on a previous call. Tag it as a transport failure so the failover - // wrapper can take over instead of surfacing to the user as a final - // error. Going through markTransportFailure (not probe.onError) - // keeps the classification explicit -- probe.onError always means - // server-emitted QUERY_ERROR. - probe.markTransportFailure(tf.status, tf.message); + if (tf.isProtocol) { + probe.onError(tf.status, tf.message); + } else { + probe.markTransportFailure(tf.status, tf.message); + } return; } bindValues.reset(); @@ -1649,16 +1638,12 @@ private void executeOnce(String sql, QwpBindSetter binds, FailoverProbeHandler p probe.onExecDone(ev.opType, ev.rowsAffected); return; case QueryEvent.KIND_ERROR: - // Server-emitted QUERY_ERROR. Connection remains healthy; - // pass straight through to the user. Never triggers failover. probe.onError(ev.errorStatus, ev.errorMessage); return; case QueryEvent.KIND_TRANSPORT_ERROR: - // Transport-level failure -- replay candidate. probe.markTransportFailure(ev.errorStatus, ev.errorMessage); return; case QueryEvent.KIND_PROTOCOL_ERROR: - // Permanent protocol disagreement -- surface to user, no failover. probe.onError(ev.errorStatus, ev.errorMessage); return; default: @@ -1839,9 +1824,14 @@ private void reconnectViaTracker() { */ @SuppressWarnings("unused") void recordTerminalFailure(byte status, String message) { + recordTerminalFailure(status, message, false); + } + + @SuppressWarnings("unused") + void recordTerminalFailure(byte status, String message, boolean isProtocol) { GenerationListener listener = currentGenerationListener; if (listener != null) { - listener.onTerminalFailure(status, message); + listener.onTerminalFailure(status, message, isProtocol); } } @@ -1941,11 +1931,11 @@ private static final class GenerationListener implements QwpEgressIoThread.Termi private volatile boolean orphaned; @Override - public void onTerminalFailure(byte status, String message) { + public void onTerminalFailure(byte status, String message, boolean isProtocol) { if (orphaned) { return; } - target.compareAndSet(null, new TerminalFailure(status, message)); + target.compareAndSet(null, new TerminalFailure(status, message, isProtocol)); } TerminalFailure get() { @@ -2010,12 +2000,14 @@ public void onTextMessage(long payloadPtr, int payloadLen) { } private static final class TerminalFailure { + final boolean isProtocol; final String message; final byte status; - TerminalFailure(byte status, String message) { + TerminalFailure(byte status, String message, boolean isProtocol) { this.status = status; this.message = message; + this.isProtocol = isProtocol; } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUpgradeFailures.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUpgradeFailures.java new file mode 100644 index 00000000..8052cc1f --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUpgradeFailures.java @@ -0,0 +1,56 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.http.client.WebSocketClient; + +final class QwpUpgradeFailures { + private QwpUpgradeFailures() { + } + + /** + * Inspects {@code client}'s rejected-upgrade state and returns a typed + * exception if the failure is classifiable as a role reject ({@code 421} + + * {@code X-QuestDB-Role}) or a credential failure ({@code 401}/{@code 403}). + * Falls through to {@code ex} for any other status, including {@code 404} + * (per-endpoint path mismatch) and unknown codes. + */ + static HttpClientException classify(WebSocketClient client, String host, int port, HttpClientException ex) { + String role = client.getUpgradeRejectRole(); + if (role != null) { + QwpIngressRoleRejectedException re = new QwpIngressRoleRejectedException(role, host, port); + re.initCause(ex); + return re; + } + int status = client.getUpgradeStatusCode(); + if (status == 401 || status == 403) { + QwpAuthFailedException ae = new QwpAuthFailedException(status, host, port); + ae.initCause(ex); + return ae; + } + return ex; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index e01bd468..8528cf87 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1671,7 +1671,7 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { * Should be called once, immediately after {@code connect()} returns. * Subsequent calls add more drainers to the same pool. */ - public void startOrphanDrainers( + public synchronized void startOrphanDrainers( io.questdb.client.std.ObjList orphanSlotPaths, int maxBackgroundDrainers, long segmentSizeBytes, @@ -1690,7 +1690,7 @@ public void startOrphanDrainers( io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer drainer = new io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer( slot, segmentSizeBytes, sfMaxTotalBytes, - this::buildAndConnect, + newReconnectFactory(), reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis); @@ -1885,19 +1885,30 @@ private void atNanos(long timestampNanos) { } /** - * Build and connect a fresh WebSocket client using the sender's - * persistent config (host/port/TLS/auth/durable-ack flag). Used both - * for the initial connect and as the reconnect factory passed to the - * cursor I/O loop. Throws {@link LineSenderException} on any failure - * — the I/O loop's reconnect path treats a throw as fatal for that - * attempt (and, in the follow-up commit, schedules a backoff retry - * within the per-outage time cap). + * Returns a {@link CursorWebSocketSendLoop.ReconnectFactory} that, on each + * call, performs the multi-endpoint walk and returns a freshly connected + * {@link WebSocketClient}. Each factory holds private "previously-bound + * endpoint" state, so foreground and drainer reconnects do not corrupt + * each other's host-tracker priorities. */ - private synchronized WebSocketClient buildAndConnect() { - int previousIdx = currentEndpointIdx; + public CursorWebSocketSendLoop.ReconnectFactory newReconnectFactory() { + return new ReconnectSupplier(); + } + + private final class ReconnectSupplier implements CursorWebSocketSendLoop.ReconnectFactory { + private int previousIdx = -1; + + @Override + public WebSocketClient reconnect() { + return buildAndConnect(this); + } + } + + private synchronized WebSocketClient buildAndConnect(ReconnectSupplier ctx) { + int previousIdx = ctx.previousIdx; if (previousIdx >= 0) { hostTracker.recordMidStreamFailure(previousIdx); - currentEndpointIdx = -1; + ctx.previousIdx = -1; } if (hostTracker.isRoundExhausted()) { hostTracker.beginRound(true); @@ -1924,27 +1935,19 @@ private synchronized WebSocketClient buildAndConnect() { int upgradeTimeoutMs = (int) Math.min(authTimeoutMs, Integer.MAX_VALUE); newClient.upgrade(WRITE_PATH, upgradeTimeoutMs, authorizationHeader); } catch (HttpClientException e) { - String role = newClient.getUpgradeRejectRole(); - int status = newClient.getUpgradeStatusCode(); + HttpClientException classified = QwpUpgradeFailures.classify(newClient, ep.host, ep.port, e); newClient.close(); - if (role != null) { - boolean isTransient = QwpIngressRoleRejectedException.ROLE_PRIMARY_CATCHUP.equalsIgnoreCase(role); - hostTracker.recordRoleReject(idx, isTransient); - QwpIngressRoleRejectedException re = new QwpIngressRoleRejectedException(role, ep.host, ep.port); - re.initCause(e); + if (classified instanceof QwpIngressRoleRejectedException) { + QwpIngressRoleRejectedException re = (QwpIngressRoleRejectedException) classified; + hostTracker.recordRoleReject(idx, re.isTransient()); lastError = re; continue; } - if (status == 401 || status == 403) { - QwpAuthFailedException ae = new QwpAuthFailedException(status, ep.host, ep.port); - ae.initCause(e); - throw ae; + if (classified instanceof QwpAuthFailedException) { + throw (QwpAuthFailedException) classified; } hostTracker.recordTransportError(idx); - lastError = e; - if (terminalUpgradeError == null && isUpgradeFailedSentinel(e)) { - terminalUpgradeError = e; - } + lastError = classified; continue; } catch (Exception e) { newClient.close(); @@ -1969,6 +1972,7 @@ private synchronized WebSocketClient buildAndConnect() { continue; } hostTracker.recordSuccess(idx); + ctx.previousIdx = idx; currentEndpointIdx = idx; return newClient; } @@ -1994,11 +1998,6 @@ private synchronized WebSocketClient buildAndConnect() { throw ex; } - private static boolean isUpgradeFailedSentinel(Throwable e) { - String msg = e.getMessage(); - return msg != null && msg.contains("WebSocket upgrade failed:"); - } - private void checkConnectionError() { LineSenderException error = connectionError.get(); if (error != null) { @@ -2125,10 +2124,11 @@ private void ensureConnected() { if (cursorEngine == null) { throw new LineSenderException("cursor engine must be attached before connect"); } + CursorWebSocketSendLoop.ReconnectFactory reconnectFactory = newReconnectFactory(); switch (initialConnectMode) { case SYNC: client = CursorWebSocketSendLoop.connectWithRetry( - this::buildAndConnect, + reconnectFactory, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, @@ -2147,7 +2147,13 @@ private void ensureConnected() { break; case OFF: default: - client = buildAndConnect(); + try { + client = reconnectFactory.reconnect(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new LineSenderException(e).put("Failed to connect"); + } break; } @@ -2155,7 +2161,7 @@ private void ensureConnected() { cursorSendLoop = new CursorWebSocketSendLoop( client, cursorEngine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, - this::buildAndConnect, + reconnectFactory, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressIoThreadCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressIoThreadCloseRaceTest.java index 56703328..bc0504e0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressIoThreadCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressIoThreadCloseRaceTest.java @@ -60,7 +60,7 @@ public void testReleaseBufferRacesClosePoolSafely() throws Exception { // and must leave freeBuffers empty + all buffers closed by // the time both threads exit. QwpEgressIoThread io = new QwpEgressIoThread(null, /*bufferPoolSize=*/ 2, - (status, message) -> { /* unused */ }); + (status, message, isProtocol) -> { /* unused */ }); QwpBatchBuffer b0 = new QwpBatchBuffer(64); QwpBatchBuffer b1 = new QwpBatchBuffer(64); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java index a28b8300..c564889a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java @@ -78,7 +78,7 @@ public void testInterruptDuringOnBatchKeepsIoThreadParkedUntilRelease() throws E int payloadCap = 256; long staging = Unsafe.malloc(payloadCap, MemoryTag.NATIVE_DEFAULT); QwpEgressIoThread io = new QwpEgressIoThread(null, /*bufferPoolSize=*/ 2, - (status, message) -> { + (status, message, isProtocol) -> { // Terminal failures during this test would mean the // decode itself rejected our frame -- fail loudly. throw new AssertionError("unexpected terminal failure during decode: " + message); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpProtocolErrorRoutingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpProtocolErrorRoutingTest.java new file mode 100644 index 00000000..6fcdb4ae --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpProtocolErrorRoutingTest.java @@ -0,0 +1,75 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QueryEvent; +import io.questdb.client.cutlass.qwp.client.QwpEgressIoThread; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +public class QwpProtocolErrorRoutingTest { + + @Test + public void testQueryEventAsProtocolError() { + QueryEvent ev = new QueryEvent().asProtocolError((byte) 0x05, "unsupported version 99"); + Assert.assertEquals(QueryEvent.KIND_PROTOCOL_ERROR, ev.kind); + Assert.assertEquals((byte) 0x05, ev.errorStatus); + Assert.assertEquals("unsupported version 99", ev.errorMessage); + Assert.assertNull("buffer must be null on protocol error", ev.buffer); + } + + @Test + public void testProtocolFailurePropagatesViaListener() { + AtomicReference capturedMessage = new AtomicReference<>(); + AtomicReference capturedStatus = new AtomicReference<>(); + AtomicBoolean capturedIsProtocol = new AtomicBoolean(); + + QwpEgressIoThread.TerminalFailureListener listener = (status, message, isProtocol) -> { + capturedStatus.set(status); + capturedMessage.set(message); + capturedIsProtocol.set(isProtocol); + }; + + listener.onTerminalFailure((byte) 0x05, "version mismatch", true); + + Assert.assertEquals(Byte.valueOf((byte) 0x05), capturedStatus.get()); + Assert.assertEquals("version mismatch", capturedMessage.get()); + Assert.assertTrue("isProtocol=true must propagate", capturedIsProtocol.get()); + } + + @Test + public void testTransportFailurePassesIsProtocolFalse() { + AtomicBoolean capturedIsProtocol = new AtomicBoolean(true); + QwpEgressIoThread.TerminalFailureListener listener = + (status, message, isProtocol) -> capturedIsProtocol.set(isProtocol); + + listener.onTerminalFailure((byte) 0x01, "decode failure", false); + + Assert.assertFalse("transport-error path must pass isProtocol=false", capturedIsProtocol.get()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientExecutingFlagTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientExecutingFlagTest.java index 86f3c98a..59686257 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientExecutingFlagTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientExecutingFlagTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientMultiHostFailoverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientMultiHostFailoverTest.java new file mode 100644 index 00000000..fc0e46c4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientMultiHostFailoverTest.java @@ -0,0 +1,162 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; + +public class QwpQueryClientMultiHostFailoverTest { + + @Test(timeout = 10_000) + public void testReplicaThen401_FailsFastWithAuthOnSecond() throws Exception { + try (FakeStatusServer replica = new FakeStatusServer(421, "X-QuestDB-Role: REPLICA"); + FakeStatusServer auth = new FakeStatusServer(401, null)) { + replica.start(); + auth.start(); + + String cfg = "ws::addr=127.0.0.1:" + replica.port() + ",127.0.0.1:" + auth.port() + + ";lb_strategy=first;auth_timeout_ms=2000;failover=off;target=any;"; + try (QwpQueryClient client = QwpQueryClient.fromConfig(cfg)) { + try { + client.connect(); + Assert.fail("expected connect to throw QwpAuthFailedException"); + } catch (QwpAuthFailedException ae) { + Assert.assertEquals(401, ae.getStatusCode()); + Assert.assertTrue("first endpoint should have been probed", + replica.connections.get() >= 1); + Assert.assertTrue("auth endpoint must have been probed before short-circuit", + auth.connections.get() >= 1); + } + } + } + } + + @Test(timeout = 10_000) + public void testAllReplica_FailsWithRoleSummary() throws Exception { + try (FakeStatusServer r1 = new FakeStatusServer(421, "X-QuestDB-Role: REPLICA"); + FakeStatusServer r2 = new FakeStatusServer(421, "X-QuestDB-Role: REPLICA")) { + r1.start(); + r2.start(); + + String cfg = "ws::addr=127.0.0.1:" + r1.port() + ",127.0.0.1:" + r2.port() + + ";lb_strategy=first;auth_timeout_ms=2000;failover=off;target=any;"; + try (QwpQueryClient client = QwpQueryClient.fromConfig(cfg)) { + try { + client.connect(); + Assert.fail("expected connect to throw when all endpoints role-reject"); + } catch (HttpClientException ex) { + Assert.assertFalse("should not be classified as auth fail", + ex instanceof QwpAuthFailedException); + } + Assert.assertTrue("both replica endpoints should be probed", + r1.connections.get() >= 1 && r2.connections.get() >= 1); + } + } + } + + private static final class FakeStatusServer implements AutoCloseable { + final AtomicInteger connections = new AtomicInteger(); + private final String roleHeader; + private final ServerSocket socket; + private final int statusCode; + private volatile boolean running = true; + + FakeStatusServer(int statusCode, String roleHeader) throws IOException { + this.statusCode = statusCode; + this.roleHeader = roleHeader; + this.socket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); + } + + int port() { + return socket.getLocalPort(); + } + + void start() { + Thread t = new Thread(this::loop, "fake-status-" + statusCode); + t.setDaemon(true); + t.start(); + } + + @Override + public void close() throws IOException { + running = false; + socket.close(); + } + + private void loop() { + while (running) { + try { + Socket s = socket.accept(); + Thread h = new Thread(() -> handle(s), "fake-status-handler-" + statusCode); + h.setDaemon(true); + h.start(); + } catch (IOException e) { + if (!running) return; + } + } + } + + private void handle(Socket s) { + try (Socket sock = s) { + connections.incrementAndGet(); + byte[] discard = new byte[8192]; + int n = sock.getInputStream().read(discard); + if (n < 0) return; + StringBuilder resp = new StringBuilder(); + resp.append("HTTP/1.1 ").append(statusCode).append(' ').append(reason(statusCode)).append("\r\n"); + if (roleHeader != null) { + resp.append(roleHeader).append("\r\n"); + } + resp.append("Content-Length: 0\r\nConnection: close\r\n\r\n"); + OutputStream out = sock.getOutputStream(); + out.write(resp.toString().getBytes(StandardCharsets.US_ASCII)); + out.flush(); + } catch (Exception ignored) { + } + } + + private static String reason(int code) { + switch (code) { + case 401: + return "Unauthorized"; + case 421: + return "Misdirected Request"; + default: + return "Status"; + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUpgradeStatusTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUpgradeStatusTest.java index 63344166..e8cf93ff 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUpgradeStatusTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUpgradeStatusTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/*+***************************************************************************** * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) * | | | | | | |/ _ \/ __| __| | | | _ \ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java index dc4ee05a..160a3d17 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java @@ -205,7 +205,7 @@ public void testExecDoneTruncatedRowsAffectedVarintIsRejected() throws Exception AtomicReference failure = new AtomicReference<>(); TestUtils.assertMemoryLeak(() -> { QwpEgressIoThread io = new QwpEgressIoThread(null, /*bufferPoolSize=*/ 2, - (status, message) -> failure.compareAndSet(null, message)); + (status, message, isProtocol) -> failure.compareAndSet(null, message)); try { int cap = 64; long buf = Unsafe.malloc(cap, MemoryTag.NATIVE_DEFAULT); @@ -436,7 +436,7 @@ public void testResultEndTruncatedFinalSeqVarintIsRejected() throws Exception { AtomicReference failure = new AtomicReference<>(); TestUtils.assertMemoryLeak(() -> { QwpEgressIoThread io = new QwpEgressIoThread(null, /*bufferPoolSize=*/ 2, - (status, message) -> failure.compareAndSet(null, message)); + (status, message, isProtocol) -> failure.compareAndSet(null, message)); try { int cap = 64; long buf = Unsafe.malloc(cap, MemoryTag.NATIVE_DEFAULT); @@ -469,7 +469,7 @@ public void testResultEndTruncatedTotalRowsVarintIsRejected() throws Exception { AtomicReference failure = new AtomicReference<>(); TestUtils.assertMemoryLeak(() -> { QwpEgressIoThread io = new QwpEgressIoThread(null, /*bufferPoolSize=*/ 2, - (status, message) -> failure.compareAndSet(null, message)); + (status, message, isProtocol) -> failure.compareAndSet(null, message)); try { int cap = 64; long buf = Unsafe.malloc(cap, MemoryTag.NATIVE_DEFAULT); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpRoleRejectCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpRoleRejectCloseRaceTest.java new file mode 100644 index 00000000..b15ccd38 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpRoleRejectCloseRaceTest.java @@ -0,0 +1,135 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; + +public class QwpRoleRejectCloseRaceTest { + + @Test(timeout = 15_000) + public void testCloseDuringRoleRejectBackoffReturnsPromptly() throws Exception { + try (RoleRejectServer server = new RoleRejectServer()) { + server.start(); + + String cfg = "ws::addr=127.0.0.1:" + server.port() + + ";reconnect_backoff_initial_millis=4000" + + ";reconnect_backoff_max_millis=4000" + + ";auth_timeout_ms=2000" + + ";auto_flush_rows=1" + + ";close_flush_timeout_millis=0" + + ";initial_connect_retry=async;"; + + Sender sender = Sender.fromConfig(cfg); + try { + // Push a row so the I/O thread starts attempting connect; the + // first attempt will hit the role reject and enter the parkNanos + // backoff branch. + sender.table("t").longColumn("v", 1L).atNow(); + waitFor(() -> server.upgrades.get() >= 1, 5_000); + Thread.sleep(100); + } finally { + long start = System.currentTimeMillis(); + sender.close(); + long elapsed = System.currentTimeMillis() - start; + Assert.assertTrue( + "close() during role-reject backoff must return promptly (got " + elapsed + "ms)", + elapsed < 2_000); + } + } + } + + private static void waitFor(java.util.function.BooleanSupplier cond, long timeoutMs) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + if (cond.getAsBoolean()) return; + Thread.sleep(20); + } + } + + private static final class RoleRejectServer implements AutoCloseable { + final AtomicInteger upgrades = new AtomicInteger(); + private final ServerSocket socket; + private volatile boolean running = true; + + RoleRejectServer() throws IOException { + this.socket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); + } + + int port() { + return socket.getLocalPort(); + } + + void start() { + Thread t = new Thread(this::loop, "role-reject-server"); + t.setDaemon(true); + t.start(); + } + + @Override + public void close() throws IOException { + running = false; + socket.close(); + } + + private void loop() { + while (running) { + try { + Socket s = socket.accept(); + Thread h = new Thread(() -> handle(s), "role-reject-handler"); + h.setDaemon(true); + h.start(); + } catch (IOException e) { + if (!running) return; + } + } + } + + private void handle(Socket s) { + try (Socket sock = s) { + byte[] discard = new byte[8192]; + int n = sock.getInputStream().read(discard); + if (n < 0) return; + upgrades.incrementAndGet(); + String resp = "HTTP/1.1 421 Misdirected Request\r\n" + + "X-QuestDB-Role: PRIMARY_CATCHUP\r\n" + + "Content-Length: 0\r\nConnection: close\r\n\r\n"; + OutputStream out = sock.getOutputStream(); + out.write(resp.getBytes(StandardCharsets.US_ASCII)); + out.flush(); + } catch (Exception ignored) { + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderMultiEndpointTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderMultiEndpointTest.java new file mode 100644 index 00000000..9715cf22 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderMultiEndpointTest.java @@ -0,0 +1,206 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.line.LineSenderException; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.concurrent.atomic.AtomicInteger; + +public class QwpWebSocketSenderMultiEndpointTest { + + private static final String WEBSOCKET_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + + @Test(timeout = 10_000) + public void testReplicaThenPrimaryWalksToSecond() throws Exception { + try (FakeUpgradeServer replica = new FakeUpgradeServer(Mode.REPLICA_REJECT); + FakeUpgradeServer primary = new FakeUpgradeServer(Mode.UPGRADE_OK)) { + replica.start(); + primary.start(); + + String cfg = "ws::addr=127.0.0.1:" + replica.port() + ",127.0.0.1:" + primary.port() + + ";lb_strategy=first;auth_timeout_ms=2000;"; + try (Sender ignored = Sender.fromConfig(cfg)) { + Assert.assertEquals("primary should accept exactly one upgrade", + 1, primary.upgradeCount.get()); + Assert.assertTrue("replica should be probed at least once", + replica.upgradeCount.get() >= 1); + } + } + } + + @Test(timeout = 10_000) + public void test401TerminatesAcrossEndpoints() throws Exception { + try (FakeUpgradeServer auth = new FakeUpgradeServer(Mode.AUTH_401); + FakeUpgradeServer healthy = new FakeUpgradeServer(Mode.UPGRADE_OK)) { + auth.start(); + healthy.start(); + + String cfg = "ws::addr=127.0.0.1:" + auth.port() + ",127.0.0.1:" + healthy.port() + + ";lb_strategy=first;auth_timeout_ms=2000;"; + try { + Sender.fromConfig(cfg).close(); + Assert.fail("expected auth-fail to terminate connect across endpoints"); + } catch (HttpClientException e) { + Assert.assertTrue("expected message to mention 401, got: " + e.getMessage(), + e.getMessage().contains("401")); + } + Assert.assertEquals("healthy peer must NOT be probed when 401 short-circuits", + 0, healthy.upgradeCount.get()); + } + } + + @Test(timeout = 10_000) + public void testAllReplica_AuthFailNotEmitted() throws Exception { + try (FakeUpgradeServer r1 = new FakeUpgradeServer(Mode.REPLICA_REJECT); + FakeUpgradeServer r2 = new FakeUpgradeServer(Mode.REPLICA_REJECT)) { + r1.start(); + r2.start(); + + String cfg = "ws::addr=127.0.0.1:" + r1.port() + ",127.0.0.1:" + r2.port() + + ";lb_strategy=first;auth_timeout_ms=2000;"; + try { + Sender.fromConfig(cfg).close(); + Assert.fail("expected connect to fail when all endpoints reject by role"); + } catch (LineSenderException e) { + Assert.assertTrue("expected role-reject summary, got: " + e.getMessage(), + e.getMessage().contains("rejected the upgrade by role")); + } + } + } + + private enum Mode { + UPGRADE_OK, + REPLICA_REJECT, + AUTH_401, + } + + private static final class FakeUpgradeServer implements AutoCloseable { + final AtomicInteger upgradeCount = new AtomicInteger(); + private final Mode mode; + private final ServerSocket socket; + private volatile boolean running = true; + + FakeUpgradeServer(Mode mode) throws IOException { + this.mode = mode; + this.socket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); + } + + int port() { + return socket.getLocalPort(); + } + + void start() { + Thread t = new Thread(this::loop, "fake-upgrade-" + mode); + t.setDaemon(true); + t.start(); + } + + @Override + public void close() throws IOException { + running = false; + socket.close(); + } + + private void loop() { + while (running) { + try { + Socket s = socket.accept(); + Thread h = new Thread(() -> handle(s), "fake-upgrade-handler-" + mode); + h.setDaemon(true); + h.start(); + } catch (IOException e) { + if (!running) return; + } + } + } + + private void handle(Socket s) { + try (Socket sock = s) { + BufferedReader in = new BufferedReader(new InputStreamReader( + sock.getInputStream(), StandardCharsets.US_ASCII)); + String secKey = null; + String line; + while ((line = in.readLine()) != null && !line.isEmpty()) { + if (line.regionMatches(true, 0, "Sec-WebSocket-Key:", 0, 18)) { + secKey = line.substring(18).trim(); + } + } + upgradeCount.incrementAndGet(); + OutputStream out = sock.getOutputStream(); + switch (mode) { + case UPGRADE_OK: + out.write(("HTTP/1.1 101 Switching Protocols\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + "Sec-WebSocket-Accept: " + computeAcceptKey(secKey) + "\r\n\r\n" + ).getBytes(StandardCharsets.US_ASCII)); + out.flush(); + // Hold the socket open so the sender can complete its post-upgrade + // setup; the test closes the sender which triggers a close handshake. + Thread.sleep(500); + break; + case REPLICA_REJECT: + out.write(("HTTP/1.1 421 Misdirected Request\r\n" + + "X-QuestDB-Role: REPLICA\r\n" + + "Content-Length: 0\r\nConnection: close\r\n\r\n" + ).getBytes(StandardCharsets.US_ASCII)); + out.flush(); + break; + case AUTH_401: + out.write(("HTTP/1.1 401 Unauthorized\r\n" + + "Content-Length: 0\r\nConnection: close\r\n\r\n" + ).getBytes(StandardCharsets.US_ASCII)); + out.flush(); + break; + } + } catch (Exception ignored) { + } + } + + private static String computeAcceptKey(String secKey) { + try { + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + byte[] digest = sha1.digest((secKey + WEBSOCKET_GUID).getBytes(StandardCharsets.US_ASCII)); + return Base64.getEncoder().encodeToString(digest); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } +} From ec238946ee40a43aa18692e00c9aca6817d2a816 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 7 May 2026 15:10:48 +0800 Subject: [PATCH 8/8] final code review --- .../main/java/io/questdb/client/Sender.java | 5 +- .../cutlass/qwp/client/QwpEgressIoThread.java | 1 + .../qwp/client/QwpHostHealthTracker.java | 6 +- .../cutlass/qwp/client/QwpQueryClient.java | 90 ++++++++++--------- .../qwp/client/QwpWebSocketSender.java | 20 ++--- .../LineSenderBuilderWebSocketTest.java | 12 +++ .../qwp/client/QwpHostHealthTrackerTest.java | 11 +++ .../QwpQueryClientMultiHostFailoverTest.java | 29 ++++++ .../QwpResultBatchDecoderHardeningTest.java | 28 ++++++ 9 files changed, 147 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 927e8298..36514e5e 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -838,10 +838,13 @@ private void addAddressEntry(CharSequence src, int start, int end, int defaultPo .put(src.subSequence(start, end)).put("]"); } int hostEnd = colon < 0 ? end : colon; + int portStart = colon < 0 ? end : colon + 1; + while (hostEnd > start && Character.isWhitespace(src.charAt(hostEnd - 1))) hostEnd--; + while (portStart < end && Character.isWhitespace(src.charAt(portStart))) portStart++; int parsedPort = -1; if (colon >= 0) { try { - parsedPort = Numbers.parseInt(src, colon + 1, end); + parsedPort = Numbers.parseInt(src, portStart, end); if (parsedPort < 1 || parsedPort > 65535) { throw new LineSenderException("invalid port [port=").put(parsedPort).put("]"); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java index dd4c5ef1..e0387816 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java @@ -533,6 +533,7 @@ private void emitError(byte status, String message) { private void emitTerminalProtocolError(String message) { notifyTerminalFailure(message, true); events.offer(new QueryEvent().asProtocolError(WebSocketResponse.STATUS_INTERNAL_ERROR, message)); + shutdown = true; } private void emitTerminalTransportError(String message) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java index 2fd899b9..45a134af 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java @@ -143,14 +143,16 @@ public int pickNext() { } /** - * Demotes a previously-healthy host on send/receive failure so a subsequent - * sticky-Healthy reset doesn't preserve it as the priority entry. + * Demotes a previously-healthy host on send/receive failure and marks it + * attempted in the current round so a subsequent {@link #pickNext()} + * inside the same round does not pick it again. */ public void recordMidStreamFailure(int idx) { synchronized (lock) { if (states[idx] == HostState.HEALTHY) { states[idx] = HostState.TRANSPORT_ERROR; } + attemptedThisRound[idx] = true; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index 40a9f064..d4377ddc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -218,7 +218,6 @@ public class QwpQueryClient implements QuietCloseable { // payload before it parks, and the client auto-replenishes by the size of // each batch as the user releases it. private long initialCreditBytes; - private String lbStrategy = LB_RANDOM; // Volatile so a cancel() call from a thread other than the one that ran // connect() sees the published reference (and a concurrent null-out from // close() is observed without a stale-reference race). The thread-safety @@ -233,6 +232,7 @@ public class QwpQueryClient implements QuietCloseable { private volatile QwpEgressIoThread ioThread; private volatile Thread ioThreadHandle; private boolean lastCloseTimedOut; + private String lbStrategy = LB_RANDOM; // Client preference for server-side per-batch row cap. 0 means "unset", // server uses its default. Set via {@code max_batch_rows=N} in the // connection string or {@link #withMaxBatchRows}. Smaller values give @@ -757,6 +757,7 @@ public synchronized void connect() { if (connected) { return; } + lastCloseTimedOut = false; if (hostTracker == null) { if (LB_RANDOM.equals(lbStrategy) && endpoints.size() > 1) { List shuffled = new ArrayList<>(endpoints); @@ -771,15 +772,9 @@ public synchronized void connect() { QwpServerInfo lastObservedMismatch = null; boolean sawV1Mismatch = false; Throwable lastTransportError = null; - boolean retriedAfterReset = false; while (true) { int i = hostTracker.pickNext(); if (i < 0) { - if (!retriedAfterReset) { - hostTracker.beginRound(true); - retriedAfterReset = true; - continue; - } break; } Endpoint ep = endpoints.get(i); @@ -820,6 +815,7 @@ public synchronized void connect() { cleanupFailedConnect(); continue; } + spawnIoThread(); hostTracker.recordSuccess(i); currentEndpointIndex = i; connected = true; @@ -1120,9 +1116,11 @@ public void withFailover(boolean enabled) { * Configures the exponential backoff applied between failover reconnect * attempts. {@code initialMs} is the delay before the first retry (the * second overall execute attempt); each subsequent retry doubles the - * delay up to {@code maxMs}. A zero {@code initialMs} disables backoff - * entirely -- retries fire back to back, which is fine for fast LAN - * clusters but risks hammering a struggling one during a real outage. + * delay up to {@code maxMs}. Setting either {@code initialMs} or + * {@code maxMs} to 0 disables backoff entirely -- retries fire back to + * back, bounded only by {@link #withFailoverMaxAttempts} and + * {@link #withFailoverMaxDuration}. Fine for fast LAN clusters, but + * risks hammering a struggling one during a real outage. * Defaults: initial {@value #DEFAULT_FAILOVER_INITIAL_BACKOFF_MS} ms, * max {@value #DEFAULT_FAILOVER_MAX_BACKOFF_MS} ms. */ @@ -1377,7 +1375,7 @@ private static List parseEndpointList(String value) { host = entry; port = DEFAULT_WS_PORT; } else { - host = entry.substring(0, colon); + host = entry.substring(0, colon).trim(); port = parsePort(entry.substring(colon + 1), entry); } } @@ -1551,7 +1549,9 @@ private void connectToEndpoint(Endpoint ep) { if (!"raw".equals(compressionPreference)) { probeZstdAvailable(); } + } + private void spawnIoThread() { // Wire a fresh generation-scoped listener into this I/O thread. Each // listener owns its own terminal-failure latch, so even if a dying // I/O thread slips a late onTerminalFailure callback past the orphan @@ -1675,32 +1675,37 @@ private void executeOnce(String sql, QwpBindSetter binds, FailoverProbeHandler p * the caller doesn't inherit a half-open socket. */ private void probeZstdAvailable() { - long dctx; + long dctx = 0; try { - dctx = Zstd.createDCtx(); - } catch (UnsatisfiedLinkError e) { - LOG.error("zstd JNI symbols missing from libquestdb; aborting connect", e); - if (webSocketClient != null) { - webSocketClient.close(); - webSocketClient = null; + try { + dctx = Zstd.createDCtx(); + } catch (UnsatisfiedLinkError e) { + LOG.error("zstd JNI symbols missing from libquestdb; aborting connect", e); + if (webSocketClient != null) { + webSocketClient.close(); + webSocketClient = null; + } + throw new HttpClientException("this client build does not support zstd compression -- " + + "libquestdb was built without the zstd submodule. Rebuild the native library " + + "with 'git submodule update --init --recursive' and 'cmake --build', or set " + + "compression=raw on the connection string to skip the probe. " + + "[cause=" + e.getMessage() + "]"); } - throw new HttpClientException("this client build does not support zstd compression -- " - + "libquestdb was built without the zstd submodule. Rebuild the native library " - + "with 'git submodule update --init --recursive' and 'cmake --build', or set " - + "compression=raw on the connection string to skip the probe. " - + "[cause=" + e.getMessage() + "]"); - } - if (dctx == 0) { - LOG.error("zstd createDCtx returned 0 (native allocation failure); aborting connect"); - if (webSocketClient != null) { - webSocketClient.close(); - webSocketClient = null; + if (dctx == 0) { + LOG.error("zstd createDCtx returned 0 (native allocation failure); aborting connect"); + if (webSocketClient != null) { + webSocketClient.close(); + webSocketClient = null; + } + throw new HttpClientException("zstd decompression context allocation failed; " + + "cannot accept compressed batches. Set compression=raw on the connection " + + "string to disable compression, or retry once memory pressure subsides."); + } + } finally { + if (dctx != 0) { + Zstd.freeDCtx(dctx); } - throw new HttpClientException("zstd decompression context allocation failed; " - + "cannot accept compressed batches. Set compression=raw on the connection " - + "string to disable compression, or retry once memory pressure subsides."); } - Zstd.freeDCtx(dctx); } private QwpServerInfo receiveServerInfoSync() { @@ -1730,15 +1735,14 @@ private QwpServerInfo receiveServerInfoSync() { } /** - * Walks the endpoint list starting at the entry right after the one that - * just failed, wrapping around so every other endpoint gets a - * try. The failed endpoint itself is deliberately not - * retried: a transport failure is likely to repeat immediately on the same - * socket ({@code PeerDisconnectedException} from a server that's still - * accepting new connections but torn the old one down, for example), and - * a retry would just burn an attempt. The outer {@link #execute} loop - * can revisit the failed endpoint on a subsequent failover attempt if - * every other endpoint is also unreachable. + * Walks the endpoint list by tracker priority (HEALTHY → UNKNOWN → + * TRANSIENT_REJECT → TRANSPORT_ERROR → TOPOLOGY_REJECT). The mid-stream + * failed endpoint was demoted by {@link QwpHostHealthTracker#recordMidStreamFailure} + * before this method is entered, so in multi-host configurations a different + * endpoint is preferred; with a single configured endpoint the same host is + * the only option. After the first round exhausts, classifications other + * than HEALTHY are forgotten and the list is walked once more so a long-lived + * client recovers from topology changes. *

* On success, leaves the client in the same state {@link #connect()} * produces: {@code connected=true}, {@code ioThread} spawned, @@ -1747,6 +1751,7 @@ private QwpServerInfo receiveServerInfoSync() { */ private void reconnectViaTracker() { int total = endpoints.size(); + lastCloseTimedOut = false; hostTracker.beginRound(false); QwpServerInfo lastMismatch = null; boolean sawV1Mismatch = false; @@ -1793,6 +1798,7 @@ private void reconnectViaTracker() { cleanupFailedConnect(); continue; } + spawnIoThread(); hostTracker.recordSuccess(i); currentEndpointIndex = i; connected = true; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8528cf87..d81ae041 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -143,6 +143,7 @@ public class QwpWebSocketSender implements Sender { // null means plain text (no TLS) private final ClientTlsConfiguration tlsConfig; private MicrobatchBuffer activeBuffer; + private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; // Double-buffering for async I/O private MicrobatchBuffer buffer0; // Cached column references to avoid repeated hashmap lookups @@ -171,6 +172,12 @@ public class QwpWebSocketSender implements Sender { // alongside the cursor send loop in close(). private io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerPool drainerPool; + // Keepalive PING cadence used by the I/O loop while + // request_durable_ack=on AND there are pending durable-ack + // confirmations. Default mirrors the loop's spec value; 0 or negative + // disables keepalive PINGs entirely. + private long durableAckKeepaliveIntervalMillis = + CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS; private SenderErrorDispatcher errorDispatcher; // Async-delivery sink for SenderError notifications. Default-constructed // here with the loud-not-silent default handler; a builder hook can swap @@ -209,14 +216,7 @@ public class QwpWebSocketSender implements Sender { // values; Sender.build can override via the new connect overload. private long reconnectMaxDurationMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_DURATION_MILLIS; - private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; private boolean requestDurableAck; - // Keepalive PING cadence used by the I/O loop while - // request_durable_ack=on AND there are pending durable-ack - // confirmations. Default mirrors the loop's spec value; 0 or negative - // disables keepalive PINGs entirely. - private long durableAckKeepaliveIntervalMillis = - CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS; private QwpWebSocketSender( List endpoints, @@ -1888,8 +1888,8 @@ private void atNanos(long timestampNanos) { * Returns a {@link CursorWebSocketSendLoop.ReconnectFactory} that, on each * call, performs the multi-endpoint walk and returns a freshly connected * {@link WebSocketClient}. Each factory holds private "previously-bound - * endpoint" state, so foreground and drainer reconnects do not corrupt - * each other's host-tracker priorities. + * endpoint" state for mid-stream-failure attribution; the host tracker + * itself is shared across factories. */ public CursorWebSocketSendLoop.ReconnectFactory newReconnectFactory() { return new ReconnectSupplier(); @@ -1957,7 +1957,7 @@ private synchronized WebSocketClient buildAndConnect(ReconnectSupplier ctx) { } if (requestDurableAck && !newClient.isServerDurableAckEnabled()) { newClient.close(); - hostTracker.recordTransportError(idx); + hostTracker.recordRoleReject(idx, false); LineSenderException ackErr = new LineSenderException( "WebSocket upgrade failed: server does not support durable ack [host=") .put(ep.host).put(", port=").put(ep.port) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index bab5834a..f392700a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -433,6 +433,18 @@ public void testWsConfigString_dupAddr_bothDefaultPort_fails() { assertBadConfig("ws::addr=a,a;", "duplicated addresses are not allowed"); } + @Test + public void testWsConfigString_addrWithTrailingHostWhitespace_trimmed() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=localhost :9000;"); + Assert.assertNotNull(builder); + } + + @Test + public void testWsConfigString_addrWithLeadingPortWhitespace_trimmed() { + Sender.LineSenderBuilder builder = Sender.builder("ws::addr=localhost: 9000;"); + Assert.assertNotNull(builder); + } + @Test public void testFullAsyncConfiguration() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java index a98fe068..2903eb15 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpHostHealthTrackerTest.java @@ -98,6 +98,17 @@ public void testMidStreamFailure_HealthyDemoted() { Assert.assertEquals(QwpHostHealthTracker.HostState.TRANSPORT_ERROR, t.getState(0)); } + @Test + public void testMidStreamFailure_MarksAttempted() { + QwpHostHealthTracker t = new QwpHostHealthTracker(2); + t.recordSuccess(0); + t.beginRound(false); + t.recordMidStreamFailure(0); + Assert.assertEquals(1, t.pickNext()); + t.recordTransportError(1); + Assert.assertEquals(-1, t.pickNext()); + } + @Test public void testMidStreamFailure_NonHealthyUnchanged() { QwpHostHealthTracker t = new QwpHostHealthTracker(2); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientMultiHostFailoverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientMultiHostFailoverTest.java index fc0e46c4..84ae0526 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientMultiHostFailoverTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientMultiHostFailoverTest.java @@ -87,6 +87,35 @@ public void testAllReplica_FailsWithRoleSummary() throws Exception { } } + @Test(timeout = 10_000) + public void testConnectDoesNotDoubleWalkOnFirstFailure() throws Exception { + try (FakeStatusServer r1 = new FakeStatusServer(421, "X-QuestDB-Role: REPLICA"); + FakeStatusServer r2 = new FakeStatusServer(421, "X-QuestDB-Role: REPLICA"); + FakeStatusServer r3 = new FakeStatusServer(421, "X-QuestDB-Role: REPLICA")) { + r1.start(); + r2.start(); + r3.start(); + + String cfg = "ws::addr=127.0.0.1:" + r1.port() + + ",127.0.0.1:" + r2.port() + + ",127.0.0.1:" + r3.port() + + ";lb_strategy=first;auth_timeout_ms=2000;failover=off;target=any;"; + try (QwpQueryClient client = QwpQueryClient.fromConfig(cfg)) { + try { + client.connect(); + Assert.fail("expected connect to throw when all endpoints role-reject"); + } catch (HttpClientException ignored) { + } + } + Assert.assertEquals("first endpoint must only be probed once on initial connect", + 1, r1.connections.get()); + Assert.assertEquals("second endpoint must only be probed once on initial connect", + 1, r2.connections.get()); + Assert.assertEquals("third endpoint must only be probed once on initial connect", + 1, r3.connections.get()); + } + } + private static final class FakeStatusServer implements AutoCloseable { final AtomicInteger connections = new AtomicInteger(); private final String roleHeader; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java index 160a3d17..aa3a193a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java @@ -29,6 +29,7 @@ import io.questdb.client.cutlass.qwp.client.QwpDecodeException; import io.questdb.client.cutlass.qwp.client.QwpEgressIoThread; import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpProtocolVersionException; import io.questdb.client.cutlass.qwp.client.QwpResultBatchDecoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.MemoryTag; @@ -109,6 +110,33 @@ public void testArrayValidDimensionsAreAccepted() throws Exception { }); } + @Test + public void testUnsupportedVersionThrowsProtocolVersionException() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(128); + long staging = Unsafe.malloc(128, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeMinimalResultBatch(staging, 0L); + Unsafe.getUnsafe().putByte(staging + 4, (byte) 99); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must throw on unsupported version"); + } catch (QwpProtocolVersionException expected) { + Assert.assertTrue("error must reference unsupported version: " + expected.getMessage(), + expected.getMessage().contains("unsupported version")); + Assert.assertTrue("must extend QwpDecodeException", + expected instanceof QwpDecodeException); + } + } finally { + Unsafe.free(staging, 128, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + /** * Regression: a delta SYMBOL dict entry whose length exceeds * {@link Integer#MAX_VALUE} must be rejected. Prior to the fix, the